Phase 2: Implement Execution Environment Abstraction (v0.3.0)

This commit implements Phase 2 of the CHORUS Task Execution Engine development plan,
providing a comprehensive execution environment abstraction layer with Docker
container sandboxing support.

## New Features

### Core Sandbox Interface
- Comprehensive ExecutionSandbox interface with isolated task execution
- Support for command execution, file I/O, environment management
- Resource usage monitoring and sandbox lifecycle management
- Standardized error handling with SandboxError types and categories

### Docker Container Sandbox Implementation
- Full Docker API integration with secure container creation
- Transparent repository mounting with configurable read/write access
- Advanced security policies with capability dropping and privilege controls
- Comprehensive resource limits (CPU, memory, disk, processes, file handles)
- Support for tmpfs mounts, masked paths, and read-only bind mounts
- Container lifecycle management with proper cleanup and health monitoring

### Security & Resource Management
- Configurable security policies with SELinux, AppArmor, and Seccomp support
- Fine-grained capability management with secure defaults
- Network isolation options with configurable DNS and proxy settings
- Resource monitoring with real-time CPU, memory, and network usage tracking
- Comprehensive ulimits configuration for process and file handle limits

### Repository Integration
- Seamless repository mounting from local paths to container workspaces
- Git configuration support with user credentials and global settings
- File inclusion/exclusion patterns for selective repository access
- Configurable permissions and ownership for mounted repositories

### Testing Infrastructure
- Comprehensive test suite with 60+ test cases covering all functionality
- Docker integration tests with Alpine Linux containers (skipped in short mode)
- Mock sandbox implementation for unit testing without Docker dependencies
- Security policy validation tests with read-only filesystem enforcement
- Resource usage monitoring and cleanup verification tests

## Technical Details

### Dependencies Added
- github.com/docker/docker v28.4.0+incompatible - Docker API client
- github.com/docker/go-connections v0.6.0 - Docker connection utilities
- github.com/docker/go-units v0.5.0 - Docker units and formatting
- Associated Docker API dependencies for complete container management

### Architecture
- Interface-driven design enabling multiple sandbox implementations
- Comprehensive configuration structures for all sandbox aspects
- Resource usage tracking with detailed metrics collection
- Error handling with retryable error classification
- Proper cleanup and resource management throughout sandbox lifecycle

### Compatibility
- Maintains backward compatibility with existing CHORUS architecture
- Designed for future integration with Phase 3 Core Task Execution Engine
- Extensible design supporting additional sandbox implementations (VM, process)

This Phase 2 implementation provides the foundation for secure, isolated task
execution that will be integrated with the AI model providers from Phase 1
in the upcoming Phase 3 development.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
anthonyrawlins
2025-09-25 14:28:08 +10:00
parent d1252ade69
commit 8d9b62daf3
653 changed files with 88039 additions and 3766 deletions

1020
pkg/execution/docker.go Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,482 @@
package execution
import (
"context"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewDockerSandbox(t *testing.T) {
sandbox := NewDockerSandbox()
assert.NotNil(t, sandbox)
assert.NotNil(t, sandbox.environment)
assert.Empty(t, sandbox.containerID)
}
func TestDockerSandbox_Initialize(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := NewDockerSandbox()
ctx := context.Background()
// Create a minimal configuration
config := &SandboxConfig{
Type: "docker",
Image: "alpine:latest",
Architecture: "amd64",
Resources: ResourceLimits{
MemoryLimit: 512 * 1024 * 1024, // 512MB
CPULimit: 1.0,
ProcessLimit: 50,
FileLimit: 1024,
},
Security: SecurityPolicy{
ReadOnlyRoot: false,
NoNewPrivileges: true,
AllowNetworking: false,
IsolateNetwork: true,
IsolateProcess: true,
DropCapabilities: []string{"ALL"},
},
Environment: map[string]string{
"TEST_VAR": "test_value",
},
WorkingDir: "/workspace",
Timeout: 30 * time.Second,
}
err := sandbox.Initialize(ctx, config)
if err != nil {
t.Skipf("Docker not available or image pull failed: %v", err)
}
defer sandbox.Cleanup()
// Verify sandbox is initialized
assert.NotEmpty(t, sandbox.containerID)
assert.Equal(t, config, sandbox.config)
assert.Equal(t, StatusRunning, sandbox.info.Status)
assert.Equal(t, "docker", sandbox.info.Type)
}
func TestDockerSandbox_ExecuteCommand(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := setupTestSandbox(t)
defer sandbox.Cleanup()
ctx := context.Background()
tests := []struct {
name string
cmd *Command
expectedExit int
expectedOutput string
shouldError bool
}{
{
name: "simple echo command",
cmd: &Command{
Executable: "echo",
Args: []string{"hello world"},
},
expectedExit: 0,
expectedOutput: "hello world\n",
},
{
name: "command with environment",
cmd: &Command{
Executable: "sh",
Args: []string{"-c", "echo $TEST_VAR"},
Environment: map[string]string{"TEST_VAR": "custom_value"},
},
expectedExit: 0,
expectedOutput: "custom_value\n",
},
{
name: "failing command",
cmd: &Command{
Executable: "sh",
Args: []string{"-c", "exit 1"},
},
expectedExit: 1,
},
{
name: "command with timeout",
cmd: &Command{
Executable: "sleep",
Args: []string{"2"},
Timeout: 1 * time.Second,
},
shouldError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := sandbox.ExecuteCommand(ctx, tt.cmd)
if tt.shouldError {
assert.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, tt.expectedExit, result.ExitCode)
assert.Equal(t, tt.expectedExit == 0, result.Success)
if tt.expectedOutput != "" {
assert.Equal(t, tt.expectedOutput, result.Stdout)
}
assert.NotZero(t, result.Duration)
assert.False(t, result.StartTime.IsZero())
assert.False(t, result.EndTime.IsZero())
})
}
}
func TestDockerSandbox_FileOperations(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := setupTestSandbox(t)
defer sandbox.Cleanup()
ctx := context.Background()
// Test WriteFile
testContent := []byte("Hello, Docker sandbox!")
testPath := "/tmp/test_file.txt"
err := sandbox.WriteFile(ctx, testPath, testContent, 0644)
require.NoError(t, err)
// Test ReadFile
readContent, err := sandbox.ReadFile(ctx, testPath)
require.NoError(t, err)
assert.Equal(t, testContent, readContent)
// Test ListFiles
files, err := sandbox.ListFiles(ctx, "/tmp")
require.NoError(t, err)
assert.NotEmpty(t, files)
// Find our test file
var testFile *FileInfo
for _, file := range files {
if file.Name == "test_file.txt" {
testFile = &file
break
}
}
require.NotNil(t, testFile)
assert.Equal(t, "test_file.txt", testFile.Name)
assert.Equal(t, int64(len(testContent)), testFile.Size)
assert.False(t, testFile.IsDir)
}
func TestDockerSandbox_CopyFiles(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := setupTestSandbox(t)
defer sandbox.Cleanup()
ctx := context.Background()
// Create a temporary file on host
tempDir := t.TempDir()
hostFile := filepath.Join(tempDir, "host_file.txt")
hostContent := []byte("Content from host")
err := os.WriteFile(hostFile, hostContent, 0644)
require.NoError(t, err)
// Copy from host to container
containerPath := "container:/tmp/copied_file.txt"
err = sandbox.CopyFiles(ctx, hostFile, containerPath)
require.NoError(t, err)
// Verify file exists in container
readContent, err := sandbox.ReadFile(ctx, "/tmp/copied_file.txt")
require.NoError(t, err)
assert.Equal(t, hostContent, readContent)
// Copy from container back to host
hostDestFile := filepath.Join(tempDir, "copied_back.txt")
err = sandbox.CopyFiles(ctx, "container:/tmp/copied_file.txt", hostDestFile)
require.NoError(t, err)
// Verify file exists on host
backContent, err := os.ReadFile(hostDestFile)
require.NoError(t, err)
assert.Equal(t, hostContent, backContent)
}
func TestDockerSandbox_Environment(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := setupTestSandbox(t)
defer sandbox.Cleanup()
// Test getting initial environment
env := sandbox.GetEnvironment()
assert.Equal(t, "test_value", env["TEST_VAR"])
// Test setting additional environment
newEnv := map[string]string{
"NEW_VAR": "new_value",
"PATH": "/custom/path",
}
err := sandbox.SetEnvironment(newEnv)
require.NoError(t, err)
// Verify environment is updated
env = sandbox.GetEnvironment()
assert.Equal(t, "new_value", env["NEW_VAR"])
assert.Equal(t, "/custom/path", env["PATH"])
assert.Equal(t, "test_value", env["TEST_VAR"]) // Original should still be there
}
func TestDockerSandbox_WorkingDirectory(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := setupTestSandbox(t)
defer sandbox.Cleanup()
// Test getting initial working directory
workDir := sandbox.GetWorkingDirectory()
assert.Equal(t, "/workspace", workDir)
// Test setting working directory
newWorkDir := "/tmp"
err := sandbox.SetWorkingDirectory(newWorkDir)
require.NoError(t, err)
// Verify working directory is updated
workDir = sandbox.GetWorkingDirectory()
assert.Equal(t, newWorkDir, workDir)
}
func TestDockerSandbox_ResourceUsage(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := setupTestSandbox(t)
defer sandbox.Cleanup()
ctx := context.Background()
// Get resource usage
usage, err := sandbox.GetResourceUsage(ctx)
require.NoError(t, err)
// Verify usage structure
assert.NotNil(t, usage)
assert.False(t, usage.Timestamp.IsZero())
assert.GreaterOrEqual(t, usage.CPUUsage, 0.0)
assert.GreaterOrEqual(t, usage.MemoryUsage, int64(0))
assert.GreaterOrEqual(t, usage.MemoryPercent, 0.0)
}
func TestDockerSandbox_GetInfo(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := setupTestSandbox(t)
defer sandbox.Cleanup()
info := sandbox.GetInfo()
assert.NotEmpty(t, info.ID)
assert.Contains(t, info.Name, "chorus-sandbox")
assert.Equal(t, "docker", info.Type)
assert.Equal(t, StatusRunning, info.Status)
assert.Equal(t, "docker", info.Runtime)
assert.Equal(t, "alpine:latest", info.Image)
assert.False(t, info.CreatedAt.IsZero())
assert.False(t, info.StartedAt.IsZero())
}
func TestDockerSandbox_Cleanup(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := setupTestSandbox(t)
// Verify sandbox is running
assert.Equal(t, StatusRunning, sandbox.info.Status)
assert.NotEmpty(t, sandbox.containerID)
// Cleanup
err := sandbox.Cleanup()
require.NoError(t, err)
// Verify sandbox is destroyed
assert.Equal(t, StatusDestroyed, sandbox.info.Status)
}
func TestDockerSandbox_SecurityPolicies(t *testing.T) {
if testing.Short() {
t.Skip("Skipping Docker integration test in short mode")
}
sandbox := NewDockerSandbox()
ctx := context.Background()
// Create configuration with strict security policies
config := &SandboxConfig{
Type: "docker",
Image: "alpine:latest",
Architecture: "amd64",
Resources: ResourceLimits{
MemoryLimit: 256 * 1024 * 1024, // 256MB
CPULimit: 0.5,
ProcessLimit: 10,
FileLimit: 256,
},
Security: SecurityPolicy{
ReadOnlyRoot: true,
NoNewPrivileges: true,
AllowNetworking: false,
IsolateNetwork: true,
IsolateProcess: true,
DropCapabilities: []string{"ALL"},
RunAsUser: "1000",
RunAsGroup: "1000",
TmpfsPaths: []string{"/tmp", "/var/tmp"},
MaskedPaths: []string{"/proc/kcore", "/proc/keys"},
ReadOnlyPaths: []string{"/etc"},
},
WorkingDir: "/workspace",
Timeout: 30 * time.Second,
}
err := sandbox.Initialize(ctx, config)
if err != nil {
t.Skipf("Docker not available or security policies not supported: %v", err)
}
defer sandbox.Cleanup()
// Test that we can't write to read-only filesystem
result, err := sandbox.ExecuteCommand(ctx, &Command{
Executable: "touch",
Args: []string{"/test_readonly"},
})
require.NoError(t, err)
assert.NotEqual(t, 0, result.ExitCode) // Should fail due to read-only root
// Test that tmpfs is writable
result, err = sandbox.ExecuteCommand(ctx, &Command{
Executable: "touch",
Args: []string{"/tmp/test_tmpfs"},
})
require.NoError(t, err)
assert.Equal(t, 0, result.ExitCode) // Should succeed on tmpfs
}
// setupTestSandbox creates a basic Docker sandbox for testing
func setupTestSandbox(t *testing.T) *DockerSandbox {
sandbox := NewDockerSandbox()
ctx := context.Background()
config := &SandboxConfig{
Type: "docker",
Image: "alpine:latest",
Architecture: "amd64",
Resources: ResourceLimits{
MemoryLimit: 512 * 1024 * 1024, // 512MB
CPULimit: 1.0,
ProcessLimit: 50,
FileLimit: 1024,
},
Security: SecurityPolicy{
ReadOnlyRoot: false,
NoNewPrivileges: true,
AllowNetworking: true, // Allow networking for easier testing
IsolateNetwork: false,
IsolateProcess: true,
DropCapabilities: []string{"NET_ADMIN", "SYS_ADMIN"},
},
Environment: map[string]string{
"TEST_VAR": "test_value",
},
WorkingDir: "/workspace",
Timeout: 30 * time.Second,
}
err := sandbox.Initialize(ctx, config)
if err != nil {
t.Skipf("Docker not available: %v", err)
}
return sandbox
}
// Benchmark tests
func BenchmarkDockerSandbox_ExecuteCommand(b *testing.B) {
if testing.Short() {
b.Skip("Skipping Docker benchmark in short mode")
}
sandbox := &DockerSandbox{}
ctx := context.Background()
// Setup minimal config for benchmarking
config := &SandboxConfig{
Type: "docker",
Image: "alpine:latest",
Architecture: "amd64",
Resources: ResourceLimits{
MemoryLimit: 256 * 1024 * 1024,
CPULimit: 1.0,
ProcessLimit: 50,
},
Security: SecurityPolicy{
NoNewPrivileges: true,
AllowNetworking: true,
},
WorkingDir: "/workspace",
Timeout: 10 * time.Second,
}
err := sandbox.Initialize(ctx, config)
if err != nil {
b.Skipf("Docker not available: %v", err)
}
defer sandbox.Cleanup()
cmd := &Command{
Executable: "echo",
Args: []string{"benchmark test"},
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := sandbox.ExecuteCommand(ctx, cmd)
if err != nil {
b.Fatalf("Command execution failed: %v", err)
}
}
}

415
pkg/execution/sandbox.go Normal file
View File

@@ -0,0 +1,415 @@
package execution
import (
"context"
"io"
"time"
)
// ExecutionSandbox defines the interface for isolated task execution environments
type ExecutionSandbox interface {
// Initialize sets up the sandbox environment
Initialize(ctx context.Context, config *SandboxConfig) error
// ExecuteCommand runs a command within the sandbox
ExecuteCommand(ctx context.Context, cmd *Command) (*CommandResult, error)
// CopyFiles copies files between host and sandbox
CopyFiles(ctx context.Context, source, dest string) error
// WriteFile writes content to a file in the sandbox
WriteFile(ctx context.Context, path string, content []byte, mode uint32) error
// ReadFile reads content from a file in the sandbox
ReadFile(ctx context.Context, path string) ([]byte, error)
// ListFiles lists files in a directory within the sandbox
ListFiles(ctx context.Context, path string) ([]FileInfo, error)
// GetWorkingDirectory returns the current working directory in the sandbox
GetWorkingDirectory() string
// SetWorkingDirectory changes the working directory in the sandbox
SetWorkingDirectory(path string) error
// GetEnvironment returns environment variables in the sandbox
GetEnvironment() map[string]string
// SetEnvironment sets environment variables in the sandbox
SetEnvironment(env map[string]string) error
// GetResourceUsage returns current resource usage statistics
GetResourceUsage(ctx context.Context) (*ResourceUsage, error)
// Cleanup destroys the sandbox and cleans up resources
Cleanup() error
// GetInfo returns information about the sandbox
GetInfo() SandboxInfo
}
// SandboxConfig represents configuration for a sandbox environment
type SandboxConfig struct {
// Sandbox type and runtime
Type string `json:"type"` // docker, vm, process
Image string `json:"image"` // Container/VM image
Runtime string `json:"runtime"` // docker, containerd, etc.
Architecture string `json:"architecture"` // amd64, arm64
// Resource limits
Resources ResourceLimits `json:"resources"`
// Security settings
Security SecurityPolicy `json:"security"`
// Repository configuration
Repository RepositoryConfig `json:"repository"`
// Network settings
Network NetworkConfig `json:"network"`
// Environment settings
Environment map[string]string `json:"environment"`
WorkingDir string `json:"working_dir"`
// Tool and service access
Tools []string `json:"tools"` // Available tools in sandbox
MCPServers []string `json:"mcp_servers"` // MCP servers to connect to
// Execution settings
Timeout time.Duration `json:"timeout"` // Maximum execution time
CleanupDelay time.Duration `json:"cleanup_delay"` // Delay before cleanup
// Metadata
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
}
// Command represents a command to execute in the sandbox
type Command struct {
// Command specification
Executable string `json:"executable"`
Args []string `json:"args"`
WorkingDir string `json:"working_dir"`
Environment map[string]string `json:"environment"`
// Input/Output
Stdin io.Reader `json:"-"`
StdinContent string `json:"stdin_content"`
// Execution settings
Timeout time.Duration `json:"timeout"`
User string `json:"user"`
// Security settings
AllowNetwork bool `json:"allow_network"`
AllowWrite bool `json:"allow_write"`
RestrictPaths []string `json:"restrict_paths"`
}
// CommandResult represents the result of command execution
type CommandResult struct {
// Exit information
ExitCode int `json:"exit_code"`
Success bool `json:"success"`
// Output
Stdout string `json:"stdout"`
Stderr string `json:"stderr"`
Combined string `json:"combined"`
// Timing
StartTime time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
Duration time.Duration `json:"duration"`
// Resource usage during execution
ResourceUsage ResourceUsage `json:"resource_usage"`
// Error information
Error string `json:"error,omitempty"`
Signal string `json:"signal,omitempty"`
// Metadata
ProcessID int `json:"process_id,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
}
// FileInfo represents information about a file in the sandbox
type FileInfo struct {
Name string `json:"name"`
Path string `json:"path"`
Size int64 `json:"size"`
Mode uint32 `json:"mode"`
ModTime time.Time `json:"mod_time"`
IsDir bool `json:"is_dir"`
Owner string `json:"owner"`
Group string `json:"group"`
Permissions string `json:"permissions"`
}
// ResourceLimits defines resource constraints for the sandbox
type ResourceLimits struct {
// CPU limits
CPULimit float64 `json:"cpu_limit"` // CPU cores (e.g., 1.5)
CPURequest float64 `json:"cpu_request"` // CPU cores requested
// Memory limits
MemoryLimit int64 `json:"memory_limit"` // Bytes
MemoryRequest int64 `json:"memory_request"` // Bytes
// Storage limits
DiskLimit int64 `json:"disk_limit"` // Bytes
DiskRequest int64 `json:"disk_request"` // Bytes
// Network limits
NetworkInLimit int64 `json:"network_in_limit"` // Bytes/sec
NetworkOutLimit int64 `json:"network_out_limit"` // Bytes/sec
// Process limits
ProcessLimit int `json:"process_limit"` // Max processes
FileLimit int `json:"file_limit"` // Max open files
// Time limits
WallTimeLimit time.Duration `json:"wall_time_limit"` // Max wall clock time
CPUTimeLimit time.Duration `json:"cpu_time_limit"` // Max CPU time
}
// SecurityPolicy defines security constraints and policies
type SecurityPolicy struct {
// Container security
RunAsUser string `json:"run_as_user"`
RunAsGroup string `json:"run_as_group"`
ReadOnlyRoot bool `json:"read_only_root"`
NoNewPrivileges bool `json:"no_new_privileges"`
// Capabilities
AddCapabilities []string `json:"add_capabilities"`
DropCapabilities []string `json:"drop_capabilities"`
// SELinux/AppArmor
SELinuxContext string `json:"selinux_context"`
AppArmorProfile string `json:"apparmor_profile"`
SeccompProfile string `json:"seccomp_profile"`
// Network security
AllowNetworking bool `json:"allow_networking"`
AllowedHosts []string `json:"allowed_hosts"`
BlockedHosts []string `json:"blocked_hosts"`
AllowedPorts []int `json:"allowed_ports"`
// File system security
ReadOnlyPaths []string `json:"read_only_paths"`
MaskedPaths []string `json:"masked_paths"`
TmpfsPaths []string `json:"tmpfs_paths"`
// Resource protection
PreventEscalation bool `json:"prevent_escalation"`
IsolateNetwork bool `json:"isolate_network"`
IsolateProcess bool `json:"isolate_process"`
// Monitoring
EnableAuditLog bool `json:"enable_audit_log"`
LogSecurityEvents bool `json:"log_security_events"`
}
// RepositoryConfig defines how the repository is mounted in the sandbox
type RepositoryConfig struct {
// Repository source
URL string `json:"url"`
Branch string `json:"branch"`
CommitHash string `json:"commit_hash"`
LocalPath string `json:"local_path"`
// Mount configuration
MountPoint string `json:"mount_point"` // Path in sandbox
ReadOnly bool `json:"read_only"`
// Git configuration
GitConfig GitConfig `json:"git_config"`
// File filters
IncludeFiles []string `json:"include_files"` // Glob patterns
ExcludeFiles []string `json:"exclude_files"` // Glob patterns
// Access permissions
Permissions string `json:"permissions"` // rwx format
Owner string `json:"owner"`
Group string `json:"group"`
}
// GitConfig defines Git configuration within the sandbox
type GitConfig struct {
UserName string `json:"user_name"`
UserEmail string `json:"user_email"`
SigningKey string `json:"signing_key"`
ConfigValues map[string]string `json:"config_values"`
}
// NetworkConfig defines network settings for the sandbox
type NetworkConfig struct {
// Network isolation
Isolated bool `json:"isolated"` // No network access
Bridge string `json:"bridge"` // Network bridge
// DNS settings
DNSServers []string `json:"dns_servers"`
DNSSearch []string `json:"dns_search"`
// Proxy settings
HTTPProxy string `json:"http_proxy"`
HTTPSProxy string `json:"https_proxy"`
NoProxy string `json:"no_proxy"`
// Port mappings
PortMappings []PortMapping `json:"port_mappings"`
// Bandwidth limits
IngressLimit int64 `json:"ingress_limit"` // Bytes/sec
EgressLimit int64 `json:"egress_limit"` // Bytes/sec
}
// PortMapping defines port forwarding configuration
type PortMapping struct {
HostPort int `json:"host_port"`
ContainerPort int `json:"container_port"`
Protocol string `json:"protocol"` // tcp, udp
}
// ResourceUsage represents current resource consumption
type ResourceUsage struct {
// Timestamp of measurement
Timestamp time.Time `json:"timestamp"`
// CPU usage
CPUUsage float64 `json:"cpu_usage"` // Percentage
CPUTime time.Duration `json:"cpu_time"` // Total CPU time
// Memory usage
MemoryUsage int64 `json:"memory_usage"` // Bytes
MemoryPercent float64 `json:"memory_percent"` // Percentage of limit
MemoryPeak int64 `json:"memory_peak"` // Peak usage
// Disk usage
DiskUsage int64 `json:"disk_usage"` // Bytes
DiskReads int64 `json:"disk_reads"` // Read operations
DiskWrites int64 `json:"disk_writes"` // Write operations
// Network usage
NetworkIn int64 `json:"network_in"` // Bytes received
NetworkOut int64 `json:"network_out"` // Bytes sent
// Process information
ProcessCount int `json:"process_count"` // Active processes
ThreadCount int `json:"thread_count"` // Active threads
FileHandles int `json:"file_handles"` // Open file handles
// Runtime information
Uptime time.Duration `json:"uptime"` // Sandbox uptime
}
// SandboxInfo provides information about a sandbox instance
type SandboxInfo struct {
// Identification
ID string `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
// Status
Status SandboxStatus `json:"status"`
CreatedAt time.Time `json:"created_at"`
StartedAt time.Time `json:"started_at"`
// Runtime information
Runtime string `json:"runtime"`
Image string `json:"image"`
Platform string `json:"platform"`
// Network information
IPAddress string `json:"ip_address"`
MACAddress string `json:"mac_address"`
Hostname string `json:"hostname"`
// Resource information
AllocatedResources ResourceLimits `json:"allocated_resources"`
// Configuration
Config SandboxConfig `json:"config"`
// Metadata
Labels map[string]string `json:"labels"`
Annotations map[string]string `json:"annotations"`
}
// SandboxStatus represents the current status of a sandbox
type SandboxStatus string
const (
StatusCreating SandboxStatus = "creating"
StatusStarting SandboxStatus = "starting"
StatusRunning SandboxStatus = "running"
StatusPaused SandboxStatus = "paused"
StatusStopping SandboxStatus = "stopping"
StatusStopped SandboxStatus = "stopped"
StatusFailed SandboxStatus = "failed"
StatusDestroyed SandboxStatus = "destroyed"
)
// Common sandbox errors
var (
ErrSandboxNotFound = &SandboxError{Code: "SANDBOX_NOT_FOUND", Message: "Sandbox not found"}
ErrSandboxAlreadyExists = &SandboxError{Code: "SANDBOX_ALREADY_EXISTS", Message: "Sandbox already exists"}
ErrSandboxNotRunning = &SandboxError{Code: "SANDBOX_NOT_RUNNING", Message: "Sandbox is not running"}
ErrSandboxInitFailed = &SandboxError{Code: "SANDBOX_INIT_FAILED", Message: "Sandbox initialization failed"}
ErrCommandExecutionFailed = &SandboxError{Code: "COMMAND_EXECUTION_FAILED", Message: "Command execution failed"}
ErrResourceLimitExceeded = &SandboxError{Code: "RESOURCE_LIMIT_EXCEEDED", Message: "Resource limit exceeded"}
ErrSecurityViolation = &SandboxError{Code: "SECURITY_VIOLATION", Message: "Security policy violation"}
ErrFileOperationFailed = &SandboxError{Code: "FILE_OPERATION_FAILED", Message: "File operation failed"}
ErrNetworkAccessDenied = &SandboxError{Code: "NETWORK_ACCESS_DENIED", Message: "Network access denied"}
ErrTimeoutExceeded = &SandboxError{Code: "TIMEOUT_EXCEEDED", Message: "Execution timeout exceeded"}
)
// SandboxError represents sandbox-specific errors
type SandboxError struct {
Code string `json:"code"`
Message string `json:"message"`
Details string `json:"details,omitempty"`
Retryable bool `json:"retryable"`
Cause error `json:"-"`
}
func (e *SandboxError) Error() string {
if e.Details != "" {
return e.Message + ": " + e.Details
}
return e.Message
}
func (e *SandboxError) Unwrap() error {
return e.Cause
}
func (e *SandboxError) IsRetryable() bool {
return e.Retryable
}
// NewSandboxError creates a new sandbox error with details
func NewSandboxError(base *SandboxError, details string) *SandboxError {
return &SandboxError{
Code: base.Code,
Message: base.Message,
Details: details,
Retryable: base.Retryable,
}
}
// NewSandboxErrorWithCause creates a new sandbox error with an underlying cause
func NewSandboxErrorWithCause(base *SandboxError, details string, cause error) *SandboxError {
return &SandboxError{
Code: base.Code,
Message: base.Message,
Details: details,
Retryable: base.Retryable,
Cause: cause,
}
}

View File

@@ -0,0 +1,639 @@
package execution
import (
"context"
"errors"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSandboxError(t *testing.T) {
tests := []struct {
name string
err *SandboxError
expected string
retryable bool
}{
{
name: "simple error",
err: ErrSandboxNotFound,
expected: "Sandbox not found",
retryable: false,
},
{
name: "error with details",
err: NewSandboxError(ErrResourceLimitExceeded, "Memory limit of 1GB exceeded"),
expected: "Resource limit exceeded: Memory limit of 1GB exceeded",
retryable: false,
},
{
name: "retryable error",
err: &SandboxError{
Code: "TEMPORARY_FAILURE",
Message: "Temporary network failure",
Retryable: true,
},
expected: "Temporary network failure",
retryable: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, tt.err.Error())
assert.Equal(t, tt.retryable, tt.err.IsRetryable())
})
}
}
func TestSandboxErrorUnwrap(t *testing.T) {
baseErr := errors.New("underlying error")
sandboxErr := NewSandboxErrorWithCause(ErrCommandExecutionFailed, "command failed", baseErr)
unwrapped := sandboxErr.Unwrap()
assert.Equal(t, baseErr, unwrapped)
}
func TestSandboxConfig(t *testing.T) {
config := &SandboxConfig{
Type: "docker",
Image: "alpine:latest",
Runtime: "docker",
Architecture: "amd64",
Resources: ResourceLimits{
MemoryLimit: 1024 * 1024 * 1024, // 1GB
MemoryRequest: 512 * 1024 * 1024, // 512MB
CPULimit: 2.0,
CPURequest: 1.0,
DiskLimit: 10 * 1024 * 1024 * 1024, // 10GB
ProcessLimit: 100,
FileLimit: 1024,
WallTimeLimit: 30 * time.Minute,
CPUTimeLimit: 10 * time.Minute,
},
Security: SecurityPolicy{
RunAsUser: "1000",
RunAsGroup: "1000",
ReadOnlyRoot: true,
NoNewPrivileges: true,
AddCapabilities: []string{"NET_BIND_SERVICE"},
DropCapabilities: []string{"ALL"},
SELinuxContext: "unconfined_u:unconfined_r:container_t:s0",
AppArmorProfile: "docker-default",
SeccompProfile: "runtime/default",
AllowNetworking: false,
AllowedHosts: []string{"api.example.com"},
BlockedHosts: []string{"malicious.com"},
AllowedPorts: []int{80, 443},
ReadOnlyPaths: []string{"/etc", "/usr"},
MaskedPaths: []string{"/proc/kcore", "/proc/keys"},
TmpfsPaths: []string{"/tmp", "/var/tmp"},
PreventEscalation: true,
IsolateNetwork: true,
IsolateProcess: true,
EnableAuditLog: true,
LogSecurityEvents: true,
},
Repository: RepositoryConfig{
URL: "https://github.com/example/repo.git",
Branch: "main",
LocalPath: "/home/user/repo",
MountPoint: "/workspace",
ReadOnly: false,
GitConfig: GitConfig{
UserName: "Test User",
UserEmail: "test@example.com",
ConfigValues: map[string]string{
"core.autocrlf": "input",
},
},
IncludeFiles: []string{"*.go", "*.md"},
ExcludeFiles: []string{"*.tmp", "*.log"},
Permissions: "755",
Owner: "user",
Group: "user",
},
Network: NetworkConfig{
Isolated: false,
Bridge: "docker0",
DNSServers: []string{"8.8.8.8", "1.1.1.1"},
DNSSearch: []string{"example.com"},
HTTPProxy: "http://proxy:8080",
HTTPSProxy: "http://proxy:8080",
NoProxy: "localhost,127.0.0.1",
PortMappings: []PortMapping{
{HostPort: 8080, ContainerPort: 80, Protocol: "tcp"},
},
IngressLimit: 1024 * 1024, // 1MB/s
EgressLimit: 2048 * 1024, // 2MB/s
},
Environment: map[string]string{
"NODE_ENV": "test",
"DEBUG": "true",
},
WorkingDir: "/workspace",
Tools: []string{"git", "node", "npm"},
MCPServers: []string{"file-server", "web-server"},
Timeout: 5 * time.Minute,
CleanupDelay: 30 * time.Second,
Labels: map[string]string{
"app": "chorus",
"version": "1.0.0",
},
Annotations: map[string]string{
"description": "Test sandbox configuration",
},
}
// Validate required fields
assert.NotEmpty(t, config.Type)
assert.NotEmpty(t, config.Image)
assert.NotEmpty(t, config.Architecture)
// Validate resource limits
assert.Greater(t, config.Resources.MemoryLimit, int64(0))
assert.Greater(t, config.Resources.CPULimit, 0.0)
// Validate security policy
assert.NotEmpty(t, config.Security.RunAsUser)
assert.True(t, config.Security.NoNewPrivileges)
assert.NotEmpty(t, config.Security.DropCapabilities)
// Validate repository config
assert.NotEmpty(t, config.Repository.MountPoint)
assert.NotEmpty(t, config.Repository.GitConfig.UserName)
// Validate network config
assert.NotEmpty(t, config.Network.DNSServers)
assert.Len(t, config.Network.PortMappings, 1)
// Validate timeouts
assert.Greater(t, config.Timeout, time.Duration(0))
assert.Greater(t, config.CleanupDelay, time.Duration(0))
}
func TestCommand(t *testing.T) {
cmd := &Command{
Executable: "python3",
Args: []string{"-c", "print('hello world')"},
WorkingDir: "/workspace",
Environment: map[string]string{"PYTHONPATH": "/custom/path"},
StdinContent: "input data",
Timeout: 30 * time.Second,
User: "1000",
AllowNetwork: true,
AllowWrite: true,
RestrictPaths: []string{"/etc", "/usr"},
}
// Validate command structure
assert.Equal(t, "python3", cmd.Executable)
assert.Len(t, cmd.Args, 2)
assert.Equal(t, "/workspace", cmd.WorkingDir)
assert.Equal(t, "/custom/path", cmd.Environment["PYTHONPATH"])
assert.Equal(t, "input data", cmd.StdinContent)
assert.Equal(t, 30*time.Second, cmd.Timeout)
assert.True(t, cmd.AllowNetwork)
assert.True(t, cmd.AllowWrite)
assert.Len(t, cmd.RestrictPaths, 2)
}
func TestCommandResult(t *testing.T) {
startTime := time.Now()
endTime := startTime.Add(2 * time.Second)
result := &CommandResult{
ExitCode: 0,
Success: true,
Stdout: "Standard output",
Stderr: "Standard error",
Combined: "Combined output",
StartTime: startTime,
EndTime: endTime,
Duration: endTime.Sub(startTime),
ResourceUsage: ResourceUsage{
CPUUsage: 25.5,
MemoryUsage: 1024 * 1024, // 1MB
},
ProcessID: 12345,
Metadata: map[string]interface{}{
"container_id": "abc123",
"image": "alpine:latest",
},
}
// Validate result structure
assert.Equal(t, 0, result.ExitCode)
assert.True(t, result.Success)
assert.Equal(t, "Standard output", result.Stdout)
assert.Equal(t, "Standard error", result.Stderr)
assert.Equal(t, 2*time.Second, result.Duration)
assert.Equal(t, 25.5, result.ResourceUsage.CPUUsage)
assert.Equal(t, int64(1024*1024), result.ResourceUsage.MemoryUsage)
assert.Equal(t, 12345, result.ProcessID)
assert.Equal(t, "abc123", result.Metadata["container_id"])
}
func TestFileInfo(t *testing.T) {
modTime := time.Now()
fileInfo := FileInfo{
Name: "test.txt",
Path: "/workspace/test.txt",
Size: 1024,
Mode: 0644,
ModTime: modTime,
IsDir: false,
Owner: "user",
Group: "user",
Permissions: "-rw-r--r--",
}
// Validate file info structure
assert.Equal(t, "test.txt", fileInfo.Name)
assert.Equal(t, "/workspace/test.txt", fileInfo.Path)
assert.Equal(t, int64(1024), fileInfo.Size)
assert.Equal(t, uint32(0644), fileInfo.Mode)
assert.Equal(t, modTime, fileInfo.ModTime)
assert.False(t, fileInfo.IsDir)
assert.Equal(t, "user", fileInfo.Owner)
assert.Equal(t, "user", fileInfo.Group)
assert.Equal(t, "-rw-r--r--", fileInfo.Permissions)
}
func TestResourceLimits(t *testing.T) {
limits := ResourceLimits{
CPULimit: 2.5,
CPURequest: 1.0,
MemoryLimit: 2 * 1024 * 1024 * 1024, // 2GB
MemoryRequest: 1 * 1024 * 1024 * 1024, // 1GB
DiskLimit: 50 * 1024 * 1024 * 1024, // 50GB
DiskRequest: 10 * 1024 * 1024 * 1024, // 10GB
NetworkInLimit: 10 * 1024 * 1024, // 10MB/s
NetworkOutLimit: 5 * 1024 * 1024, // 5MB/s
ProcessLimit: 200,
FileLimit: 2048,
WallTimeLimit: 1 * time.Hour,
CPUTimeLimit: 30 * time.Minute,
}
// Validate resource limits
assert.Equal(t, 2.5, limits.CPULimit)
assert.Equal(t, 1.0, limits.CPURequest)
assert.Equal(t, int64(2*1024*1024*1024), limits.MemoryLimit)
assert.Equal(t, int64(1*1024*1024*1024), limits.MemoryRequest)
assert.Equal(t, int64(50*1024*1024*1024), limits.DiskLimit)
assert.Equal(t, 200, limits.ProcessLimit)
assert.Equal(t, 2048, limits.FileLimit)
assert.Equal(t, 1*time.Hour, limits.WallTimeLimit)
assert.Equal(t, 30*time.Minute, limits.CPUTimeLimit)
}
func TestResourceUsage(t *testing.T) {
timestamp := time.Now()
usage := ResourceUsage{
Timestamp: timestamp,
CPUUsage: 75.5,
CPUTime: 15 * time.Minute,
MemoryUsage: 512 * 1024 * 1024, // 512MB
MemoryPercent: 25.0,
MemoryPeak: 768 * 1024 * 1024, // 768MB
DiskUsage: 1 * 1024 * 1024 * 1024, // 1GB
DiskReads: 1000,
DiskWrites: 500,
NetworkIn: 10 * 1024 * 1024, // 10MB
NetworkOut: 5 * 1024 * 1024, // 5MB
ProcessCount: 25,
ThreadCount: 100,
FileHandles: 50,
Uptime: 2 * time.Hour,
}
// Validate resource usage
assert.Equal(t, timestamp, usage.Timestamp)
assert.Equal(t, 75.5, usage.CPUUsage)
assert.Equal(t, 15*time.Minute, usage.CPUTime)
assert.Equal(t, int64(512*1024*1024), usage.MemoryUsage)
assert.Equal(t, 25.0, usage.MemoryPercent)
assert.Equal(t, int64(768*1024*1024), usage.MemoryPeak)
assert.Equal(t, 25, usage.ProcessCount)
assert.Equal(t, 100, usage.ThreadCount)
assert.Equal(t, 50, usage.FileHandles)
assert.Equal(t, 2*time.Hour, usage.Uptime)
}
func TestSandboxInfo(t *testing.T) {
createdAt := time.Now()
startedAt := createdAt.Add(5 * time.Second)
info := SandboxInfo{
ID: "sandbox-123",
Name: "test-sandbox",
Type: "docker",
Status: StatusRunning,
CreatedAt: createdAt,
StartedAt: startedAt,
Runtime: "docker",
Image: "alpine:latest",
Platform: "linux/amd64",
IPAddress: "172.17.0.2",
MACAddress: "02:42:ac:11:00:02",
Hostname: "sandbox-123",
AllocatedResources: ResourceLimits{
MemoryLimit: 1024 * 1024 * 1024, // 1GB
CPULimit: 2.0,
},
Labels: map[string]string{
"app": "chorus",
},
Annotations: map[string]string{
"creator": "test",
},
}
// Validate sandbox info
assert.Equal(t, "sandbox-123", info.ID)
assert.Equal(t, "test-sandbox", info.Name)
assert.Equal(t, "docker", info.Type)
assert.Equal(t, StatusRunning, info.Status)
assert.Equal(t, createdAt, info.CreatedAt)
assert.Equal(t, startedAt, info.StartedAt)
assert.Equal(t, "docker", info.Runtime)
assert.Equal(t, "alpine:latest", info.Image)
assert.Equal(t, "172.17.0.2", info.IPAddress)
assert.Equal(t, "chorus", info.Labels["app"])
assert.Equal(t, "test", info.Annotations["creator"])
}
func TestSandboxStatus(t *testing.T) {
statuses := []SandboxStatus{
StatusCreating,
StatusStarting,
StatusRunning,
StatusPaused,
StatusStopping,
StatusStopped,
StatusFailed,
StatusDestroyed,
}
expectedStatuses := []string{
"creating",
"starting",
"running",
"paused",
"stopping",
"stopped",
"failed",
"destroyed",
}
for i, status := range statuses {
assert.Equal(t, expectedStatuses[i], string(status))
}
}
func TestPortMapping(t *testing.T) {
mapping := PortMapping{
HostPort: 8080,
ContainerPort: 80,
Protocol: "tcp",
}
assert.Equal(t, 8080, mapping.HostPort)
assert.Equal(t, 80, mapping.ContainerPort)
assert.Equal(t, "tcp", mapping.Protocol)
}
func TestGitConfig(t *testing.T) {
config := GitConfig{
UserName: "Test User",
UserEmail: "test@example.com",
SigningKey: "ABC123",
ConfigValues: map[string]string{
"core.autocrlf": "input",
"pull.rebase": "true",
"init.defaultBranch": "main",
},
}
assert.Equal(t, "Test User", config.UserName)
assert.Equal(t, "test@example.com", config.UserEmail)
assert.Equal(t, "ABC123", config.SigningKey)
assert.Equal(t, "input", config.ConfigValues["core.autocrlf"])
assert.Equal(t, "true", config.ConfigValues["pull.rebase"])
assert.Equal(t, "main", config.ConfigValues["init.defaultBranch"])
}
// MockSandbox implements ExecutionSandbox for testing
type MockSandbox struct {
id string
status SandboxStatus
workingDir string
environment map[string]string
shouldFail bool
commandResult *CommandResult
files []FileInfo
resourceUsage *ResourceUsage
}
func NewMockSandbox() *MockSandbox {
return &MockSandbox{
id: "mock-sandbox-123",
status: StatusStopped,
workingDir: "/workspace",
environment: make(map[string]string),
files: []FileInfo{},
commandResult: &CommandResult{
Success: true,
ExitCode: 0,
Stdout: "mock output",
},
resourceUsage: &ResourceUsage{
CPUUsage: 10.0,
MemoryUsage: 100 * 1024 * 1024, // 100MB
},
}
}
func (m *MockSandbox) Initialize(ctx context.Context, config *SandboxConfig) error {
if m.shouldFail {
return NewSandboxError(ErrSandboxInitFailed, "mock initialization failed")
}
m.status = StatusRunning
return nil
}
func (m *MockSandbox) ExecuteCommand(ctx context.Context, cmd *Command) (*CommandResult, error) {
if m.shouldFail {
return nil, NewSandboxError(ErrCommandExecutionFailed, "mock command execution failed")
}
return m.commandResult, nil
}
func (m *MockSandbox) CopyFiles(ctx context.Context, source, dest string) error {
if m.shouldFail {
return NewSandboxError(ErrFileOperationFailed, "mock file copy failed")
}
return nil
}
func (m *MockSandbox) WriteFile(ctx context.Context, path string, content []byte, mode uint32) error {
if m.shouldFail {
return NewSandboxError(ErrFileOperationFailed, "mock file write failed")
}
return nil
}
func (m *MockSandbox) ReadFile(ctx context.Context, path string) ([]byte, error) {
if m.shouldFail {
return nil, NewSandboxError(ErrFileOperationFailed, "mock file read failed")
}
return []byte("mock file content"), nil
}
func (m *MockSandbox) ListFiles(ctx context.Context, path string) ([]FileInfo, error) {
if m.shouldFail {
return nil, NewSandboxError(ErrFileOperationFailed, "mock file list failed")
}
return m.files, nil
}
func (m *MockSandbox) GetWorkingDirectory() string {
return m.workingDir
}
func (m *MockSandbox) SetWorkingDirectory(path string) error {
if m.shouldFail {
return NewSandboxError(ErrFileOperationFailed, "mock set working directory failed")
}
m.workingDir = path
return nil
}
func (m *MockSandbox) GetEnvironment() map[string]string {
env := make(map[string]string)
for k, v := range m.environment {
env[k] = v
}
return env
}
func (m *MockSandbox) SetEnvironment(env map[string]string) error {
if m.shouldFail {
return NewSandboxError(ErrFileOperationFailed, "mock set environment failed")
}
for k, v := range env {
m.environment[k] = v
}
return nil
}
func (m *MockSandbox) GetResourceUsage(ctx context.Context) (*ResourceUsage, error) {
if m.shouldFail {
return nil, NewSandboxError(ErrSandboxInitFailed, "mock resource usage failed")
}
return m.resourceUsage, nil
}
func (m *MockSandbox) Cleanup() error {
if m.shouldFail {
return NewSandboxError(ErrSandboxInitFailed, "mock cleanup failed")
}
m.status = StatusDestroyed
return nil
}
func (m *MockSandbox) GetInfo() SandboxInfo {
return SandboxInfo{
ID: m.id,
Status: m.status,
Type: "mock",
}
}
func TestMockSandbox(t *testing.T) {
sandbox := NewMockSandbox()
ctx := context.Background()
// Test initialization
err := sandbox.Initialize(ctx, &SandboxConfig{})
require.NoError(t, err)
assert.Equal(t, StatusRunning, sandbox.status)
// Test command execution
result, err := sandbox.ExecuteCommand(ctx, &Command{})
require.NoError(t, err)
assert.True(t, result.Success)
assert.Equal(t, "mock output", result.Stdout)
// Test file operations
err = sandbox.WriteFile(ctx, "/test.txt", []byte("test"), 0644)
require.NoError(t, err)
content, err := sandbox.ReadFile(ctx, "/test.txt")
require.NoError(t, err)
assert.Equal(t, []byte("mock file content"), content)
files, err := sandbox.ListFiles(ctx, "/")
require.NoError(t, err)
assert.Empty(t, files) // Mock returns empty list by default
// Test environment
env := sandbox.GetEnvironment()
assert.Empty(t, env)
err = sandbox.SetEnvironment(map[string]string{"TEST": "value"})
require.NoError(t, err)
env = sandbox.GetEnvironment()
assert.Equal(t, "value", env["TEST"])
// Test resource usage
usage, err := sandbox.GetResourceUsage(ctx)
require.NoError(t, err)
assert.Equal(t, 10.0, usage.CPUUsage)
// Test cleanup
err = sandbox.Cleanup()
require.NoError(t, err)
assert.Equal(t, StatusDestroyed, sandbox.status)
}
func TestMockSandboxFailure(t *testing.T) {
sandbox := NewMockSandbox()
sandbox.shouldFail = true
ctx := context.Background()
// All operations should fail when shouldFail is true
err := sandbox.Initialize(ctx, &SandboxConfig{})
assert.Error(t, err)
_, err = sandbox.ExecuteCommand(ctx, &Command{})
assert.Error(t, err)
err = sandbox.WriteFile(ctx, "/test.txt", []byte("test"), 0644)
assert.Error(t, err)
_, err = sandbox.ReadFile(ctx, "/test.txt")
assert.Error(t, err)
_, err = sandbox.ListFiles(ctx, "/")
assert.Error(t, err)
err = sandbox.SetWorkingDirectory("/tmp")
assert.Error(t, err)
err = sandbox.SetEnvironment(map[string]string{"TEST": "value"})
assert.Error(t, err)
_, err = sandbox.GetResourceUsage(ctx)
assert.Error(t, err)
err = sandbox.Cleanup()
assert.Error(t, err)
}