Phase 2: Implement Execution Environment Abstraction (v0.3.0)

This commit implements Phase 2 of the CHORUS Task Execution Engine development plan,
providing a comprehensive execution environment abstraction layer with Docker
container sandboxing support.

## New Features

### Core Sandbox Interface
- Comprehensive ExecutionSandbox interface with isolated task execution
- Support for command execution, file I/O, environment management
- Resource usage monitoring and sandbox lifecycle management
- Standardized error handling with SandboxError types and categories

### Docker Container Sandbox Implementation
- Full Docker API integration with secure container creation
- Transparent repository mounting with configurable read/write access
- Advanced security policies with capability dropping and privilege controls
- Comprehensive resource limits (CPU, memory, disk, processes, file handles)
- Support for tmpfs mounts, masked paths, and read-only bind mounts
- Container lifecycle management with proper cleanup and health monitoring

### Security & Resource Management
- Configurable security policies with SELinux, AppArmor, and Seccomp support
- Fine-grained capability management with secure defaults
- Network isolation options with configurable DNS and proxy settings
- Resource monitoring with real-time CPU, memory, and network usage tracking
- Comprehensive ulimits configuration for process and file handle limits

### Repository Integration
- Seamless repository mounting from local paths to container workspaces
- Git configuration support with user credentials and global settings
- File inclusion/exclusion patterns for selective repository access
- Configurable permissions and ownership for mounted repositories

### Testing Infrastructure
- Comprehensive test suite with 60+ test cases covering all functionality
- Docker integration tests with Alpine Linux containers (skipped in short mode)
- Mock sandbox implementation for unit testing without Docker dependencies
- Security policy validation tests with read-only filesystem enforcement
- Resource usage monitoring and cleanup verification tests

## Technical Details

### Dependencies Added
- github.com/docker/docker v28.4.0+incompatible - Docker API client
- github.com/docker/go-connections v0.6.0 - Docker connection utilities
- github.com/docker/go-units v0.5.0 - Docker units and formatting
- Associated Docker API dependencies for complete container management

### Architecture
- Interface-driven design enabling multiple sandbox implementations
- Comprehensive configuration structures for all sandbox aspects
- Resource usage tracking with detailed metrics collection
- Error handling with retryable error classification
- Proper cleanup and resource management throughout sandbox lifecycle

### Compatibility
- Maintains backward compatibility with existing CHORUS architecture
- Designed for future integration with Phase 3 Core Task Execution Engine
- Extensible design supporting additional sandbox implementations (VM, process)

This Phase 2 implementation provides the foundation for secure, isolated task
execution that will be integrated with the AI model providers from Phase 1
in the upcoming Phase 3 development.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
anthonyrawlins
2025-09-25 14:28:08 +10:00
parent d1252ade69
commit 8d9b62daf3
653 changed files with 88039 additions and 3766 deletions

View File

@@ -13,6 +13,7 @@
package protodesc
import (
"google.golang.org/protobuf/internal/editionssupport"
"google.golang.org/protobuf/internal/errors"
"google.golang.org/protobuf/internal/filedesc"
"google.golang.org/protobuf/internal/pragma"
@@ -91,15 +92,17 @@ func (o FileOptions) New(fd *descriptorpb.FileDescriptorProto, r Resolver) (prot
switch fd.GetSyntax() {
case "proto2", "":
f.L1.Syntax = protoreflect.Proto2
f.L1.Edition = filedesc.EditionProto2
case "proto3":
f.L1.Syntax = protoreflect.Proto3
f.L1.Edition = filedesc.EditionProto3
case "editions":
f.L1.Syntax = protoreflect.Editions
f.L1.Edition = fromEditionProto(fd.GetEdition())
default:
return nil, errors.New("invalid syntax: %q", fd.GetSyntax())
}
if f.L1.Syntax == protoreflect.Editions && (fd.GetEdition() < SupportedEditionsMinimum || fd.GetEdition() > SupportedEditionsMaximum) {
if f.L1.Syntax == protoreflect.Editions && (fd.GetEdition() < editionssupport.Minimum || fd.GetEdition() > editionssupport.Maximum) {
return nil, errors.New("use of edition %v not yet supported by the Go Protobuf runtime", fd.GetEdition())
}
f.L1.Path = fd.GetName()
@@ -114,9 +117,7 @@ func (o FileOptions) New(fd *descriptorpb.FileDescriptorProto, r Resolver) (prot
opts = proto.Clone(opts).(*descriptorpb.FileOptions)
f.L2.Options = func() protoreflect.ProtoMessage { return opts }
}
if f.L1.Syntax == protoreflect.Editions {
initFileDescFromFeatureSet(f, fd.GetOptions().GetFeatures())
}
initFileDescFromFeatureSet(f, fd.GetOptions().GetFeatures())
f.L2.Imports = make(filedesc.FileImports, len(fd.GetDependency()))
for _, i := range fd.GetPublicDependency() {
@@ -219,10 +220,10 @@ func (o FileOptions) New(fd *descriptorpb.FileDescriptorProto, r Resolver) (prot
if err := validateEnumDeclarations(f.L1.Enums.List, fd.GetEnumType()); err != nil {
return nil, err
}
if err := validateMessageDeclarations(f.L1.Messages.List, fd.GetMessageType()); err != nil {
if err := validateMessageDeclarations(f, f.L1.Messages.List, fd.GetMessageType()); err != nil {
return nil, err
}
if err := validateExtensionDeclarations(f.L1.Extensions.List, fd.GetExtension()); err != nil {
if err := validateExtensionDeclarations(f, f.L1.Extensions.List, fd.GetExtension()); err != nil {
return nil, err
}

View File

@@ -69,9 +69,7 @@ func (r descsByName) initMessagesDeclarations(mds []*descriptorpb.DescriptorProt
if m.L0, err = r.makeBase(m, parent, md.GetName(), i, sb); err != nil {
return nil, err
}
if m.Base.L0.ParentFile.Syntax() == protoreflect.Editions {
m.L1.EditionFeatures = mergeEditionFeatures(parent, md.GetOptions().GetFeatures())
}
m.L1.EditionFeatures = mergeEditionFeatures(parent, md.GetOptions().GetFeatures())
if opts := md.GetOptions(); opts != nil {
opts = proto.Clone(opts).(*descriptorpb.MessageOptions)
m.L2.Options = func() protoreflect.ProtoMessage { return opts }
@@ -146,13 +144,15 @@ func (r descsByName) initFieldsFromDescriptorProto(fds []*descriptorpb.FieldDesc
if f.L0, err = r.makeBase(f, parent, fd.GetName(), i, sb); err != nil {
return nil, err
}
f.L1.EditionFeatures = mergeEditionFeatures(parent, fd.GetOptions().GetFeatures())
f.L1.IsProto3Optional = fd.GetProto3Optional()
if opts := fd.GetOptions(); opts != nil {
opts = proto.Clone(opts).(*descriptorpb.FieldOptions)
f.L1.Options = func() protoreflect.ProtoMessage { return opts }
f.L1.IsWeak = opts.GetWeak()
f.L1.HasPacked = opts.Packed != nil
f.L1.IsPacked = opts.GetPacked()
if opts.Packed != nil {
f.L1.EditionFeatures.IsPacked = opts.GetPacked()
}
}
f.L1.Number = protoreflect.FieldNumber(fd.GetNumber())
f.L1.Cardinality = protoreflect.Cardinality(fd.GetLabel())
@@ -163,32 +163,12 @@ func (r descsByName) initFieldsFromDescriptorProto(fds []*descriptorpb.FieldDesc
f.L1.StringName.InitJSON(fd.GetJsonName())
}
if f.Base.L0.ParentFile.Syntax() == protoreflect.Editions {
f.L1.EditionFeatures = mergeEditionFeatures(parent, fd.GetOptions().GetFeatures())
if f.L1.EditionFeatures.IsLegacyRequired {
f.L1.Cardinality = protoreflect.Required
}
if f.L1.EditionFeatures.IsLegacyRequired {
f.L1.Cardinality = protoreflect.Required
}
// We reuse the existing field because the old option `[packed =
// true]` is mutually exclusive with the editions feature.
if canBePacked(fd) {
f.L1.HasPacked = true
f.L1.IsPacked = f.L1.EditionFeatures.IsPacked
}
// We pretend this option is always explicitly set because the only
// use of HasEnforceUTF8 is to determine whether to use EnforceUTF8
// or to return the appropriate default.
// When using editions we either parse the option or resolve the
// appropriate default here (instead of later when this option is
// requested from the descriptor).
// In proto2/proto3 syntax HasEnforceUTF8 might be false.
f.L1.HasEnforceUTF8 = true
f.L1.EnforceUTF8 = f.L1.EditionFeatures.IsUTF8Validated
if f.L1.Kind == protoreflect.MessageKind && f.L1.EditionFeatures.IsDelimitedEncoded {
f.L1.Kind = protoreflect.GroupKind
}
if f.L1.Kind == protoreflect.MessageKind && f.L1.EditionFeatures.IsDelimitedEncoded {
f.L1.Kind = protoreflect.GroupKind
}
}
return fs, nil
@@ -201,12 +181,10 @@ func (r descsByName) initOneofsFromDescriptorProto(ods []*descriptorpb.OneofDesc
if o.L0, err = r.makeBase(o, parent, od.GetName(), i, sb); err != nil {
return nil, err
}
o.L1.EditionFeatures = mergeEditionFeatures(parent, od.GetOptions().GetFeatures())
if opts := od.GetOptions(); opts != nil {
opts = proto.Clone(opts).(*descriptorpb.OneofOptions)
o.L1.Options = func() protoreflect.ProtoMessage { return opts }
if parent.Syntax() == protoreflect.Editions {
o.L1.EditionFeatures = mergeEditionFeatures(parent, opts.GetFeatures())
}
}
}
return os, nil
@@ -220,10 +198,13 @@ func (r descsByName) initExtensionDeclarations(xds []*descriptorpb.FieldDescript
if x.L0, err = r.makeBase(x, parent, xd.GetName(), i, sb); err != nil {
return nil, err
}
x.L1.EditionFeatures = mergeEditionFeatures(parent, xd.GetOptions().GetFeatures())
if opts := xd.GetOptions(); opts != nil {
opts = proto.Clone(opts).(*descriptorpb.FieldOptions)
x.L2.Options = func() protoreflect.ProtoMessage { return opts }
x.L2.IsPacked = opts.GetPacked()
if opts.Packed != nil {
x.L1.EditionFeatures.IsPacked = opts.GetPacked()
}
}
x.L1.Number = protoreflect.FieldNumber(xd.GetNumber())
x.L1.Cardinality = protoreflect.Cardinality(xd.GetLabel())

View File

@@ -46,6 +46,11 @@ func (r *resolver) resolveMessageDependencies(ms []filedesc.Message, mds []*desc
if f.L1.Kind, f.L1.Enum, f.L1.Message, err = r.findTarget(f.Kind(), f.Parent().FullName(), partialName(fd.GetTypeName()), f.IsWeak()); err != nil {
return errors.New("message field %q cannot resolve type: %v", f.FullName(), err)
}
if f.L1.Kind == protoreflect.GroupKind && (f.IsMap() || f.IsMapEntry()) {
// A map field might inherit delimited encoding from a file-wide default feature.
// But maps never actually use delimited encoding. (At least for now...)
f.L1.Kind = protoreflect.MessageKind
}
if fd.DefaultValue != nil {
v, ev, err := unmarshalDefault(fd.GetDefaultValue(), f, r.allowUnresolvable)
if err != nil {

View File

@@ -45,11 +45,11 @@ func validateEnumDeclarations(es []filedesc.Enum, eds []*descriptorpb.EnumDescri
if allowAlias && !foundAlias {
return errors.New("enum %q allows aliases, but none were found", e.FullName())
}
if e.Syntax() == protoreflect.Proto3 {
if !e.IsClosed() {
if v := e.Values().Get(0); v.Number() != 0 {
return errors.New("enum %q using proto3 semantics must have zero number for the first value", v.FullName())
return errors.New("enum %q using open semantics must have zero number for the first value", v.FullName())
}
// Verify that value names in proto3 do not conflict if the
// Verify that value names in open enums do not conflict if the
// case-insensitive prefix is removed.
// See protoc v3.8.0: src/google/protobuf/descriptor.cc:4991-5055
names := map[string]protoreflect.EnumValueDescriptor{}
@@ -58,7 +58,7 @@ func validateEnumDeclarations(es []filedesc.Enum, eds []*descriptorpb.EnumDescri
v1 := e.Values().Get(i)
s := strs.EnumValueName(strs.TrimEnumPrefix(string(v1.Name()), prefix))
if v2, ok := names[s]; ok && v1.Number() != v2.Number() {
return errors.New("enum %q using proto3 semantics has conflict: %q with %q", e.FullName(), v1.Name(), v2.Name())
return errors.New("enum %q using open semantics has conflict: %q with %q", e.FullName(), v1.Name(), v2.Name())
}
names[s] = v1
}
@@ -80,7 +80,9 @@ func validateEnumDeclarations(es []filedesc.Enum, eds []*descriptorpb.EnumDescri
return nil
}
func validateMessageDeclarations(ms []filedesc.Message, mds []*descriptorpb.DescriptorProto) error {
func validateMessageDeclarations(file *filedesc.File, ms []filedesc.Message, mds []*descriptorpb.DescriptorProto) error {
// There are a few limited exceptions only for proto3
isProto3 := file.L1.Edition == fromEditionProto(descriptorpb.Edition_EDITION_PROTO3)
for i, md := range mds {
m := &ms[i]
@@ -107,25 +109,13 @@ func validateMessageDeclarations(ms []filedesc.Message, mds []*descriptorpb.Desc
if isMessageSet && !flags.ProtoLegacy {
return errors.New("message %q is a MessageSet, which is a legacy proto1 feature that is no longer supported", m.FullName())
}
if isMessageSet && (m.Syntax() == protoreflect.Proto3 || m.Fields().Len() > 0 || m.ExtensionRanges().Len() == 0) {
if isMessageSet && (isProto3 || m.Fields().Len() > 0 || m.ExtensionRanges().Len() == 0) {
return errors.New("message %q is an invalid proto1 MessageSet", m.FullName())
}
if m.Syntax() == protoreflect.Proto3 {
if isProto3 {
if m.ExtensionRanges().Len() > 0 {
return errors.New("message %q using proto3 semantics cannot have extension ranges", m.FullName())
}
// Verify that field names in proto3 do not conflict if lowercased
// with all underscores removed.
// See protoc v3.8.0: src/google/protobuf/descriptor.cc:5830-5847
names := map[string]protoreflect.FieldDescriptor{}
for i := 0; i < m.Fields().Len(); i++ {
f1 := m.Fields().Get(i)
s := strings.Replace(strings.ToLower(string(f1.Name())), "_", "", -1)
if f2, ok := names[s]; ok {
return errors.New("message %q using proto3 semantics has conflict: %q with %q", m.FullName(), f1.Name(), f2.Name())
}
names[s] = f1
}
}
for j, fd := range md.GetField() {
@@ -149,7 +139,7 @@ func validateMessageDeclarations(ms []filedesc.Message, mds []*descriptorpb.Desc
return errors.New("message field %q may not have extendee: %q", f.FullName(), fd.GetExtendee())
}
if f.L1.IsProto3Optional {
if f.Syntax() != protoreflect.Proto3 {
if !isProto3 {
return errors.New("message field %q under proto3 optional semantics must be specified in the proto3 syntax", f.FullName())
}
if f.Cardinality() != protoreflect.Optional {
@@ -162,26 +152,29 @@ func validateMessageDeclarations(ms []filedesc.Message, mds []*descriptorpb.Desc
if f.IsWeak() && !flags.ProtoLegacy {
return errors.New("message field %q is a weak field, which is a legacy proto1 feature that is no longer supported", f.FullName())
}
if f.IsWeak() && (f.Syntax() != protoreflect.Proto2 || !isOptionalMessage(f) || f.ContainingOneof() != nil) {
if f.IsWeak() && (!f.HasPresence() || !isOptionalMessage(f) || f.ContainingOneof() != nil) {
return errors.New("message field %q may only be weak for an optional message", f.FullName())
}
if f.IsPacked() && !isPackable(f) {
return errors.New("message field %q is not packable", f.FullName())
}
if err := checkValidGroup(f); err != nil {
if err := checkValidGroup(file, f); err != nil {
return errors.New("message field %q is an invalid group: %v", f.FullName(), err)
}
if err := checkValidMap(f); err != nil {
return errors.New("message field %q is an invalid map: %v", f.FullName(), err)
}
if f.Syntax() == protoreflect.Proto3 {
if isProto3 {
if f.Cardinality() == protoreflect.Required {
return errors.New("message field %q using proto3 semantics cannot be required", f.FullName())
}
if f.Enum() != nil && !f.Enum().IsPlaceholder() && f.Enum().Syntax() != protoreflect.Proto3 {
return errors.New("message field %q using proto3 semantics may only depend on a proto3 enum", f.FullName())
if f.Enum() != nil && !f.Enum().IsPlaceholder() && f.Enum().IsClosed() {
return errors.New("message field %q using proto3 semantics may only depend on open enums", f.FullName())
}
}
if f.Cardinality() == protoreflect.Optional && !f.HasPresence() && f.Enum() != nil && !f.Enum().IsPlaceholder() && f.Enum().IsClosed() {
return errors.New("message field %q with implicit presence may only use open enums", f.FullName())
}
}
seenSynthetic := false // synthetic oneofs for proto3 optional must come after real oneofs
for j := range md.GetOneofDecl() {
@@ -215,17 +208,17 @@ func validateMessageDeclarations(ms []filedesc.Message, mds []*descriptorpb.Desc
if err := validateEnumDeclarations(m.L1.Enums.List, md.GetEnumType()); err != nil {
return err
}
if err := validateMessageDeclarations(m.L1.Messages.List, md.GetNestedType()); err != nil {
if err := validateMessageDeclarations(file, m.L1.Messages.List, md.GetNestedType()); err != nil {
return err
}
if err := validateExtensionDeclarations(m.L1.Extensions.List, md.GetExtension()); err != nil {
if err := validateExtensionDeclarations(file, m.L1.Extensions.List, md.GetExtension()); err != nil {
return err
}
}
return nil
}
func validateExtensionDeclarations(xs []filedesc.Extension, xds []*descriptorpb.FieldDescriptorProto) error {
func validateExtensionDeclarations(f *filedesc.File, xs []filedesc.Extension, xds []*descriptorpb.FieldDescriptorProto) error {
for i, xd := range xds {
x := &xs[i]
// NOTE: Avoid using the IsValid method since extensions to MessageSet
@@ -267,13 +260,13 @@ func validateExtensionDeclarations(xs []filedesc.Extension, xds []*descriptorpb.
if x.IsPacked() && !isPackable(x) {
return errors.New("extension field %q is not packable", x.FullName())
}
if err := checkValidGroup(x); err != nil {
if err := checkValidGroup(f, x); err != nil {
return errors.New("extension field %q is an invalid group: %v", x.FullName(), err)
}
if md := x.Message(); md != nil && md.IsMapEntry() {
return errors.New("extension field %q cannot be a map entry", x.FullName())
}
if x.Syntax() == protoreflect.Proto3 {
if f.L1.Edition == fromEditionProto(descriptorpb.Edition_EDITION_PROTO3) {
switch x.ContainingMessage().FullName() {
case (*descriptorpb.FileOptions)(nil).ProtoReflect().Descriptor().FullName():
case (*descriptorpb.EnumOptions)(nil).ProtoReflect().Descriptor().FullName():
@@ -309,21 +302,25 @@ func isPackable(fd protoreflect.FieldDescriptor) bool {
// checkValidGroup reports whether fd is a valid group according to the same
// rules that protoc imposes.
func checkValidGroup(fd protoreflect.FieldDescriptor) error {
func checkValidGroup(f *filedesc.File, fd protoreflect.FieldDescriptor) error {
md := fd.Message()
switch {
case fd.Kind() != protoreflect.GroupKind:
return nil
case fd.Syntax() == protoreflect.Proto3:
case f.L1.Edition == fromEditionProto(descriptorpb.Edition_EDITION_PROTO3):
return errors.New("invalid under proto3 semantics")
case md == nil || md.IsPlaceholder():
return errors.New("message must be resolvable")
case fd.FullName().Parent() != md.FullName().Parent():
return errors.New("message and field must be declared in the same scope")
case !unicode.IsUpper(rune(md.Name()[0])):
return errors.New("message name must start with an uppercase")
case fd.Name() != protoreflect.Name(strings.ToLower(string(md.Name()))):
return errors.New("field name must be lowercased form of the message name")
}
if f.L1.Edition < fromEditionProto(descriptorpb.Edition_EDITION_2023) {
switch {
case fd.FullName().Parent() != md.FullName().Parent():
return errors.New("message and field must be declared in the same scope")
case !unicode.IsUpper(rune(md.Name()[0])):
return errors.New("message name must start with an uppercase")
case fd.Name() != protoreflect.Name(strings.ToLower(string(md.Name()))):
return errors.New("field name must be lowercased form of the message name")
}
}
return nil
}

View File

@@ -17,11 +17,6 @@ import (
gofeaturespb "google.golang.org/protobuf/types/gofeaturespb"
)
const (
SupportedEditionsMinimum = descriptorpb.Edition_EDITION_PROTO2
SupportedEditionsMaximum = descriptorpb.Edition_EDITION_2023
)
var defaults = &descriptorpb.FeatureSetDefaults{}
var defaultsCacheMu sync.Mutex
var defaultsCache = make(map[filedesc.Edition]*descriptorpb.FeatureSet)
@@ -67,18 +62,20 @@ func getFeatureSetFor(ed filedesc.Edition) *descriptorpb.FeatureSet {
fmt.Fprintf(os.Stderr, "internal error: unsupported edition %v (did you forget to update the embedded defaults (i.e. the bootstrap descriptor proto)?)\n", edpb)
os.Exit(1)
}
fs := defaults.GetDefaults()[0].GetFeatures()
fsed := defaults.GetDefaults()[0]
// Using a linear search for now.
// Editions are guaranteed to be sorted and thus we could use a binary search.
// Given that there are only a handful of editions (with one more per year)
// there is not much reason to use a binary search.
for _, def := range defaults.GetDefaults() {
if def.GetEdition() <= edpb {
fs = def.GetFeatures()
fsed = def
} else {
break
}
}
fs := proto.Clone(fsed.GetFixedFeatures()).(*descriptorpb.FeatureSet)
proto.Merge(fs, fsed.GetOverridableFeatures())
defaultsCache[ed] = fs
return fs
}

View File

@@ -73,6 +73,16 @@ func ToFileDescriptorProto(file protoreflect.FileDescriptor) *descriptorpb.FileD
if syntax := file.Syntax(); syntax != protoreflect.Proto2 && syntax.IsValid() {
p.Syntax = proto.String(file.Syntax().String())
}
if file.Syntax() == protoreflect.Editions {
desc := file
if fileImportDesc, ok := file.(protoreflect.FileImport); ok {
desc = fileImportDesc.FileDescriptor
}
if editionsInterface, ok := desc.(interface{ Edition() int32 }); ok {
p.Edition = descriptorpb.Edition(editionsInterface.Edition()).Enum()
}
}
return p
}
@@ -153,6 +163,18 @@ func ToFieldDescriptorProto(field protoreflect.FieldDescriptor) *descriptorpb.Fi
if field.Syntax() == protoreflect.Proto3 && field.HasOptionalKeyword() {
p.Proto3Optional = proto.Bool(true)
}
if field.Syntax() == protoreflect.Editions {
// Editions have no group keyword, this type is only set so that downstream users continue
// treating this as delimited encoding.
if p.GetType() == descriptorpb.FieldDescriptorProto_TYPE_GROUP {
p.Type = descriptorpb.FieldDescriptorProto_TYPE_MESSAGE.Enum()
}
// Editions have no required keyword, this label is only set so that downstream users continue
// treating it as required.
if p.GetLabel() == descriptorpb.FieldDescriptorProto_LABEL_REQUIRED {
p.Label = descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum()
}
}
if field.HasDefault() {
def, err := defval.Marshal(field.Default(), field.DefaultEnumValue(), field.Kind(), defval.Descriptor)
if err != nil && field.DefaultEnumValue() != nil {