Integrate BACKBEAT SDK and resolve KACHING license validation

Major integrations and fixes:
- Added BACKBEAT SDK integration for P2P operation timing
- Implemented beat-aware status tracking for distributed operations
- Added Docker secrets support for secure license management
- Resolved KACHING license validation via HTTPS/TLS
- Updated docker-compose configuration for clean stack deployment
- Disabled rollback policies to prevent deployment failures
- Added license credential storage (CHORUS-DEV-MULTI-001)

Technical improvements:
- BACKBEAT P2P operation tracking with phase management
- Enhanced configuration system with file-based secrets
- Improved error handling for license validation
- Clean separation of KACHING and CHORUS deployment stacks

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

vendor/go.uber.org/fx/internal/fxclock/clock.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2021 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package fxclock
+
+import (
+	"context"
+	"time"
+)
+
+// Clock defines how Fx accesses time.
+// The interface is pretty minimal but it matches github.com/benbjohnson/clock.
+// We intentionally don't use that interface directly;
+// this keeps it a test dependency for us.
+type Clock interface {
+	Now() time.Time
+	Since(time.Time) time.Duration
+	Sleep(time.Duration)
+	WithTimeout(context.Context, time.Duration) (context.Context, context.CancelFunc)
+}
+
+// System is the default implementation of Clock based on real time.
+var System Clock = systemClock{}
+
+type systemClock struct{}
+
+func (systemClock) Now() time.Time {
+	return time.Now()
+}
+
+func (systemClock) Since(t time.Time) time.Duration {
+	return time.Since(t)
+}
+
+func (systemClock) Sleep(d time.Duration) {
+	time.Sleep(d)
+}
+
+func (systemClock) WithTimeout(ctx context.Context, d time.Duration) (context.Context, context.CancelFunc) {
+	return context.WithTimeout(ctx, d)
+}

vendor/go.uber.org/fx/internal/fxlog/default.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2021 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package fxlog
+
+import (
+	"io"
+
+	"go.uber.org/fx/fxevent"
+)
+
+// DefaultLogger constructs a Logger out of io.Writer.
+func DefaultLogger(w io.Writer) fxevent.Logger {
+	return &fxevent.ConsoleLogger{W: w}
+}

vendor/go.uber.org/fx/internal/fxlog/spy.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2020-2021 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package fxlog
+
+import (
+	"reflect"
+	"sync"
+
+	"go.uber.org/fx/fxevent"
+)
+
+// Events is a list of events captured by fxlog.Spy.
+type Events []fxevent.Event
+
+// Len returns the number of events in this list.
+func (es Events) Len() int { return len(es) }
+
+// SelectByTypeName returns a new list with only events matching the specified
+// type.
+func (es Events) SelectByTypeName(name string) Events {
+	var out Events
+	for _, e := range es {
+		if reflect.TypeOf(e).Elem().Name() == name {
+			out = append(out, e)
+		}
+	}
+	return out
+}
+
+// Spy is an Fx event logger that captures emitted events and/or logged
+// statements. It may be used in tests of Fx logs.
+type Spy struct {
+	mu     sync.RWMutex
+	events Events
+}
+
+var _ fxevent.Logger = &Spy{}
+
+// LogEvent appends an Event.
+func (s *Spy) LogEvent(event fxevent.Event) {
+	s.mu.Lock()
+	s.events = append(s.events, event)
+	s.mu.Unlock()
+}
+
+// Events returns all captured events.
+func (s *Spy) Events() Events {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	events := make(Events, len(s.events))
+	copy(events, s.events)
+	return events
+}
+
+// EventTypes returns all captured event types.
+func (s *Spy) EventTypes() []string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	types := make([]string, len(s.events))
+	for i, e := range s.events {
+		types[i] = reflect.TypeOf(e).Elem().Name()
+	}
+	return types
+}
+
+// Reset clears all messages and events from the Spy.
+func (s *Spy) Reset() {
+	s.mu.Lock()
+	s.events = s.events[:0]
+	s.mu.Unlock()
+}

vendor/go.uber.org/fx/internal/fxreflect/fxreflect.go

@@ -0,0 +1,86 @@
+// Copyright (c) 2019-2021 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package fxreflect
+
+import (
+	"fmt"
+	"net/url"
+	"reflect"
+	"regexp"
+	"runtime"
+	"strings"
+)
+
+// Match from beginning of the line until the first `vendor/` (non-greedy)
+var vendorRe = regexp.MustCompile("^.*?/vendor/")
+
+// sanitize makes the function name suitable for logging display. It removes
+// url-encoded elements from the `dot.git` package names and shortens the
+// vendored paths.
+func sanitize(function string) string {
+	// Use the stdlib to un-escape any package import paths which can happen
+	// in the case of the "dot-git" postfix. Seems like a bug in stdlib =/
+	if unescaped, err := url.QueryUnescape(function); err == nil {
+		function = unescaped
+	}
+
+	// strip everything prior to the vendor
+	return vendorRe.ReplaceAllString(function, "vendor/")
+}
+
+// Caller returns the formatted calling func name
+func Caller() string {
+	return CallerStack(1, 0).CallerName()
+}
+
+// FuncName returns a funcs formatted name
+func FuncName(fn interface{}) string {
+	fnV := reflect.ValueOf(fn)
+	if fnV.Kind() != reflect.Func {
+		return fmt.Sprint(fn)
+	}
+
+	function := runtime.FuncForPC(fnV.Pointer()).Name()
+	return fmt.Sprintf("%s()", sanitize(function))
+}
+
+// Ascend the call stack until we leave the Fx production code. This allows us
+// to avoid hard-coding a frame skip, which makes this code work well even
+// when it's wrapped.
+func shouldIgnoreFrame(f Frame) bool {
+	// Treat test files as leafs.
+	if strings.Contains(f.File, "_test.go") {
+		return false
+	}
+
+	// The unique, fully-qualified name for all functions begins with
+	// "{{importPath}}.". We'll ignore Fx and its subpackages.
+	s := strings.TrimPrefix(f.Function, "go.uber.org/fx")
+	if len(s) > 0 && s[0] == '.' || s[0] == '/' {
+		// We want to match,
+		//   go.uber.org/fx.Foo
+		//   go.uber.org/fx/something.Foo
+		// But not, go.uber.org/fxfoo
+		return true
+	}
+
+	return false
+}

vendor/go.uber.org/fx/internal/fxreflect/stack.go

@@ -0,0 +1,158 @@
+// Copyright (c) 2019 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package fxreflect
+
+import (
+	"fmt"
+	"io"
+	"runtime"
+	"strings"
+)
+
+// Frame holds information about a single frame in the call stack.
+type Frame struct {
+	// Unique, package path-qualified name for the function of this call
+	// frame.
+	Function string
+
+	// File and line number of our location in the frame.
+	//
+	// Note that the line number does not refer to where the function was
+	// defined but where in the function the next call was made.
+	File string
+	Line int
+}
+
+func (f Frame) String() string {
+	// This takes the following forms.
+	//  (path/to/file.go)
+	//  (path/to/file.go:42)
+	//  path/to/package.MyFunction
+	//  path/to/package.MyFunction (path/to/file.go)
+	//  path/to/package.MyFunction (path/to/file.go:42)
+
+	var sb strings.Builder
+	sb.WriteString(f.Function)
+	if len(f.File) > 0 {
+		if sb.Len() > 0 {
+			sb.WriteRune(' ')
+		}
+		fmt.Fprintf(&sb, "(%v", f.File)
+		if f.Line > 0 {
+			fmt.Fprintf(&sb, ":%d", f.Line)
+		}
+		sb.WriteRune(')')
+	}
+
+	if sb.Len() == 0 {
+		return "unknown"
+	}
+
+	return sb.String()
+}
+
+const _defaultCallersDepth = 8
+
+// Stack is a stack of call frames.
+//
+// Formatted with %v, the output is in a single-line, in the form,
+//
+//	foo/bar.Baz() (path/to/foo.go:42); bar/baz.Qux() (bar/baz/qux.go:12); ...
+//
+// Formatted with %+v, the output is in the form,
+//
+//	foo/bar.Baz()
+//		path/to/foo.go:42
+//	bar/baz.Qux()
+//		bar/baz/qux.go:12
+type Stack []Frame
+
+// String returns a single-line, semi-colon representation of a Stack.
+// For a list of strings where each represents one frame, use Strings.
+// For a cleaner multi-line representation, use %+v.
+func (fs Stack) String() string {
+	return strings.Join(fs.Strings(), "; ")
+}
+
+// Strings returns a list of strings, each representing a frame in the stack.
+// Each line will be in the form,
+//
+//	foo/bar.Baz() (path/to/foo.go:42)
+func (fs Stack) Strings() []string {
+	items := make([]string, len(fs))
+	for i, f := range fs {
+		items[i] = f.String()
+	}
+	return items
+}
+
+// Format implements fmt.Formatter to handle "%+v".
+func (fs Stack) Format(w fmt.State, c rune) {
+	if !w.Flag('+') {
+		// Without %+v, fall back to String().
+		io.WriteString(w, fs.String())
+		return
+	}
+
+	for _, f := range fs {
+		fmt.Fprintln(w, f.Function)
+		fmt.Fprintf(w, "\t%v:%v\n", f.File, f.Line)
+	}
+}
+
+// CallerName returns the name of the first caller in this stack that isn't
+// owned by the Fx library.
+func (fs Stack) CallerName() string {
+	for _, f := range fs {
+		if shouldIgnoreFrame(f) {
+			continue
+		}
+		return f.Function
+	}
+	return "n/a"
+}
+
+// CallerStack returns the call stack for the calling function, up to depth frames
+// deep, skipping the provided number of frames, not including Callers itself.
+//
+// If zero, depth defaults to 8.
+func CallerStack(skip, depth int) Stack {
+	if depth <= 0 {
+		depth = _defaultCallersDepth
+	}
+
+	pcs := make([]uintptr, depth)
+
+	// +2 to skip this frame and runtime.Callers.
+	n := runtime.Callers(skip+2, pcs)
+	pcs = pcs[:n] // truncate to number of frames actually read
+
+	result := make([]Frame, 0, n)
+	frames := runtime.CallersFrames(pcs)
+	for f, more := frames.Next(); more; f, more = frames.Next() {
+		result = append(result, Frame{
+			Function: sanitize(f.Function),
+			File:     f.File,
+			Line:     f.Line,
+		})
+	}
+	return result
+}

vendor/go.uber.org/fx/internal/lifecycle/lifecycle.go

@@ -0,0 +1,401 @@
+// Copyright (c) 2022 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"reflect"
+	"strings"
+	"sync"
+	"time"
+
+	"go.uber.org/fx/fxevent"
+	"go.uber.org/fx/internal/fxclock"
+	"go.uber.org/fx/internal/fxreflect"
+	"go.uber.org/multierr"
+)
+
+// Reflection types for each of the supported hook function signatures. These
+// are used in cases in which the Callable constraint matches a user-defined
+// function type that cannot be converted to an underlying function type with
+// a conventional conversion or type switch.
+var (
+	_reflFunc             = reflect.TypeOf(Func(nil))
+	_reflErrorFunc        = reflect.TypeOf(ErrorFunc(nil))
+	_reflContextFunc      = reflect.TypeOf(ContextFunc(nil))
+	_reflContextErrorFunc = reflect.TypeOf(ContextErrorFunc(nil))
+)
+
+// Discrete function signatures that are allowed as part of a [Callable].
+type (
+	// A Func can be converted to a ContextErrorFunc.
+	Func = func()
+	// An ErrorFunc can be converted to a ContextErrorFunc.
+	ErrorFunc = func() error
+	// A ContextFunc can be converted to a ContextErrorFunc.
+	ContextFunc = func(context.Context)
+	// A ContextErrorFunc is used as a [Hook.OnStart] or [Hook.OnStop]
+	// function.
+	ContextErrorFunc = func(context.Context) error
+)
+
+// A Callable is a constraint that matches functions that are, or can be
+// converted to, functions suitable for a Hook.
+//
+// Callable must be identical to [fx.HookFunc].
+type Callable interface {
+	~Func | ~ErrorFunc | ~ContextFunc | ~ContextErrorFunc
+}
+
+// Wrap wraps x into a ContextErrorFunc suitable for a Hook.
+func Wrap[T Callable](x T) (ContextErrorFunc, string) {
+	if x == nil {
+		return nil, ""
+	}
+
+	switch fn := any(x).(type) {
+	case Func:
+		return func(context.Context) error {
+			fn()
+			return nil
+		}, fxreflect.FuncName(x)
+	case ErrorFunc:
+		return func(context.Context) error {
+			return fn()
+		}, fxreflect.FuncName(x)
+	case ContextFunc:
+		return func(ctx context.Context) error {
+			fn(ctx)
+			return nil
+		}, fxreflect.FuncName(x)
+	case ContextErrorFunc:
+		return fn, fxreflect.FuncName(x)
+	}
+
+	// Since (1) we're already using reflect in Fx, (2) we're not particularly
+	// concerned with performance, and (3) unsafe would require discrete build
+	// targets for appengine (etc), just use reflect to convert user-defined
+	// function types to their underlying function types and then call Wrap
+	// again with the converted value.
+	reflVal := reflect.ValueOf(x)
+	switch {
+	case reflVal.CanConvert(_reflFunc):
+		return Wrap(reflVal.Convert(_reflFunc).Interface().(Func))
+	case reflVal.CanConvert(_reflErrorFunc):
+		return Wrap(reflVal.Convert(_reflErrorFunc).Interface().(ErrorFunc))
+	case reflVal.CanConvert(_reflContextFunc):
+		return Wrap(reflVal.Convert(_reflContextFunc).Interface().(ContextFunc))
+	default:
+		// Is already convertible to ContextErrorFunc.
+		return Wrap(reflVal.Convert(_reflContextErrorFunc).Interface().(ContextErrorFunc))
+	}
+}
+
+// A Hook is a pair of start and stop callbacks, either of which can be nil,
+// plus a string identifying the supplier of the hook.
+type Hook struct {
+	OnStart     func(context.Context) error
+	OnStop      func(context.Context) error
+	OnStartName string
+	OnStopName  string
+
+	callerFrame fxreflect.Frame
+}
+
+type appState int
+
+const (
+	stopped appState = iota
+	starting
+	incompleteStart
+	started
+	stopping
+)
+
+func (as appState) String() string {
+	switch as {
+	case stopped:
+		return "stopped"
+	case starting:
+		return "starting"
+	case incompleteStart:
+		return "incompleteStart"
+	case started:
+		return "started"
+	case stopping:
+		return "stopping"
+	default:
+		return "invalidState"
+	}
+}
+
+// Lifecycle coordinates application lifecycle hooks.
+type Lifecycle struct {
+	clock        fxclock.Clock
+	logger       fxevent.Logger
+	state        appState
+	hooks        []Hook
+	numStarted   int
+	startRecords HookRecords
+	stopRecords  HookRecords
+	runningHook  Hook
+	mu           sync.Mutex
+}
+
+// New constructs a new Lifecycle.
+func New(logger fxevent.Logger, clock fxclock.Clock) *Lifecycle {
+	return &Lifecycle{logger: logger, clock: clock}
+}
+
+// Append adds a Hook to the lifecycle.
+func (l *Lifecycle) Append(hook Hook) {
+	// Save the caller's stack frame to report file/line number.
+	if f := fxreflect.CallerStack(2, 0); len(f) > 0 {
+		hook.callerFrame = f[0]
+	}
+	l.hooks = append(l.hooks, hook)
+}
+
+// Start runs all OnStart hooks, returning immediately if it encounters an
+// error.
+func (l *Lifecycle) Start(ctx context.Context) error {
+	if ctx == nil {
+		return errors.New("called OnStart with nil context")
+	}
+
+	l.mu.Lock()
+	if l.state != stopped {
+		defer l.mu.Unlock()
+		return fmt.Errorf("attempted to start lifecycle when in state: %v", l.state)
+	}
+	l.numStarted = 0
+	l.state = starting
+
+	l.startRecords = make(HookRecords, 0, len(l.hooks))
+	l.mu.Unlock()
+
+	var returnState appState = incompleteStart
+	defer func() {
+		l.mu.Lock()
+		l.state = returnState
+		l.mu.Unlock()
+	}()
+
+	for _, hook := range l.hooks {
+		// if ctx has cancelled, bail out of the loop.
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+
+		if hook.OnStart != nil {
+			l.mu.Lock()
+			l.runningHook = hook
+			l.mu.Unlock()
+
+			runtime, err := l.runStartHook(ctx, hook)
+			if err != nil {
+				return err
+			}
+
+			l.mu.Lock()
+			l.startRecords = append(l.startRecords, HookRecord{
+				CallerFrame: hook.callerFrame,
+				Func:        hook.OnStart,
+				Runtime:     runtime,
+			})
+			l.mu.Unlock()
+		}
+		l.numStarted++
+	}
+
+	returnState = started
+	return nil
+}
+
+func (l *Lifecycle) runStartHook(ctx context.Context, hook Hook) (runtime time.Duration, err error) {
+	funcName := hook.OnStartName
+	if len(funcName) == 0 {
+		funcName = fxreflect.FuncName(hook.OnStart)
+	}
+
+	l.logger.LogEvent(&fxevent.OnStartExecuting{
+		CallerName:   hook.callerFrame.Function,
+		FunctionName: funcName,
+	})
+	defer func() {
+		l.logger.LogEvent(&fxevent.OnStartExecuted{
+			CallerName:   hook.callerFrame.Function,
+			FunctionName: funcName,
+			Runtime:      runtime,
+			Err:          err,
+		})
+	}()
+
+	begin := l.clock.Now()
+	err = hook.OnStart(ctx)
+	return l.clock.Since(begin), err
+}
+
+// Stop runs any OnStop hooks whose OnStart counterpart succeeded. OnStop
+// hooks run in reverse order.
+func (l *Lifecycle) Stop(ctx context.Context) error {
+	if ctx == nil {
+		return errors.New("called OnStop with nil context")
+	}
+
+	l.mu.Lock()
+	if l.state != started && l.state != incompleteStart && l.state != starting {
+		defer l.mu.Unlock()
+		return nil
+	}
+	l.state = stopping
+	l.mu.Unlock()
+
+	defer func() {
+		l.mu.Lock()
+		l.state = stopped
+		l.mu.Unlock()
+	}()
+
+	l.mu.Lock()
+	l.stopRecords = make(HookRecords, 0, l.numStarted)
+	// Take a snapshot of hook state to avoid races.
+	allHooks := l.hooks[:]
+	numStarted := l.numStarted
+	l.mu.Unlock()
+
+	// Run backward from last successful OnStart.
+	var errs []error
+	for ; numStarted > 0; numStarted-- {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		hook := allHooks[numStarted-1]
+		if hook.OnStop == nil {
+			continue
+		}
+
+		l.mu.Lock()
+		l.runningHook = hook
+		l.mu.Unlock()
+
+		runtime, err := l.runStopHook(ctx, hook)
+		if err != nil {
+			// For best-effort cleanup, keep going after errors.
+			errs = append(errs, err)
+		}
+
+		l.mu.Lock()
+		l.stopRecords = append(l.stopRecords, HookRecord{
+			CallerFrame: hook.callerFrame,
+			Func:        hook.OnStop,
+			Runtime:     runtime,
+		})
+		l.mu.Unlock()
+	}
+
+	return multierr.Combine(errs...)
+}
+
+func (l *Lifecycle) runStopHook(ctx context.Context, hook Hook) (runtime time.Duration, err error) {
+	funcName := hook.OnStopName
+	if len(funcName) == 0 {
+		funcName = fxreflect.FuncName(hook.OnStop)
+	}
+
+	l.logger.LogEvent(&fxevent.OnStopExecuting{
+		CallerName:   hook.callerFrame.Function,
+		FunctionName: funcName,
+	})
+	defer func() {
+		l.logger.LogEvent(&fxevent.OnStopExecuted{
+			CallerName:   hook.callerFrame.Function,
+			FunctionName: funcName,
+			Runtime:      runtime,
+			Err:          err,
+		})
+	}()
+
+	begin := l.clock.Now()
+	err = hook.OnStop(ctx)
+	return l.clock.Since(begin), err
+}
+
+// RunningHookCaller returns the name of the hook that was running when a Start/Stop
+// hook timed out.
+func (l *Lifecycle) RunningHookCaller() string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.runningHook.callerFrame.Function
+}
+
+// HookRecord keeps track of each Hook's execution time, the caller that appended the Hook, and function that ran as the Hook.
+type HookRecord struct {
+	CallerFrame fxreflect.Frame             // stack frame of the caller
+	Func        func(context.Context) error // function that ran as sanitized name
+	Runtime     time.Duration               // how long the hook ran
+}
+
+// HookRecords is a Stringer wrapper of HookRecord slice.
+type HookRecords []HookRecord
+
+func (rs HookRecords) Len() int {
+	return len(rs)
+}
+
+func (rs HookRecords) Less(i, j int) bool {
+	// Sort by runtime, greater ones at top.
+	return rs[i].Runtime > rs[j].Runtime
+}
+
+func (rs HookRecords) Swap(i, j int) {
+	rs[i], rs[j] = rs[j], rs[i]
+}
+
+// Used for logging startup errors.
+func (rs HookRecords) String() string {
+	var b strings.Builder
+	for _, r := range rs {
+		fmt.Fprintf(&b, "%s took %v from %s",
+			fxreflect.FuncName(r.Func), r.Runtime, r.CallerFrame)
+	}
+	return b.String()
+}
+
+// Format implements fmt.Formatter to handle "%+v".
+func (rs HookRecords) Format(w fmt.State, c rune) {
+	if !w.Flag('+') {
+		// Without %+v, fall back to String().
+		io.WriteString(w, rs.String())
+		return
+	}
+
+	for _, r := range rs {
+		fmt.Fprintf(w, "\n%s took %v from:\n\t%+v",
+			fxreflect.FuncName(r.Func),
+			r.Runtime,
+			r.CallerFrame)
+	}
+	fmt.Fprintf(w, "\n")
+}