Major integrations and fixes: - Added BACKBEAT SDK integration for P2P operation timing - Implemented beat-aware status tracking for distributed operations - Added Docker secrets support for secure license management - Resolved KACHING license validation via HTTPS/TLS - Updated docker-compose configuration for clean stack deployment - Disabled rollback policies to prevent deployment failures - Added license credential storage (CHORUS-DEV-MULTI-001) Technical improvements: - BACKBEAT P2P operation tracking with phase management - Enhanced configuration system with file-based secrets - Improved error handling for license validation - Clean separation of KACHING and CHORUS deployment stacks 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
123 lines
5.0 KiB
Go
123 lines
5.0 KiB
Go
package storage
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
)
|
|
|
|
/*
|
|
This file contains equivalents of every method that can be feature-detected on a storage system.
|
|
You can always call these functions, and give them the most basic storage interface,
|
|
and they'll attempt to feature-detect their way to the best possible implementation of the behavior,
|
|
or they'll fall back to synthesizing the same behavior from more basic interfaces.
|
|
|
|
Long story short: you can always use these functions as an end user, and get the behavior you want --
|
|
regardless of how much explicit support the storage implementation has for the exact behavior you requested.
|
|
*/
|
|
|
|
func Has(ctx context.Context, store Storage, key string) (bool, error) {
|
|
// Okay, not much going on here -- this function is only here for consistency of style.
|
|
return store.Has(ctx, key)
|
|
}
|
|
|
|
func Get(ctx context.Context, store ReadableStorage, key string) ([]byte, error) {
|
|
// Okay, not much going on here -- this function is only here for consistency of style.
|
|
return store.Get(ctx, key)
|
|
}
|
|
|
|
func Put(ctx context.Context, store WritableStorage, key string, content []byte) error {
|
|
// Okay, not much going on here -- this function is only here for consistency of style.
|
|
return store.Put(ctx, key, content)
|
|
}
|
|
|
|
// GetStream returns a streaming reader.
|
|
// This function will feature-detect the StreamingReadableStorage interface, and use that if possible;
|
|
// otherwise it will fall back to using basic ReadableStorage methods transparently
|
|
// (at the cost of loading all the data into memory at once and up front).
|
|
func GetStream(ctx context.Context, store ReadableStorage, key string) (io.ReadCloser, error) {
|
|
// Prefer the feature itself, first.
|
|
if streamable, ok := store.(StreamingReadableStorage); ok {
|
|
return streamable.GetStream(ctx, key)
|
|
}
|
|
// Fallback to basic.
|
|
blob, err := store.Get(ctx, key)
|
|
return noopCloser{bytes.NewReader(blob)}, err
|
|
}
|
|
|
|
// PutStream returns an io.Writer and a WriteCommitter callback.
|
|
// (See the docs on StreamingWritableStorage.PutStream for details on what that means.)
|
|
// This function will feature-detect the StreamingWritableStorage interface, and use that if possible;
|
|
// otherwise it will fall back to using basic WritableStorage methods transparently
|
|
// (at the cost of needing to buffer all of the content in memory while the write is in progress).
|
|
func PutStream(ctx context.Context, store WritableStorage) (io.Writer, func(key string) error, error) {
|
|
// Prefer the feature itself, first.
|
|
if streamable, ok := store.(StreamingWritableStorage); ok {
|
|
return streamable.PutStream(ctx)
|
|
}
|
|
// Fallback to basic.
|
|
var buf bytes.Buffer
|
|
var written bool
|
|
return &buf, func(key string) error {
|
|
if written {
|
|
return fmt.Errorf("WriteCommitter already used")
|
|
}
|
|
written = true
|
|
return store.Put(ctx, key, buf.Bytes())
|
|
}, nil
|
|
}
|
|
|
|
// PutVec is an API for writing several slices of bytes at once into storage.
|
|
// This kind of API can be useful for maximizing performance in scenarios where
|
|
// data is already loaded completely into memory, but scattered across several non-contiguous regions.
|
|
// This function will feature-detect the VectorWritableStorage interface, and use that if possible;
|
|
// otherwise it will fall back to using StreamingWritableStorage,
|
|
// or failing that, fall further back to basic WritableStorage methods, transparently.
|
|
func PutVec(ctx context.Context, store WritableStorage, key string, blobVec [][]byte) error {
|
|
// Prefer the feature itself, first.
|
|
if putvable, ok := store.(VectorWritableStorage); ok {
|
|
return putvable.PutVec(ctx, key, blobVec)
|
|
}
|
|
// Fallback to streaming mode.
|
|
// ... or, fallback to basic, and use emulated streaming. Still presumably preferable to doing a big giant memcopy.
|
|
// Conveniently, the PutStream function makes that transparent for our implementation, too.
|
|
wr, wrcommit, err := PutStream(ctx, store)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, blob := range blobVec {
|
|
_, err := wr.Write(blob)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return wrcommit(key)
|
|
}
|
|
|
|
// Peek accessess the same data as Get, but indicates that the caller promises not to mutate the returned byte slice.
|
|
// (By contrast, Get is expected to return a safe copy.)
|
|
// This function will feature-detect the PeekableStorage interface, and use that if possible;
|
|
// otherwise it will fall back to using basic ReadableStorage methods transparently
|
|
// (meaning that a no-copy fastpath simply wasn't available).
|
|
//
|
|
// An io.Closer is returned along with the byte slice.
|
|
// The Close method on the Closer must be called when the caller is done with the byte slice;
|
|
// otherwise, memory leaks may result.
|
|
// (Implementers of this interface may be expecting to reuse the byte slice after Close is called.)
|
|
func Peek(ctx context.Context, store ReadableStorage, key string) ([]byte, io.Closer, error) {
|
|
// Prefer the feature itself, first.
|
|
if peekable, ok := store.(PeekableStorage); ok {
|
|
return peekable.Peek(ctx, key)
|
|
}
|
|
// Fallback to basic.
|
|
bs, err := store.Get(ctx, key)
|
|
return bs, noopCloser{nil}, err
|
|
}
|
|
|
|
type noopCloser struct {
|
|
io.Reader
|
|
}
|
|
|
|
func (noopCloser) Close() error { return nil }
|