Integrate BACKBEAT SDK and resolve KACHING license validation
Major integrations and fixes: - Added BACKBEAT SDK integration for P2P operation timing - Implemented beat-aware status tracking for distributed operations - Added Docker secrets support for secure license management - Resolved KACHING license validation via HTTPS/TLS - Updated docker-compose configuration for clean stack deployment - Disabled rollback policies to prevent deployment failures - Added license credential storage (CHORUS-DEV-MULTI-001) Technical improvements: - BACKBEAT P2P operation tracking with phase management - Enhanced configuration system with file-based secrets - Improved error handling for license validation - Clean separation of KACHING and CHORUS deployment stacks 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
120
vendor/github.com/blevesearch/go-faiss/index_io.go
generated
vendored
Normal file
120
vendor/github.com/blevesearch/go-faiss/index_io.go
generated
vendored
Normal file
@@ -0,0 +1,120 @@
|
||||
package faiss
|
||||
|
||||
/*
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <faiss/c_api/index_io_c.h>
|
||||
#include <faiss/c_api/index_io_c_ex.h>
|
||||
*/
|
||||
import "C"
|
||||
import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// WriteIndex writes an index to a file.
|
||||
func WriteIndex(idx Index, filename string) error {
|
||||
cfname := C.CString(filename)
|
||||
defer C.free(unsafe.Pointer(cfname))
|
||||
if c := C.faiss_write_index_fname(idx.cPtr(), cfname); c != 0 {
|
||||
return getLastError()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func WriteIndexIntoBuffer(idx Index) ([]byte, error) {
|
||||
// the values to be returned by the faiss APIs
|
||||
tempBuf := (*C.uchar)(nil)
|
||||
bufSize := C.size_t(0)
|
||||
|
||||
if c := C.faiss_write_index_buf(
|
||||
idx.cPtr(),
|
||||
&bufSize,
|
||||
&tempBuf,
|
||||
); c != 0 {
|
||||
C.faiss_free_buf(&tempBuf)
|
||||
return nil, getLastError()
|
||||
}
|
||||
|
||||
// at this point, the idx has a valid ref count. furthermore, the index is
|
||||
// something that's present on the C memory space, so not available to go's
|
||||
// GC. needs to be freed when its of no more use.
|
||||
|
||||
// todo: add checksum.
|
||||
// the content populated in the tempBuf is converted from *C.uchar to unsafe.Pointer
|
||||
// and then the pointer is casted into a large byte slice which is then sliced
|
||||
// to a length and capacity equal to bufSize returned across the cgo interface.
|
||||
// NOTE: it still points to the C memory though
|
||||
// the bufSize is of type size_t which is equivalent to a uint in golang, so
|
||||
// the conversion is safe.
|
||||
val := unsafe.Slice((*byte)(unsafe.Pointer(tempBuf)), uint(bufSize))
|
||||
|
||||
// NOTE: This method is compatible with 64-bit systems but may encounter issues on 32-bit systems.
|
||||
// leading to vector indexing being supported only for 64-bit systems.
|
||||
// This limitation arises because the maximum allowed length of a slice on 32-bit systems
|
||||
// is math.MaxInt32 (2^31-1), whereas the maximum value of a size_t in C++ is math.MaxUInt32
|
||||
// (4^31-1), exceeding the maximum allowed size of a slice in Go.
|
||||
// Consequently, the bufSize returned by faiss_write_index_buf might exceed the
|
||||
// maximum allowed size of a slice in Go, leading to a panic when attempting to
|
||||
// create the following slice rv.
|
||||
rv := make([]byte, uint(bufSize))
|
||||
// an explicit copy is necessary to free the memory on C heap and then return
|
||||
// the rv back to the caller which is definitely on goruntime space (which will
|
||||
// GC'd later on).
|
||||
//
|
||||
// an optimization over here - create buffer pool which can be used to make the
|
||||
// memory allocations cheaper. specifically two separate pools can be utilized,
|
||||
// one for C pointers and another for goruntime. within the faiss_write_index_buf
|
||||
// a cheaper calloc rather than malloc can be used to make any extra allocations
|
||||
// cheaper.
|
||||
copy(rv, val)
|
||||
|
||||
// safe to free the c memory allocated (tempBuf) while serializing the index (must be done
|
||||
// within C runtime for it was allocated there);
|
||||
// rv is from go runtime - so different address space altogether
|
||||
C.faiss_free_buf(&tempBuf)
|
||||
|
||||
// p.s: no need to free "val" since the underlying memory is same as tempBuf (deferred free)
|
||||
val = nil
|
||||
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
func ReadIndexFromBuffer(buf []byte, ioflags int) (*IndexImpl, error) {
|
||||
ptr := (*C.uchar)(unsafe.Pointer(&buf[0]))
|
||||
size := C.size_t(len(buf))
|
||||
|
||||
// the idx var has C.FaissIndex within the struct which is nil as of now.
|
||||
var idx faissIndex
|
||||
if c := C.faiss_read_index_buf(ptr,
|
||||
size,
|
||||
C.int(ioflags),
|
||||
&idx.idx); c != 0 {
|
||||
return nil, getLastError()
|
||||
}
|
||||
|
||||
ptr = nil
|
||||
|
||||
// after exiting the faiss_read_index_buf, the ref count to the memory allocated
|
||||
// for the freshly created faiss::index becomes 1 (held by idx.idx of type C.FaissIndex)
|
||||
// this is allocated on the C heap, so not available for golang's GC. hence needs
|
||||
// to be cleaned up after the index is longer being used - to be done at zap layer.
|
||||
return &IndexImpl{&idx}, nil
|
||||
}
|
||||
|
||||
const (
|
||||
IOFlagMmap = C.FAISS_IO_FLAG_MMAP
|
||||
IOFlagReadOnly = C.FAISS_IO_FLAG_READ_ONLY
|
||||
IOFlagReadMmap = C.FAISS_IO_FLAG_READ_MMAP | C.FAISS_IO_FLAG_ONDISK_IVF
|
||||
IOFlagSkipPrefetch = C.FAISS_IO_FLAG_SKIP_PREFETCH
|
||||
)
|
||||
|
||||
// ReadIndex reads an index from a file.
|
||||
func ReadIndex(filename string, ioflags int) (*IndexImpl, error) {
|
||||
cfname := C.CString(filename)
|
||||
defer C.free(unsafe.Pointer(cfname))
|
||||
var idx faissIndex
|
||||
if c := C.faiss_read_index_fname(cfname, C.int(ioflags), &idx.idx); c != 0 {
|
||||
return nil, getLastError()
|
||||
}
|
||||
return &IndexImpl{&idx}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user