Integrate BACKBEAT SDK and resolve KACHING license validation

Major integrations and fixes:
- Added BACKBEAT SDK integration for P2P operation timing
- Implemented beat-aware status tracking for distributed operations
- Added Docker secrets support for secure license management
- Resolved KACHING license validation via HTTPS/TLS
- Updated docker-compose configuration for clean stack deployment
- Disabled rollback policies to prevent deployment failures
- Added license credential storage (CHORUS-DEV-MULTI-001)

Technical improvements:
- BACKBEAT P2P operation tracking with phase management
- Enhanced configuration system with file-based secrets
- Improved error handling for license validation
- Clean separation of KACHING and CHORUS deployment stacks

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

vendor/github.com/blevesearch/go-faiss/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Paul Ouellette
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/blevesearch/go-faiss/README.md

@@ -0,0 +1,43 @@
+# go-faiss
+
+[![Go Reference](https://pkg.go.dev/badge/github.com/DataIntelligenceCrew/go-faiss.svg)](https://pkg.go.dev/github.com/DataIntelligenceCrew/go-faiss)
+
+Go bindings for [Faiss](https://github.com/facebookresearch/faiss), a library for vector similarity search.
+
+## Install
+
+First you will need to build and install Faiss:
+
+```
+git clone https://github.com/blevesearch/faiss.git
+cd faiss
+cmake -B build -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_C_API=ON -DBUILD_SHARED_LIBS=ON .
+make -C build
+sudo make -C build install
+```
+
+On osX ARM64, the instructions needed to be slightly adjusted based on https://github.com/facebookresearch/faiss/issues/2111:
+
+```
+LDFLAGS="-L/opt/homebrew/opt/llvm/lib" CPPFLAGS="-I/opt/homebrew/opt/llvm/include" CXX=/opt/homebrew/opt/llvm/bin/clang++ CC=/opt/homebrew/opt/llvm/bin/clang cmake -B build -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_C_API=ON -DBUILD_SHARED_LIBS=ON .
+// set FAISS_ENABLE_PYTHON to OFF in CMakeLists.txt to ignore libpython dylib
+make -C build
+sudo make -C build install
+```
+
+Building will produce the dynamic library `faiss_c`.
+You will need to install it in a place where your system will find it (e.g. `/usr/local/lib` on mac or `/usr/lib` on Linux).
+You can do this with:
+
+    sudo cp build/c_api/libfaiss_c.so /usr/local/lib
+
+Now you can install the Go module:
+
+    go get github.com/blevesearch/go-faiss
+
+## Usage
+
+API documentation is available at <https://pkg.go.dev/github.com/DataIntelligenceCrew/go-faiss>.
+See the [Faiss wiki](https://github.com/facebookresearch/faiss/wiki) for more information.
+
+Examples can be found in the [_example](_example) directory.

vendor/github.com/blevesearch/go-faiss/autotune.go

@@ -0,0 +1,44 @@
+package faiss
+
+/*
+#include <stdlib.h>
+#include <faiss/c_api/AutoTune_c.h>
+*/
+import "C"
+import (
+	"unsafe"
+)
+
+type ParameterSpace struct {
+	ps *C.FaissParameterSpace
+}
+
+// NewParameterSpace creates a new ParameterSpace.
+func NewParameterSpace() (*ParameterSpace, error) {
+	var ps *C.FaissParameterSpace
+	if c := C.faiss_ParameterSpace_new(&ps); c != 0 {
+		return nil, getLastError()
+	}
+	return &ParameterSpace{ps}, nil
+}
+
+// SetIndexParameter sets one of the parameters.
+func (p *ParameterSpace) SetIndexParameter(idx Index, name string, val float64) error {
+	cname := C.CString(name)
+
+	defer func() {
+		C.free(unsafe.Pointer(cname))
+	}()
+
+	c := C.faiss_ParameterSpace_set_index_parameter(
+		p.ps, idx.cPtr(), cname, C.double(val))
+	if c != 0 {
+		return getLastError()
+	}
+	return nil
+}
+
+// Delete frees the memory associated with p.
+func (p *ParameterSpace) Delete() {
+	C.faiss_ParameterSpace_free(p.ps)
+}

vendor/github.com/blevesearch/go-faiss/faiss.go

@@ -0,0 +1,41 @@
+// Package faiss provides bindings to Faiss, a library for vector similarity
+// search.
+// More detailed documentation can be found at the Faiss wiki:
+// https://github.com/facebookresearch/faiss/wiki.
+package faiss
+
+/*
+#cgo LDFLAGS: -lfaiss_c
+
+#include <faiss/c_api/Index_c.h>
+#include <faiss/c_api/error_c.h>
+#include <faiss/c_api/utils/distances_c.h>
+*/
+import "C"
+import "errors"
+
+func getLastError() error {
+	return errors.New(C.GoString(C.faiss_get_last_error()))
+}
+
+// Metric type
+const (
+	MetricInnerProduct  = C.METRIC_INNER_PRODUCT
+	MetricL2            = C.METRIC_L2
+	MetricL1            = C.METRIC_L1
+	MetricLinf          = C.METRIC_Linf
+	MetricLp            = C.METRIC_Lp
+	MetricCanberra      = C.METRIC_Canberra
+	MetricBrayCurtis    = C.METRIC_BrayCurtis
+	MetricJensenShannon = C.METRIC_JensenShannon
+)
+
+// In-place normalization of provided vector (single)
+func NormalizeVector(vector []float32) []float32 {
+	C.faiss_fvec_renorm_L2(
+		C.size_t(len(vector)),
+		1, // number of vectors
+		(*C.float)(&vector[0]))
+
+	return vector
+}

vendor/github.com/blevesearch/go-faiss/index.go

@@ -0,0 +1,512 @@
+package faiss
+
+/*
+#include <stdlib.h>
+#include <faiss/c_api/Index_c.h>
+#include <faiss/c_api/IndexIVF_c.h>
+#include <faiss/c_api/IndexIVF_c_ex.h>
+#include <faiss/c_api/Index_c_ex.h>
+#include <faiss/c_api/impl/AuxIndexStructures_c.h>
+#include <faiss/c_api/index_factory_c.h>
+#include <faiss/c_api/MetaIndexes_c.h>
+*/
+import "C"
+import (
+	"encoding/json"
+	"fmt"
+	"unsafe"
+)
+
+// Index is a Faiss index.
+//
+// Note that some index implementations do not support all methods.
+// Check the Faiss wiki to see what operations an index supports.
+type Index interface {
+	// D returns the dimension of the indexed vectors.
+	D() int
+
+	// IsTrained returns true if the index has been trained or does not require
+	// training.
+	IsTrained() bool
+
+	// Ntotal returns the number of indexed vectors.
+	Ntotal() int64
+
+	// MetricType returns the metric type of the index.
+	MetricType() int
+
+	// Train trains the index on a representative set of vectors.
+	Train(x []float32) error
+
+	// Add adds vectors to the index.
+	Add(x []float32) error
+
+	// AddWithIDs is like Add, but stores xids instead of sequential IDs.
+	AddWithIDs(x []float32, xids []int64) error
+
+	// Returns true if the index is an IVF index.
+	IsIVFIndex() bool
+
+	// Applicable only to IVF indexes: Returns a map where the keys
+	// are cluster IDs and the values represent the count of input vectors that belong
+	// to each cluster.
+	// This method only considers the given vecIDs and does not account for all
+	// vectors in the index.
+	// Example:
+	// If vecIDs = [1, 2, 3, 4, 5], and:
+	// - Vectors 1 and 2 belong to cluster 1
+	// - Vectors 3, 4, and 5 belong to cluster 2
+	// The output will be: map[1:2, 2:3]
+	ObtainClusterVectorCountsFromIVFIndex(vecIDs []int64) (map[int64]int64, error)
+
+	// Applicable only to IVF indexes: Returns the centroid IDs in decreasing order
+	// of proximity to query 'x' and their distance from 'x'
+	ObtainClustersWithDistancesFromIVFIndex(x []float32, centroidIDs []int64) (
+		[]int64, []float32, error)
+
+	// Search queries the index with the vectors in x.
+	// Returns the IDs of the k nearest neighbors for each query vector and the
+	// corresponding distances.
+	Search(x []float32, k int64) (distances []float32, labels []int64, err error)
+
+	SearchWithoutIDs(x []float32, k int64, exclude []int64, params json.RawMessage) (distances []float32,
+		labels []int64, err error)
+
+	SearchWithIDs(x []float32, k int64, include []int64, params json.RawMessage) (distances []float32,
+		labels []int64, err error)
+
+	// Applicable only to IVF indexes: Search clusters whose IDs are in eligibleCentroidIDs
+	SearchClustersFromIVFIndex(selector Selector, eligibleCentroidIDs []int64,
+		minEligibleCentroids int, k int64, x, centroidDis []float32,
+		params json.RawMessage) ([]float32, []int64, error)
+
+	Reconstruct(key int64) ([]float32, error)
+
+	ReconstructBatch(keys []int64, recons []float32) ([]float32, error)
+
+	MergeFrom(other Index, add_id int64) error
+
+	// RangeSearch queries the index with the vectors in x.
+	// Returns all vectors with distance < radius.
+	RangeSearch(x []float32, radius float32) (*RangeSearchResult, error)
+
+	// Reset removes all vectors from the index.
+	Reset() error
+
+	// RemoveIDs removes the vectors specified by sel from the index.
+	// Returns the number of elements removed and error.
+	RemoveIDs(sel *IDSelector) (int, error)
+
+	// Close frees the memory used by the index.
+	Close()
+
+	// consults the C++ side to get the size of the index
+	Size() uint64
+
+	cPtr() *C.FaissIndex
+}
+
+type faissIndex struct {
+	idx *C.FaissIndex
+}
+
+func (idx *faissIndex) cPtr() *C.FaissIndex {
+	return idx.idx
+}
+
+func (idx *faissIndex) Size() uint64 {
+	size := C.faiss_Index_size(idx.idx)
+	return uint64(size)
+}
+
+func (idx *faissIndex) D() int {
+	return int(C.faiss_Index_d(idx.idx))
+}
+
+func (idx *faissIndex) IsTrained() bool {
+	return C.faiss_Index_is_trained(idx.idx) != 0
+}
+
+func (idx *faissIndex) Ntotal() int64 {
+	return int64(C.faiss_Index_ntotal(idx.idx))
+}
+
+func (idx *faissIndex) MetricType() int {
+	return int(C.faiss_Index_metric_type(idx.idx))
+}
+
+func (idx *faissIndex) Train(x []float32) error {
+	n := len(x) / idx.D()
+	if c := C.faiss_Index_train(idx.idx, C.idx_t(n), (*C.float)(&x[0])); c != 0 {
+		return getLastError()
+	}
+	return nil
+}
+
+func (idx *faissIndex) Add(x []float32) error {
+	n := len(x) / idx.D()
+	if c := C.faiss_Index_add(idx.idx, C.idx_t(n), (*C.float)(&x[0])); c != 0 {
+		return getLastError()
+	}
+	return nil
+}
+
+func (idx *faissIndex) ObtainClusterVectorCountsFromIVFIndex(vecIDs []int64) (map[int64]int64, error) {
+	if !idx.IsIVFIndex() {
+		return nil, fmt.Errorf("index is not an IVF index")
+	}
+	clusterIDs := make([]int64, len(vecIDs))
+	if c := C.faiss_get_lists_for_keys(
+		idx.idx,
+		(*C.idx_t)(unsafe.Pointer(&vecIDs[0])),
+		(C.size_t)(len(vecIDs)),
+		(*C.idx_t)(unsafe.Pointer(&clusterIDs[0])),
+	); c != 0 {
+		return nil, getLastError()
+	}
+	rv := make(map[int64]int64, len(vecIDs))
+	for _, v := range clusterIDs {
+		rv[v]++
+	}
+	return rv, nil
+}
+
+func (idx *faissIndex) IsIVFIndex() bool {
+	if ivfIdx := C.faiss_IndexIVF_cast(idx.cPtr()); ivfIdx == nil {
+		return false
+	}
+	return true
+}
+
+func (idx *faissIndex) ObtainClustersWithDistancesFromIVFIndex(x []float32, centroidIDs []int64) (
+	[]int64, []float32, error) {
+	// Selector to include only the centroids whose IDs are part of 'centroidIDs'.
+	includeSelector, err := NewIDSelectorBatch(centroidIDs)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer includeSelector.Delete()
+
+	params, err := NewSearchParams(idx, json.RawMessage{}, includeSelector.Get(), nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer params.Delete()
+
+	// Populate these with the centroids and their distances.
+	centroids := make([]int64, len(centroidIDs))
+	centroidDistances := make([]float32, len(centroidIDs))
+
+	n := len(x) / idx.D()
+
+	c := C.faiss_Search_closest_eligible_centroids(
+		idx.idx,
+		(C.idx_t)(n),
+		(*C.float)(&x[0]),
+		(C.idx_t)(len(centroidIDs)),
+		(*C.float)(&centroidDistances[0]),
+		(*C.idx_t)(&centroids[0]),
+		params.sp)
+	if c != 0 {
+		return nil, nil, getLastError()
+	}
+
+	return centroids, centroidDistances, nil
+}
+
+func (idx *faissIndex) SearchClustersFromIVFIndex(selector Selector,
+	eligibleCentroidIDs []int64, minEligibleCentroids int, k int64, x,
+	centroidDis []float32, params json.RawMessage) ([]float32, []int64, error) {
+
+	tempParams := &defaultSearchParamsIVF{
+		Nlist: len(eligibleCentroidIDs),
+		// Have to override nprobe so that more clusters will be searched for this
+		// query, if required.
+		Nprobe: minEligibleCentroids,
+	}
+
+	searchParams, err := NewSearchParams(idx, params, selector.Get(), tempParams)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer searchParams.Delete()
+
+	n := len(x) / idx.D()
+
+	distances := make([]float32, int64(n)*k)
+	labels := make([]int64, int64(n)*k)
+
+	effectiveNprobe := getNProbeFromSearchParams(searchParams)
+	eligibleCentroidIDs = eligibleCentroidIDs[:effectiveNprobe]
+	centroidDis = centroidDis[:effectiveNprobe]
+
+	if c := C.faiss_IndexIVF_search_preassigned_with_params(
+		idx.idx,
+		(C.idx_t)(n),
+		(*C.float)(&x[0]),
+		(C.idx_t)(k),
+		(*C.idx_t)(&eligibleCentroidIDs[0]),
+		(*C.float)(&centroidDis[0]),
+		(*C.float)(&distances[0]),
+		(*C.idx_t)(&labels[0]),
+		(C.int)(0),
+		searchParams.sp); c != 0 {
+		return nil, nil, getLastError()
+	}
+
+	return distances, labels, nil
+}
+
+func (idx *faissIndex) AddWithIDs(x []float32, xids []int64) error {
+	n := len(x) / idx.D()
+	if c := C.faiss_Index_add_with_ids(
+		idx.idx,
+		C.idx_t(n),
+		(*C.float)(&x[0]),
+		(*C.idx_t)(&xids[0]),
+	); c != 0 {
+		return getLastError()
+	}
+	return nil
+}
+
+func (idx *faissIndex) Search(x []float32, k int64) (
+	distances []float32, labels []int64, err error,
+) {
+	n := len(x) / idx.D()
+	distances = make([]float32, int64(n)*k)
+	labels = make([]int64, int64(n)*k)
+	if c := C.faiss_Index_search(
+		idx.idx,
+		C.idx_t(n),
+		(*C.float)(&x[0]),
+		C.idx_t(k),
+		(*C.float)(&distances[0]),
+		(*C.idx_t)(&labels[0]),
+	); c != 0 {
+		err = getLastError()
+	}
+
+	return
+}
+
+func (idx *faissIndex) SearchWithoutIDs(x []float32, k int64, exclude []int64, params json.RawMessage) (
+	distances []float32, labels []int64, err error,
+) {
+	if params == nil && len(exclude) == 0 {
+		return idx.Search(x, k)
+	}
+
+	var selector *C.FaissIDSelector
+	if len(exclude) > 0 {
+		excludeSelector, err := NewIDSelectorNot(exclude)
+		if err != nil {
+			return nil, nil, err
+		}
+		selector = excludeSelector.Get()
+		defer excludeSelector.Delete()
+	}
+
+	searchParams, err := NewSearchParams(idx, params, selector, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer searchParams.Delete()
+
+	distances, labels, err = idx.searchWithParams(x, k, searchParams.sp)
+
+	return
+}
+
+func (idx *faissIndex) SearchWithIDs(x []float32, k int64, include []int64,
+	params json.RawMessage) (distances []float32, labels []int64, err error,
+) {
+	includeSelector, err := NewIDSelectorBatch(include)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer includeSelector.Delete()
+
+	searchParams, err := NewSearchParams(idx, params, includeSelector.Get(), nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer searchParams.Delete()
+
+	distances, labels, err = idx.searchWithParams(x, k, searchParams.sp)
+	return
+}
+
+func (idx *faissIndex) Reconstruct(key int64) (recons []float32, err error) {
+	rv := make([]float32, idx.D())
+	if c := C.faiss_Index_reconstruct(
+		idx.idx,
+		C.idx_t(key),
+		(*C.float)(&rv[0]),
+	); c != 0 {
+		err = getLastError()
+	}
+
+	return rv, err
+}
+
+func (idx *faissIndex) ReconstructBatch(keys []int64, recons []float32) ([]float32, error) {
+	var err error
+	n := int64(len(keys))
+	if c := C.faiss_Index_reconstruct_batch(
+		idx.idx,
+		C.idx_t(n),
+		(*C.idx_t)(&keys[0]),
+		(*C.float)(&recons[0]),
+	); c != 0 {
+		err = getLastError()
+	}
+
+	return recons, err
+}
+
+func (i *IndexImpl) MergeFrom(other Index, add_id int64) error {
+	if impl, ok := other.(*IndexImpl); ok {
+		return i.Index.MergeFrom(impl.Index, add_id)
+	}
+	return fmt.Errorf("merge not support")
+}
+
+func (idx *faissIndex) MergeFrom(other Index, add_id int64) (err error) {
+	otherIdx, ok := other.(*faissIndex)
+	if !ok {
+		return fmt.Errorf("merge api not supported")
+	}
+
+	if c := C.faiss_Index_merge_from(
+		idx.idx,
+		otherIdx.idx,
+		(C.idx_t)(add_id),
+	); c != 0 {
+		err = getLastError()
+	}
+
+	return err
+}
+
+func (idx *faissIndex) RangeSearch(x []float32, radius float32) (
+	*RangeSearchResult, error,
+) {
+	n := len(x) / idx.D()
+	var rsr *C.FaissRangeSearchResult
+	if c := C.faiss_RangeSearchResult_new(&rsr, C.idx_t(n)); c != 0 {
+		return nil, getLastError()
+	}
+	if c := C.faiss_Index_range_search(
+		idx.idx,
+		C.idx_t(n),
+		(*C.float)(&x[0]),
+		C.float(radius),
+		rsr,
+	); c != 0 {
+		return nil, getLastError()
+	}
+	return &RangeSearchResult{rsr}, nil
+}
+
+func (idx *faissIndex) Reset() error {
+	if c := C.faiss_Index_reset(idx.idx); c != 0 {
+		return getLastError()
+	}
+	return nil
+}
+
+func (idx *faissIndex) RemoveIDs(sel *IDSelector) (int, error) {
+	var nRemoved C.size_t
+	if c := C.faiss_Index_remove_ids(idx.idx, sel.sel, &nRemoved); c != 0 {
+		return 0, getLastError()
+	}
+	return int(nRemoved), nil
+}
+
+func (idx *faissIndex) Close() {
+	C.faiss_Index_free(idx.idx)
+}
+
+func (idx *faissIndex) searchWithParams(x []float32, k int64, searchParams *C.FaissSearchParameters) (
+	distances []float32, labels []int64, err error,
+) {
+	n := len(x) / idx.D()
+	distances = make([]float32, int64(n)*k)
+	labels = make([]int64, int64(n)*k)
+
+	if c := C.faiss_Index_search_with_params(
+		idx.idx,
+		C.idx_t(n),
+		(*C.float)(&x[0]),
+		C.idx_t(k),
+		searchParams,
+		(*C.float)(&distances[0]),
+		(*C.idx_t)(&labels[0]),
+	); c != 0 {
+		err = getLastError()
+	}
+
+	return
+}
+
+// -----------------------------------------------------------------------------
+
+// RangeSearchResult is the result of a range search.
+type RangeSearchResult struct {
+	rsr *C.FaissRangeSearchResult
+}
+
+// Nq returns the number of queries.
+func (r *RangeSearchResult) Nq() int {
+	return int(C.faiss_RangeSearchResult_nq(r.rsr))
+}
+
+// Lims returns a slice containing start and end indices for queries in the
+// distances and labels slices returned by Labels.
+func (r *RangeSearchResult) Lims() []int {
+	var lims *C.size_t
+	C.faiss_RangeSearchResult_lims(r.rsr, &lims)
+	length := r.Nq() + 1
+	return (*[1 << 30]int)(unsafe.Pointer(lims))[:length:length]
+}
+
+// Labels returns the unsorted IDs and respective distances for each query.
+// The result for query i is labels[lims[i]:lims[i+1]].
+func (r *RangeSearchResult) Labels() (labels []int64, distances []float32) {
+	lims := r.Lims()
+	length := lims[len(lims)-1]
+	var clabels *C.idx_t
+	var cdist *C.float
+	C.faiss_RangeSearchResult_labels(r.rsr, &clabels, &cdist)
+	labels = (*[1 << 30]int64)(unsafe.Pointer(clabels))[:length:length]
+	distances = (*[1 << 30]float32)(unsafe.Pointer(cdist))[:length:length]
+	return
+}
+
+// Delete frees the memory associated with r.
+func (r *RangeSearchResult) Delete() {
+	C.faiss_RangeSearchResult_free(r.rsr)
+}
+
+// IndexImpl is an abstract structure for an index.
+type IndexImpl struct {
+	Index
+}
+
+// IndexFactory builds a composite index.
+// description is a comma-separated list of components.
+func IndexFactory(d int, description string, metric int) (*IndexImpl, error) {
+	cdesc := C.CString(description)
+	defer C.free(unsafe.Pointer(cdesc))
+	var idx faissIndex
+	c := C.faiss_index_factory(&idx.idx, C.int(d), cdesc, C.FaissMetricType(metric))
+	if c != 0 {
+		return nil, getLastError()
+	}
+	return &IndexImpl{&idx}, nil
+}
+
+func SetOMPThreads(n uint) {
+	C.faiss_set_omp_threads(C.uint(n))
+}

vendor/github.com/blevesearch/go-faiss/index_flat.go

@@ -0,0 +1,56 @@
+package faiss
+
+/*
+#include <faiss/c_api/IndexFlat_c.h>
+#include <faiss/c_api/Index_c.h>
+*/
+import "C"
+import "unsafe"
+
+// IndexFlat is an index that stores the full vectors and performs exhaustive
+// search.
+type IndexFlat struct {
+	Index
+}
+
+// NewIndexFlat creates a new flat index.
+func NewIndexFlat(d int, metric int) (*IndexFlat, error) {
+	var idx faissIndex
+	if c := C.faiss_IndexFlat_new_with(
+		&idx.idx,
+		C.idx_t(d),
+		C.FaissMetricType(metric),
+	); c != 0 {
+		return nil, getLastError()
+	}
+	return &IndexFlat{&idx}, nil
+}
+
+// NewIndexFlatIP creates a new flat index with the inner product metric type.
+func NewIndexFlatIP(d int) (*IndexFlat, error) {
+	return NewIndexFlat(d, MetricInnerProduct)
+}
+
+// NewIndexFlatL2 creates a new flat index with the L2 metric type.
+func NewIndexFlatL2(d int) (*IndexFlat, error) {
+	return NewIndexFlat(d, MetricL2)
+}
+
+// Xb returns the index's vectors.
+// The returned slice becomes invalid after any add or remove operation.
+func (idx *IndexFlat) Xb() []float32 {
+	var size C.size_t
+	var ptr *C.float
+	C.faiss_IndexFlat_xb(idx.cPtr(), &ptr, &size)
+	return (*[1 << 30]float32)(unsafe.Pointer(ptr))[:size:size]
+}
+
+// AsFlat casts idx to a flat index.
+// AsFlat panics if idx is not a flat index.
+func (idx *IndexImpl) AsFlat() *IndexFlat {
+	ptr := C.faiss_IndexFlat_cast(idx.cPtr())
+	if ptr == nil {
+		panic("index is not a flat index")
+	}
+	return &IndexFlat{&faissIndex{ptr}}
+}

vendor/github.com/blevesearch/go-faiss/index_io.go

@@ -0,0 +1,120 @@
+package faiss
+
+/*
+#include <stdlib.h>
+#include <stdio.h>
+#include <faiss/c_api/index_io_c.h>
+#include <faiss/c_api/index_io_c_ex.h>
+*/
+import "C"
+import (
+	"unsafe"
+)
+
+// WriteIndex writes an index to a file.
+func WriteIndex(idx Index, filename string) error {
+	cfname := C.CString(filename)
+	defer C.free(unsafe.Pointer(cfname))
+	if c := C.faiss_write_index_fname(idx.cPtr(), cfname); c != 0 {
+		return getLastError()
+	}
+	return nil
+}
+
+func WriteIndexIntoBuffer(idx Index) ([]byte, error) {
+	// the values to be returned by the faiss APIs
+	tempBuf := (*C.uchar)(nil)
+	bufSize := C.size_t(0)
+
+	if c := C.faiss_write_index_buf(
+		idx.cPtr(),
+		&bufSize,
+		&tempBuf,
+	); c != 0 {
+		C.faiss_free_buf(&tempBuf)
+		return nil, getLastError()
+	}
+
+	// at this point, the idx has a valid ref count. furthermore, the index is
+	// something that's present on the C memory space, so not available to go's
+	// GC. needs to be freed when its of no more use.
+
+	// todo: add checksum.
+	// the content populated in the tempBuf is converted from *C.uchar to unsafe.Pointer
+	// and then the pointer is casted into a large byte slice which is then sliced
+	// to a length and capacity equal to bufSize returned across the cgo interface.
+	// NOTE: it still points to the C memory though
+	// the bufSize is of type size_t  which is equivalent to a uint in golang, so
+	// the conversion is safe.
+	val := unsafe.Slice((*byte)(unsafe.Pointer(tempBuf)), uint(bufSize))
+
+	// NOTE: This method is compatible with 64-bit systems but may encounter issues on 32-bit systems.
+	// leading to vector indexing being supported only for 64-bit systems.
+	// This limitation arises because the maximum allowed length of a slice on 32-bit systems
+	// is math.MaxInt32 (2^31-1), whereas the maximum value of a size_t in C++ is math.MaxUInt32
+	// (4^31-1), exceeding the maximum allowed size of a slice in Go.
+	// Consequently, the bufSize returned by faiss_write_index_buf might exceed the
+	// maximum allowed size of a slice in Go, leading to a panic when attempting to
+	// create the following slice rv.
+	rv := make([]byte, uint(bufSize))
+	// an explicit copy is necessary to free the memory on C heap and then return
+	// the rv back to the caller which is definitely on goruntime space (which will
+	// GC'd later on).
+	//
+	// an optimization over here - create buffer pool which can be used to make the
+	// memory allocations cheaper. specifically two separate pools can be utilized,
+	// one for C pointers and another for goruntime. within the faiss_write_index_buf
+	// a cheaper calloc rather than malloc can be used to make any extra allocations
+	// cheaper.
+	copy(rv, val)
+
+	// safe to free the c memory allocated (tempBuf) while serializing the index (must be done
+	// within C runtime for it was allocated there);
+	// rv is from go runtime - so different address space altogether
+	C.faiss_free_buf(&tempBuf)
+
+	// p.s: no need to free "val" since the underlying memory is same as tempBuf (deferred free)
+	val = nil
+
+	return rv, nil
+}
+
+func ReadIndexFromBuffer(buf []byte, ioflags int) (*IndexImpl, error) {
+	ptr := (*C.uchar)(unsafe.Pointer(&buf[0]))
+	size := C.size_t(len(buf))
+
+	// the idx var has C.FaissIndex within the struct which is nil as of now.
+	var idx faissIndex
+	if c := C.faiss_read_index_buf(ptr,
+		size,
+		C.int(ioflags),
+		&idx.idx); c != 0 {
+		return nil, getLastError()
+	}
+
+	ptr = nil
+
+	// after exiting the faiss_read_index_buf, the ref count to the memory allocated
+	// for the freshly created faiss::index becomes 1 (held by idx.idx of type C.FaissIndex)
+	// this is allocated on the C heap, so not available for golang's GC. hence needs
+	// to be cleaned up after the index is longer being used - to be done at zap layer.
+	return &IndexImpl{&idx}, nil
+}
+
+const (
+	IOFlagMmap         = C.FAISS_IO_FLAG_MMAP
+	IOFlagReadOnly     = C.FAISS_IO_FLAG_READ_ONLY
+	IOFlagReadMmap     = C.FAISS_IO_FLAG_READ_MMAP | C.FAISS_IO_FLAG_ONDISK_IVF
+	IOFlagSkipPrefetch = C.FAISS_IO_FLAG_SKIP_PREFETCH
+)
+
+// ReadIndex reads an index from a file.
+func ReadIndex(filename string, ioflags int) (*IndexImpl, error) {
+	cfname := C.CString(filename)
+	defer C.free(unsafe.Pointer(cfname))
+	var idx faissIndex
+	if c := C.faiss_read_index_fname(cfname, C.int(ioflags), &idx.idx); c != 0 {
+		return nil, getLastError()
+	}
+	return &IndexImpl{&idx}, nil
+}

vendor/github.com/blevesearch/go-faiss/index_ivf.go

@@ -0,0 +1,61 @@
+package faiss
+
+/*
+#include <faiss/c_api/IndexIVFFlat_c.h>
+#include <faiss/c_api/MetaIndexes_c.h>
+#include <faiss/c_api/Index_c.h>
+#include <faiss/c_api/IndexIVF_c.h>
+#include <faiss/c_api/IndexIVF_c_ex.h>
+*/
+import "C"
+import (
+	"fmt"
+)
+
+func (idx *IndexImpl) SetDirectMap(mapType int) (err error) {
+
+	ivfPtr := C.faiss_IndexIVF_cast(idx.cPtr())
+	if ivfPtr == nil {
+		return fmt.Errorf("index is not of ivf type")
+	}
+	if c := C.faiss_IndexIVF_set_direct_map(
+		ivfPtr,
+		C.int(mapType),
+	); c != 0 {
+		err = getLastError()
+	}
+	return err
+}
+
+func (idx *IndexImpl) GetSubIndex() (*IndexImpl, error) {
+
+	ptr := C.faiss_IndexIDMap2_cast(idx.cPtr())
+	if ptr == nil {
+		return nil, fmt.Errorf("index is not a id map")
+	}
+
+	subIdx := C.faiss_IndexIDMap2_sub_index(ptr)
+	if subIdx == nil {
+		return nil, fmt.Errorf("couldn't retrieve the sub index")
+	}
+
+	return &IndexImpl{&faissIndex{subIdx}}, nil
+}
+
+// pass nprobe to be set as index time option for IVF indexes only.
+// varying nprobe impacts recall but with an increase in latency.
+func (idx *IndexImpl) SetNProbe(nprobe int32) {
+	ivfPtr := C.faiss_IndexIVF_cast(idx.cPtr())
+	if ivfPtr == nil {
+		return
+	}
+	C.faiss_IndexIVF_set_nprobe(ivfPtr, C.size_t(nprobe))
+}
+
+func (idx *IndexImpl) GetNProbe() int32 {
+	ivfPtr := C.faiss_IndexIVF_cast(idx.cPtr())
+	if ivfPtr == nil {
+		return 0
+	}
+	return int32(C.faiss_IndexIVF_nprobe(ivfPtr))
+}

vendor/github.com/blevesearch/go-faiss/search_params.go

@@ -0,0 +1,113 @@
+package faiss
+
+/*
+#include <faiss/c_api/Index_c.h>
+#include <faiss/c_api/IndexIVF_c.h>
+#include <faiss/c_api/impl/AuxIndexStructures_c.h>
+*/
+import "C"
+import (
+	"encoding/json"
+	"fmt"
+)
+
+type SearchParams struct {
+	sp *C.FaissSearchParameters
+}
+
+// Delete frees the memory associated with s.
+func (s *SearchParams) Delete() {
+	if s == nil || s.sp == nil {
+		return
+	}
+	C.faiss_SearchParameters_free(s.sp)
+}
+
+type searchParamsIVF struct {
+	NprobePct   float32 `json:"ivf_nprobe_pct,omitempty"`
+	MaxCodesPct float32 `json:"ivf_max_codes_pct,omitempty"`
+}
+
+// IVF Parameters used to override the index-time defaults for a specific query.
+// Serve as the 'new' defaults for this query, unless overridden by search-time
+// params.
+type defaultSearchParamsIVF struct {
+	Nprobe int `json:"ivf_nprobe,omitempty"`
+	Nlist  int `json:"ivf_nlist,omitempty"`
+}
+
+func (s *searchParamsIVF) Validate() error {
+	if s.NprobePct < 0 || s.NprobePct > 100 {
+		return fmt.Errorf("invalid IVF search params, ivf_nprobe_pct:%v, "+
+			"should be in range [0, 100]", s.NprobePct)
+	}
+
+	if s.MaxCodesPct < 0 || s.MaxCodesPct > 100 {
+		return fmt.Errorf("invalid IVF search params, ivf_max_codes_pct:%v, "+
+			"should be in range [0, 100]", s.MaxCodesPct)
+	}
+
+	return nil
+}
+
+func getNProbeFromSearchParams(params *SearchParams) int32 {
+	return int32(C.faiss_SearchParametersIVF_nprobe(params.sp))
+}
+
+// Returns a valid SearchParams object,
+// thus caller must clean up the object
+// by invoking Delete() method.
+func NewSearchParams(idx Index, params json.RawMessage, sel *C.FaissIDSelector,
+	defaultParams *defaultSearchParamsIVF) (*SearchParams, error) {
+	rv := &SearchParams{}
+	if c := C.faiss_SearchParameters_new(&rv.sp, sel); c != 0 {
+		return nil, fmt.Errorf("failed to create faiss search params")
+	}
+	// check if the index is IVF and set the search params
+	if ivfIdx := C.faiss_IndexIVF_cast(idx.cPtr()); ivfIdx != nil {
+		rv.sp = C.faiss_SearchParametersIVF_cast(rv.sp)
+		if len(params) == 0 && sel == nil {
+			return rv, nil
+		}
+		var nlist, nprobe, nvecs, maxCodes int
+		nlist = int(C.faiss_IndexIVF_nlist(ivfIdx))
+		nprobe = int(C.faiss_IndexIVF_nprobe(ivfIdx))
+		nvecs = int(C.faiss_Index_ntotal(idx.cPtr()))
+		if defaultParams != nil {
+			if defaultParams.Nlist > 0 {
+				nlist = defaultParams.Nlist
+			}
+			if defaultParams.Nprobe > 0 {
+				nprobe = defaultParams.Nprobe
+			}
+		}
+		var ivfParams searchParamsIVF
+		if len(params) > 0 {
+			if err := json.Unmarshal(params, &ivfParams); err != nil {
+				rv.Delete()
+				return nil, fmt.Errorf("failed to unmarshal IVF search params, "+
+					"err:%v", err)
+			}
+			if err := ivfParams.Validate(); err != nil {
+				rv.Delete()
+				return nil, err
+			}
+		}
+		if ivfParams.NprobePct > 0 {
+			nprobe = max(int(float32(nlist)*(ivfParams.NprobePct/100)), 1)
+		}
+		if ivfParams.MaxCodesPct > 0 {
+			maxCodes = int(float32(nvecs) * (ivfParams.MaxCodesPct / 100))
+		} // else, maxCodes will be set to the default value of 0, which means no limit
+		if c := C.faiss_SearchParametersIVF_new_with(
+			&rv.sp,
+			sel,
+			C.size_t(nprobe),
+			C.size_t(maxCodes),
+		); c != 0 {
+			rv.Delete()
+			return nil, fmt.Errorf("failed to create faiss IVF search params")
+		}
+	}
+	return rv, nil
+}

vendor/github.com/blevesearch/go-faiss/selector.go

@@ -0,0 +1,95 @@
+package faiss
+
+/*
+#include <faiss/c_api/impl/AuxIndexStructures_c.h>
+*/
+import "C"
+
+type Selector interface {
+	Get() *C.FaissIDSelector
+	Delete()
+}
+
+// IDSelector represents a set of IDs to remove.
+type IDSelector struct {
+	sel *C.FaissIDSelector
+}
+
+// Delete frees the memory associated with s.
+func (s *IDSelector) Delete() {
+	if s == nil || s.sel == nil {
+		return
+	}
+
+	C.faiss_IDSelector_free(s.sel)
+}
+
+func (s *IDSelector) Get() *C.FaissIDSelector {
+	return s.sel
+}
+
+type IDSelectorNot struct {
+	sel      *C.FaissIDSelector
+	batchSel *C.FaissIDSelector
+}
+
+// Delete frees the memory associated with s.
+func (s *IDSelectorNot) Delete() {
+	if s == nil {
+		return
+	}
+
+	if s.sel != nil {
+		C.faiss_IDSelector_free(s.sel)
+	}
+	if s.batchSel != nil {
+		C.faiss_IDSelector_free(s.batchSel)
+	}
+}
+
+func (s *IDSelectorNot) Get() *C.FaissIDSelector {
+	return s.sel
+}
+
+// NewIDSelectorRange creates a selector that removes IDs on [imin, imax).
+func NewIDSelectorRange(imin, imax int64) (Selector, error) {
+	var sel *C.FaissIDSelectorRange
+	c := C.faiss_IDSelectorRange_new(&sel, C.idx_t(imin), C.idx_t(imax))
+	if c != 0 {
+		return nil, getLastError()
+	}
+	return &IDSelector{(*C.FaissIDSelector)(sel)}, nil
+}
+
+// NewIDSelectorBatch creates a new batch selector.
+func NewIDSelectorBatch(indices []int64) (Selector, error) {
+	var sel *C.FaissIDSelectorBatch
+	if c := C.faiss_IDSelectorBatch_new(
+		&sel,
+		C.size_t(len(indices)),
+		(*C.idx_t)(&indices[0]),
+	); c != 0 {
+		return nil, getLastError()
+	}
+	return &IDSelector{(*C.FaissIDSelector)(sel)}, nil
+}
+
+// NewIDSelectorNot creates a new Not selector, wrapped around a
+// batch selector, with the IDs in 'exclude'.
+func NewIDSelectorNot(exclude []int64) (Selector, error) {
+	batchSelector, err := NewIDSelectorBatch(exclude)
+	if err != nil {
+		return nil, err
+	}
+
+	var sel *C.FaissIDSelectorNot
+	if c := C.faiss_IDSelectorNot_new(
+		&sel,
+		batchSelector.Get(),
+	); c != 0 {
+		batchSelector.Delete()
+		return nil, getLastError()
+	}
+	return &IDSelectorNot{sel: (*C.FaissIDSelector)(sel),
+		batchSel: batchSelector.Get()}, nil
+}