tony/CHORUS
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

chore: align slurp config and scaffolding

Browse Source
This commit is contained in:
anthonyrawlins
2025-09-27 21:03:12 +10:00
parent acc4361463
commit 4a77862289
47 changed files with 5133 additions and 4274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
pkg/slurp/storage/index_manager.go
View File

@@ -9,22 +9,23 @@ import (
"sync"
"time"
slurpContext "chorus/pkg/slurp/context"
"chorus/pkg/ucxl"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
"github.com/blevesearch/bleve/v2/analysis/lang/en"
"github.com/blevesearch/bleve/v2/mapping"
"chorus/pkg/ucxl"
slurpContext "chorus/pkg/slurp/context"
"github.com/blevesearch/bleve/v2/search/query"
)
// IndexManagerImpl implements the IndexManager interface using Bleve
type IndexManagerImpl struct {
mu sync.RWMutex
indexes map[string]bleve.Index
stats map[string]*IndexStatistics
basePath string
nodeID string
options *IndexManagerOptions
mu sync.RWMutex
indexes map[string]bleve.Index
stats map[string]*IndexStatistics
basePath string
nodeID string
options *IndexManagerOptions
}
// IndexManagerOptions configures index manager behavior
@@ -60,11 +61,11 @@ func NewIndexManager(basePath, nodeID string, options *IndexManagerOptions) (*In
}
im := &IndexManagerImpl{
indexes: make(map[string]bleve.Index),
stats: make(map[string]*IndexStatistics),
basePath: basePath,
nodeID: nodeID,
options: options,
indexes: make(map[string]bleve.Index),
stats: make(map[string]*IndexStatistics),
basePath: basePath,
nodeID: nodeID,
options: options,
}
// Start background optimization if enabled
@@ -356,11 +357,11 @@ func (im *IndexManagerImpl) createIndexMapping(config *IndexConfig) (mapping.Ind
fieldMapping.Analyzer = analyzer
fieldMapping.Store = true
fieldMapping.Index = true
if im.options.EnableHighlighting {
fieldMapping.IncludeTermVectors = true
}
docMapping.AddFieldMappingsAt(field, fieldMapping)
}
@@ -432,31 +433,31 @@ func (im *IndexManagerImpl) createIndexDocument(data interface{}) (map[string]in
return doc, nil
}
func (im *IndexManagerImpl) buildSearchRequest(query *SearchQuery) (*bleve.SearchRequest, error) {
// Build Bleve search request from our search query
var bleveQuery bleve.Query
func (im *IndexManagerImpl) buildSearchRequest(searchQuery *SearchQuery) (*bleve.SearchRequest, error) {
// Build Bleve search request from our search query (SEC-SLURP-1.1 search path)
var bleveQuery query.Query
if query.Query == "" {
if searchQuery.Query == "" {
// Match all query
bleveQuery = bleve.NewMatchAllQuery()
} else {
// Text search query
if query.FuzzyMatch {
if searchQuery.FuzzyMatch {
// Use fuzzy query
bleveQuery = bleve.NewFuzzyQuery(query.Query)
bleveQuery = bleve.NewFuzzyQuery(searchQuery.Query)
} else {
// Use match query for better scoring
bleveQuery = bleve.NewMatchQuery(query.Query)
bleveQuery = bleve.NewMatchQuery(searchQuery.Query)
}
}
// Add filters
var conjuncts []bleve.Query
var conjuncts []query.Query
conjuncts = append(conjuncts, bleveQuery)
// Technology filters
if len(query.Technologies) > 0 {
for _, tech := range query.Technologies {
if len(searchQuery.Technologies) > 0 {
for _, tech := range searchQuery.Technologies {
techQuery := bleve.NewTermQuery(tech)
techQuery.SetField("technologies_facet")
conjuncts = append(conjuncts, techQuery)
@@ -464,8 +465,8 @@ func (im *IndexManagerImpl) buildSearchRequest(query *SearchQuery) (*bleve.Searc
}
// Tag filters
if len(query.Tags) > 0 {
for _, tag := range query.Tags {
if len(searchQuery.Tags) > 0 {
for _, tag := range searchQuery.Tags {
tagQuery := bleve.NewTermQuery(tag)
tagQuery.SetField("tags_facet")
conjuncts = append(conjuncts, tagQuery)
@@ -479,20 +480,20 @@ func (im *IndexManagerImpl) buildSearchRequest(query *SearchQuery) (*bleve.Searc
// Create search request
searchRequest := bleve.NewSearchRequest(bleveQuery)
// Set result options
if query.Limit > 0 && query.Limit <= im.options.MaxResults {
searchRequest.Size = query.Limit
if searchQuery.Limit > 0 && searchQuery.Limit <= im.options.MaxResults {
searchRequest.Size = searchQuery.Limit
} else {
searchRequest.Size = im.options.MaxResults
}
if query.Offset > 0 {
searchRequest.From = query.Offset
if searchQuery.Offset > 0 {
searchRequest.From = searchQuery.Offset
}
// Enable highlighting if requested
if query.HighlightTerms && im.options.EnableHighlighting {
if searchQuery.HighlightTerms && im.options.EnableHighlighting {
searchRequest.Highlight = bleve.NewHighlight()
searchRequest.Highlight.AddField("content")
searchRequest.Highlight.AddField("summary")
@@ -500,9 +501,9 @@ func (im *IndexManagerImpl) buildSearchRequest(query *SearchQuery) (*bleve.Searc
}
// Add facets if requested
if len(query.Facets) > 0 && im.options.EnableFaceting {
if len(searchQuery.Facets) > 0 && im.options.EnableFaceting {
searchRequest.Facets = make(bleve.FacetsRequest)
for _, facet := range query.Facets {
for _, facet := range searchQuery.Facets {
switch facet {
case "technologies":
searchRequest.Facets["technologies"] = bleve.NewFacetRequest("technologies_facet", 10)
@@ -535,7 +536,7 @@ func (im *IndexManagerImpl) convertSearchResults(
searchHit := &SearchResult{
MatchScore: hit.Score,
MatchedFields: make([]string, 0),
Highlights: make(map[string][]string),
Highlights: make(map[string][]string),
Rank: i + 1,
}
@@ -558,8 +559,8 @@ func (im *IndexManagerImpl) convertSearchResults(
// Parse UCXL address
if ucxlStr, ok := hit.Fields["ucxl_address"].(string); ok {
if addr, err := ucxl.ParseAddress(ucxlStr); err == nil {
contextNode.UCXLAddress = addr
if addr, err := ucxl.Parse(ucxlStr); err == nil {
contextNode.UCXLAddress = *addr
}
}
@@ -572,8 +573,10 @@ func (im *IndexManagerImpl) convertSearchResults(
results.Facets = make(map[string]map[string]int)
for facetName, facetResult := range searchResult.Facets {
facetCounts := make(map[string]int)
for _, term := range facetResult.Terms {
facetCounts[term.Term] = term.Count
if facetResult.Terms != nil {
for _, term := range facetResult.Terms.Terms() {
facetCounts[term.Term] = term.Count
}
}
results.Facets[facetName] = facetCounts
}