Integrate BACKBEAT SDK and resolve KACHING license validation
Major integrations and fixes: - Added BACKBEAT SDK integration for P2P operation timing - Implemented beat-aware status tracking for distributed operations - Added Docker secrets support for secure license management - Resolved KACHING license validation via HTTPS/TLS - Updated docker-compose configuration for clean stack deployment - Disabled rollback policies to prevent deployment failures - Added license credential storage (CHORUS-DEV-MULTI-001) Technical improvements: - BACKBEAT P2P operation tracking with phase management - Enhanced configuration system with file-based secrets - Improved error handling for license validation - Clean separation of KACHING and CHORUS deployment stacks 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
53
vendor/github.com/blevesearch/bleve/v2/search/searcher/optimize_knn.go
generated
vendored
Normal file
53
vendor/github.com/blevesearch/bleve/v2/search/searcher/optimize_knn.go
generated
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
// Copyright (c) 2023 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build vectors
|
||||
// +build vectors
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
func optimizeKNN(ctx context.Context, indexReader index.IndexReader,
|
||||
qsearchers []search.Searcher) error {
|
||||
var octx index.VectorOptimizableContext
|
||||
var err error
|
||||
|
||||
for _, searcher := range qsearchers {
|
||||
// Only applicable to KNN Searchers.
|
||||
o, ok := searcher.(index.VectorOptimizable)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
octx, err = o.VectorOptimize(ctx, octx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// No KNN searchers.
|
||||
if octx == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Postings lists and iterators replaced in the pointer to the
|
||||
// vector reader
|
||||
return octx.Finish()
|
||||
}
|
||||
31
vendor/github.com/blevesearch/bleve/v2/search/searcher/optimize_no_knn.go
generated
vendored
Normal file
31
vendor/github.com/blevesearch/bleve/v2/search/searcher/optimize_no_knn.go
generated
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
// Copyright (c) 2023 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build !vectors
|
||||
// +build !vectors
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
func optimizeKNN(ctx context.Context, indexReader index.IndexReader,
|
||||
qsearchers []search.Searcher) error {
|
||||
// No-op
|
||||
return nil
|
||||
}
|
||||
55
vendor/github.com/blevesearch/bleve/v2/search/searcher/ordered_searchers_list.go
generated
vendored
Normal file
55
vendor/github.com/blevesearch/bleve/v2/search/searcher/ordered_searchers_list.go
generated
vendored
Normal file
@@ -0,0 +1,55 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
)
|
||||
|
||||
type OrderedSearcherList []search.Searcher
|
||||
|
||||
// sort.Interface
|
||||
|
||||
func (otrl OrderedSearcherList) Len() int {
|
||||
return len(otrl)
|
||||
}
|
||||
|
||||
func (otrl OrderedSearcherList) Less(i, j int) bool {
|
||||
return otrl[i].Count() < otrl[j].Count()
|
||||
}
|
||||
|
||||
func (otrl OrderedSearcherList) Swap(i, j int) {
|
||||
otrl[i], otrl[j] = otrl[j], otrl[i]
|
||||
}
|
||||
|
||||
type OrderedPositionalSearcherList struct {
|
||||
searchers []search.Searcher
|
||||
index []int
|
||||
}
|
||||
|
||||
// sort.Interface
|
||||
|
||||
func (otrl OrderedPositionalSearcherList) Len() int {
|
||||
return len(otrl.searchers)
|
||||
}
|
||||
|
||||
func (otrl OrderedPositionalSearcherList) Less(i, j int) bool {
|
||||
return otrl.searchers[i].Count() < otrl.searchers[j].Count()
|
||||
}
|
||||
|
||||
func (otrl OrderedPositionalSearcherList) Swap(i, j int) {
|
||||
otrl.searchers[i], otrl.searchers[j] = otrl.searchers[j], otrl.searchers[i]
|
||||
otrl.index[i], otrl.index[j] = otrl.index[j], otrl.index[i]
|
||||
}
|
||||
451
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_boolean.go
generated
vendored
Normal file
451
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_boolean.go
generated
vendored
Normal file
@@ -0,0 +1,451 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/scorer"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeBooleanSearcher int
|
||||
|
||||
func init() {
|
||||
var bs BooleanSearcher
|
||||
reflectStaticSizeBooleanSearcher = int(reflect.TypeOf(bs).Size())
|
||||
}
|
||||
|
||||
type BooleanSearcher struct {
|
||||
indexReader index.IndexReader
|
||||
mustSearcher search.Searcher
|
||||
shouldSearcher search.Searcher
|
||||
mustNotSearcher search.Searcher
|
||||
queryNorm float64
|
||||
currMust *search.DocumentMatch
|
||||
currShould *search.DocumentMatch
|
||||
currMustNot *search.DocumentMatch
|
||||
currentID index.IndexInternalID
|
||||
min uint64
|
||||
scorer *scorer.ConjunctionQueryScorer
|
||||
matches []*search.DocumentMatch
|
||||
initialized bool
|
||||
done bool
|
||||
}
|
||||
|
||||
func NewBooleanSearcher(ctx context.Context, indexReader index.IndexReader, mustSearcher search.Searcher, shouldSearcher search.Searcher, mustNotSearcher search.Searcher, options search.SearcherOptions) (*BooleanSearcher, error) {
|
||||
// build our searcher
|
||||
rv := BooleanSearcher{
|
||||
indexReader: indexReader,
|
||||
mustSearcher: mustSearcher,
|
||||
shouldSearcher: shouldSearcher,
|
||||
mustNotSearcher: mustNotSearcher,
|
||||
scorer: scorer.NewConjunctionQueryScorer(options),
|
||||
matches: make([]*search.DocumentMatch, 2),
|
||||
}
|
||||
rv.computeQueryNorm()
|
||||
return &rv, nil
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) Size() int {
|
||||
sizeInBytes := reflectStaticSizeBooleanSearcher + size.SizeOfPtr
|
||||
|
||||
if s.mustSearcher != nil {
|
||||
sizeInBytes += s.mustSearcher.Size()
|
||||
}
|
||||
|
||||
if s.shouldSearcher != nil {
|
||||
sizeInBytes += s.shouldSearcher.Size()
|
||||
}
|
||||
|
||||
if s.mustNotSearcher != nil {
|
||||
sizeInBytes += s.mustNotSearcher.Size()
|
||||
}
|
||||
|
||||
sizeInBytes += s.scorer.Size()
|
||||
|
||||
for _, entry := range s.matches {
|
||||
if entry != nil {
|
||||
sizeInBytes += entry.Size()
|
||||
}
|
||||
}
|
||||
|
||||
return sizeInBytes
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) computeQueryNorm() {
|
||||
// first calculate sum of squared weights
|
||||
sumOfSquaredWeights := 0.0
|
||||
if s.mustSearcher != nil {
|
||||
sumOfSquaredWeights += s.mustSearcher.Weight()
|
||||
}
|
||||
if s.shouldSearcher != nil {
|
||||
sumOfSquaredWeights += s.shouldSearcher.Weight()
|
||||
}
|
||||
|
||||
// now compute query norm from this
|
||||
s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)
|
||||
// finally tell all the downstream searchers the norm
|
||||
if s.mustSearcher != nil {
|
||||
s.mustSearcher.SetQueryNorm(s.queryNorm)
|
||||
}
|
||||
if s.shouldSearcher != nil {
|
||||
s.shouldSearcher.SetQueryNorm(s.queryNorm)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) initSearchers(ctx *search.SearchContext) error {
|
||||
var err error
|
||||
// get all searchers pointing at their first match
|
||||
if s.mustSearcher != nil {
|
||||
if s.currMust != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currMust)
|
||||
}
|
||||
s.currMust, err = s.mustSearcher.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if s.shouldSearcher != nil {
|
||||
if s.currShould != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currShould)
|
||||
}
|
||||
s.currShould, err = s.shouldSearcher.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if s.mustNotSearcher != nil {
|
||||
if s.currMustNot != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currMustNot)
|
||||
}
|
||||
s.currMustNot, err = s.mustNotSearcher.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if s.mustSearcher != nil && s.currMust != nil {
|
||||
s.currentID = s.currMust.IndexInternalID
|
||||
} else if s.mustSearcher == nil && s.currShould != nil {
|
||||
s.currentID = s.currShould.IndexInternalID
|
||||
} else {
|
||||
s.currentID = nil
|
||||
}
|
||||
|
||||
s.initialized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) advanceNextMust(ctx *search.SearchContext, skipReturn *search.DocumentMatch) error {
|
||||
var err error
|
||||
|
||||
if s.mustSearcher != nil {
|
||||
if s.currMust != skipReturn {
|
||||
ctx.DocumentMatchPool.Put(s.currMust)
|
||||
}
|
||||
s.currMust, err = s.mustSearcher.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if s.currShould != skipReturn {
|
||||
ctx.DocumentMatchPool.Put(s.currShould)
|
||||
}
|
||||
s.currShould, err = s.shouldSearcher.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if s.mustSearcher != nil && s.currMust != nil {
|
||||
s.currentID = s.currMust.IndexInternalID
|
||||
} else if s.mustSearcher == nil && s.currShould != nil {
|
||||
s.currentID = s.currShould.IndexInternalID
|
||||
} else {
|
||||
s.currentID = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) Weight() float64 {
|
||||
var rv float64
|
||||
if s.mustSearcher != nil {
|
||||
rv += s.mustSearcher.Weight()
|
||||
}
|
||||
if s.shouldSearcher != nil {
|
||||
rv += s.shouldSearcher.Weight()
|
||||
}
|
||||
|
||||
return rv
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) SetQueryNorm(qnorm float64) {
|
||||
if s.mustSearcher != nil {
|
||||
s.mustSearcher.SetQueryNorm(qnorm)
|
||||
}
|
||||
if s.shouldSearcher != nil {
|
||||
s.shouldSearcher.SetQueryNorm(qnorm)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
|
||||
if s.done {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var err error
|
||||
var rv *search.DocumentMatch
|
||||
|
||||
for s.currentID != nil {
|
||||
if s.currMustNot != nil {
|
||||
cmp := s.currMustNot.IndexInternalID.Compare(s.currentID)
|
||||
if cmp < 0 {
|
||||
ctx.DocumentMatchPool.Put(s.currMustNot)
|
||||
// advance must not searcher to our candidate entry
|
||||
s.currMustNot, err = s.mustNotSearcher.Advance(ctx, s.currentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.currMustNot != nil && s.currMustNot.IndexInternalID.Equals(s.currentID) {
|
||||
// the candidate is excluded
|
||||
err = s.advanceNextMust(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
} else if cmp == 0 {
|
||||
// the candidate is excluded
|
||||
err = s.advanceNextMust(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
shouldCmpOrNil := 1 // NOTE: shouldCmp will also be 1 when currShould == nil.
|
||||
if s.currShould != nil {
|
||||
shouldCmpOrNil = s.currShould.IndexInternalID.Compare(s.currentID)
|
||||
}
|
||||
|
||||
if shouldCmpOrNil < 0 {
|
||||
ctx.DocumentMatchPool.Put(s.currShould)
|
||||
// advance should searcher to our candidate entry
|
||||
s.currShould, err = s.shouldSearcher.Advance(ctx, s.currentID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.currShould != nil && s.currShould.IndexInternalID.Equals(s.currentID) {
|
||||
// score bonus matches should
|
||||
var cons []*search.DocumentMatch
|
||||
if s.currMust != nil {
|
||||
cons = s.matches
|
||||
cons[0] = s.currMust
|
||||
cons[1] = s.currShould
|
||||
} else {
|
||||
cons = s.matches[0:1]
|
||||
cons[0] = s.currShould
|
||||
}
|
||||
rv = s.scorer.Score(ctx, cons)
|
||||
err = s.advanceNextMust(ctx, rv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
} else if s.shouldSearcher.Min() == 0 {
|
||||
// match is OK anyway
|
||||
cons := s.matches[0:1]
|
||||
cons[0] = s.currMust
|
||||
rv = s.scorer.Score(ctx, cons)
|
||||
err = s.advanceNextMust(ctx, rv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
} else if shouldCmpOrNil == 0 {
|
||||
// score bonus matches should
|
||||
var cons []*search.DocumentMatch
|
||||
if s.currMust != nil {
|
||||
cons = s.matches
|
||||
cons[0] = s.currMust
|
||||
cons[1] = s.currShould
|
||||
} else {
|
||||
cons = s.matches[0:1]
|
||||
cons[0] = s.currShould
|
||||
}
|
||||
rv = s.scorer.Score(ctx, cons)
|
||||
err = s.advanceNextMust(ctx, rv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
} else if s.shouldSearcher == nil || s.shouldSearcher.Min() == 0 {
|
||||
// match is OK anyway
|
||||
cons := s.matches[0:1]
|
||||
cons[0] = s.currMust
|
||||
rv = s.scorer.Score(ctx, cons)
|
||||
err = s.advanceNextMust(ctx, rv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
err = s.advanceNextMust(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if rv == nil {
|
||||
s.done = true
|
||||
}
|
||||
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
|
||||
if s.done {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Advance the searcher only if the cursor is trailing the lookup ID
|
||||
if s.currentID == nil || s.currentID.Compare(ID) < 0 {
|
||||
var err error
|
||||
if s.mustSearcher != nil {
|
||||
if s.currMust != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currMust)
|
||||
}
|
||||
s.currMust, err = s.mustSearcher.Advance(ctx, ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if s.shouldSearcher != nil {
|
||||
if s.currShould != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currShould)
|
||||
}
|
||||
s.currShould, err = s.shouldSearcher.Advance(ctx, ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if s.mustNotSearcher != nil {
|
||||
// Additional check for mustNotSearcher, whose cursor isn't tracked by
|
||||
// currentID to prevent it from moving when the searcher's tracked
|
||||
// position is already ahead of or at the requested ID.
|
||||
if s.currMustNot == nil || s.currMustNot.IndexInternalID.Compare(ID) < 0 {
|
||||
if s.currMustNot != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currMustNot)
|
||||
}
|
||||
s.currMustNot, err = s.mustNotSearcher.Advance(ctx, ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if s.mustSearcher != nil && s.currMust != nil {
|
||||
s.currentID = s.currMust.IndexInternalID
|
||||
} else if s.mustSearcher == nil && s.currShould != nil {
|
||||
s.currentID = s.currShould.IndexInternalID
|
||||
} else {
|
||||
s.currentID = nil
|
||||
}
|
||||
}
|
||||
|
||||
return s.Next(ctx)
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) Count() uint64 {
|
||||
|
||||
// for now return a worst case
|
||||
var sum uint64
|
||||
if s.mustSearcher != nil {
|
||||
sum += s.mustSearcher.Count()
|
||||
}
|
||||
if s.shouldSearcher != nil {
|
||||
sum += s.shouldSearcher.Count()
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) Close() error {
|
||||
var err0, err1, err2 error
|
||||
if s.mustSearcher != nil {
|
||||
err0 = s.mustSearcher.Close()
|
||||
}
|
||||
if s.shouldSearcher != nil {
|
||||
err1 = s.shouldSearcher.Close()
|
||||
}
|
||||
if s.mustNotSearcher != nil {
|
||||
err2 = s.mustNotSearcher.Close()
|
||||
}
|
||||
if err0 != nil {
|
||||
return err0
|
||||
}
|
||||
if err1 != nil {
|
||||
return err1
|
||||
}
|
||||
if err2 != nil {
|
||||
return err2
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) Min() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *BooleanSearcher) DocumentMatchPoolSize() int {
|
||||
rv := 3
|
||||
if s.mustSearcher != nil {
|
||||
rv += s.mustSearcher.DocumentMatchPoolSize()
|
||||
}
|
||||
if s.shouldSearcher != nil {
|
||||
rv += s.shouldSearcher.DocumentMatchPoolSize()
|
||||
}
|
||||
if s.mustNotSearcher != nil {
|
||||
rv += s.mustNotSearcher.DocumentMatchPoolSize()
|
||||
}
|
||||
return rv
|
||||
}
|
||||
285
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_conjunction.go
generated
vendored
Normal file
285
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_conjunction.go
generated
vendored
Normal file
@@ -0,0 +1,285 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/scorer"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeConjunctionSearcher int
|
||||
|
||||
func init() {
|
||||
var cs ConjunctionSearcher
|
||||
reflectStaticSizeConjunctionSearcher = int(reflect.TypeOf(cs).Size())
|
||||
}
|
||||
|
||||
type ConjunctionSearcher struct {
|
||||
indexReader index.IndexReader
|
||||
searchers []search.Searcher
|
||||
queryNorm float64
|
||||
currs []*search.DocumentMatch
|
||||
maxIDIdx int
|
||||
scorer *scorer.ConjunctionQueryScorer
|
||||
initialized bool
|
||||
options search.SearcherOptions
|
||||
bytesRead uint64
|
||||
}
|
||||
|
||||
func NewConjunctionSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
qsearchers []search.Searcher, options search.SearcherOptions) (
|
||||
search.Searcher, error,
|
||||
) {
|
||||
// build the sorted downstream searchers
|
||||
searchers := make(OrderedSearcherList, len(qsearchers))
|
||||
copy(searchers, qsearchers)
|
||||
sort.Sort(searchers)
|
||||
|
||||
// attempt the "unadorned" conjunction optimization only when we
|
||||
// do not need extra information like freq-norm's or term vectors
|
||||
if len(searchers) > 1 &&
|
||||
options.Score == "none" && !options.IncludeTermVectors {
|
||||
rv, err := optimizeCompositeSearcher(ctx, "conjunction:unadorned",
|
||||
indexReader, searchers, options)
|
||||
if err != nil || rv != nil {
|
||||
return rv, err
|
||||
}
|
||||
}
|
||||
|
||||
// build our searcher
|
||||
rv := ConjunctionSearcher{
|
||||
indexReader: indexReader,
|
||||
options: options,
|
||||
searchers: searchers,
|
||||
currs: make([]*search.DocumentMatch, len(searchers)),
|
||||
scorer: scorer.NewConjunctionQueryScorer(options),
|
||||
}
|
||||
rv.computeQueryNorm()
|
||||
|
||||
// attempt push-down conjunction optimization when there's >1 searchers
|
||||
if len(searchers) > 1 {
|
||||
rv, err := optimizeCompositeSearcher(ctx, "conjunction",
|
||||
indexReader, searchers, options)
|
||||
if err != nil || rv != nil {
|
||||
return rv, err
|
||||
}
|
||||
}
|
||||
|
||||
return &rv, nil
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) computeQueryNorm() {
|
||||
// first calculate sum of squared weights
|
||||
sumOfSquaredWeights := 0.0
|
||||
for _, searcher := range s.searchers {
|
||||
sumOfSquaredWeights += searcher.Weight()
|
||||
}
|
||||
// now compute query norm from this
|
||||
s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)
|
||||
// finally tell all the downstream searchers the norm
|
||||
for _, searcher := range s.searchers {
|
||||
searcher.SetQueryNorm(s.queryNorm)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) Size() int {
|
||||
sizeInBytes := reflectStaticSizeConjunctionSearcher + size.SizeOfPtr +
|
||||
s.scorer.Size()
|
||||
|
||||
for _, entry := range s.searchers {
|
||||
sizeInBytes += entry.Size()
|
||||
}
|
||||
|
||||
for _, entry := range s.currs {
|
||||
if entry != nil {
|
||||
sizeInBytes += entry.Size()
|
||||
}
|
||||
}
|
||||
|
||||
return sizeInBytes
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) initSearchers(ctx *search.SearchContext) error {
|
||||
var err error
|
||||
// get all searchers pointing at their first match
|
||||
for i, searcher := range s.searchers {
|
||||
if s.currs[i] != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currs[i])
|
||||
}
|
||||
s.currs[i], err = searcher.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.initialized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) Weight() float64 {
|
||||
var rv float64
|
||||
for _, searcher := range s.searchers {
|
||||
rv += searcher.Weight()
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) SetQueryNorm(qnorm float64) {
|
||||
for _, searcher := range s.searchers {
|
||||
searcher.SetQueryNorm(qnorm)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var rv *search.DocumentMatch
|
||||
var err error
|
||||
OUTER:
|
||||
for s.maxIDIdx < len(s.currs) && s.currs[s.maxIDIdx] != nil {
|
||||
maxID := s.currs[s.maxIDIdx].IndexInternalID
|
||||
|
||||
i := 0
|
||||
for i < len(s.currs) {
|
||||
if s.currs[i] == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if i == s.maxIDIdx {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
cmp := maxID.Compare(s.currs[i].IndexInternalID)
|
||||
if cmp == 0 {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if cmp < 0 {
|
||||
// maxID < currs[i], so we found a new maxIDIdx
|
||||
s.maxIDIdx = i
|
||||
|
||||
// advance the positions where [0 <= x < i], since we
|
||||
// know they were equal to the former max entry
|
||||
maxID = s.currs[s.maxIDIdx].IndexInternalID
|
||||
for x := 0; x < i; x++ {
|
||||
err = s.advanceChild(ctx, x, maxID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
continue OUTER
|
||||
}
|
||||
|
||||
// maxID > currs[i], so need to advance searchers[i]
|
||||
err = s.advanceChild(ctx, i, maxID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// don't bump i, so that we'll examine the just-advanced
|
||||
// currs[i] again
|
||||
}
|
||||
|
||||
// if we get here, a doc matched all readers, so score and add it
|
||||
rv = s.scorer.Score(ctx, s.currs)
|
||||
|
||||
// we know all the searchers are pointing at the same thing
|
||||
// so they all need to be bumped
|
||||
for i, searcher := range s.searchers {
|
||||
if s.currs[i] != rv {
|
||||
ctx.DocumentMatchPool.Put(s.currs[i])
|
||||
}
|
||||
s.currs[i], err = searcher.Next(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// don't continue now, wait for the next call to Next()
|
||||
break
|
||||
}
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for i := range s.searchers {
|
||||
if s.currs[i] != nil && s.currs[i].IndexInternalID.Compare(ID) >= 0 {
|
||||
continue
|
||||
}
|
||||
err := s.advanceChild(ctx, i, ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return s.Next(ctx)
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) advanceChild(ctx *search.SearchContext, i int, ID index.IndexInternalID) (err error) {
|
||||
if s.currs[i] != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currs[i])
|
||||
}
|
||||
s.currs[i], err = s.searchers[i].Advance(ctx, ID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) Count() uint64 {
|
||||
// for now return a worst case
|
||||
var sum uint64
|
||||
for _, searcher := range s.searchers {
|
||||
sum += searcher.Count()
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) Close() (rv error) {
|
||||
for _, searcher := range s.searchers {
|
||||
err := searcher.Close()
|
||||
if err != nil && rv == nil {
|
||||
rv = err
|
||||
}
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) Min() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *ConjunctionSearcher) DocumentMatchPoolSize() int {
|
||||
rv := len(s.currs)
|
||||
for _, s := range s.searchers {
|
||||
rv += s.DocumentMatchPoolSize()
|
||||
}
|
||||
return rv
|
||||
}
|
||||
131
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction.go
generated
vendored
Normal file
131
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction.go
generated
vendored
Normal file
@@ -0,0 +1,131 @@
|
||||
// Copyright (c) 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
// DisjunctionMaxClauseCount is a compile time setting that applications can
|
||||
// adjust to non-zero value to cause the DisjunctionSearcher to return an
|
||||
// error instead of exeucting searches when the size exceeds this value.
|
||||
var DisjunctionMaxClauseCount = 0
|
||||
|
||||
// DisjunctionHeapTakeover is a compile time setting that applications can
|
||||
// adjust to control when the DisjunctionSearcher will switch from a simple
|
||||
// slice implementation to a heap implementation.
|
||||
var DisjunctionHeapTakeover = 10
|
||||
|
||||
func NewDisjunctionSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
qsearchers []search.Searcher, min float64, options search.SearcherOptions) (
|
||||
search.Searcher, error) {
|
||||
return newDisjunctionSearcher(ctx, indexReader, qsearchers, min, options, true)
|
||||
}
|
||||
|
||||
func optionsDisjunctionOptimizable(options search.SearcherOptions) bool {
|
||||
rv := options.Score == "none" && !options.IncludeTermVectors
|
||||
return rv
|
||||
}
|
||||
|
||||
func newDisjunctionSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
qsearchers []search.Searcher, min float64, options search.SearcherOptions,
|
||||
limit bool) (search.Searcher, error) {
|
||||
|
||||
var disjOverKNN bool
|
||||
if ctx != nil {
|
||||
disjOverKNN, _ = ctx.Value(search.IncludeScoreBreakdownKey).(bool)
|
||||
}
|
||||
if disjOverKNN {
|
||||
// The KNN Searcher optimization is a necessary pre-req for the KNN Searchers,
|
||||
// not an optional optimization like for, say term searchers.
|
||||
// It's an optimization to repeat search an open vector index when applicable,
|
||||
// rather than individually opening and searching a vector index.
|
||||
err := optimizeKNN(ctx, indexReader, qsearchers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// attempt the "unadorned" disjunction optimization only when we
|
||||
// do not need extra information like freq-norm's or term vectors
|
||||
// and the requested min is simple
|
||||
if len(qsearchers) > 1 && min <= 1 &&
|
||||
optionsDisjunctionOptimizable(options) {
|
||||
rv, err := optimizeCompositeSearcher(ctx, "disjunction:unadorned",
|
||||
indexReader, qsearchers, options)
|
||||
if err != nil || rv != nil {
|
||||
return rv, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(qsearchers) > DisjunctionHeapTakeover {
|
||||
return newDisjunctionHeapSearcher(ctx, indexReader, qsearchers, min, options,
|
||||
limit)
|
||||
}
|
||||
return newDisjunctionSliceSearcher(ctx, indexReader, qsearchers, min, options,
|
||||
limit)
|
||||
}
|
||||
|
||||
func optimizeCompositeSearcher(ctx context.Context, optimizationKind string,
|
||||
indexReader index.IndexReader, qsearchers []search.Searcher,
|
||||
options search.SearcherOptions) (search.Searcher, error) {
|
||||
var octx index.OptimizableContext
|
||||
|
||||
for _, searcher := range qsearchers {
|
||||
o, ok := searcher.(index.Optimizable)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var err error
|
||||
octx, err = o.Optimize(optimizationKind, octx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if octx == nil {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
optimized, err := octx.Finish()
|
||||
if err != nil || optimized == nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tfr, ok := optimized.(index.TermFieldReader)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return newTermSearcherFromReader(ctx, indexReader, tfr,
|
||||
[]byte(optimizationKind), "*", 1.0, options)
|
||||
}
|
||||
|
||||
func tooManyClauses(count int) bool {
|
||||
if DisjunctionMaxClauseCount != 0 && count > DisjunctionMaxClauseCount {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func tooManyClausesErr(field string, count int) error {
|
||||
return fmt.Errorf("TooManyClauses over field: `%s` [%d > maxClauseCount,"+
|
||||
" which is set to %d]", field, count, DisjunctionMaxClauseCount)
|
||||
}
|
||||
367
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go
generated
vendored
Normal file
367
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_heap.go
generated
vendored
Normal file
@@ -0,0 +1,367 @@
|
||||
// Copyright (c) 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/heap"
|
||||
"context"
|
||||
"math"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/scorer"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeDisjunctionHeapSearcher int
|
||||
var reflectStaticSizeSearcherCurr int
|
||||
|
||||
func init() {
|
||||
var dhs DisjunctionHeapSearcher
|
||||
reflectStaticSizeDisjunctionHeapSearcher = int(reflect.TypeOf(dhs).Size())
|
||||
|
||||
var sc SearcherCurr
|
||||
reflectStaticSizeSearcherCurr = int(reflect.TypeOf(sc).Size())
|
||||
}
|
||||
|
||||
type SearcherCurr struct {
|
||||
searcher search.Searcher
|
||||
curr *search.DocumentMatch
|
||||
matchingIdx int
|
||||
}
|
||||
|
||||
type DisjunctionHeapSearcher struct {
|
||||
indexReader index.IndexReader
|
||||
|
||||
numSearchers int
|
||||
scorer *scorer.DisjunctionQueryScorer
|
||||
min int
|
||||
queryNorm float64
|
||||
retrieveScoreBreakdown bool
|
||||
initialized bool
|
||||
searchers []search.Searcher
|
||||
heap []*SearcherCurr
|
||||
|
||||
matching []*search.DocumentMatch
|
||||
matchingIdxs []int
|
||||
matchingCurrs []*SearcherCurr
|
||||
|
||||
bytesRead uint64
|
||||
}
|
||||
|
||||
func newDisjunctionHeapSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
searchers []search.Searcher, min float64, options search.SearcherOptions,
|
||||
limit bool) (
|
||||
*DisjunctionHeapSearcher, error) {
|
||||
if limit && tooManyClauses(len(searchers)) {
|
||||
return nil, tooManyClausesErr("", len(searchers))
|
||||
}
|
||||
var retrieveScoreBreakdown bool
|
||||
if ctx != nil {
|
||||
retrieveScoreBreakdown, _ = ctx.Value(search.IncludeScoreBreakdownKey).(bool)
|
||||
}
|
||||
|
||||
// build our searcher
|
||||
rv := DisjunctionHeapSearcher{
|
||||
indexReader: indexReader,
|
||||
searchers: searchers,
|
||||
numSearchers: len(searchers),
|
||||
scorer: scorer.NewDisjunctionQueryScorer(options),
|
||||
min: int(min),
|
||||
matching: make([]*search.DocumentMatch, len(searchers)),
|
||||
matchingCurrs: make([]*SearcherCurr, len(searchers)),
|
||||
matchingIdxs: make([]int, len(searchers)),
|
||||
retrieveScoreBreakdown: retrieveScoreBreakdown,
|
||||
heap: make([]*SearcherCurr, 0, len(searchers)),
|
||||
}
|
||||
rv.computeQueryNorm()
|
||||
return &rv, nil
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) computeQueryNorm() {
|
||||
// first calculate sum of squared weights
|
||||
sumOfSquaredWeights := 0.0
|
||||
for _, searcher := range s.searchers {
|
||||
sumOfSquaredWeights += searcher.Weight()
|
||||
}
|
||||
// now compute query norm from this
|
||||
s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)
|
||||
// finally tell all the downstream searchers the norm
|
||||
for _, searcher := range s.searchers {
|
||||
searcher.SetQueryNorm(s.queryNorm)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Size() int {
|
||||
sizeInBytes := reflectStaticSizeDisjunctionHeapSearcher + size.SizeOfPtr +
|
||||
s.scorer.Size()
|
||||
|
||||
for _, entry := range s.searchers {
|
||||
sizeInBytes += entry.Size()
|
||||
}
|
||||
|
||||
for _, entry := range s.matching {
|
||||
if entry != nil {
|
||||
sizeInBytes += entry.Size()
|
||||
}
|
||||
}
|
||||
|
||||
// for matchingCurrs and heap, just use static size * len
|
||||
// since searchers and document matches already counted above
|
||||
sizeInBytes += len(s.matchingCurrs) * reflectStaticSizeSearcherCurr
|
||||
sizeInBytes += len(s.heap) * reflectStaticSizeSearcherCurr
|
||||
sizeInBytes += len(s.matchingIdxs) * size.SizeOfInt
|
||||
|
||||
return sizeInBytes
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) initSearchers(ctx *search.SearchContext) error {
|
||||
// alloc a single block of SearcherCurrs
|
||||
block := make([]SearcherCurr, len(s.searchers))
|
||||
|
||||
// get all searchers pointing at their first match
|
||||
for i, searcher := range s.searchers {
|
||||
curr, err := searcher.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if curr != nil {
|
||||
block[i].searcher = searcher
|
||||
block[i].curr = curr
|
||||
block[i].matchingIdx = i
|
||||
heap.Push(s, &block[i])
|
||||
}
|
||||
}
|
||||
|
||||
err := s.updateMatches()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.initialized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) updateMatches() error {
|
||||
matching := s.matching[:0]
|
||||
matchingCurrs := s.matchingCurrs[:0]
|
||||
matchingIdxs := s.matchingIdxs[:0]
|
||||
|
||||
if len(s.heap) > 0 {
|
||||
|
||||
// top of the heap is our next hit
|
||||
next := heap.Pop(s).(*SearcherCurr)
|
||||
matching = append(matching, next.curr)
|
||||
matchingCurrs = append(matchingCurrs, next)
|
||||
matchingIdxs = append(matchingIdxs, next.matchingIdx)
|
||||
|
||||
// now as long as top of heap matches, keep popping
|
||||
for len(s.heap) > 0 && bytes.Compare(next.curr.IndexInternalID, s.heap[0].curr.IndexInternalID) == 0 {
|
||||
next = heap.Pop(s).(*SearcherCurr)
|
||||
matching = append(matching, next.curr)
|
||||
matchingCurrs = append(matchingCurrs, next)
|
||||
matchingIdxs = append(matchingIdxs, next.matchingIdx)
|
||||
}
|
||||
}
|
||||
|
||||
s.matching = matching
|
||||
s.matchingCurrs = matchingCurrs
|
||||
s.matchingIdxs = matchingIdxs
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Weight() float64 {
|
||||
var rv float64
|
||||
for _, searcher := range s.searchers {
|
||||
rv += searcher.Weight()
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) SetQueryNorm(qnorm float64) {
|
||||
for _, searcher := range s.searchers {
|
||||
searcher.SetQueryNorm(qnorm)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Next(ctx *search.SearchContext) (
|
||||
*search.DocumentMatch, error) {
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var rv *search.DocumentMatch
|
||||
found := false
|
||||
for !found && len(s.matching) > 0 {
|
||||
if len(s.matching) >= s.min {
|
||||
found = true
|
||||
if s.retrieveScoreBreakdown {
|
||||
// just return score and expl breakdown here, since it is a disjunction over knn searchers,
|
||||
// and the final score and expl is calculated in the knn collector
|
||||
rv = s.scorer.ScoreAndExplBreakdown(ctx, s.matching, s.matchingIdxs, nil, s.numSearchers)
|
||||
} else {
|
||||
// score this match
|
||||
rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers)
|
||||
}
|
||||
}
|
||||
|
||||
// invoke next on all the matching searchers
|
||||
for _, matchingCurr := range s.matchingCurrs {
|
||||
if matchingCurr.curr != rv {
|
||||
ctx.DocumentMatchPool.Put(matchingCurr.curr)
|
||||
}
|
||||
curr, err := matchingCurr.searcher.Next(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if curr != nil {
|
||||
matchingCurr.curr = curr
|
||||
heap.Push(s, matchingCurr)
|
||||
}
|
||||
}
|
||||
|
||||
err := s.updateMatches()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Advance(ctx *search.SearchContext,
|
||||
ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// if there is anything in matching, toss it back onto the heap
|
||||
for _, matchingCurr := range s.matchingCurrs {
|
||||
heap.Push(s, matchingCurr)
|
||||
}
|
||||
s.matching = s.matching[:0]
|
||||
s.matchingCurrs = s.matchingCurrs[:0]
|
||||
|
||||
// find all searchers that actually need to be advanced
|
||||
// advance them, using s.matchingCurrs as temp storage
|
||||
for len(s.heap) > 0 && bytes.Compare(s.heap[0].curr.IndexInternalID, ID) < 0 {
|
||||
searcherCurr := heap.Pop(s).(*SearcherCurr)
|
||||
ctx.DocumentMatchPool.Put(searcherCurr.curr)
|
||||
curr, err := searcherCurr.searcher.Advance(ctx, ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if curr != nil {
|
||||
searcherCurr.curr = curr
|
||||
s.matchingCurrs = append(s.matchingCurrs, searcherCurr)
|
||||
}
|
||||
}
|
||||
// now all of the searchers that we advanced have to be pushed back
|
||||
for _, matchingCurr := range s.matchingCurrs {
|
||||
heap.Push(s, matchingCurr)
|
||||
}
|
||||
// reset our temp space
|
||||
s.matchingCurrs = s.matchingCurrs[:0]
|
||||
|
||||
err := s.updateMatches()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s.Next(ctx)
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Count() uint64 {
|
||||
// for now return a worst case
|
||||
var sum uint64
|
||||
for _, searcher := range s.searchers {
|
||||
sum += searcher.Count()
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Close() (rv error) {
|
||||
for _, searcher := range s.searchers {
|
||||
err := searcher.Close()
|
||||
if err != nil && rv == nil {
|
||||
rv = err
|
||||
}
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Min() int {
|
||||
return s.min
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) DocumentMatchPoolSize() int {
|
||||
rv := len(s.searchers)
|
||||
for _, s := range s.searchers {
|
||||
rv += s.DocumentMatchPoolSize()
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
// a disjunction searcher implements the index.Optimizable interface
|
||||
// but only activates on an edge case where the disjunction is a
|
||||
// wrapper around a single Optimizable child searcher
|
||||
func (s *DisjunctionHeapSearcher) Optimize(kind string, octx index.OptimizableContext) (
|
||||
index.OptimizableContext, error) {
|
||||
if len(s.searchers) == 1 {
|
||||
o, ok := s.searchers[0].(index.Optimizable)
|
||||
if ok {
|
||||
return o.Optimize(kind, octx)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// heap impl
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Len() int { return len(s.heap) }
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Less(i, j int) bool {
|
||||
if s.heap[i].curr == nil {
|
||||
return true
|
||||
} else if s.heap[j].curr == nil {
|
||||
return false
|
||||
}
|
||||
return bytes.Compare(s.heap[i].curr.IndexInternalID, s.heap[j].curr.IndexInternalID) < 0
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Swap(i, j int) {
|
||||
s.heap[i], s.heap[j] = s.heap[j], s.heap[i]
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Push(x interface{}) {
|
||||
s.heap = append(s.heap, x.(*SearcherCurr))
|
||||
}
|
||||
|
||||
func (s *DisjunctionHeapSearcher) Pop() interface{} {
|
||||
old := s.heap
|
||||
n := len(old)
|
||||
x := old[n-1]
|
||||
s.heap = old[0 : n-1]
|
||||
return x
|
||||
}
|
||||
334
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go
generated
vendored
Normal file
334
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_disjunction_slice.go
generated
vendored
Normal file
@@ -0,0 +1,334 @@
|
||||
// Copyright (c) 2018 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/scorer"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeDisjunctionSliceSearcher int
|
||||
|
||||
func init() {
|
||||
var ds DisjunctionSliceSearcher
|
||||
reflectStaticSizeDisjunctionSliceSearcher = int(reflect.TypeOf(ds).Size())
|
||||
}
|
||||
|
||||
type DisjunctionSliceSearcher struct {
|
||||
indexReader index.IndexReader
|
||||
searchers []search.Searcher
|
||||
originalPos []int
|
||||
numSearchers int
|
||||
queryNorm float64
|
||||
retrieveScoreBreakdown bool
|
||||
currs []*search.DocumentMatch
|
||||
scorer *scorer.DisjunctionQueryScorer
|
||||
min int
|
||||
matching []*search.DocumentMatch
|
||||
matchingIdxs []int
|
||||
initialized bool
|
||||
bytesRead uint64
|
||||
}
|
||||
|
||||
func newDisjunctionSliceSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
qsearchers []search.Searcher, min float64, options search.SearcherOptions,
|
||||
limit bool) (
|
||||
*DisjunctionSliceSearcher, error,
|
||||
) {
|
||||
if limit && tooManyClauses(len(qsearchers)) {
|
||||
return nil, tooManyClausesErr("", len(qsearchers))
|
||||
}
|
||||
|
||||
var searchers OrderedSearcherList
|
||||
var originalPos []int
|
||||
var retrieveScoreBreakdown bool
|
||||
if ctx != nil {
|
||||
retrieveScoreBreakdown, _ = ctx.Value(search.IncludeScoreBreakdownKey).(bool)
|
||||
}
|
||||
|
||||
if retrieveScoreBreakdown {
|
||||
// needed only when kNN is in picture
|
||||
sortedSearchers := &OrderedPositionalSearcherList{
|
||||
searchers: make([]search.Searcher, len(qsearchers)),
|
||||
index: make([]int, len(qsearchers)),
|
||||
}
|
||||
for i, searcher := range qsearchers {
|
||||
sortedSearchers.searchers[i] = searcher
|
||||
sortedSearchers.index[i] = i
|
||||
}
|
||||
sort.Sort(sortedSearchers)
|
||||
searchers = sortedSearchers.searchers
|
||||
originalPos = sortedSearchers.index
|
||||
} else {
|
||||
searchers = make(OrderedSearcherList, len(qsearchers))
|
||||
copy(searchers, qsearchers)
|
||||
sort.Sort(searchers)
|
||||
}
|
||||
|
||||
rv := DisjunctionSliceSearcher{
|
||||
indexReader: indexReader,
|
||||
searchers: searchers,
|
||||
originalPos: originalPos,
|
||||
numSearchers: len(searchers),
|
||||
currs: make([]*search.DocumentMatch, len(searchers)),
|
||||
scorer: scorer.NewDisjunctionQueryScorer(options),
|
||||
min: int(min),
|
||||
retrieveScoreBreakdown: retrieveScoreBreakdown,
|
||||
|
||||
matching: make([]*search.DocumentMatch, len(searchers)),
|
||||
matchingIdxs: make([]int, len(searchers)),
|
||||
}
|
||||
rv.computeQueryNorm()
|
||||
return &rv, nil
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) computeQueryNorm() {
|
||||
// first calculate sum of squared weights
|
||||
sumOfSquaredWeights := 0.0
|
||||
for _, searcher := range s.searchers {
|
||||
sumOfSquaredWeights += searcher.Weight()
|
||||
}
|
||||
// now compute query norm from this
|
||||
s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)
|
||||
// finally tell all the downstream searchers the norm
|
||||
for _, searcher := range s.searchers {
|
||||
searcher.SetQueryNorm(s.queryNorm)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) Size() int {
|
||||
sizeInBytes := reflectStaticSizeDisjunctionSliceSearcher + size.SizeOfPtr +
|
||||
s.scorer.Size()
|
||||
|
||||
for _, entry := range s.searchers {
|
||||
sizeInBytes += entry.Size()
|
||||
}
|
||||
|
||||
for _, entry := range s.currs {
|
||||
if entry != nil {
|
||||
sizeInBytes += entry.Size()
|
||||
}
|
||||
}
|
||||
|
||||
for _, entry := range s.matching {
|
||||
if entry != nil {
|
||||
sizeInBytes += entry.Size()
|
||||
}
|
||||
}
|
||||
|
||||
sizeInBytes += len(s.matchingIdxs) * size.SizeOfInt
|
||||
sizeInBytes += len(s.originalPos) * size.SizeOfInt
|
||||
|
||||
return sizeInBytes
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) initSearchers(ctx *search.SearchContext) error {
|
||||
var err error
|
||||
// get all searchers pointing at their first match
|
||||
for i, searcher := range s.searchers {
|
||||
if s.currs[i] != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currs[i])
|
||||
}
|
||||
s.currs[i], err = searcher.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = s.updateMatches()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.initialized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) updateMatches() error {
|
||||
matching := s.matching[:0]
|
||||
matchingIdxs := s.matchingIdxs[:0]
|
||||
|
||||
for i := 0; i < len(s.currs); i++ {
|
||||
curr := s.currs[i]
|
||||
if curr == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(matching) > 0 {
|
||||
cmp := curr.IndexInternalID.Compare(matching[0].IndexInternalID)
|
||||
if cmp > 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if cmp < 0 {
|
||||
matching = matching[:0]
|
||||
matchingIdxs = matchingIdxs[:0]
|
||||
}
|
||||
}
|
||||
matching = append(matching, curr)
|
||||
matchingIdxs = append(matchingIdxs, i)
|
||||
}
|
||||
|
||||
s.matching = matching
|
||||
s.matchingIdxs = matchingIdxs
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) Weight() float64 {
|
||||
var rv float64
|
||||
for _, searcher := range s.searchers {
|
||||
rv += searcher.Weight()
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) SetQueryNorm(qnorm float64) {
|
||||
for _, searcher := range s.searchers {
|
||||
searcher.SetQueryNorm(qnorm)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) Next(ctx *search.SearchContext) (
|
||||
*search.DocumentMatch, error,
|
||||
) {
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var err error
|
||||
var rv *search.DocumentMatch
|
||||
|
||||
found := false
|
||||
for !found && len(s.matching) > 0 {
|
||||
if len(s.matching) >= s.min {
|
||||
found = true
|
||||
if s.retrieveScoreBreakdown {
|
||||
// just return score and expl breakdown here, since it is a disjunction over knn searchers,
|
||||
// and the final score and expl is calculated in the knn collector
|
||||
rv = s.scorer.ScoreAndExplBreakdown(ctx, s.matching, s.matchingIdxs, s.originalPos, s.numSearchers)
|
||||
} else {
|
||||
// score this match
|
||||
rv = s.scorer.Score(ctx, s.matching, len(s.matching), s.numSearchers)
|
||||
}
|
||||
}
|
||||
|
||||
// invoke next on all the matching searchers
|
||||
for _, i := range s.matchingIdxs {
|
||||
searcher := s.searchers[i]
|
||||
if s.currs[i] != rv {
|
||||
ctx.DocumentMatchPool.Put(s.currs[i])
|
||||
}
|
||||
s.currs[i], err = searcher.Next(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err = s.updateMatches()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) Advance(ctx *search.SearchContext,
|
||||
ID index.IndexInternalID,
|
||||
) (*search.DocumentMatch, error) {
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// get all searchers pointing at their first match
|
||||
var err error
|
||||
for i, searcher := range s.searchers {
|
||||
if s.currs[i] != nil {
|
||||
if s.currs[i].IndexInternalID.Compare(ID) >= 0 {
|
||||
continue
|
||||
}
|
||||
ctx.DocumentMatchPool.Put(s.currs[i])
|
||||
}
|
||||
s.currs[i], err = searcher.Advance(ctx, ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
err = s.updateMatches()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s.Next(ctx)
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) Count() uint64 {
|
||||
// for now return a worst case
|
||||
var sum uint64
|
||||
for _, searcher := range s.searchers {
|
||||
sum += searcher.Count()
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) Close() (rv error) {
|
||||
for _, searcher := range s.searchers {
|
||||
err := searcher.Close()
|
||||
if err != nil && rv == nil {
|
||||
rv = err
|
||||
}
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) Min() int {
|
||||
return s.min
|
||||
}
|
||||
|
||||
func (s *DisjunctionSliceSearcher) DocumentMatchPoolSize() int {
|
||||
rv := len(s.currs)
|
||||
for _, s := range s.searchers {
|
||||
rv += s.DocumentMatchPoolSize()
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
// a disjunction searcher implements the index.Optimizable interface
|
||||
// but only activates on an edge case where the disjunction is a
|
||||
// wrapper around a single Optimizable child searcher
|
||||
func (s *DisjunctionSliceSearcher) Optimize(kind string, octx index.OptimizableContext) (
|
||||
index.OptimizableContext, error,
|
||||
) {
|
||||
if len(s.searchers) == 1 {
|
||||
o, ok := s.searchers[0].(index.Optimizable)
|
||||
if ok {
|
||||
return o.Optimize(kind, octx)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
110
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_docid.go
generated
vendored
Normal file
110
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_docid.go
generated
vendored
Normal file
@@ -0,0 +1,110 @@
|
||||
// Copyright (c) 2015 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/scorer"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeDocIDSearcher int
|
||||
|
||||
func init() {
|
||||
var ds DocIDSearcher
|
||||
reflectStaticSizeDocIDSearcher = int(reflect.TypeOf(ds).Size())
|
||||
}
|
||||
|
||||
// DocIDSearcher returns documents matching a predefined set of identifiers.
|
||||
type DocIDSearcher struct {
|
||||
reader index.DocIDReader
|
||||
scorer *scorer.ConstantScorer
|
||||
count int
|
||||
}
|
||||
|
||||
func NewDocIDSearcher(ctx context.Context, indexReader index.IndexReader, ids []string, boost float64,
|
||||
options search.SearcherOptions) (searcher *DocIDSearcher, err error) {
|
||||
|
||||
reader, err := indexReader.DocIDReaderOnly(ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scorer := scorer.NewConstantScorer(1.0, boost, options)
|
||||
return &DocIDSearcher{
|
||||
scorer: scorer,
|
||||
reader: reader,
|
||||
count: len(ids),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *DocIDSearcher) Size() int {
|
||||
return reflectStaticSizeDocIDSearcher + size.SizeOfPtr +
|
||||
s.reader.Size() +
|
||||
s.scorer.Size()
|
||||
}
|
||||
|
||||
func (s *DocIDSearcher) Count() uint64 {
|
||||
return uint64(s.count)
|
||||
}
|
||||
|
||||
func (s *DocIDSearcher) Weight() float64 {
|
||||
return s.scorer.Weight()
|
||||
}
|
||||
|
||||
func (s *DocIDSearcher) SetQueryNorm(qnorm float64) {
|
||||
s.scorer.SetQueryNorm(qnorm)
|
||||
}
|
||||
|
||||
func (s *DocIDSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
docidMatch, err := s.reader.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if docidMatch == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
docMatch := s.scorer.Score(ctx, docidMatch)
|
||||
return docMatch, nil
|
||||
}
|
||||
|
||||
func (s *DocIDSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
docidMatch, err := s.reader.Advance(ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if docidMatch == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
docMatch := s.scorer.Score(ctx, docidMatch)
|
||||
return docMatch, nil
|
||||
}
|
||||
|
||||
func (s *DocIDSearcher) Close() error {
|
||||
return s.reader.Close()
|
||||
}
|
||||
|
||||
func (s *DocIDSearcher) Min() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *DocIDSearcher) DocumentMatchPoolSize() int {
|
||||
return 1
|
||||
}
|
||||
104
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_filter.go
generated
vendored
Normal file
104
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_filter.go
generated
vendored
Normal file
@@ -0,0 +1,104 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeFilteringSearcher int
|
||||
|
||||
func init() {
|
||||
var fs FilteringSearcher
|
||||
reflectStaticSizeFilteringSearcher = int(reflect.TypeOf(fs).Size())
|
||||
}
|
||||
|
||||
// FilterFunc defines a function which can filter documents
|
||||
// returning true means keep the document
|
||||
// returning false means do not keep the document
|
||||
type FilterFunc func(d *search.DocumentMatch) bool
|
||||
|
||||
// FilteringSearcher wraps any other searcher, but checks any Next/Advance
|
||||
// call against the supplied FilterFunc
|
||||
type FilteringSearcher struct {
|
||||
child search.Searcher
|
||||
accept FilterFunc
|
||||
}
|
||||
|
||||
func NewFilteringSearcher(ctx context.Context, s search.Searcher, filter FilterFunc) *FilteringSearcher {
|
||||
return &FilteringSearcher{
|
||||
child: s,
|
||||
accept: filter,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Size() int {
|
||||
return reflectStaticSizeFilteringSearcher + size.SizeOfPtr +
|
||||
f.child.Size()
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
next, err := f.child.Next(ctx)
|
||||
for next != nil && err == nil {
|
||||
if f.accept(next) {
|
||||
return next, nil
|
||||
}
|
||||
next, err = f.child.Next(ctx)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
adv, err := f.child.Advance(ctx, ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if adv == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if f.accept(adv) {
|
||||
return adv, nil
|
||||
}
|
||||
return f.Next(ctx)
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Close() error {
|
||||
return f.child.Close()
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Weight() float64 {
|
||||
return f.child.Weight()
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) SetQueryNorm(n float64) {
|
||||
f.child.SetQueryNorm(n)
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Count() uint64 {
|
||||
return f.child.Count()
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) Min() int {
|
||||
return f.child.Min()
|
||||
}
|
||||
|
||||
func (f *FilteringSearcher) DocumentMatchPoolSize() int {
|
||||
return f.child.DocumentMatchPoolSize()
|
||||
}
|
||||
250
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go
generated
vendored
Normal file
250
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_fuzzy.go
generated
vendored
Normal file
@@ -0,0 +1,250 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var MaxFuzziness = 2
|
||||
|
||||
// AutoFuzzinessHighThreshold is the threshold for the term length
|
||||
// above which the fuzziness is set to MaxFuzziness when the fuzziness
|
||||
// mode is set to AutoFuzziness.
|
||||
var AutoFuzzinessHighThreshold = 5
|
||||
|
||||
// AutoFuzzinessLowThreshold is the threshold for the term length
|
||||
// below which the fuzziness is set to zero when the fuzziness mode
|
||||
// is set to AutoFuzziness.
|
||||
// For terms with length between AutoFuzzinessLowThreshold and
|
||||
// AutoFuzzinessHighThreshold, the fuzziness is set to
|
||||
// MaxFuzziness - 1.
|
||||
var AutoFuzzinessLowThreshold = 2
|
||||
|
||||
func NewFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term string,
|
||||
prefix, fuzziness int, field string, boost float64,
|
||||
options search.SearcherOptions) (search.Searcher, error) {
|
||||
|
||||
if fuzziness > MaxFuzziness {
|
||||
return nil, fmt.Errorf("fuzziness exceeds max (%d)", MaxFuzziness)
|
||||
}
|
||||
|
||||
if fuzziness < 0 {
|
||||
return nil, fmt.Errorf("invalid fuzziness, negative")
|
||||
}
|
||||
if fuzziness == 0 {
|
||||
// no fuzziness, just do a term search
|
||||
// check if the call is made from a phrase searcher
|
||||
// and if so, add the term to the fuzzy term matches
|
||||
// since the fuzzy candidate terms are not collected
|
||||
// for a term search, and the only candidate term is
|
||||
// the term itself
|
||||
if ctx != nil {
|
||||
fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey)
|
||||
if fuzzyTermMatches != nil {
|
||||
fuzzyTermMatches.(map[string][]string)[term] = []string{term}
|
||||
}
|
||||
}
|
||||
return NewTermSearcher(ctx, indexReader, term, field, boost, options)
|
||||
}
|
||||
|
||||
// Note: we don't byte slice the term for a prefix because of runes.
|
||||
prefixTerm := ""
|
||||
for i, r := range term {
|
||||
if i < prefix {
|
||||
prefixTerm += string(r)
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
fuzzyCandidates, err := findFuzzyCandidateTerms(ctx, indexReader, term, fuzziness,
|
||||
field, prefixTerm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var candidates []string
|
||||
var editDistances []uint8
|
||||
var dictBytesRead uint64
|
||||
if fuzzyCandidates != nil {
|
||||
candidates = fuzzyCandidates.candidates
|
||||
editDistances = fuzzyCandidates.editDistances
|
||||
dictBytesRead = fuzzyCandidates.bytesRead
|
||||
}
|
||||
|
||||
if ctx != nil {
|
||||
reportIOStats(ctx, dictBytesRead)
|
||||
search.RecordSearchCost(ctx, search.AddM, dictBytesRead)
|
||||
fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey)
|
||||
if fuzzyTermMatches != nil {
|
||||
fuzzyTermMatches.(map[string][]string)[term] = candidates
|
||||
}
|
||||
}
|
||||
// check if the candidates are empty or have one term which is the term itself
|
||||
if len(candidates) == 0 || (len(candidates) == 1 && candidates[0] == term) {
|
||||
if ctx != nil {
|
||||
fuzzyTermMatches := ctx.Value(search.FuzzyMatchPhraseKey)
|
||||
if fuzzyTermMatches != nil {
|
||||
fuzzyTermMatches.(map[string][]string)[term] = []string{term}
|
||||
}
|
||||
}
|
||||
return NewTermSearcher(ctx, indexReader, term, field, boost, options)
|
||||
}
|
||||
|
||||
return NewMultiTermSearcherBoosted(ctx, indexReader, candidates, field,
|
||||
boost, editDistances, options, true)
|
||||
}
|
||||
|
||||
func GetAutoFuzziness(term string) int {
|
||||
termLength := len(term)
|
||||
if termLength > AutoFuzzinessHighThreshold {
|
||||
return MaxFuzziness
|
||||
} else if termLength > AutoFuzzinessLowThreshold {
|
||||
return MaxFuzziness - 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func NewAutoFuzzySearcher(ctx context.Context, indexReader index.IndexReader, term string,
|
||||
prefix int, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) {
|
||||
return NewFuzzySearcher(ctx, indexReader, term, prefix, GetAutoFuzziness(term), field, boost, options)
|
||||
}
|
||||
|
||||
type fuzzyCandidates struct {
|
||||
candidates []string
|
||||
editDistances []uint8
|
||||
bytesRead uint64
|
||||
}
|
||||
|
||||
func reportIOStats(ctx context.Context, bytesRead uint64) {
|
||||
// The fuzzy, regexp like queries essentially load a dictionary,
|
||||
// which potentially incurs a cost that must be accounted by
|
||||
// using the callback to report the value.
|
||||
if ctx != nil {
|
||||
statsCallbackFn := ctx.Value(search.SearchIOStatsCallbackKey)
|
||||
if statsCallbackFn != nil {
|
||||
statsCallbackFn.(search.SearchIOStatsCallbackFunc)(bytesRead)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func findFuzzyCandidateTerms(ctx context.Context, indexReader index.IndexReader, term string,
|
||||
fuzziness int, field, prefixTerm string) (rv *fuzzyCandidates, err error) {
|
||||
rv = &fuzzyCandidates{
|
||||
candidates: make([]string, 0),
|
||||
editDistances: make([]uint8, 0),
|
||||
}
|
||||
|
||||
// in case of advanced reader implementations directly call
|
||||
// the levenshtein automaton based iterator to collect the
|
||||
// candidate terms
|
||||
if ir, ok := indexReader.(index.IndexReaderFuzzy); ok {
|
||||
termSet := make(map[string]struct{})
|
||||
addCandidateTerm := func(term string, editDistance uint8) error {
|
||||
if _, exists := termSet[term]; !exists {
|
||||
termSet[term] = struct{}{}
|
||||
rv.candidates = append(rv.candidates, term)
|
||||
rv.editDistances = append(rv.editDistances, editDistance)
|
||||
if tooManyClauses(len(rv.candidates)) {
|
||||
return tooManyClausesErr(field, len(rv.candidates))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fieldDict, a, err := ir.FieldDictFuzzyAutomaton(field, term, fuzziness, prefixTerm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if cerr := fieldDict.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
tfd, err := fieldDict.Next()
|
||||
for err == nil && tfd != nil {
|
||||
err = addCandidateTerm(tfd.Term, tfd.EditDistance)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tfd, err = fieldDict.Next()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ctx != nil {
|
||||
if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {
|
||||
if ts, exists := fts[field]; exists {
|
||||
for term := range ts {
|
||||
if _, exists := termSet[term]; exists {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(term, prefixTerm) {
|
||||
continue
|
||||
}
|
||||
match, editDistance := a.MatchAndDistance(term)
|
||||
if match {
|
||||
err = addCandidateTerm(term, editDistance)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rv.bytesRead = fieldDict.BytesRead()
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
var fieldDict index.FieldDict
|
||||
if len(prefixTerm) > 0 {
|
||||
fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm))
|
||||
} else {
|
||||
fieldDict, err = indexReader.FieldDict(field)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if cerr := fieldDict.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
|
||||
// enumerate terms and check levenshtein distance
|
||||
var reuse []int
|
||||
tfd, err := fieldDict.Next()
|
||||
for err == nil && tfd != nil {
|
||||
var ld int
|
||||
var exceeded bool
|
||||
ld, exceeded, reuse = search.LevenshteinDistanceMaxReuseSlice(term, tfd.Term, fuzziness, reuse)
|
||||
if !exceeded && ld <= fuzziness {
|
||||
rv.candidates = append(rv.candidates, tfd.Term)
|
||||
rv.editDistances = append(rv.editDistances, uint8(ld))
|
||||
if tooManyClauses(len(rv.candidates)) {
|
||||
return nil, tooManyClausesErr(field, len(rv.candidates))
|
||||
}
|
||||
}
|
||||
tfd, err = fieldDict.Next()
|
||||
}
|
||||
|
||||
rv.bytesRead = fieldDict.BytesRead()
|
||||
return rv, err
|
||||
}
|
||||
306
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoboundingbox.go
generated
vendored
Normal file
306
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoboundingbox.go
generated
vendored
Normal file
@@ -0,0 +1,306 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/document"
|
||||
"github.com/blevesearch/bleve/v2/geo"
|
||||
"github.com/blevesearch/bleve/v2/numeric"
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
type filterFunc func(key []byte) bool
|
||||
|
||||
var (
|
||||
GeoBitsShift1 = geo.GeoBits << 1
|
||||
GeoBitsShift1Minus1 = GeoBitsShift1 - 1
|
||||
)
|
||||
|
||||
func NewGeoBoundingBoxSearcher(ctx context.Context, indexReader index.IndexReader, minLon, minLat,
|
||||
maxLon, maxLat float64, field string, boost float64,
|
||||
options search.SearcherOptions, checkBoundaries bool) (
|
||||
search.Searcher, error,
|
||||
) {
|
||||
if tp, ok := indexReader.(index.SpatialIndexPlugin); ok {
|
||||
sp, err := tp.GetSpatialAnalyzerPlugin("s2")
|
||||
if err == nil {
|
||||
terms := sp.GetQueryTokens(geo.NewBoundedRectangle(minLat,
|
||||
minLon, maxLat, maxLon))
|
||||
boxSearcher, err := NewMultiTermSearcher(ctx, indexReader,
|
||||
terms, field, boost, options, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dvReader, err := indexReader.DocValueReader([]string{field})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewFilteringSearcher(ctx, boxSearcher, buildRectFilter(ctx, dvReader,
|
||||
field, minLon, minLat, maxLon, maxLat)), nil
|
||||
}
|
||||
}
|
||||
|
||||
// indexes without the spatial plugin override would continue here.
|
||||
|
||||
// track list of opened searchers, for cleanup on early exit
|
||||
var openedSearchers []search.Searcher
|
||||
cleanupOpenedSearchers := func() {
|
||||
for _, s := range openedSearchers {
|
||||
_ = s.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// do math to produce list of terms needed for this search
|
||||
onBoundaryTerms, notOnBoundaryTerms, err := ComputeGeoRange(context.TODO(), 0, GeoBitsShift1Minus1,
|
||||
minLon, minLat, maxLon, maxLat, checkBoundaries, indexReader, field)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var onBoundarySearcher search.Searcher
|
||||
dvReader, err := indexReader.DocValueReader([]string{field})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(onBoundaryTerms) > 0 {
|
||||
rawOnBoundarySearcher, err := NewMultiTermSearcherBytes(ctx, indexReader,
|
||||
onBoundaryTerms, field, boost, options, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// add filter to check points near the boundary
|
||||
onBoundarySearcher = NewFilteringSearcher(ctx, rawOnBoundarySearcher,
|
||||
buildRectFilter(ctx, dvReader, field, minLon, minLat, maxLon, maxLat))
|
||||
openedSearchers = append(openedSearchers, onBoundarySearcher)
|
||||
}
|
||||
|
||||
var notOnBoundarySearcher search.Searcher
|
||||
if len(notOnBoundaryTerms) > 0 {
|
||||
var err error
|
||||
notOnBoundarySearcher, err = NewMultiTermSearcherBytes(ctx, indexReader,
|
||||
notOnBoundaryTerms, field, boost, options, false)
|
||||
if err != nil {
|
||||
cleanupOpenedSearchers()
|
||||
return nil, err
|
||||
}
|
||||
openedSearchers = append(openedSearchers, notOnBoundarySearcher)
|
||||
}
|
||||
|
||||
if onBoundarySearcher != nil && notOnBoundarySearcher != nil {
|
||||
rv, err := NewDisjunctionSearcher(ctx, indexReader,
|
||||
[]search.Searcher{
|
||||
onBoundarySearcher,
|
||||
notOnBoundarySearcher,
|
||||
},
|
||||
0, options)
|
||||
if err != nil {
|
||||
cleanupOpenedSearchers()
|
||||
return nil, err
|
||||
}
|
||||
return rv, nil
|
||||
} else if onBoundarySearcher != nil {
|
||||
return onBoundarySearcher, nil
|
||||
} else if notOnBoundarySearcher != nil {
|
||||
return notOnBoundarySearcher, nil
|
||||
}
|
||||
|
||||
return NewMatchNoneSearcher(indexReader)
|
||||
}
|
||||
|
||||
var (
|
||||
geoMaxShift = document.GeoPrecisionStep * 4
|
||||
geoDetailLevel = ((geo.GeoBits << 1) - geoMaxShift) / 2
|
||||
)
|
||||
|
||||
type closeFunc func() error
|
||||
|
||||
func ComputeGeoRange(ctx context.Context, term uint64, shift uint,
|
||||
sminLon, sminLat, smaxLon, smaxLat float64, checkBoundaries bool,
|
||||
indexReader index.IndexReader, field string) (
|
||||
onBoundary [][]byte, notOnBoundary [][]byte, err error,
|
||||
) {
|
||||
isIndexed, closeF, err := buildIsIndexedFunc(ctx, indexReader, field)
|
||||
if closeF != nil {
|
||||
defer func() {
|
||||
cerr := closeF()
|
||||
if cerr != nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
grc := &geoRangeCompute{
|
||||
preallocBytesLen: 32,
|
||||
preallocBytes: make([]byte, 32),
|
||||
sminLon: sminLon,
|
||||
sminLat: sminLat,
|
||||
smaxLon: smaxLon,
|
||||
smaxLat: smaxLat,
|
||||
checkBoundaries: checkBoundaries,
|
||||
isIndexed: isIndexed,
|
||||
}
|
||||
|
||||
grc.computeGeoRange(term, shift)
|
||||
|
||||
return grc.onBoundary, grc.notOnBoundary, nil
|
||||
}
|
||||
|
||||
func buildIsIndexedFunc(ctx context.Context, indexReader index.IndexReader, field string) (isIndexed filterFunc, closeF closeFunc, err error) {
|
||||
if irr, ok := indexReader.(index.IndexReaderContains); ok {
|
||||
fieldDict, err := irr.FieldDictContains(field)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
isIndexed = func(term []byte) bool {
|
||||
found, err := fieldDict.Contains(term)
|
||||
return err == nil && found
|
||||
}
|
||||
|
||||
closeF = func() error {
|
||||
if fd, ok := fieldDict.(index.FieldDict); ok {
|
||||
err := fd.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
} else if indexReader != nil {
|
||||
isIndexed = func(term []byte) bool {
|
||||
reader, err := indexReader.TermFieldReader(ctx, term, field, false, false, false)
|
||||
if err != nil || reader == nil {
|
||||
return false
|
||||
}
|
||||
if reader.Count() == 0 {
|
||||
_ = reader.Close()
|
||||
return false
|
||||
}
|
||||
_ = reader.Close()
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
isIndexed = func([]byte) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return isIndexed, closeF, err
|
||||
}
|
||||
|
||||
func buildRectFilter(ctx context.Context, dvReader index.DocValueReader, field string,
|
||||
minLon, minLat, maxLon, maxLat float64,
|
||||
) FilterFunc {
|
||||
return func(d *search.DocumentMatch) bool {
|
||||
// check geo matches against all numeric type terms indexed
|
||||
var lons, lats []float64
|
||||
var found bool
|
||||
err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) {
|
||||
// only consider the values which are shifted 0
|
||||
prefixCoded := numeric.PrefixCoded(term)
|
||||
shift, err := prefixCoded.Shift()
|
||||
if err == nil && shift == 0 {
|
||||
var i64 int64
|
||||
i64, err = prefixCoded.Int64()
|
||||
if err == nil {
|
||||
lons = append(lons, geo.MortonUnhashLon(uint64(i64)))
|
||||
lats = append(lats, geo.MortonUnhashLat(uint64(i64)))
|
||||
found = true
|
||||
}
|
||||
}
|
||||
})
|
||||
if err == nil && found {
|
||||
bytes := dvReader.BytesRead()
|
||||
if bytes > 0 {
|
||||
reportIOStats(ctx, bytes)
|
||||
search.RecordSearchCost(ctx, search.AddM, bytes)
|
||||
}
|
||||
for i := range lons {
|
||||
if geo.BoundingBoxContains(lons[i], lats[i],
|
||||
minLon, minLat, maxLon, maxLat) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type geoRangeCompute struct {
|
||||
preallocBytesLen int
|
||||
preallocBytes []byte
|
||||
sminLon, sminLat, smaxLon, smaxLat float64
|
||||
checkBoundaries bool
|
||||
onBoundary, notOnBoundary [][]byte
|
||||
isIndexed func(term []byte) bool
|
||||
}
|
||||
|
||||
func (grc *geoRangeCompute) makePrefixCoded(in int64, shift uint) (rv numeric.PrefixCoded) {
|
||||
if len(grc.preallocBytes) <= 0 {
|
||||
grc.preallocBytesLen = grc.preallocBytesLen * 2
|
||||
grc.preallocBytes = make([]byte, grc.preallocBytesLen)
|
||||
}
|
||||
|
||||
rv, grc.preallocBytes, _ = numeric.NewPrefixCodedInt64Prealloc(in, shift, grc.preallocBytes)
|
||||
|
||||
return rv
|
||||
}
|
||||
|
||||
func (grc *geoRangeCompute) computeGeoRange(term uint64, shift uint) {
|
||||
split := term | uint64(0x1)<<shift
|
||||
var upperMax uint64
|
||||
if shift < 63 {
|
||||
upperMax = term | ((uint64(1) << (shift + 1)) - 1)
|
||||
} else {
|
||||
upperMax = 0xffffffffffffffff
|
||||
}
|
||||
lowerMax := split - 1
|
||||
grc.relateAndRecurse(term, lowerMax, shift)
|
||||
grc.relateAndRecurse(split, upperMax, shift)
|
||||
}
|
||||
|
||||
func (grc *geoRangeCompute) relateAndRecurse(start, end uint64, res uint) {
|
||||
minLon := geo.MortonUnhashLon(start)
|
||||
minLat := geo.MortonUnhashLat(start)
|
||||
maxLon := geo.MortonUnhashLon(end)
|
||||
maxLat := geo.MortonUnhashLat(end)
|
||||
|
||||
level := (GeoBitsShift1 - res) >> 1
|
||||
|
||||
within := res%document.GeoPrecisionStep == 0 &&
|
||||
geo.RectWithin(minLon, minLat, maxLon, maxLat,
|
||||
grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat)
|
||||
if within || (level == geoDetailLevel &&
|
||||
geo.RectIntersects(minLon, minLat, maxLon, maxLat,
|
||||
grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat)) {
|
||||
codedTerm := grc.makePrefixCoded(int64(start), res)
|
||||
if grc.isIndexed(codedTerm) {
|
||||
if !within && grc.checkBoundaries {
|
||||
grc.onBoundary = append(grc.onBoundary, codedTerm)
|
||||
} else {
|
||||
grc.notOnBoundary = append(grc.notOnBoundary, codedTerm)
|
||||
}
|
||||
}
|
||||
} else if level < geoDetailLevel &&
|
||||
geo.RectIntersects(minLon, minLat, maxLon, maxLat,
|
||||
grc.sminLon, grc.sminLat, grc.smaxLon, grc.smaxLat) {
|
||||
grc.computeGeoRange(start, res-1)
|
||||
}
|
||||
}
|
||||
151
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopointdistance.go
generated
vendored
Normal file
151
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopointdistance.go
generated
vendored
Normal file
@@ -0,0 +1,151 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/geo"
|
||||
"github.com/blevesearch/bleve/v2/numeric"
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
func NewGeoPointDistanceSearcher(ctx context.Context, indexReader index.IndexReader, centerLon,
|
||||
centerLat, dist float64, field string, boost float64,
|
||||
options search.SearcherOptions) (search.Searcher, error) {
|
||||
var rectSearcher search.Searcher
|
||||
if tp, ok := indexReader.(index.SpatialIndexPlugin); ok {
|
||||
sp, err := tp.GetSpatialAnalyzerPlugin("s2")
|
||||
if err == nil {
|
||||
terms := sp.GetQueryTokens(geo.NewPointDistance(centerLat,
|
||||
centerLon, dist))
|
||||
rectSearcher, err = NewMultiTermSearcher(ctx, indexReader, terms,
|
||||
field, boost, options, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// indexes without the spatial plugin override would get
|
||||
// initialized here.
|
||||
if rectSearcher == nil {
|
||||
// compute bounding box containing the circle
|
||||
topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err :=
|
||||
geo.RectFromPointDistance(centerLon, centerLat, dist)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// build a searcher for the box
|
||||
rectSearcher, err = boxSearcher(ctx, indexReader,
|
||||
topLeftLon, topLeftLat, bottomRightLon, bottomRightLat,
|
||||
field, boost, options, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
dvReader, err := indexReader.DocValueReader([]string{field})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// wrap it in a filtering searcher which checks the actual distance
|
||||
return NewFilteringSearcher(ctx, rectSearcher,
|
||||
buildDistFilter(ctx, dvReader, field, centerLon, centerLat, dist)), nil
|
||||
}
|
||||
|
||||
// boxSearcher builds a searcher for the described bounding box
|
||||
// if the desired box crosses the dateline, it is automatically split into
|
||||
// two boxes joined through a disjunction searcher
|
||||
func boxSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
topLeftLon, topLeftLat, bottomRightLon, bottomRightLat float64,
|
||||
field string, boost float64, options search.SearcherOptions, checkBoundaries bool) (
|
||||
search.Searcher, error) {
|
||||
if bottomRightLon < topLeftLon {
|
||||
// cross date line, rewrite as two parts
|
||||
|
||||
leftSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader,
|
||||
-180, bottomRightLat, bottomRightLon, topLeftLat,
|
||||
field, boost, options, checkBoundaries)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rightSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader,
|
||||
topLeftLon, bottomRightLat, 180, topLeftLat, field, boost, options,
|
||||
checkBoundaries)
|
||||
if err != nil {
|
||||
_ = leftSearcher.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
boxSearcher, err := NewDisjunctionSearcher(ctx, indexReader,
|
||||
[]search.Searcher{leftSearcher, rightSearcher}, 0, options)
|
||||
if err != nil {
|
||||
_ = leftSearcher.Close()
|
||||
_ = rightSearcher.Close()
|
||||
return nil, err
|
||||
}
|
||||
return boxSearcher, nil
|
||||
}
|
||||
|
||||
// build geoboundingbox searcher for that bounding box
|
||||
boxSearcher, err := NewGeoBoundingBoxSearcher(ctx, indexReader,
|
||||
topLeftLon, bottomRightLat, bottomRightLon, topLeftLat, field, boost,
|
||||
options, checkBoundaries)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return boxSearcher, nil
|
||||
}
|
||||
|
||||
func buildDistFilter(ctx context.Context, dvReader index.DocValueReader, field string,
|
||||
centerLon, centerLat, maxDist float64) FilterFunc {
|
||||
return func(d *search.DocumentMatch) bool {
|
||||
// check geo matches against all numeric type terms indexed
|
||||
var lons, lats []float64
|
||||
var found bool
|
||||
|
||||
err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) {
|
||||
// only consider the values which are shifted 0
|
||||
prefixCoded := numeric.PrefixCoded(term)
|
||||
shift, err := prefixCoded.Shift()
|
||||
if err == nil && shift == 0 {
|
||||
i64, err := prefixCoded.Int64()
|
||||
if err == nil {
|
||||
lons = append(lons, geo.MortonUnhashLon(uint64(i64)))
|
||||
lats = append(lats, geo.MortonUnhashLat(uint64(i64)))
|
||||
found = true
|
||||
}
|
||||
}
|
||||
})
|
||||
if err == nil && found {
|
||||
bytes := dvReader.BytesRead()
|
||||
if bytes > 0 {
|
||||
reportIOStats(ctx, bytes)
|
||||
search.RecordSearchCost(ctx, search.AddM, bytes)
|
||||
}
|
||||
for i := range lons {
|
||||
dist := geo.Haversin(lons[i], lats[i], centerLon, centerLat)
|
||||
if dist <= maxDist/1000 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
149
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopolygon.go
generated
vendored
Normal file
149
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geopolygon.go
generated
vendored
Normal file
@@ -0,0 +1,149 @@
|
||||
// Copyright (c) 2019 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/geo"
|
||||
"github.com/blevesearch/bleve/v2/numeric"
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
func NewGeoBoundedPolygonSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
coordinates []geo.Point, field string, boost float64,
|
||||
options search.SearcherOptions) (search.Searcher, error) {
|
||||
if len(coordinates) < 3 {
|
||||
return nil, fmt.Errorf("Too few points specified for the polygon boundary")
|
||||
}
|
||||
|
||||
var rectSearcher search.Searcher
|
||||
if sr, ok := indexReader.(index.SpatialIndexPlugin); ok {
|
||||
tp, err := sr.GetSpatialAnalyzerPlugin("s2")
|
||||
if err == nil {
|
||||
terms := tp.GetQueryTokens(geo.NewBoundedPolygon(coordinates))
|
||||
rectSearcher, err = NewMultiTermSearcher(ctx, indexReader, terms,
|
||||
field, boost, options, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// indexes without the spatial plugin override would get
|
||||
// initialized here.
|
||||
if rectSearcher == nil {
|
||||
// compute the bounding box enclosing the polygon
|
||||
topLeftLon, topLeftLat, bottomRightLon, bottomRightLat, err :=
|
||||
geo.BoundingRectangleForPolygon(coordinates)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// build a searcher for the bounding box on the polygon
|
||||
rectSearcher, err = boxSearcher(ctx, indexReader,
|
||||
topLeftLon, topLeftLat, bottomRightLon, bottomRightLat,
|
||||
field, boost, options, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
dvReader, err := indexReader.DocValueReader([]string{field})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// wrap it in a filtering searcher that checks for the polygon inclusivity
|
||||
return NewFilteringSearcher(ctx, rectSearcher,
|
||||
buildPolygonFilter(ctx, dvReader, field, coordinates)), nil
|
||||
}
|
||||
|
||||
const float64EqualityThreshold = 1e-6
|
||||
|
||||
func almostEqual(a, b float64) bool {
|
||||
return math.Abs(a-b) <= float64EqualityThreshold
|
||||
}
|
||||
|
||||
// buildPolygonFilter returns true if the point lies inside the
|
||||
// polygon. It is based on the ray-casting technique as referred
|
||||
// here: https://wrf.ecse.rpi.edu/nikola/pubdetails/pnpoly.html
|
||||
func buildPolygonFilter(ctx context.Context, dvReader index.DocValueReader, field string,
|
||||
coordinates []geo.Point) FilterFunc {
|
||||
return func(d *search.DocumentMatch) bool {
|
||||
// check geo matches against all numeric type terms indexed
|
||||
var lons, lats []float64
|
||||
var found bool
|
||||
|
||||
err := dvReader.VisitDocValues(d.IndexInternalID, func(field string, term []byte) {
|
||||
// only consider the values which are shifted 0
|
||||
prefixCoded := numeric.PrefixCoded(term)
|
||||
shift, err := prefixCoded.Shift()
|
||||
if err == nil && shift == 0 {
|
||||
i64, err := prefixCoded.Int64()
|
||||
if err == nil {
|
||||
lons = append(lons, geo.MortonUnhashLon(uint64(i64)))
|
||||
lats = append(lats, geo.MortonUnhashLat(uint64(i64)))
|
||||
found = true
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Note: this approach works for points which are strictly inside
|
||||
// the polygon. ie it might fail for certain points on the polygon boundaries.
|
||||
if err == nil && found {
|
||||
bytes := dvReader.BytesRead()
|
||||
if bytes > 0 {
|
||||
reportIOStats(ctx, bytes)
|
||||
search.RecordSearchCost(ctx, search.AddM, bytes)
|
||||
}
|
||||
nVertices := len(coordinates)
|
||||
if len(coordinates) < 3 {
|
||||
return false
|
||||
}
|
||||
rayIntersectsSegment := func(point, a, b geo.Point) bool {
|
||||
return (a.Lat > point.Lat) != (b.Lat > point.Lat) &&
|
||||
point.Lon < (b.Lon-a.Lon)*(point.Lat-a.Lat)/(b.Lat-a.Lat)+a.Lon
|
||||
}
|
||||
|
||||
for i := range lons {
|
||||
pt := geo.Point{Lon: lons[i], Lat: lats[i]}
|
||||
inside := rayIntersectsSegment(pt, coordinates[len(coordinates)-1], coordinates[0])
|
||||
// check for a direct vertex match
|
||||
if almostEqual(coordinates[0].Lat, lats[i]) &&
|
||||
almostEqual(coordinates[0].Lon, lons[i]) {
|
||||
return true
|
||||
}
|
||||
|
||||
for j := 1; j < nVertices; j++ {
|
||||
if almostEqual(coordinates[j].Lat, lats[i]) &&
|
||||
almostEqual(coordinates[j].Lon, lons[i]) {
|
||||
return true
|
||||
}
|
||||
if rayIntersectsSegment(pt, coordinates[j-1], coordinates[j]) {
|
||||
inside = !inside
|
||||
}
|
||||
}
|
||||
if inside {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
135
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoshape.go
generated
vendored
Normal file
135
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_geoshape.go
generated
vendored
Normal file
@@ -0,0 +1,135 @@
|
||||
// Copyright (c) 2022 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/geo"
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
"github.com/blevesearch/geo/geojson"
|
||||
"github.com/blevesearch/geo/s2"
|
||||
)
|
||||
|
||||
func NewGeoShapeSearcher(ctx context.Context, indexReader index.IndexReader, shape index.GeoJSON,
|
||||
relation string, field string, boost float64,
|
||||
options search.SearcherOptions,
|
||||
) (search.Searcher, error) {
|
||||
var err error
|
||||
var spatialPlugin index.SpatialAnalyzerPlugin
|
||||
|
||||
// check for the spatial plugin from the index.
|
||||
if sr, ok := indexReader.(index.SpatialIndexPlugin); ok {
|
||||
spatialPlugin, _ = sr.GetSpatialAnalyzerPlugin("s2")
|
||||
}
|
||||
|
||||
if spatialPlugin == nil {
|
||||
// fallback to the default spatial plugin(s2).
|
||||
spatialPlugin = geo.GetSpatialAnalyzerPlugin("s2")
|
||||
}
|
||||
|
||||
// obtain the query tokens.
|
||||
terms := spatialPlugin.GetQueryTokens(shape)
|
||||
mSearcher, err := NewMultiTermSearcher(ctx, indexReader, terms,
|
||||
field, boost, options, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dvReader, err := indexReader.DocValueReader([]string{field})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewFilteringSearcher(ctx, mSearcher, buildRelationFilterOnShapes(ctx, dvReader, field, relation, shape)), nil
|
||||
}
|
||||
|
||||
// Using the same term splitter slice used in the doc values in zap.
|
||||
// TODO: This needs to be revisited whenever we change the zap
|
||||
// implementation of doc values.
|
||||
var termSeparatorSplitSlice = []byte{0xff}
|
||||
|
||||
func buildRelationFilterOnShapes(ctx context.Context, dvReader index.DocValueReader, field string,
|
||||
relation string, shape index.GeoJSON,
|
||||
) FilterFunc {
|
||||
// this is for accumulating the shape's actual complete value
|
||||
// spread across multiple docvalue visitor callbacks.
|
||||
var dvShapeValue []byte
|
||||
var startReading, finishReading bool
|
||||
var reader *bytes.Reader
|
||||
|
||||
var bufPool *s2.GeoBufferPool
|
||||
if bufPoolCallback, ok := ctx.Value(search.GeoBufferPoolCallbackKey).(search.GeoBufferPoolCallbackFunc); ok {
|
||||
bufPool = bufPoolCallback()
|
||||
}
|
||||
|
||||
return func(d *search.DocumentMatch) bool {
|
||||
var found bool
|
||||
|
||||
err := dvReader.VisitDocValues(d.IndexInternalID,
|
||||
func(field string, term []byte) {
|
||||
// only consider the values which are GlueBytes prefixed or
|
||||
// if it had already started reading the shape bytes from previous callbacks.
|
||||
if startReading || len(term) > geo.GlueBytesOffset {
|
||||
|
||||
if !startReading && bytes.Equal(geo.GlueBytes, term[:geo.GlueBytesOffset]) {
|
||||
startReading = true
|
||||
|
||||
if bytes.Equal(geo.GlueBytes, term[len(term)-geo.GlueBytesOffset:]) {
|
||||
term = term[:len(term)-geo.GlueBytesOffset]
|
||||
finishReading = true
|
||||
}
|
||||
|
||||
dvShapeValue = append(dvShapeValue, term[geo.GlueBytesOffset:]...)
|
||||
|
||||
} else if startReading && !finishReading {
|
||||
if len(term) > geo.GlueBytesOffset &&
|
||||
bytes.Equal(geo.GlueBytes, term[len(term)-geo.GlueBytesOffset:]) {
|
||||
term = term[:len(term)-geo.GlueBytesOffset]
|
||||
finishReading = true
|
||||
}
|
||||
|
||||
term = append(termSeparatorSplitSlice, term...)
|
||||
dvShapeValue = append(dvShapeValue, term...)
|
||||
}
|
||||
|
||||
// apply the filter once the entire docvalue is finished reading.
|
||||
if finishReading {
|
||||
v, err := geojson.FilterGeoShapesOnRelation(shape, dvShapeValue, relation, &reader, bufPool)
|
||||
if err == nil && v {
|
||||
found = true
|
||||
}
|
||||
|
||||
dvShapeValue = dvShapeValue[:0]
|
||||
startReading = false
|
||||
finishReading = false
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if err == nil && found {
|
||||
bytes := dvReader.BytesRead()
|
||||
if bytes > 0 {
|
||||
reportIOStats(ctx, bytes)
|
||||
search.RecordSearchCost(ctx, search.AddM, bytes)
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
}
|
||||
68
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_ip_range.go
generated
vendored
Normal file
68
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_ip_range.go
generated
vendored
Normal file
@@ -0,0 +1,68 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
// netLimits returns the lo and hi bounds inside the network.
|
||||
func netLimits(n *net.IPNet) (lo net.IP, hi net.IP) {
|
||||
ones, bits := n.Mask.Size()
|
||||
netNum := n.IP
|
||||
if bits == net.IPv4len*8 {
|
||||
netNum = netNum.To16()
|
||||
ones += 8 * (net.IPv6len - net.IPv4len)
|
||||
}
|
||||
mask := net.CIDRMask(ones, 8*net.IPv6len)
|
||||
lo = make(net.IP, net.IPv6len)
|
||||
hi = make(net.IP, net.IPv6len)
|
||||
for i := 0; i < net.IPv6len; i++ {
|
||||
lo[i] = netNum[i] & mask[i]
|
||||
hi[i] = lo[i] | ^mask[i]
|
||||
}
|
||||
return lo, hi
|
||||
}
|
||||
|
||||
func NewIPRangeSearcher(ctx context.Context, indexReader index.IndexReader, ipNet *net.IPNet,
|
||||
field string, boost float64, options search.SearcherOptions) (
|
||||
search.Searcher, error) {
|
||||
|
||||
lo, hi := netLimits(ipNet)
|
||||
fieldDict, err := indexReader.FieldDictRange(field, lo, hi)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer fieldDict.Close()
|
||||
|
||||
var terms []string
|
||||
tfd, err := fieldDict.Next()
|
||||
for err == nil && tfd != nil {
|
||||
terms = append(terms, tfd.Term)
|
||||
if tooManyClauses(len(terms)) {
|
||||
return nil, tooManyClausesErr(field, len(terms))
|
||||
}
|
||||
tfd, err = fieldDict.Next()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true)
|
||||
}
|
||||
145
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_knn.go
generated
vendored
Normal file
145
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_knn.go
generated
vendored
Normal file
@@ -0,0 +1,145 @@
|
||||
// Copyright (c) 2023 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build vectors
|
||||
// +build vectors
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/mapping"
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/scorer"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeKNNSearcher int
|
||||
|
||||
func init() {
|
||||
var ks KNNSearcher
|
||||
reflectStaticSizeKNNSearcher = int(reflect.TypeOf(ks).Size())
|
||||
}
|
||||
|
||||
type KNNSearcher struct {
|
||||
field string
|
||||
vector []float32
|
||||
k int64
|
||||
indexReader index.IndexReader
|
||||
vectorReader index.VectorReader
|
||||
scorer *scorer.KNNQueryScorer
|
||||
count uint64
|
||||
vd index.VectorDoc
|
||||
}
|
||||
|
||||
func NewKNNSearcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping,
|
||||
options search.SearcherOptions, field string, vector []float32, k int64,
|
||||
boost float64, similarityMetric string, searchParams json.RawMessage,
|
||||
eligibleSelector index.EligibleDocumentSelector) (
|
||||
search.Searcher, error) {
|
||||
|
||||
if vr, ok := i.(index.VectorIndexReader); ok {
|
||||
vectorReader, err := vr.VectorReader(ctx, vector, field, k, searchParams, eligibleSelector)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
knnScorer := scorer.NewKNNQueryScorer(vector, field, boost,
|
||||
options, similarityMetric)
|
||||
return &KNNSearcher{
|
||||
indexReader: i,
|
||||
vectorReader: vectorReader,
|
||||
field: field,
|
||||
vector: vector,
|
||||
k: k,
|
||||
scorer: knnScorer,
|
||||
}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) VectorOptimize(ctx context.Context, octx index.VectorOptimizableContext) (
|
||||
index.VectorOptimizableContext, error) {
|
||||
o, ok := s.vectorReader.(index.VectorOptimizable)
|
||||
if ok {
|
||||
return o.VectorOptimize(ctx, octx)
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (
|
||||
*search.DocumentMatch, error) {
|
||||
knnMatch, err := s.vectorReader.Next(s.vd.Reset())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if knnMatch == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
docMatch := s.scorer.Score(ctx, knnMatch)
|
||||
|
||||
return docMatch, nil
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) Close() error {
|
||||
return s.vectorReader.Close()
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) Count() uint64 {
|
||||
return s.vectorReader.Count()
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) DocumentMatchPoolSize() int {
|
||||
return 1
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) Min() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
knnMatch, err := s.vectorReader.Next(s.vd.Reset())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if knnMatch == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
docMatch := s.scorer.Score(ctx, knnMatch)
|
||||
|
||||
return docMatch, nil
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) SetQueryNorm(qnorm float64) {
|
||||
s.scorer.SetQueryNorm(qnorm)
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) Size() int {
|
||||
return reflectStaticSizeKNNSearcher + size.SizeOfPtr +
|
||||
s.vectorReader.Size() +
|
||||
s.vd.Size() +
|
||||
s.scorer.Size()
|
||||
}
|
||||
|
||||
func (s *KNNSearcher) Weight() float64 {
|
||||
return s.scorer.Weight()
|
||||
}
|
||||
123
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_all.go
generated
vendored
Normal file
123
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_all.go
generated
vendored
Normal file
@@ -0,0 +1,123 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/scorer"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeMatchAllSearcher int
|
||||
|
||||
func init() {
|
||||
var mas MatchAllSearcher
|
||||
reflectStaticSizeMatchAllSearcher = int(reflect.TypeOf(mas).Size())
|
||||
}
|
||||
|
||||
type MatchAllSearcher struct {
|
||||
indexReader index.IndexReader
|
||||
reader index.DocIDReader
|
||||
scorer *scorer.ConstantScorer
|
||||
count uint64
|
||||
}
|
||||
|
||||
func NewMatchAllSearcher(ctx context.Context, indexReader index.IndexReader, boost float64, options search.SearcherOptions) (*MatchAllSearcher, error) {
|
||||
reader, err := indexReader.DocIDReaderAll()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
count, err := indexReader.DocCount()
|
||||
if err != nil {
|
||||
_ = reader.Close()
|
||||
return nil, err
|
||||
}
|
||||
scorer := scorer.NewConstantScorer(1.0, boost, options)
|
||||
|
||||
return &MatchAllSearcher{
|
||||
indexReader: indexReader,
|
||||
reader: reader,
|
||||
scorer: scorer,
|
||||
count: count,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *MatchAllSearcher) Size() int {
|
||||
return reflectStaticSizeMatchAllSearcher + size.SizeOfPtr +
|
||||
s.reader.Size() +
|
||||
s.scorer.Size()
|
||||
}
|
||||
|
||||
func (s *MatchAllSearcher) Count() uint64 {
|
||||
return s.count
|
||||
}
|
||||
|
||||
func (s *MatchAllSearcher) Weight() float64 {
|
||||
return s.scorer.Weight()
|
||||
}
|
||||
|
||||
func (s *MatchAllSearcher) SetQueryNorm(qnorm float64) {
|
||||
s.scorer.SetQueryNorm(qnorm)
|
||||
}
|
||||
|
||||
func (s *MatchAllSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
id, err := s.reader.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if id == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// score match
|
||||
docMatch := s.scorer.Score(ctx, id)
|
||||
// return doc match
|
||||
return docMatch, nil
|
||||
|
||||
}
|
||||
|
||||
func (s *MatchAllSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
id, err := s.reader.Advance(ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if id == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// score match
|
||||
docMatch := s.scorer.Score(ctx, id)
|
||||
|
||||
// return doc match
|
||||
return docMatch, nil
|
||||
}
|
||||
|
||||
func (s *MatchAllSearcher) Close() error {
|
||||
return s.reader.Close()
|
||||
}
|
||||
|
||||
func (s *MatchAllSearcher) Min() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *MatchAllSearcher) DocumentMatchPoolSize() int {
|
||||
return 1
|
||||
}
|
||||
76
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_none.go
generated
vendored
Normal file
76
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_match_none.go
generated
vendored
Normal file
@@ -0,0 +1,76 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeMatchNoneSearcher int
|
||||
|
||||
func init() {
|
||||
var mns MatchNoneSearcher
|
||||
reflectStaticSizeMatchNoneSearcher = int(reflect.TypeOf(mns).Size())
|
||||
}
|
||||
|
||||
type MatchNoneSearcher struct {
|
||||
indexReader index.IndexReader
|
||||
}
|
||||
|
||||
func NewMatchNoneSearcher(indexReader index.IndexReader) (*MatchNoneSearcher, error) {
|
||||
return &MatchNoneSearcher{
|
||||
indexReader: indexReader,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *MatchNoneSearcher) Size() int {
|
||||
return reflectStaticSizeMatchNoneSearcher + size.SizeOfPtr
|
||||
}
|
||||
|
||||
func (s *MatchNoneSearcher) Count() uint64 {
|
||||
return uint64(0)
|
||||
}
|
||||
|
||||
func (s *MatchNoneSearcher) Weight() float64 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
func (s *MatchNoneSearcher) SetQueryNorm(qnorm float64) {
|
||||
|
||||
}
|
||||
|
||||
func (s *MatchNoneSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *MatchNoneSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *MatchNoneSearcher) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *MatchNoneSearcher) Min() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *MatchNoneSearcher) DocumentMatchPoolSize() int {
|
||||
return 0
|
||||
}
|
||||
268
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_multi_term.go
generated
vendored
Normal file
268
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_multi_term.go
generated
vendored
Normal file
@@ -0,0 +1,268 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
func NewMultiTermSearcher(ctx context.Context, indexReader index.IndexReader, terms []string,
|
||||
field string, boost float64, options search.SearcherOptions, limit bool) (
|
||||
search.Searcher, error) {
|
||||
|
||||
if tooManyClauses(len(terms)) {
|
||||
if optionsDisjunctionOptimizable(options) {
|
||||
return optimizeMultiTermSearcher(ctx, indexReader, terms, field, boost, options)
|
||||
}
|
||||
if limit {
|
||||
return nil, tooManyClausesErr(field, len(terms))
|
||||
}
|
||||
}
|
||||
|
||||
qsearchers, err := makeBatchSearchers(ctx, indexReader, terms, field, boost, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// build disjunction searcher of these ranges
|
||||
return newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost,
|
||||
options, limit)
|
||||
}
|
||||
|
||||
// Works similarly to the multi term searcher but additionally boosts individual terms based on
|
||||
// their edit distance from the query terms
|
||||
func NewMultiTermSearcherBoosted(ctx context.Context, indexReader index.IndexReader, terms []string,
|
||||
field string, boost float64, editDistances []uint8, options search.SearcherOptions, limit bool) (
|
||||
search.Searcher, error) {
|
||||
|
||||
if tooManyClauses(len(terms)) {
|
||||
if optionsDisjunctionOptimizable(options) {
|
||||
return optimizeMultiTermSearcher(ctx, indexReader, terms, field, boost, options)
|
||||
}
|
||||
if limit {
|
||||
return nil, tooManyClausesErr(field, len(terms))
|
||||
}
|
||||
}
|
||||
|
||||
qsearchers, err := makeBatchSearchersBoosted(ctx, indexReader, terms, field, boost, editDistances, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// build disjunction searcher of these ranges
|
||||
return newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost,
|
||||
options, limit)
|
||||
}
|
||||
|
||||
func NewMultiTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte,
|
||||
field string, boost float64, options search.SearcherOptions, limit bool) (
|
||||
search.Searcher, error) {
|
||||
|
||||
if tooManyClauses(len(terms)) {
|
||||
if optionsDisjunctionOptimizable(options) {
|
||||
return optimizeMultiTermSearcherBytes(ctx, indexReader, terms, field, boost, options)
|
||||
}
|
||||
|
||||
if limit {
|
||||
return nil, tooManyClausesErr(field, len(terms))
|
||||
}
|
||||
}
|
||||
|
||||
qsearchers, err := makeBatchSearchersBytes(ctx, indexReader, terms, field, boost, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// build disjunction searcher of these ranges
|
||||
return newMultiTermSearcherInternal(ctx, indexReader, qsearchers, field, boost,
|
||||
options, limit)
|
||||
}
|
||||
|
||||
func newMultiTermSearcherInternal(ctx context.Context, indexReader index.IndexReader,
|
||||
searchers []search.Searcher, field string, boost float64,
|
||||
options search.SearcherOptions, limit bool) (
|
||||
search.Searcher, error) {
|
||||
|
||||
// build disjunction searcher of these ranges
|
||||
searcher, err := newDisjunctionSearcher(ctx, indexReader, searchers, 0, options,
|
||||
limit)
|
||||
if err != nil {
|
||||
for _, s := range searchers {
|
||||
_ = s.Close()
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return searcher, nil
|
||||
}
|
||||
|
||||
func optimizeMultiTermSearcher(ctx context.Context, indexReader index.IndexReader, terms []string,
|
||||
field string, boost float64, options search.SearcherOptions) (
|
||||
search.Searcher, error) {
|
||||
var finalSearcher search.Searcher
|
||||
for len(terms) > 0 {
|
||||
var batchTerms []string
|
||||
if len(terms) > DisjunctionMaxClauseCount {
|
||||
batchTerms = terms[:DisjunctionMaxClauseCount]
|
||||
terms = terms[DisjunctionMaxClauseCount:]
|
||||
} else {
|
||||
batchTerms = terms
|
||||
terms = nil
|
||||
}
|
||||
batch, err := makeBatchSearchers(ctx, indexReader, batchTerms, field, boost, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if finalSearcher != nil {
|
||||
batch = append(batch, finalSearcher)
|
||||
}
|
||||
cleanup := func() {
|
||||
for _, searcher := range batch {
|
||||
if searcher != nil {
|
||||
_ = searcher.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
finalSearcher, err = optimizeCompositeSearcher(ctx, "disjunction:unadorned",
|
||||
indexReader, batch, options)
|
||||
// all searchers in batch should be closed, regardless of error or optimization failure
|
||||
// either we're returning, or continuing and only finalSearcher is needed for next loop
|
||||
cleanup()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if finalSearcher == nil {
|
||||
return nil, fmt.Errorf("unable to optimize")
|
||||
}
|
||||
}
|
||||
return finalSearcher, nil
|
||||
}
|
||||
|
||||
func makeBatchSearchers(ctx context.Context, indexReader index.IndexReader, terms []string, field string,
|
||||
boost float64, options search.SearcherOptions) ([]search.Searcher, error) {
|
||||
|
||||
qsearchers := make([]search.Searcher, len(terms))
|
||||
qsearchersClose := func() {
|
||||
for _, searcher := range qsearchers {
|
||||
if searcher != nil {
|
||||
_ = searcher.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, term := range terms {
|
||||
var err error
|
||||
qsearchers[i], err = NewTermSearcher(ctx, indexReader, term, field, boost, options)
|
||||
if err != nil {
|
||||
qsearchersClose()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return qsearchers, nil
|
||||
}
|
||||
|
||||
func makeBatchSearchersBoosted(ctx context.Context, indexReader index.IndexReader, terms []string, field string,
|
||||
boost float64, editDistances []uint8, options search.SearcherOptions) ([]search.Searcher, error) {
|
||||
|
||||
qsearchers := make([]search.Searcher, len(terms))
|
||||
qsearchersClose := func() {
|
||||
for _, searcher := range qsearchers {
|
||||
if searcher != nil {
|
||||
_ = searcher.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, term := range terms {
|
||||
var err error
|
||||
var editMultiplier float64
|
||||
if editDistances != nil {
|
||||
editMultiplier = 1 / float64(editDistances[i]+1)
|
||||
}
|
||||
qsearchers[i], err = NewTermSearcher(ctx, indexReader, term, field, boost*editMultiplier, options)
|
||||
if err != nil {
|
||||
qsearchersClose()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return qsearchers, nil
|
||||
}
|
||||
|
||||
func optimizeMultiTermSearcherBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte,
|
||||
field string, boost float64, options search.SearcherOptions) (
|
||||
search.Searcher, error) {
|
||||
|
||||
var finalSearcher search.Searcher
|
||||
for len(terms) > 0 {
|
||||
var batchTerms [][]byte
|
||||
if len(terms) > DisjunctionMaxClauseCount {
|
||||
batchTerms = terms[:DisjunctionMaxClauseCount]
|
||||
terms = terms[DisjunctionMaxClauseCount:]
|
||||
} else {
|
||||
batchTerms = terms
|
||||
terms = nil
|
||||
}
|
||||
batch, err := makeBatchSearchersBytes(ctx, indexReader, batchTerms, field, boost, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if finalSearcher != nil {
|
||||
batch = append(batch, finalSearcher)
|
||||
}
|
||||
cleanup := func() {
|
||||
for _, searcher := range batch {
|
||||
if searcher != nil {
|
||||
_ = searcher.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
finalSearcher, err = optimizeCompositeSearcher(ctx, "disjunction:unadorned",
|
||||
indexReader, batch, options)
|
||||
// all searchers in batch should be closed, regardless of error or optimization failure
|
||||
// either we're returning, or continuing and only finalSearcher is needed for next loop
|
||||
cleanup()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if finalSearcher == nil {
|
||||
return nil, fmt.Errorf("unable to optimize")
|
||||
}
|
||||
}
|
||||
return finalSearcher, nil
|
||||
}
|
||||
|
||||
func makeBatchSearchersBytes(ctx context.Context, indexReader index.IndexReader, terms [][]byte, field string,
|
||||
boost float64, options search.SearcherOptions) ([]search.Searcher, error) {
|
||||
|
||||
qsearchers := make([]search.Searcher, len(terms))
|
||||
qsearchersClose := func() {
|
||||
for _, searcher := range qsearchers {
|
||||
if searcher != nil {
|
||||
_ = searcher.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, term := range terms {
|
||||
var err error
|
||||
qsearchers[i], err = NewTermSearcherBytes(ctx, indexReader, term, field, boost, options)
|
||||
if err != nil {
|
||||
qsearchersClose()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return qsearchers, nil
|
||||
}
|
||||
258
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_numeric_range.go
generated
vendored
Normal file
258
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_numeric_range.go
generated
vendored
Normal file
@@ -0,0 +1,258 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/numeric"
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
func NewNumericRangeSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
min *float64, max *float64, inclusiveMin, inclusiveMax *bool, field string,
|
||||
boost float64, options search.SearcherOptions) (search.Searcher, error) {
|
||||
// account for unbounded edges
|
||||
if min == nil {
|
||||
negInf := math.Inf(-1)
|
||||
min = &negInf
|
||||
}
|
||||
if max == nil {
|
||||
Inf := math.Inf(1)
|
||||
max = &Inf
|
||||
}
|
||||
if inclusiveMin == nil {
|
||||
defaultInclusiveMin := true
|
||||
inclusiveMin = &defaultInclusiveMin
|
||||
}
|
||||
if inclusiveMax == nil {
|
||||
defaultInclusiveMax := false
|
||||
inclusiveMax = &defaultInclusiveMax
|
||||
}
|
||||
// find all the ranges
|
||||
minInt64 := numeric.Float64ToInt64(*min)
|
||||
if !*inclusiveMin && minInt64 != math.MaxInt64 {
|
||||
minInt64++
|
||||
}
|
||||
maxInt64 := numeric.Float64ToInt64(*max)
|
||||
if !*inclusiveMax && maxInt64 != math.MinInt64 {
|
||||
maxInt64--
|
||||
}
|
||||
|
||||
var fieldDict index.FieldDictContains
|
||||
var dictBytesRead uint64
|
||||
var isIndexed filterFunc
|
||||
var err error
|
||||
if irr, ok := indexReader.(index.IndexReaderContains); ok {
|
||||
fieldDict, err = irr.FieldDictContains(field)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
isIndexed = func(term []byte) bool {
|
||||
found, err := fieldDict.Contains(term)
|
||||
return err == nil && found
|
||||
}
|
||||
|
||||
dictBytesRead = fieldDict.BytesRead()
|
||||
}
|
||||
|
||||
// FIXME hard-coded precision, should match field declaration
|
||||
termRanges := splitInt64Range(minInt64, maxInt64, 4)
|
||||
terms := termRanges.Enumerate(isIndexed)
|
||||
if fieldDict != nil {
|
||||
if fd, ok := fieldDict.(index.FieldDict); ok {
|
||||
if err = fd.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(terms) < 1 {
|
||||
// reporting back the IO stats with respect to the dictionary
|
||||
// loaded, using the context
|
||||
if ctx != nil {
|
||||
reportIOStats(ctx, dictBytesRead)
|
||||
search.RecordSearchCost(ctx, search.AddM, dictBytesRead)
|
||||
}
|
||||
|
||||
// cannot return MatchNoneSearcher because of interaction with
|
||||
// commit f391b991c20f02681bacd197afc6d8aed444e132
|
||||
return NewMultiTermSearcherBytes(ctx, indexReader, terms, field,
|
||||
boost, options, true)
|
||||
}
|
||||
|
||||
// for upside_down
|
||||
if isIndexed == nil {
|
||||
terms, err = filterCandidateTerms(indexReader, terms, field)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if tooManyClauses(len(terms)) {
|
||||
return nil, tooManyClausesErr(field, len(terms))
|
||||
}
|
||||
|
||||
if ctx != nil {
|
||||
reportIOStats(ctx, dictBytesRead)
|
||||
search.RecordSearchCost(ctx, search.AddM, dictBytesRead)
|
||||
}
|
||||
|
||||
return NewMultiTermSearcherBytes(ctx, indexReader, terms, field,
|
||||
boost, options, true)
|
||||
}
|
||||
|
||||
func filterCandidateTerms(indexReader index.IndexReader,
|
||||
terms [][]byte, field string) (rv [][]byte, err error) {
|
||||
|
||||
fieldDict, err := indexReader.FieldDictRange(field, terms[0], terms[len(terms)-1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// enumerate the terms and check against list of terms
|
||||
tfd, err := fieldDict.Next()
|
||||
for err == nil && tfd != nil {
|
||||
termBytes := []byte(tfd.Term)
|
||||
i := sort.Search(len(terms), func(i int) bool { return bytes.Compare(terms[i], termBytes) >= 0 })
|
||||
if i < len(terms) && bytes.Compare(terms[i], termBytes) == 0 {
|
||||
rv = append(rv, terms[i])
|
||||
}
|
||||
terms = terms[i:]
|
||||
tfd, err = fieldDict.Next()
|
||||
}
|
||||
|
||||
if cerr := fieldDict.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
|
||||
return rv, err
|
||||
}
|
||||
|
||||
type termRange struct {
|
||||
startTerm []byte
|
||||
endTerm []byte
|
||||
}
|
||||
|
||||
func (t *termRange) Enumerate(filter filterFunc) [][]byte {
|
||||
var rv [][]byte
|
||||
next := t.startTerm
|
||||
for bytes.Compare(next, t.endTerm) <= 0 {
|
||||
if filter != nil {
|
||||
if filter(next) {
|
||||
rv = append(rv, next)
|
||||
}
|
||||
} else {
|
||||
rv = append(rv, next)
|
||||
}
|
||||
next = incrementBytes(next)
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func incrementBytes(in []byte) []byte {
|
||||
rv := make([]byte, len(in))
|
||||
copy(rv, in)
|
||||
for i := len(rv) - 1; i >= 0; i-- {
|
||||
rv[i] = rv[i] + 1
|
||||
if rv[i] != 0 {
|
||||
// didn't overflow, so stop
|
||||
break
|
||||
}
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
type termRanges []*termRange
|
||||
|
||||
func (tr termRanges) Enumerate(filter filterFunc) [][]byte {
|
||||
var rv [][]byte
|
||||
for _, tri := range tr {
|
||||
trie := tri.Enumerate(filter)
|
||||
rv = append(rv, trie...)
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func splitInt64Range(minBound, maxBound int64, precisionStep uint) termRanges {
|
||||
rv := make(termRanges, 0)
|
||||
if minBound > maxBound {
|
||||
return rv
|
||||
}
|
||||
|
||||
for shift := uint(0); ; shift += precisionStep {
|
||||
|
||||
diff := int64(1) << (shift + precisionStep)
|
||||
mask := ((int64(1) << precisionStep) - int64(1)) << shift
|
||||
hasLower := (minBound & mask) != int64(0)
|
||||
hasUpper := (maxBound & mask) != mask
|
||||
|
||||
var nextMinBound int64
|
||||
if hasLower {
|
||||
nextMinBound = (minBound + diff) &^ mask
|
||||
} else {
|
||||
nextMinBound = minBound &^ mask
|
||||
}
|
||||
var nextMaxBound int64
|
||||
if hasUpper {
|
||||
nextMaxBound = (maxBound - diff) &^ mask
|
||||
} else {
|
||||
nextMaxBound = maxBound &^ mask
|
||||
}
|
||||
|
||||
lowerWrapped := nextMinBound < minBound
|
||||
upperWrapped := nextMaxBound > maxBound
|
||||
|
||||
if shift+precisionStep >= 64 || nextMinBound > nextMaxBound ||
|
||||
lowerWrapped || upperWrapped {
|
||||
// We are in the lowest precision or the next precision is not available.
|
||||
rv = append(rv, newRange(minBound, maxBound, shift))
|
||||
// exit the split recursion loop
|
||||
break
|
||||
}
|
||||
|
||||
if hasLower {
|
||||
rv = append(rv, newRange(minBound, minBound|mask, shift))
|
||||
}
|
||||
if hasUpper {
|
||||
rv = append(rv, newRange(maxBound&^mask, maxBound, shift))
|
||||
}
|
||||
|
||||
// recurse to next precision
|
||||
minBound = nextMinBound
|
||||
maxBound = nextMaxBound
|
||||
}
|
||||
|
||||
return rv
|
||||
}
|
||||
|
||||
func newRange(minBound, maxBound int64, shift uint) *termRange {
|
||||
maxBound |= (int64(1) << shift) - int64(1)
|
||||
minBytes := numeric.MustNewPrefixCodedInt64(minBound, shift)
|
||||
maxBytes := numeric.MustNewPrefixCodedInt64(maxBound, shift)
|
||||
return newRangeBytes(minBytes, maxBytes)
|
||||
}
|
||||
|
||||
func newRangeBytes(minBytes, maxBytes []byte) *termRange {
|
||||
return &termRange{
|
||||
startTerm: minBytes,
|
||||
endTerm: maxBytes,
|
||||
}
|
||||
}
|
||||
554
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go
generated
vendored
Normal file
554
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_phrase.go
generated
vendored
Normal file
@@ -0,0 +1,554 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizePhraseSearcher int
|
||||
|
||||
func init() {
|
||||
var ps PhraseSearcher
|
||||
reflectStaticSizePhraseSearcher = int(reflect.TypeOf(ps).Size())
|
||||
}
|
||||
|
||||
type PhraseSearcher struct {
|
||||
mustSearcher search.Searcher
|
||||
queryNorm float64
|
||||
currMust *search.DocumentMatch
|
||||
terms [][]string
|
||||
path phrasePath
|
||||
paths []phrasePath
|
||||
locations []search.Location
|
||||
initialized bool
|
||||
// map a term to a list of fuzzy terms that match it
|
||||
fuzzyTermMatches map[string][]string
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) Size() int {
|
||||
sizeInBytes := reflectStaticSizePhraseSearcher + size.SizeOfPtr
|
||||
|
||||
if s.mustSearcher != nil {
|
||||
sizeInBytes += s.mustSearcher.Size()
|
||||
}
|
||||
|
||||
if s.currMust != nil {
|
||||
sizeInBytes += s.currMust.Size()
|
||||
}
|
||||
|
||||
for _, entry := range s.terms {
|
||||
sizeInBytes += size.SizeOfSlice
|
||||
for _, entry1 := range entry {
|
||||
sizeInBytes += size.SizeOfString + len(entry1)
|
||||
}
|
||||
}
|
||||
|
||||
return sizeInBytes
|
||||
}
|
||||
|
||||
func NewPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms []string,
|
||||
fuzziness int, autoFuzzy bool, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) {
|
||||
|
||||
// turn flat terms []string into [][]string
|
||||
mterms := make([][]string, len(terms))
|
||||
for i, term := range terms {
|
||||
mterms[i] = []string{term}
|
||||
}
|
||||
return NewMultiPhraseSearcher(ctx, indexReader, mterms, fuzziness, autoFuzzy, field, boost, options)
|
||||
}
|
||||
|
||||
func NewMultiPhraseSearcher(ctx context.Context, indexReader index.IndexReader, terms [][]string,
|
||||
fuzziness int, autoFuzzy bool, field string, boost float64, options search.SearcherOptions) (*PhraseSearcher, error) {
|
||||
|
||||
options.IncludeTermVectors = true
|
||||
var termPositionSearchers []search.Searcher
|
||||
var err error
|
||||
var ts search.Searcher
|
||||
// The following logic checks if fuzziness is enabled.
|
||||
// Fuzziness is considered enabled if either:
|
||||
// a. `fuzziness` is greater than 0, or
|
||||
// b. `autoFuzzy` is set to true.
|
||||
// if both conditions are true, `autoFuzzy` takes precedence.
|
||||
// If enabled, a map will be created to store the matches for fuzzy terms.
|
||||
fuzzinessEnabled := autoFuzzy || fuzziness > 0
|
||||
var fuzzyTermMatches map[string][]string
|
||||
if fuzzinessEnabled {
|
||||
fuzzyTermMatches = make(map[string][]string)
|
||||
ctx = context.WithValue(ctx, search.FuzzyMatchPhraseKey, fuzzyTermMatches)
|
||||
}
|
||||
// in case of fuzzy multi-phrase, phrase and match-phrase queries we hardcode the
|
||||
// prefix length to 0, as setting a per word matching prefix length would not
|
||||
// make sense from a user perspective.
|
||||
for _, termPos := range terms {
|
||||
if len(termPos) == 1 && termPos[0] != "" {
|
||||
// single term
|
||||
if fuzzinessEnabled {
|
||||
// fuzzy
|
||||
if autoFuzzy {
|
||||
// auto fuzzy
|
||||
ts, err = NewAutoFuzzySearcher(ctx, indexReader, termPos[0], 0, field, boost, options)
|
||||
} else {
|
||||
// non-auto fuzzy
|
||||
ts, err = NewFuzzySearcher(ctx, indexReader, termPos[0], 0, fuzziness, field, boost, options)
|
||||
}
|
||||
} else {
|
||||
// non-fuzzy
|
||||
ts, err = NewTermSearcher(ctx, indexReader, termPos[0], field, boost, options)
|
||||
}
|
||||
if err != nil {
|
||||
// close any searchers already opened
|
||||
for _, ts := range termPositionSearchers {
|
||||
_ = ts.Close()
|
||||
}
|
||||
return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err)
|
||||
}
|
||||
termPositionSearchers = append(termPositionSearchers, ts)
|
||||
} else if len(termPos) > 1 {
|
||||
// multiple terms
|
||||
var termSearchers []search.Searcher
|
||||
for _, term := range termPos {
|
||||
if term == "" {
|
||||
continue
|
||||
}
|
||||
if fuzzinessEnabled {
|
||||
// fuzzy
|
||||
if autoFuzzy {
|
||||
// auto fuzzy
|
||||
ts, err = NewAutoFuzzySearcher(ctx, indexReader, term, 0, field, boost, options)
|
||||
} else {
|
||||
// non-auto fuzzy
|
||||
ts, err = NewFuzzySearcher(ctx, indexReader, term, 0, fuzziness, field, boost, options)
|
||||
}
|
||||
} else {
|
||||
// non-fuzzy
|
||||
ts, err = NewTermSearcher(ctx, indexReader, term, field, boost, options)
|
||||
}
|
||||
if err != nil {
|
||||
// close any searchers already opened
|
||||
for _, ts := range termPositionSearchers {
|
||||
_ = ts.Close()
|
||||
}
|
||||
return nil, fmt.Errorf("phrase searcher error building term searcher: %v", err)
|
||||
}
|
||||
termSearchers = append(termSearchers, ts)
|
||||
}
|
||||
disjunction, err := NewDisjunctionSearcher(ctx, indexReader, termSearchers, 1, options)
|
||||
if err != nil {
|
||||
// close any searchers already opened
|
||||
for _, ts := range termPositionSearchers {
|
||||
_ = ts.Close()
|
||||
}
|
||||
return nil, fmt.Errorf("phrase searcher error building term position disjunction searcher: %v", err)
|
||||
}
|
||||
termPositionSearchers = append(termPositionSearchers, disjunction)
|
||||
}
|
||||
}
|
||||
|
||||
if ctx != nil {
|
||||
if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {
|
||||
if ts, exists := fts[field]; exists {
|
||||
if fuzzinessEnabled {
|
||||
for term, fuzzyTerms := range fuzzyTermMatches {
|
||||
fuzzySynonymTerms := make([]string, 0, len(fuzzyTerms))
|
||||
if s, found := ts[term]; found {
|
||||
fuzzySynonymTerms = append(fuzzySynonymTerms, s...)
|
||||
}
|
||||
for _, fuzzyTerm := range fuzzyTerms {
|
||||
if fuzzyTerm == term {
|
||||
continue
|
||||
}
|
||||
if s, found := ts[fuzzyTerm]; found {
|
||||
fuzzySynonymTerms = append(fuzzySynonymTerms, s...)
|
||||
}
|
||||
}
|
||||
if len(fuzzySynonymTerms) > 0 {
|
||||
fuzzyTermMatches[term] = append(fuzzyTermMatches[term], fuzzySynonymTerms...)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, termPos := range terms {
|
||||
for _, term := range termPos {
|
||||
if s, found := ts[term]; found {
|
||||
if fuzzyTermMatches == nil {
|
||||
fuzzyTermMatches = make(map[string][]string)
|
||||
}
|
||||
fuzzyTermMatches[term] = s
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
mustSearcher, err := NewConjunctionSearcher(ctx, indexReader, termPositionSearchers, options)
|
||||
if err != nil {
|
||||
// close any searchers already opened
|
||||
for _, ts := range termPositionSearchers {
|
||||
_ = ts.Close()
|
||||
}
|
||||
return nil, fmt.Errorf("phrase searcher error building conjunction searcher: %v", err)
|
||||
}
|
||||
|
||||
// build our searcher
|
||||
rv := PhraseSearcher{
|
||||
mustSearcher: mustSearcher,
|
||||
terms: terms,
|
||||
fuzzyTermMatches: fuzzyTermMatches,
|
||||
}
|
||||
rv.computeQueryNorm()
|
||||
return &rv, nil
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) computeQueryNorm() {
|
||||
// first calculate sum of squared weights
|
||||
sumOfSquaredWeights := 0.0
|
||||
if s.mustSearcher != nil {
|
||||
sumOfSquaredWeights += s.mustSearcher.Weight()
|
||||
}
|
||||
|
||||
// now compute query norm from this
|
||||
s.queryNorm = 1.0 / math.Sqrt(sumOfSquaredWeights)
|
||||
// finally tell all the downstream searchers the norm
|
||||
if s.mustSearcher != nil {
|
||||
s.mustSearcher.SetQueryNorm(s.queryNorm)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) initSearchers(ctx *search.SearchContext) error {
|
||||
err := s.advanceNextMust(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s.initialized = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) advanceNextMust(ctx *search.SearchContext) error {
|
||||
var err error
|
||||
|
||||
if s.mustSearcher != nil {
|
||||
if s.currMust != nil {
|
||||
ctx.DocumentMatchPool.Put(s.currMust)
|
||||
}
|
||||
s.currMust, err = s.mustSearcher.Next(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) Weight() float64 {
|
||||
return s.mustSearcher.Weight()
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) SetQueryNorm(qnorm float64) {
|
||||
s.mustSearcher.SetQueryNorm(qnorm)
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for s.currMust != nil {
|
||||
// check this match against phrase constraints
|
||||
rv := s.checkCurrMustMatch(ctx)
|
||||
|
||||
// prepare for next iteration (either loop or subsequent call to Next())
|
||||
err := s.advanceNextMust(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if match satisfied phrase constraints return it as a hit
|
||||
if rv != nil {
|
||||
return rv, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// checkCurrMustMatch is solely concerned with determining if the DocumentMatch
|
||||
// pointed to by s.currMust (which satisifies the pre-condition searcher)
|
||||
// also satisfies the phrase constraints. if so, it returns a DocumentMatch
|
||||
// for this document, otherwise nil
|
||||
func (s *PhraseSearcher) checkCurrMustMatch(ctx *search.SearchContext) *search.DocumentMatch {
|
||||
s.locations = s.currMust.Complete(s.locations)
|
||||
|
||||
locations := s.currMust.Locations
|
||||
s.currMust.Locations = nil
|
||||
|
||||
ftls := s.currMust.FieldTermLocations
|
||||
|
||||
// typically we would expect there to only actually be results in
|
||||
// one field, but we allow for this to not be the case
|
||||
// but, we note that phrase constraints can only be satisfied within
|
||||
// a single field, so we can check them each independently
|
||||
for field, tlm := range locations {
|
||||
ftls = s.checkCurrMustMatchField(ctx, field, tlm, ftls)
|
||||
}
|
||||
|
||||
if len(ftls) > 0 {
|
||||
// return match
|
||||
rv := s.currMust
|
||||
s.currMust = nil
|
||||
rv.FieldTermLocations = ftls
|
||||
return rv
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkCurrMustMatchField is solely concerned with determining if one
|
||||
// particular field within the currMust DocumentMatch Locations
|
||||
// satisfies the phrase constraints (possibly more than once). if so,
|
||||
// the matching field term locations are appended to the provided
|
||||
// slice
|
||||
func (s *PhraseSearcher) checkCurrMustMatchField(ctx *search.SearchContext,
|
||||
field string, tlm search.TermLocationMap,
|
||||
ftls []search.FieldTermLocation) []search.FieldTermLocation {
|
||||
if s.path == nil {
|
||||
s.path = make(phrasePath, 0, len(s.terms))
|
||||
}
|
||||
var tlmPtr *search.TermLocationMap = &tlm
|
||||
if s.fuzzyTermMatches != nil {
|
||||
// if fuzzy search, we need to expand the tlm to include all the fuzzy matches
|
||||
// Example - term is "foo" and fuzzy matches are "foo", "fool", "food"
|
||||
// the non expanded tlm will be:
|
||||
// foo -> Locations[foo]
|
||||
// fool -> Locations[fool]
|
||||
// food -> Locations[food]
|
||||
// the expanded tlm will be:
|
||||
// foo -> [Locations[foo], Locations[fool], Locations[food]]
|
||||
expandedTlm := make(search.TermLocationMap)
|
||||
s.expandFuzzyMatches(tlm, expandedTlm)
|
||||
tlmPtr = &expandedTlm
|
||||
}
|
||||
s.paths = findPhrasePaths(0, nil, s.terms, *tlmPtr, s.path[:0], 0, s.paths[:0])
|
||||
for _, p := range s.paths {
|
||||
for _, pp := range p {
|
||||
ftls = append(ftls, search.FieldTermLocation{
|
||||
Field: field,
|
||||
Term: pp.term,
|
||||
Location: search.Location{
|
||||
Pos: pp.loc.Pos,
|
||||
Start: pp.loc.Start,
|
||||
End: pp.loc.End,
|
||||
ArrayPositions: pp.loc.ArrayPositions,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
return ftls
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) expandFuzzyMatches(tlm search.TermLocationMap, expandedTlm search.TermLocationMap) {
|
||||
for term, fuzzyMatches := range s.fuzzyTermMatches {
|
||||
locations := tlm[term]
|
||||
for _, fuzzyMatch := range fuzzyMatches {
|
||||
if fuzzyMatch == term {
|
||||
continue
|
||||
}
|
||||
locations = append(locations, tlm[fuzzyMatch]...)
|
||||
}
|
||||
expandedTlm[term] = locations
|
||||
}
|
||||
}
|
||||
|
||||
type phrasePart struct {
|
||||
term string
|
||||
loc *search.Location
|
||||
}
|
||||
|
||||
func (p *phrasePart) String() string {
|
||||
return fmt.Sprintf("[%s %v]", p.term, p.loc)
|
||||
}
|
||||
|
||||
type phrasePath []phrasePart
|
||||
|
||||
func (p phrasePath) MergeInto(in search.TermLocationMap) {
|
||||
for _, pp := range p {
|
||||
in[pp.term] = append(in[pp.term], pp.loc)
|
||||
}
|
||||
}
|
||||
|
||||
func (p phrasePath) String() string {
|
||||
rv := "["
|
||||
for i, pp := range p {
|
||||
if i > 0 {
|
||||
rv += ", "
|
||||
}
|
||||
rv += pp.String()
|
||||
}
|
||||
rv += "]"
|
||||
return rv
|
||||
}
|
||||
|
||||
// findPhrasePaths is a function to identify phrase matches from a set
|
||||
// of known term locations. it recursive so care must be taken with
|
||||
// arguments and return values.
|
||||
//
|
||||
// prevPos - the previous location, 0 on first invocation
|
||||
//
|
||||
// ap - array positions of the first candidate phrase part to
|
||||
// which further recursive phrase parts must match,
|
||||
// nil on initial invocation or when there are no array positions
|
||||
//
|
||||
// phraseTerms - slice containing the phrase terms,
|
||||
// may contain empty string as placeholder (don't care)
|
||||
//
|
||||
// tlm - the Term Location Map containing all relevant term locations
|
||||
//
|
||||
// p - the current path being explored (appended to in recursive calls)
|
||||
// this is the primary state being built during the traversal
|
||||
//
|
||||
// remainingSlop - amount of sloppiness that's allowed, which is the
|
||||
// sum of the editDistances from each matching phrase part, where 0 means no
|
||||
// sloppiness allowed (all editDistances must be 0), decremented during recursion
|
||||
//
|
||||
// rv - the final result being appended to by all the recursive calls
|
||||
//
|
||||
// returns slice of paths, or nil if invocation did not find any successful paths
|
||||
func findPhrasePaths(prevPos uint64, ap search.ArrayPositions, phraseTerms [][]string,
|
||||
tlm search.TermLocationMap, p phrasePath, remainingSlop int, rv []phrasePath) []phrasePath {
|
||||
// no more terms
|
||||
if len(phraseTerms) < 1 {
|
||||
// snapshot or copy the recursively built phrasePath p and
|
||||
// append it to the rv, also optimizing by checking if next
|
||||
// phrasePath item in the rv (which we're about to overwrite)
|
||||
// is available for reuse
|
||||
var pcopy phrasePath
|
||||
if len(rv) < cap(rv) {
|
||||
pcopy = rv[:len(rv)+1][len(rv)][:0]
|
||||
}
|
||||
return append(rv, append(pcopy, p...))
|
||||
}
|
||||
|
||||
car := phraseTerms[0]
|
||||
cdr := phraseTerms[1:]
|
||||
|
||||
// empty term is treated as match (continue)
|
||||
if len(car) == 0 || (len(car) == 1 && car[0] == "") {
|
||||
nextPos := prevPos + 1
|
||||
if prevPos == 0 {
|
||||
// if prevPos was 0, don't set it to 1 (as thats not a real abs pos)
|
||||
nextPos = 0 // don't advance nextPos if prevPos was 0
|
||||
}
|
||||
return findPhrasePaths(nextPos, ap, cdr, tlm, p, remainingSlop, rv)
|
||||
}
|
||||
|
||||
// locations for this term
|
||||
for _, carTerm := range car {
|
||||
locations := tlm[carTerm]
|
||||
LOCATIONS_LOOP:
|
||||
for _, loc := range locations {
|
||||
if prevPos != 0 && !loc.ArrayPositions.Equals(ap) {
|
||||
// if the array positions are wrong, can't match, try next location
|
||||
continue
|
||||
}
|
||||
|
||||
// compute distance from previous phrase term
|
||||
dist := 0
|
||||
if prevPos != 0 {
|
||||
dist = editDistance(prevPos+1, loc.Pos)
|
||||
}
|
||||
|
||||
// if enough slop remaining, continue recursively
|
||||
if prevPos == 0 || (remainingSlop-dist) >= 0 {
|
||||
// skip if we've already used this term+loc already
|
||||
for _, ppart := range p {
|
||||
if ppart.term == carTerm && ppart.loc == loc {
|
||||
continue LOCATIONS_LOOP
|
||||
}
|
||||
}
|
||||
|
||||
// this location works, add it to the path (but not for empty term)
|
||||
px := append(p, phrasePart{term: carTerm, loc: loc})
|
||||
rv = findPhrasePaths(loc.Pos, loc.ArrayPositions, cdr, tlm, px, remainingSlop-dist, rv)
|
||||
}
|
||||
}
|
||||
}
|
||||
return rv
|
||||
}
|
||||
|
||||
func editDistance(p1, p2 uint64) int {
|
||||
dist := int(p1 - p2)
|
||||
if dist < 0 {
|
||||
return -dist
|
||||
}
|
||||
return dist
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
if !s.initialized {
|
||||
err := s.initSearchers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if s.currMust != nil {
|
||||
if s.currMust.IndexInternalID.Compare(ID) >= 0 {
|
||||
return s.Next(ctx)
|
||||
}
|
||||
ctx.DocumentMatchPool.Put(s.currMust)
|
||||
}
|
||||
if s.currMust == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var err error
|
||||
s.currMust, err = s.mustSearcher.Advance(ctx, ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.Next(ctx)
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) Count() uint64 {
|
||||
// for now return a worst case
|
||||
return s.mustSearcher.Count()
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) Close() error {
|
||||
if s.mustSearcher != nil {
|
||||
err := s.mustSearcher.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) Min() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *PhraseSearcher) DocumentMatchPoolSize() int {
|
||||
return s.mustSearcher.DocumentMatchPoolSize() + 1
|
||||
}
|
||||
169
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_regexp.go
generated
vendored
Normal file
169
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_regexp.go
generated
vendored
Normal file
@@ -0,0 +1,169 @@
|
||||
// Copyright (c) 2015 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
// The Regexp interface defines the subset of the regexp.Regexp API
|
||||
// methods that are used by bleve indexes, allowing callers to pass in
|
||||
// alternate implementations.
|
||||
type Regexp interface {
|
||||
FindStringIndex(s string) (loc []int)
|
||||
|
||||
LiteralPrefix() (prefix string, complete bool)
|
||||
|
||||
String() string
|
||||
}
|
||||
|
||||
// NewRegexpStringSearcher is similar to NewRegexpSearcher, but
|
||||
// additionally optimizes for index readers that handle regexp's.
|
||||
func NewRegexpStringSearcher(ctx context.Context, indexReader index.IndexReader, pattern string,
|
||||
field string, boost float64, options search.SearcherOptions) (
|
||||
search.Searcher, error) {
|
||||
ir, ok := indexReader.(index.IndexReaderRegexp)
|
||||
if !ok {
|
||||
r, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return NewRegexpSearcher(ctx, indexReader, r, field, boost, options)
|
||||
}
|
||||
|
||||
fieldDict, a, err := ir.FieldDictRegexpAutomaton(field, pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if cerr := fieldDict.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
|
||||
var termSet = make(map[string]struct{})
|
||||
var candidateTerms []string
|
||||
|
||||
tfd, err := fieldDict.Next()
|
||||
for err == nil && tfd != nil {
|
||||
if _, exists := termSet[tfd.Term]; !exists {
|
||||
termSet[tfd.Term] = struct{}{}
|
||||
candidateTerms = append(candidateTerms, tfd.Term)
|
||||
tfd, err = fieldDict.Next()
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if ctx != nil {
|
||||
if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {
|
||||
if ts, exists := fts[field]; exists {
|
||||
for term := range ts {
|
||||
if _, exists := termSet[term]; exists {
|
||||
continue
|
||||
}
|
||||
if a.MatchesRegex(term) {
|
||||
termSet[term] = struct{}{}
|
||||
candidateTerms = append(candidateTerms, term)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return NewMultiTermSearcher(ctx, indexReader, candidateTerms, field, boost,
|
||||
options, true)
|
||||
}
|
||||
|
||||
// NewRegexpSearcher creates a searcher which will match documents that
|
||||
// contain terms which match the pattern regexp. The match must be EXACT
|
||||
// matching the entire term. The provided regexp SHOULD NOT start with ^
|
||||
// or end with $ as this can intefere with the implementation. Separately,
|
||||
// matches will be checked to ensure they match the entire term.
|
||||
func NewRegexpSearcher(ctx context.Context, indexReader index.IndexReader, pattern Regexp,
|
||||
field string, boost float64, options search.SearcherOptions) (
|
||||
search.Searcher, error) {
|
||||
var candidateTerms []string
|
||||
var regexpCandidates *regexpCandidates
|
||||
prefixTerm, complete := pattern.LiteralPrefix()
|
||||
if complete {
|
||||
// there is no pattern
|
||||
candidateTerms = []string{prefixTerm}
|
||||
} else {
|
||||
var err error
|
||||
regexpCandidates, err = findRegexpCandidateTerms(indexReader, pattern, field,
|
||||
prefixTerm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var dictBytesRead uint64
|
||||
if regexpCandidates != nil {
|
||||
candidateTerms = regexpCandidates.candidates
|
||||
dictBytesRead = regexpCandidates.bytesRead
|
||||
}
|
||||
|
||||
if ctx != nil {
|
||||
reportIOStats(ctx, dictBytesRead)
|
||||
search.RecordSearchCost(ctx, search.AddM, dictBytesRead)
|
||||
}
|
||||
|
||||
return NewMultiTermSearcher(ctx, indexReader, candidateTerms, field, boost,
|
||||
options, true)
|
||||
}
|
||||
|
||||
type regexpCandidates struct {
|
||||
candidates []string
|
||||
bytesRead uint64
|
||||
}
|
||||
|
||||
func findRegexpCandidateTerms(indexReader index.IndexReader,
|
||||
pattern Regexp, field, prefixTerm string) (rv *regexpCandidates, err error) {
|
||||
rv = ®expCandidates{
|
||||
candidates: make([]string, 0),
|
||||
}
|
||||
var fieldDict index.FieldDict
|
||||
if len(prefixTerm) > 0 {
|
||||
fieldDict, err = indexReader.FieldDictPrefix(field, []byte(prefixTerm))
|
||||
} else {
|
||||
fieldDict, err = indexReader.FieldDict(field)
|
||||
}
|
||||
defer func() {
|
||||
if cerr := fieldDict.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
|
||||
// enumerate the terms and check against regexp
|
||||
tfd, err := fieldDict.Next()
|
||||
for err == nil && tfd != nil {
|
||||
matchPos := pattern.FindStringIndex(tfd.Term)
|
||||
if matchPos != nil && matchPos[0] == 0 && matchPos[1] == len(tfd.Term) {
|
||||
rv.candidates = append(rv.candidates, tfd.Term)
|
||||
if tooManyClauses(len(rv.candidates)) {
|
||||
return rv, tooManyClausesErr(field, len(rv.candidates))
|
||||
}
|
||||
}
|
||||
tfd, err = fieldDict.Next()
|
||||
}
|
||||
rv.bytesRead = fieldDict.BytesRead()
|
||||
return rv, err
|
||||
}
|
||||
282
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term.go
generated
vendored
Normal file
282
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term.go
generated
vendored
Normal file
@@ -0,0 +1,282 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
"github.com/blevesearch/bleve/v2/search/scorer"
|
||||
"github.com/blevesearch/bleve/v2/size"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
var reflectStaticSizeTermSearcher int
|
||||
|
||||
func init() {
|
||||
var ts TermSearcher
|
||||
reflectStaticSizeTermSearcher = int(reflect.TypeOf(ts).Size())
|
||||
}
|
||||
|
||||
type TermSearcher struct {
|
||||
indexReader index.IndexReader
|
||||
reader index.TermFieldReader
|
||||
scorer *scorer.TermQueryScorer
|
||||
tfd index.TermFieldDoc
|
||||
}
|
||||
|
||||
func NewTermSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
term string, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) {
|
||||
if isTermQuery(ctx) {
|
||||
ctx = context.WithValue(ctx, search.QueryTypeKey, search.Term)
|
||||
}
|
||||
return NewTermSearcherBytes(ctx, indexReader, []byte(term), field, boost, options)
|
||||
}
|
||||
|
||||
func NewTermSearcherBytes(ctx context.Context, indexReader index.IndexReader,
|
||||
term []byte, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) {
|
||||
if ctx != nil {
|
||||
if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {
|
||||
if ts, exists := fts[field]; exists {
|
||||
if s, found := ts[string(term)]; found {
|
||||
return NewSynonymSearcher(ctx, indexReader, term, s, field, boost, options)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
needFreqNorm := options.Score != "none"
|
||||
reader, err := indexReader.TermFieldReader(ctx, term, field, needFreqNorm, needFreqNorm, options.IncludeTermVectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newTermSearcherFromReader(ctx, indexReader, reader, term, field, boost, options)
|
||||
}
|
||||
|
||||
func tfIDFScoreMetrics(indexReader index.IndexReader) (uint64, error) {
|
||||
// default tf-idf stats
|
||||
count, err := indexReader.DocCount()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func bm25ScoreMetrics(ctx context.Context, field string,
|
||||
indexReader index.IndexReader) (uint64, float64, error) {
|
||||
var count uint64
|
||||
var fieldCardinality int
|
||||
var err error
|
||||
|
||||
bm25Stats, ok := ctx.Value(search.BM25StatsKey).(*search.BM25Stats)
|
||||
if !ok {
|
||||
count, err = indexReader.DocCount()
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if bm25Reader, ok := indexReader.(index.BM25Reader); ok {
|
||||
fieldCardinality, err = bm25Reader.FieldCardinality(field)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
count = uint64(bm25Stats.DocCount)
|
||||
fieldCardinality, ok = bm25Stats.FieldCardinality[field]
|
||||
if !ok {
|
||||
return 0, 0, fmt.Errorf("field stat for bm25 not present %s", field)
|
||||
}
|
||||
}
|
||||
|
||||
if count == 0 && fieldCardinality == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
return count, math.Ceil(float64(fieldCardinality) / float64(count)), nil
|
||||
}
|
||||
|
||||
func newTermSearcherFromReader(ctx context.Context, indexReader index.IndexReader,
|
||||
reader index.TermFieldReader, term []byte, field string, boost float64,
|
||||
options search.SearcherOptions) (*TermSearcher, error) {
|
||||
var count uint64
|
||||
var avgDocLength float64
|
||||
var err error
|
||||
var similarityModel string
|
||||
|
||||
// as a fallback case we track certain stats for tf-idf scoring
|
||||
if ctx != nil {
|
||||
if similarityModelCallback, ok := ctx.Value(search.
|
||||
GetScoringModelCallbackKey).(search.GetScoringModelCallbackFn); ok {
|
||||
similarityModel = similarityModelCallback()
|
||||
}
|
||||
}
|
||||
switch similarityModel {
|
||||
case index.BM25Scoring:
|
||||
count, avgDocLength, err = bm25ScoreMetrics(ctx, field, indexReader)
|
||||
if err != nil {
|
||||
_ = reader.Close()
|
||||
return nil, err
|
||||
}
|
||||
case index.TFIDFScoring:
|
||||
fallthrough
|
||||
default:
|
||||
count, err = tfIDFScoreMetrics(indexReader)
|
||||
if err != nil {
|
||||
_ = reader.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
scorer := scorer.NewTermQueryScorer(term, field, boost, count, reader.Count(), avgDocLength, options)
|
||||
return &TermSearcher{
|
||||
indexReader: indexReader,
|
||||
reader: reader,
|
||||
scorer: scorer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func NewSynonymSearcher(ctx context.Context, indexReader index.IndexReader, term []byte, synonyms []string, field string, boost float64, options search.SearcherOptions) (search.Searcher, error) {
|
||||
createTermSearcher := func(term []byte, boostVal float64) (search.Searcher, error) {
|
||||
needFreqNorm := options.Score != "none"
|
||||
reader, err := indexReader.TermFieldReader(ctx, term, field, needFreqNorm, needFreqNorm, options.IncludeTermVectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return newTermSearcherFromReader(ctx, indexReader, reader, term, field, boostVal, options)
|
||||
}
|
||||
// create a searcher for the term itself
|
||||
termSearcher, err := createTermSearcher(term, boost)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// constituent searchers of the disjunction
|
||||
qsearchers := make([]search.Searcher, 0, len(synonyms)+1)
|
||||
// helper method to close all the searchers we've created
|
||||
// in case of an error
|
||||
qsearchersClose := func() {
|
||||
for _, searcher := range qsearchers {
|
||||
if searcher != nil {
|
||||
_ = searcher.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
qsearchers = append(qsearchers, termSearcher)
|
||||
// create a searcher for each synonym
|
||||
for _, synonym := range synonyms {
|
||||
synonymSearcher, err := createTermSearcher([]byte(synonym), boost/2.0)
|
||||
if err != nil {
|
||||
qsearchersClose()
|
||||
return nil, err
|
||||
}
|
||||
qsearchers = append(qsearchers, synonymSearcher)
|
||||
}
|
||||
// create a disjunction searcher
|
||||
rv, err := NewDisjunctionSearcher(ctx, indexReader, qsearchers, 0, options)
|
||||
if err != nil {
|
||||
qsearchersClose()
|
||||
return nil, err
|
||||
}
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
func (s *TermSearcher) Size() int {
|
||||
return reflectStaticSizeTermSearcher + size.SizeOfPtr +
|
||||
s.reader.Size() +
|
||||
s.tfd.Size() +
|
||||
s.scorer.Size()
|
||||
}
|
||||
|
||||
func (s *TermSearcher) Count() uint64 {
|
||||
return s.reader.Count()
|
||||
}
|
||||
|
||||
func (s *TermSearcher) Weight() float64 {
|
||||
return s.scorer.Weight()
|
||||
}
|
||||
|
||||
func (s *TermSearcher) SetQueryNorm(qnorm float64) {
|
||||
s.scorer.SetQueryNorm(qnorm)
|
||||
}
|
||||
|
||||
func (s *TermSearcher) Next(ctx *search.SearchContext) (*search.DocumentMatch, error) {
|
||||
termMatch, err := s.reader.Next(s.tfd.Reset())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if termMatch == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// score match
|
||||
docMatch := s.scorer.Score(ctx, termMatch)
|
||||
// return doc match
|
||||
return docMatch, nil
|
||||
|
||||
}
|
||||
|
||||
func (s *TermSearcher) Advance(ctx *search.SearchContext, ID index.IndexInternalID) (*search.DocumentMatch, error) {
|
||||
termMatch, err := s.reader.Advance(ID, s.tfd.Reset())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if termMatch == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// score match
|
||||
docMatch := s.scorer.Score(ctx, termMatch)
|
||||
|
||||
// return doc match
|
||||
return docMatch, nil
|
||||
}
|
||||
|
||||
func (s *TermSearcher) Close() error {
|
||||
return s.reader.Close()
|
||||
}
|
||||
|
||||
func (s *TermSearcher) Min() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (s *TermSearcher) DocumentMatchPoolSize() int {
|
||||
return 1
|
||||
}
|
||||
|
||||
func (s *TermSearcher) Optimize(kind string, octx index.OptimizableContext) (
|
||||
index.OptimizableContext, error) {
|
||||
o, ok := s.reader.(index.Optimizable)
|
||||
if ok {
|
||||
return o.Optimize(kind, octx)
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func isTermQuery(ctx context.Context) bool {
|
||||
if ctx != nil {
|
||||
// if the ctx already has a value set for query type
|
||||
// it would've been done at a non term searcher level.
|
||||
_, ok := ctx.Value(search.QueryTypeKey).(string)
|
||||
return !ok
|
||||
}
|
||||
// if the context is nil, then don't set the query type
|
||||
return false
|
||||
}
|
||||
86
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_prefix.go
generated
vendored
Normal file
86
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_prefix.go
generated
vendored
Normal file
@@ -0,0 +1,86 @@
|
||||
// Copyright (c) 2014 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
func NewTermPrefixSearcher(ctx context.Context, indexReader index.IndexReader, prefix string,
|
||||
field string, boost float64, options search.SearcherOptions) (
|
||||
search.Searcher, error) {
|
||||
// find the terms with this prefix
|
||||
fieldDict, err := indexReader.FieldDictPrefix(field, []byte(prefix))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if cerr := fieldDict.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
|
||||
var terms []string
|
||||
var termSet = make(map[string]struct{})
|
||||
tfd, err := fieldDict.Next()
|
||||
for err == nil && tfd != nil {
|
||||
if _, exists := termSet[tfd.Term]; !exists {
|
||||
termSet[tfd.Term] = struct{}{}
|
||||
terms = append(terms, tfd.Term)
|
||||
if tooManyClauses(len(terms)) {
|
||||
return nil, tooManyClausesErr(field, len(terms))
|
||||
}
|
||||
tfd, err = fieldDict.Next()
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if ctx != nil {
|
||||
reportIOStats(ctx, fieldDict.BytesRead())
|
||||
search.RecordSearchCost(ctx, search.AddM, fieldDict.BytesRead())
|
||||
}
|
||||
|
||||
if ctx != nil {
|
||||
if fts, ok := ctx.Value(search.FieldTermSynonymMapKey).(search.FieldTermSynonymMap); ok {
|
||||
if ts, exists := fts[field]; exists {
|
||||
for term := range ts {
|
||||
if _, exists := termSet[term]; exists {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(term, prefix) {
|
||||
termSet[term] = struct{}{}
|
||||
terms = append(terms, term)
|
||||
if tooManyClauses(len(terms)) {
|
||||
return nil, tooManyClausesErr(field, len(terms))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check if the terms are empty or have one term which is the prefix itself
|
||||
if len(terms) == 0 || (len(terms) == 1 && terms[0] == prefix) {
|
||||
return NewTermSearcher(ctx, indexReader, prefix, field, boost, options)
|
||||
}
|
||||
|
||||
return NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true)
|
||||
}
|
||||
92
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_range.go
generated
vendored
Normal file
92
vendor/github.com/blevesearch/bleve/v2/search/searcher/search_term_range.go
generated
vendored
Normal file
@@ -0,0 +1,92 @@
|
||||
// Copyright (c) 2017 Couchbase, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package searcher
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
)
|
||||
|
||||
func NewTermRangeSearcher(ctx context.Context, indexReader index.IndexReader,
|
||||
min, max []byte, inclusiveMin, inclusiveMax *bool, field string,
|
||||
boost float64, options search.SearcherOptions) (search.Searcher, error) {
|
||||
|
||||
if inclusiveMin == nil {
|
||||
defaultInclusiveMin := true
|
||||
inclusiveMin = &defaultInclusiveMin
|
||||
}
|
||||
if inclusiveMax == nil {
|
||||
defaultInclusiveMax := false
|
||||
inclusiveMax = &defaultInclusiveMax
|
||||
}
|
||||
|
||||
if min == nil {
|
||||
min = []byte{}
|
||||
}
|
||||
|
||||
rangeMax := max
|
||||
if rangeMax != nil {
|
||||
// the term dictionary range end has an unfortunate implementation
|
||||
rangeMax = append(rangeMax, 0)
|
||||
}
|
||||
|
||||
// find the terms with this prefix
|
||||
fieldDict, err := indexReader.FieldDictRange(field, min, rangeMax)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if cerr := fieldDict.Close(); cerr != nil && err == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
|
||||
var terms []string
|
||||
tfd, err := fieldDict.Next()
|
||||
for err == nil && tfd != nil {
|
||||
terms = append(terms, tfd.Term)
|
||||
tfd, err = fieldDict.Next()
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(terms) < 1 {
|
||||
return NewMatchNoneSearcher(indexReader)
|
||||
}
|
||||
|
||||
if !*inclusiveMin && min != nil && string(min) == terms[0] {
|
||||
terms = terms[1:]
|
||||
// check again, as we might have removed only entry
|
||||
if len(terms) < 1 {
|
||||
return NewMatchNoneSearcher(indexReader)
|
||||
}
|
||||
}
|
||||
|
||||
// if our term list included the max, it would be the last item
|
||||
if !*inclusiveMax && max != nil && string(max) == terms[len(terms)-1] {
|
||||
terms = terms[:len(terms)-1]
|
||||
}
|
||||
|
||||
if ctx != nil {
|
||||
reportIOStats(ctx, fieldDict.BytesRead())
|
||||
search.RecordSearchCost(ctx, search.AddM, fieldDict.BytesRead())
|
||||
}
|
||||
|
||||
return NewMultiTermSearcher(ctx, indexReader, terms, field, boost, options, true)
|
||||
}
|
||||
Reference in New Issue
Block a user