index_impl.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915
  1. // Copyright (c) 2014 Couchbase, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bleve
  15. import (
  16. "context"
  17. "encoding/json"
  18. "fmt"
  19. "os"
  20. "sort"
  21. "sync"
  22. "sync/atomic"
  23. "time"
  24. "github.com/blevesearch/bleve/document"
  25. "github.com/blevesearch/bleve/index"
  26. "github.com/blevesearch/bleve/index/store"
  27. "github.com/blevesearch/bleve/index/upsidedown"
  28. "github.com/blevesearch/bleve/mapping"
  29. "github.com/blevesearch/bleve/registry"
  30. "github.com/blevesearch/bleve/search"
  31. "github.com/blevesearch/bleve/search/collector"
  32. "github.com/blevesearch/bleve/search/facet"
  33. "github.com/blevesearch/bleve/search/highlight"
  34. )
  35. type indexImpl struct {
  36. path string
  37. name string
  38. meta *indexMeta
  39. i index.Index
  40. m mapping.IndexMapping
  41. mutex sync.RWMutex
  42. open bool
  43. stats *IndexStat
  44. }
  45. const storePath = "store"
  46. var mappingInternalKey = []byte("_mapping")
  47. const SearchQueryStartCallbackKey = "_search_query_start_callback_key"
  48. const SearchQueryEndCallbackKey = "_search_query_end_callback_key"
  49. type SearchQueryStartCallbackFn func(size uint64) error
  50. type SearchQueryEndCallbackFn func(size uint64) error
  51. func indexStorePath(path string) string {
  52. return path + string(os.PathSeparator) + storePath
  53. }
  54. func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
  55. // first validate the mapping
  56. err := mapping.Validate()
  57. if err != nil {
  58. return nil, err
  59. }
  60. if kvconfig == nil {
  61. kvconfig = map[string]interface{}{}
  62. }
  63. if kvstore == "" {
  64. return nil, fmt.Errorf("bleve not configured for file based indexing")
  65. }
  66. rv := indexImpl{
  67. path: path,
  68. name: path,
  69. m: mapping,
  70. meta: newIndexMeta(indexType, kvstore, kvconfig),
  71. }
  72. rv.stats = &IndexStat{i: &rv}
  73. // at this point there is hope that we can be successful, so save index meta
  74. if path != "" {
  75. err = rv.meta.Save(path)
  76. if err != nil {
  77. return nil, err
  78. }
  79. kvconfig["create_if_missing"] = true
  80. kvconfig["error_if_exists"] = true
  81. kvconfig["path"] = indexStorePath(path)
  82. } else {
  83. kvconfig["path"] = ""
  84. }
  85. // open the index
  86. indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
  87. if indexTypeConstructor == nil {
  88. return nil, ErrorUnknownIndexType
  89. }
  90. rv.i, err = indexTypeConstructor(rv.meta.Storage, kvconfig, Config.analysisQueue)
  91. if err != nil {
  92. return nil, err
  93. }
  94. err = rv.i.Open()
  95. if err != nil {
  96. if err == index.ErrorUnknownStorageType {
  97. return nil, ErrorUnknownStorageType
  98. }
  99. return nil, err
  100. }
  101. // now persist the mapping
  102. mappingBytes, err := json.Marshal(mapping)
  103. if err != nil {
  104. return nil, err
  105. }
  106. err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
  107. if err != nil {
  108. return nil, err
  109. }
  110. // mark the index as open
  111. rv.mutex.Lock()
  112. defer rv.mutex.Unlock()
  113. rv.open = true
  114. indexStats.Register(&rv)
  115. return &rv, nil
  116. }
  117. func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *indexImpl, err error) {
  118. rv = &indexImpl{
  119. path: path,
  120. name: path,
  121. }
  122. rv.stats = &IndexStat{i: rv}
  123. rv.meta, err = openIndexMeta(path)
  124. if err != nil {
  125. return nil, err
  126. }
  127. // backwards compatibility if index type is missing
  128. if rv.meta.IndexType == "" {
  129. rv.meta.IndexType = upsidedown.Name
  130. }
  131. storeConfig := rv.meta.Config
  132. if storeConfig == nil {
  133. storeConfig = map[string]interface{}{}
  134. }
  135. storeConfig["path"] = indexStorePath(path)
  136. storeConfig["create_if_missing"] = false
  137. storeConfig["error_if_exists"] = false
  138. for rck, rcv := range runtimeConfig {
  139. storeConfig[rck] = rcv
  140. }
  141. // open the index
  142. indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
  143. if indexTypeConstructor == nil {
  144. return nil, ErrorUnknownIndexType
  145. }
  146. rv.i, err = indexTypeConstructor(rv.meta.Storage, storeConfig, Config.analysisQueue)
  147. if err != nil {
  148. return nil, err
  149. }
  150. err = rv.i.Open()
  151. if err != nil {
  152. if err == index.ErrorUnknownStorageType {
  153. return nil, ErrorUnknownStorageType
  154. }
  155. return nil, err
  156. }
  157. // now load the mapping
  158. indexReader, err := rv.i.Reader()
  159. if err != nil {
  160. return nil, err
  161. }
  162. defer func() {
  163. if cerr := indexReader.Close(); cerr != nil && err == nil {
  164. err = cerr
  165. }
  166. }()
  167. mappingBytes, err := indexReader.GetInternal(mappingInternalKey)
  168. if err != nil {
  169. return nil, err
  170. }
  171. var im *mapping.IndexMappingImpl
  172. err = json.Unmarshal(mappingBytes, &im)
  173. if err != nil {
  174. return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes))
  175. }
  176. // mark the index as open
  177. rv.mutex.Lock()
  178. defer rv.mutex.Unlock()
  179. rv.open = true
  180. // validate the mapping
  181. err = im.Validate()
  182. if err != nil {
  183. // note even if the mapping is invalid
  184. // we still return an open usable index
  185. return rv, err
  186. }
  187. rv.m = im
  188. indexStats.Register(rv)
  189. return rv, err
  190. }
  191. // Advanced returns implementation internals
  192. // necessary ONLY for advanced usage.
  193. func (i *indexImpl) Advanced() (index.Index, store.KVStore, error) {
  194. s, err := i.i.Advanced()
  195. if err != nil {
  196. return nil, nil, err
  197. }
  198. return i.i, s, nil
  199. }
  200. // Mapping returns the IndexMapping in use by this
  201. // Index.
  202. func (i *indexImpl) Mapping() mapping.IndexMapping {
  203. return i.m
  204. }
  205. // Index the object with the specified identifier.
  206. // The IndexMapping for this index will determine
  207. // how the object is indexed.
  208. func (i *indexImpl) Index(id string, data interface{}) (err error) {
  209. if id == "" {
  210. return ErrorEmptyID
  211. }
  212. i.mutex.RLock()
  213. defer i.mutex.RUnlock()
  214. if !i.open {
  215. return ErrorIndexClosed
  216. }
  217. doc := document.NewDocument(id)
  218. err = i.m.MapDocument(doc, data)
  219. if err != nil {
  220. return
  221. }
  222. err = i.i.Update(doc)
  223. return
  224. }
  225. // IndexAdvanced takes a document.Document object
  226. // skips the mapping and indexes it.
  227. func (i *indexImpl) IndexAdvanced(doc *document.Document) (err error) {
  228. if doc.ID == "" {
  229. return ErrorEmptyID
  230. }
  231. i.mutex.RLock()
  232. defer i.mutex.RUnlock()
  233. if !i.open {
  234. return ErrorIndexClosed
  235. }
  236. err = i.i.Update(doc)
  237. return
  238. }
  239. // Delete entries for the specified identifier from
  240. // the index.
  241. func (i *indexImpl) Delete(id string) (err error) {
  242. if id == "" {
  243. return ErrorEmptyID
  244. }
  245. i.mutex.RLock()
  246. defer i.mutex.RUnlock()
  247. if !i.open {
  248. return ErrorIndexClosed
  249. }
  250. err = i.i.Delete(id)
  251. return
  252. }
  253. // Batch executes multiple Index and Delete
  254. // operations at the same time. There are often
  255. // significant performance benefits when performing
  256. // operations in a batch.
  257. func (i *indexImpl) Batch(b *Batch) error {
  258. i.mutex.RLock()
  259. defer i.mutex.RUnlock()
  260. if !i.open {
  261. return ErrorIndexClosed
  262. }
  263. return i.i.Batch(b.internal)
  264. }
  265. // Document is used to find the values of all the
  266. // stored fields for a document in the index. These
  267. // stored fields are put back into a Document object
  268. // and returned.
  269. func (i *indexImpl) Document(id string) (doc *document.Document, err error) {
  270. i.mutex.RLock()
  271. defer i.mutex.RUnlock()
  272. if !i.open {
  273. return nil, ErrorIndexClosed
  274. }
  275. indexReader, err := i.i.Reader()
  276. if err != nil {
  277. return nil, err
  278. }
  279. defer func() {
  280. if cerr := indexReader.Close(); err == nil && cerr != nil {
  281. err = cerr
  282. }
  283. }()
  284. doc, err = indexReader.Document(id)
  285. if err != nil {
  286. return nil, err
  287. }
  288. return doc, nil
  289. }
  290. // DocCount returns the number of documents in the
  291. // index.
  292. func (i *indexImpl) DocCount() (count uint64, err error) {
  293. i.mutex.RLock()
  294. defer i.mutex.RUnlock()
  295. if !i.open {
  296. return 0, ErrorIndexClosed
  297. }
  298. // open a reader for this search
  299. indexReader, err := i.i.Reader()
  300. if err != nil {
  301. return 0, fmt.Errorf("error opening index reader %v", err)
  302. }
  303. defer func() {
  304. if cerr := indexReader.Close(); err == nil && cerr != nil {
  305. err = cerr
  306. }
  307. }()
  308. count, err = indexReader.DocCount()
  309. return
  310. }
  311. // Search executes a search request operation.
  312. // Returns a SearchResult object or an error.
  313. func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) {
  314. return i.SearchInContext(context.Background(), req)
  315. }
  316. var documentMatchEmptySize int
  317. var searchContextEmptySize int
  318. var facetResultEmptySize int
  319. var documentEmptySize int
  320. func init() {
  321. var dm search.DocumentMatch
  322. documentMatchEmptySize = dm.Size()
  323. var sc search.SearchContext
  324. searchContextEmptySize = sc.Size()
  325. var fr search.FacetResult
  326. facetResultEmptySize = fr.Size()
  327. var d document.Document
  328. documentEmptySize = d.Size()
  329. }
  330. // memNeededForSearch is a helper function that returns an estimate of RAM
  331. // needed to execute a search request.
  332. func memNeededForSearch(req *SearchRequest,
  333. searcher search.Searcher,
  334. topnCollector *collector.TopNCollector) uint64 {
  335. backingSize := req.Size + req.From + 1
  336. if req.Size+req.From > collector.PreAllocSizeSkipCap {
  337. backingSize = collector.PreAllocSizeSkipCap + 1
  338. }
  339. numDocMatches := backingSize + searcher.DocumentMatchPoolSize()
  340. estimate := 0
  341. // overhead, size in bytes from collector
  342. estimate += topnCollector.Size()
  343. // pre-allocing DocumentMatchPool
  344. estimate += searchContextEmptySize + numDocMatches*documentMatchEmptySize
  345. // searcher overhead
  346. estimate += searcher.Size()
  347. // overhead from results, lowestMatchOutsideResults
  348. estimate += (numDocMatches + 1) * documentMatchEmptySize
  349. // additional overhead from SearchResult
  350. estimate += reflectStaticSizeSearchResult + reflectStaticSizeSearchStatus
  351. // overhead from facet results
  352. if req.Facets != nil {
  353. estimate += len(req.Facets) * facetResultEmptySize
  354. }
  355. // highlighting, store
  356. if len(req.Fields) > 0 || req.Highlight != nil {
  357. // Size + From => number of hits
  358. estimate += (req.Size + req.From) * documentEmptySize
  359. }
  360. return uint64(estimate)
  361. }
  362. // SearchInContext executes a search request operation within the provided
  363. // Context. Returns a SearchResult object or an error.
  364. func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr *SearchResult, err error) {
  365. i.mutex.RLock()
  366. defer i.mutex.RUnlock()
  367. searchStart := time.Now()
  368. if !i.open {
  369. return nil, ErrorIndexClosed
  370. }
  371. var reverseQueryExecution bool
  372. if req.SearchBefore != nil {
  373. reverseQueryExecution = true
  374. req.Sort.Reverse()
  375. req.SearchAfter = req.SearchBefore
  376. req.SearchBefore = nil
  377. }
  378. var coll *collector.TopNCollector
  379. if req.SearchAfter != nil {
  380. coll = collector.NewTopNCollectorAfter(req.Size, req.Sort, req.SearchAfter)
  381. } else {
  382. coll = collector.NewTopNCollector(req.Size, req.From, req.Sort)
  383. }
  384. // open a reader for this search
  385. indexReader, err := i.i.Reader()
  386. if err != nil {
  387. return nil, fmt.Errorf("error opening index reader %v", err)
  388. }
  389. defer func() {
  390. if cerr := indexReader.Close(); err == nil && cerr != nil {
  391. err = cerr
  392. }
  393. }()
  394. searcher, err := req.Query.Searcher(indexReader, i.m, search.SearcherOptions{
  395. Explain: req.Explain,
  396. IncludeTermVectors: req.IncludeLocations || req.Highlight != nil,
  397. Score: req.Score,
  398. })
  399. if err != nil {
  400. return nil, err
  401. }
  402. defer func() {
  403. if serr := searcher.Close(); err == nil && serr != nil {
  404. err = serr
  405. }
  406. }()
  407. if req.Facets != nil {
  408. facetsBuilder := search.NewFacetsBuilder(indexReader)
  409. for facetName, facetRequest := range req.Facets {
  410. if facetRequest.NumericRanges != nil {
  411. // build numeric range facet
  412. facetBuilder := facet.NewNumericFacetBuilder(facetRequest.Field, facetRequest.Size)
  413. for _, nr := range facetRequest.NumericRanges {
  414. facetBuilder.AddRange(nr.Name, nr.Min, nr.Max)
  415. }
  416. facetsBuilder.Add(facetName, facetBuilder)
  417. } else if facetRequest.DateTimeRanges != nil {
  418. // build date range facet
  419. facetBuilder := facet.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size)
  420. dateTimeParser := i.m.DateTimeParserNamed("")
  421. for _, dr := range facetRequest.DateTimeRanges {
  422. start, end := dr.ParseDates(dateTimeParser)
  423. facetBuilder.AddRange(dr.Name, start, end)
  424. }
  425. facetsBuilder.Add(facetName, facetBuilder)
  426. } else {
  427. // build terms facet
  428. facetBuilder := facet.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size)
  429. facetsBuilder.Add(facetName, facetBuilder)
  430. }
  431. }
  432. coll.SetFacetsBuilder(facetsBuilder)
  433. }
  434. memNeeded := memNeededForSearch(req, searcher, coll)
  435. if cb := ctx.Value(SearchQueryStartCallbackKey); cb != nil {
  436. if cbF, ok := cb.(SearchQueryStartCallbackFn); ok {
  437. err = cbF(memNeeded)
  438. }
  439. }
  440. if err != nil {
  441. return nil, err
  442. }
  443. if cb := ctx.Value(SearchQueryEndCallbackKey); cb != nil {
  444. if cbF, ok := cb.(SearchQueryEndCallbackFn); ok {
  445. defer func() {
  446. _ = cbF(memNeeded)
  447. }()
  448. }
  449. }
  450. err = coll.Collect(ctx, searcher, indexReader)
  451. if err != nil {
  452. return nil, err
  453. }
  454. hits := coll.Results()
  455. var highlighter highlight.Highlighter
  456. if req.Highlight != nil {
  457. // get the right highlighter
  458. highlighter, err = Config.Cache.HighlighterNamed(Config.DefaultHighlighter)
  459. if err != nil {
  460. return nil, err
  461. }
  462. if req.Highlight.Style != nil {
  463. highlighter, err = Config.Cache.HighlighterNamed(*req.Highlight.Style)
  464. if err != nil {
  465. return nil, err
  466. }
  467. }
  468. if highlighter == nil {
  469. return nil, fmt.Errorf("no highlighter named `%s` registered", *req.Highlight.Style)
  470. }
  471. }
  472. for _, hit := range hits {
  473. if i.name != "" {
  474. hit.Index = i.name
  475. }
  476. err = LoadAndHighlightFields(hit, req, i.name, indexReader, highlighter)
  477. if err != nil {
  478. return nil, err
  479. }
  480. }
  481. atomic.AddUint64(&i.stats.searches, 1)
  482. searchDuration := time.Since(searchStart)
  483. atomic.AddUint64(&i.stats.searchTime, uint64(searchDuration))
  484. if Config.SlowSearchLogThreshold > 0 &&
  485. searchDuration > Config.SlowSearchLogThreshold {
  486. logger.Printf("slow search took %s - %v", searchDuration, req)
  487. }
  488. if reverseQueryExecution {
  489. // reverse the sort back to the original
  490. req.Sort.Reverse()
  491. // resort using the original order
  492. mhs := newSearchHitSorter(req.Sort, hits)
  493. sort.Sort(mhs)
  494. // reset request
  495. req.SearchBefore = req.SearchAfter
  496. req.SearchAfter = nil
  497. }
  498. return &SearchResult{
  499. Status: &SearchStatus{
  500. Total: 1,
  501. Successful: 1,
  502. },
  503. Request: req,
  504. Hits: hits,
  505. Total: coll.Total(),
  506. MaxScore: coll.MaxScore(),
  507. Took: searchDuration,
  508. Facets: coll.FacetResults(),
  509. }, nil
  510. }
  511. func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest,
  512. indexName string, r index.IndexReader,
  513. highlighter highlight.Highlighter) error {
  514. if len(req.Fields) > 0 || highlighter != nil {
  515. doc, err := r.Document(hit.ID)
  516. if err == nil && doc != nil {
  517. if len(req.Fields) > 0 {
  518. fieldsToLoad := deDuplicate(req.Fields)
  519. for _, f := range fieldsToLoad {
  520. for _, docF := range doc.Fields {
  521. if f == "*" || docF.Name() == f {
  522. var value interface{}
  523. switch docF := docF.(type) {
  524. case *document.TextField:
  525. value = string(docF.Value())
  526. case *document.NumericField:
  527. num, err := docF.Number()
  528. if err == nil {
  529. value = num
  530. }
  531. case *document.DateTimeField:
  532. datetime, err := docF.DateTime()
  533. if err == nil {
  534. value = datetime.Format(time.RFC3339)
  535. }
  536. case *document.BooleanField:
  537. boolean, err := docF.Boolean()
  538. if err == nil {
  539. value = boolean
  540. }
  541. case *document.GeoPointField:
  542. lon, err := docF.Lon()
  543. if err == nil {
  544. lat, err := docF.Lat()
  545. if err == nil {
  546. value = []float64{lon, lat}
  547. }
  548. }
  549. }
  550. if value != nil {
  551. hit.AddFieldValue(docF.Name(), value)
  552. }
  553. }
  554. }
  555. }
  556. }
  557. if highlighter != nil {
  558. highlightFields := req.Highlight.Fields
  559. if highlightFields == nil {
  560. // add all fields with matches
  561. highlightFields = make([]string, 0, len(hit.Locations))
  562. for k := range hit.Locations {
  563. highlightFields = append(highlightFields, k)
  564. }
  565. }
  566. for _, hf := range highlightFields {
  567. highlighter.BestFragmentsInField(hit, doc, hf, 1)
  568. }
  569. }
  570. } else if doc == nil {
  571. // unexpected case, a doc ID that was found as a search hit
  572. // was unable to be found during document lookup
  573. return ErrorIndexReadInconsistency
  574. }
  575. }
  576. return nil
  577. }
  578. // Fields returns the name of all the fields this
  579. // Index has operated on.
  580. func (i *indexImpl) Fields() (fields []string, err error) {
  581. i.mutex.RLock()
  582. defer i.mutex.RUnlock()
  583. if !i.open {
  584. return nil, ErrorIndexClosed
  585. }
  586. indexReader, err := i.i.Reader()
  587. if err != nil {
  588. return nil, err
  589. }
  590. defer func() {
  591. if cerr := indexReader.Close(); err == nil && cerr != nil {
  592. err = cerr
  593. }
  594. }()
  595. fields, err = indexReader.Fields()
  596. if err != nil {
  597. return nil, err
  598. }
  599. return fields, nil
  600. }
  601. func (i *indexImpl) FieldDict(field string) (index.FieldDict, error) {
  602. i.mutex.RLock()
  603. if !i.open {
  604. i.mutex.RUnlock()
  605. return nil, ErrorIndexClosed
  606. }
  607. indexReader, err := i.i.Reader()
  608. if err != nil {
  609. i.mutex.RUnlock()
  610. return nil, err
  611. }
  612. fieldDict, err := indexReader.FieldDict(field)
  613. if err != nil {
  614. i.mutex.RUnlock()
  615. return nil, err
  616. }
  617. return &indexImplFieldDict{
  618. index: i,
  619. indexReader: indexReader,
  620. fieldDict: fieldDict,
  621. }, nil
  622. }
  623. func (i *indexImpl) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {
  624. i.mutex.RLock()
  625. if !i.open {
  626. i.mutex.RUnlock()
  627. return nil, ErrorIndexClosed
  628. }
  629. indexReader, err := i.i.Reader()
  630. if err != nil {
  631. i.mutex.RUnlock()
  632. return nil, err
  633. }
  634. fieldDict, err := indexReader.FieldDictRange(field, startTerm, endTerm)
  635. if err != nil {
  636. i.mutex.RUnlock()
  637. return nil, err
  638. }
  639. return &indexImplFieldDict{
  640. index: i,
  641. indexReader: indexReader,
  642. fieldDict: fieldDict,
  643. }, nil
  644. }
  645. func (i *indexImpl) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) {
  646. i.mutex.RLock()
  647. if !i.open {
  648. i.mutex.RUnlock()
  649. return nil, ErrorIndexClosed
  650. }
  651. indexReader, err := i.i.Reader()
  652. if err != nil {
  653. i.mutex.RUnlock()
  654. return nil, err
  655. }
  656. fieldDict, err := indexReader.FieldDictPrefix(field, termPrefix)
  657. if err != nil {
  658. i.mutex.RUnlock()
  659. return nil, err
  660. }
  661. return &indexImplFieldDict{
  662. index: i,
  663. indexReader: indexReader,
  664. fieldDict: fieldDict,
  665. }, nil
  666. }
  667. func (i *indexImpl) Close() error {
  668. i.mutex.Lock()
  669. defer i.mutex.Unlock()
  670. indexStats.UnRegister(i)
  671. i.open = false
  672. return i.i.Close()
  673. }
  674. func (i *indexImpl) Stats() *IndexStat {
  675. return i.stats
  676. }
  677. func (i *indexImpl) StatsMap() map[string]interface{} {
  678. return i.stats.statsMap()
  679. }
  680. func (i *indexImpl) GetInternal(key []byte) (val []byte, err error) {
  681. i.mutex.RLock()
  682. defer i.mutex.RUnlock()
  683. if !i.open {
  684. return nil, ErrorIndexClosed
  685. }
  686. reader, err := i.i.Reader()
  687. if err != nil {
  688. return nil, err
  689. }
  690. defer func() {
  691. if cerr := reader.Close(); err == nil && cerr != nil {
  692. err = cerr
  693. }
  694. }()
  695. val, err = reader.GetInternal(key)
  696. if err != nil {
  697. return nil, err
  698. }
  699. return val, nil
  700. }
  701. func (i *indexImpl) SetInternal(key, val []byte) error {
  702. i.mutex.RLock()
  703. defer i.mutex.RUnlock()
  704. if !i.open {
  705. return ErrorIndexClosed
  706. }
  707. return i.i.SetInternal(key, val)
  708. }
  709. func (i *indexImpl) DeleteInternal(key []byte) error {
  710. i.mutex.RLock()
  711. defer i.mutex.RUnlock()
  712. if !i.open {
  713. return ErrorIndexClosed
  714. }
  715. return i.i.DeleteInternal(key)
  716. }
  717. // NewBatch creates a new empty batch.
  718. func (i *indexImpl) NewBatch() *Batch {
  719. return &Batch{
  720. index: i,
  721. internal: index.NewBatch(),
  722. }
  723. }
  724. func (i *indexImpl) Name() string {
  725. return i.name
  726. }
  727. func (i *indexImpl) SetName(name string) {
  728. indexStats.UnRegister(i)
  729. i.name = name
  730. indexStats.Register(i)
  731. }
  732. type indexImplFieldDict struct {
  733. index *indexImpl
  734. indexReader index.IndexReader
  735. fieldDict index.FieldDict
  736. }
  737. func (f *indexImplFieldDict) Next() (*index.DictEntry, error) {
  738. return f.fieldDict.Next()
  739. }
  740. func (f *indexImplFieldDict) Close() error {
  741. defer f.index.mutex.RUnlock()
  742. err := f.fieldDict.Close()
  743. if err != nil {
  744. return err
  745. }
  746. return f.indexReader.Close()
  747. }
  748. // helper function to remove duplicate entries from slice of strings
  749. func deDuplicate(fields []string) []string {
  750. entries := make(map[string]struct{})
  751. ret := []string{}
  752. for _, entry := range fields {
  753. if _, exists := entries[entry]; !exists {
  754. entries[entry] = struct{}{}
  755. ret = append(ret, entry)
  756. }
  757. }
  758. return ret
  759. }
  760. type searchHitSorter struct {
  761. hits search.DocumentMatchCollection
  762. sort search.SortOrder
  763. cachedScoring []bool
  764. cachedDesc []bool
  765. }
  766. func newSearchHitSorter(sort search.SortOrder, hits search.DocumentMatchCollection) *searchHitSorter {
  767. return &searchHitSorter{
  768. sort: sort,
  769. hits: hits,
  770. cachedScoring: sort.CacheIsScore(),
  771. cachedDesc: sort.CacheDescending(),
  772. }
  773. }
  774. func (m *searchHitSorter) Len() int { return len(m.hits) }
  775. func (m *searchHitSorter) Swap(i, j int) { m.hits[i], m.hits[j] = m.hits[j], m.hits[i] }
  776. func (m *searchHitSorter) Less(i, j int) bool {
  777. c := m.sort.Compare(m.cachedScoring, m.cachedDesc, m.hits[i], m.hits[j])
  778. return c < 0
  779. }