index_impl.go 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867
  1. // Copyright (c) 2014 Couchbase, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bleve
  15. import (
  16. "context"
  17. "encoding/json"
  18. "fmt"
  19. "os"
  20. "sync"
  21. "sync/atomic"
  22. "time"
  23. "github.com/blevesearch/bleve/document"
  24. "github.com/blevesearch/bleve/index"
  25. "github.com/blevesearch/bleve/index/store"
  26. "github.com/blevesearch/bleve/index/upsidedown"
  27. "github.com/blevesearch/bleve/mapping"
  28. "github.com/blevesearch/bleve/registry"
  29. "github.com/blevesearch/bleve/search"
  30. "github.com/blevesearch/bleve/search/collector"
  31. "github.com/blevesearch/bleve/search/facet"
  32. "github.com/blevesearch/bleve/search/highlight"
  33. )
  34. type indexImpl struct {
  35. path string
  36. name string
  37. meta *indexMeta
  38. i index.Index
  39. m mapping.IndexMapping
  40. mutex sync.RWMutex
  41. open bool
  42. stats *IndexStat
  43. }
  44. const storePath = "store"
  45. var mappingInternalKey = []byte("_mapping")
  46. const SearchQueryStartCallbackKey = "_search_query_start_callback_key"
  47. const SearchQueryEndCallbackKey = "_search_query_end_callback_key"
  48. type SearchQueryStartCallbackFn func(size uint64) error
  49. type SearchQueryEndCallbackFn func(size uint64) error
  50. func indexStorePath(path string) string {
  51. return path + string(os.PathSeparator) + storePath
  52. }
  53. func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (*indexImpl, error) {
  54. // first validate the mapping
  55. err := mapping.Validate()
  56. if err != nil {
  57. return nil, err
  58. }
  59. if kvconfig == nil {
  60. kvconfig = map[string]interface{}{}
  61. }
  62. if kvstore == "" {
  63. return nil, fmt.Errorf("bleve not configured for file based indexing")
  64. }
  65. rv := indexImpl{
  66. path: path,
  67. name: path,
  68. m: mapping,
  69. meta: newIndexMeta(indexType, kvstore, kvconfig),
  70. }
  71. rv.stats = &IndexStat{i: &rv}
  72. // at this point there is hope that we can be successful, so save index meta
  73. if path != "" {
  74. err = rv.meta.Save(path)
  75. if err != nil {
  76. return nil, err
  77. }
  78. kvconfig["create_if_missing"] = true
  79. kvconfig["error_if_exists"] = true
  80. kvconfig["path"] = indexStorePath(path)
  81. } else {
  82. kvconfig["path"] = ""
  83. }
  84. // open the index
  85. indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
  86. if indexTypeConstructor == nil {
  87. return nil, ErrorUnknownIndexType
  88. }
  89. rv.i, err = indexTypeConstructor(rv.meta.Storage, kvconfig, Config.analysisQueue)
  90. if err != nil {
  91. return nil, err
  92. }
  93. err = rv.i.Open()
  94. if err != nil {
  95. if err == index.ErrorUnknownStorageType {
  96. return nil, ErrorUnknownStorageType
  97. }
  98. return nil, err
  99. }
  100. // now persist the mapping
  101. mappingBytes, err := json.Marshal(mapping)
  102. if err != nil {
  103. return nil, err
  104. }
  105. err = rv.i.SetInternal(mappingInternalKey, mappingBytes)
  106. if err != nil {
  107. return nil, err
  108. }
  109. // mark the index as open
  110. rv.mutex.Lock()
  111. defer rv.mutex.Unlock()
  112. rv.open = true
  113. indexStats.Register(&rv)
  114. return &rv, nil
  115. }
  116. func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *indexImpl, err error) {
  117. rv = &indexImpl{
  118. path: path,
  119. name: path,
  120. }
  121. rv.stats = &IndexStat{i: rv}
  122. rv.meta, err = openIndexMeta(path)
  123. if err != nil {
  124. return nil, err
  125. }
  126. // backwards compatibility if index type is missing
  127. if rv.meta.IndexType == "" {
  128. rv.meta.IndexType = upsidedown.Name
  129. }
  130. storeConfig := rv.meta.Config
  131. if storeConfig == nil {
  132. storeConfig = map[string]interface{}{}
  133. }
  134. storeConfig["path"] = indexStorePath(path)
  135. storeConfig["create_if_missing"] = false
  136. storeConfig["error_if_exists"] = false
  137. for rck, rcv := range runtimeConfig {
  138. storeConfig[rck] = rcv
  139. }
  140. // open the index
  141. indexTypeConstructor := registry.IndexTypeConstructorByName(rv.meta.IndexType)
  142. if indexTypeConstructor == nil {
  143. return nil, ErrorUnknownIndexType
  144. }
  145. rv.i, err = indexTypeConstructor(rv.meta.Storage, storeConfig, Config.analysisQueue)
  146. if err != nil {
  147. return nil, err
  148. }
  149. err = rv.i.Open()
  150. if err != nil {
  151. if err == index.ErrorUnknownStorageType {
  152. return nil, ErrorUnknownStorageType
  153. }
  154. return nil, err
  155. }
  156. // now load the mapping
  157. indexReader, err := rv.i.Reader()
  158. if err != nil {
  159. return nil, err
  160. }
  161. defer func() {
  162. if cerr := indexReader.Close(); cerr != nil && err == nil {
  163. err = cerr
  164. }
  165. }()
  166. mappingBytes, err := indexReader.GetInternal(mappingInternalKey)
  167. if err != nil {
  168. return nil, err
  169. }
  170. var im *mapping.IndexMappingImpl
  171. err = json.Unmarshal(mappingBytes, &im)
  172. if err != nil {
  173. return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes))
  174. }
  175. // mark the index as open
  176. rv.mutex.Lock()
  177. defer rv.mutex.Unlock()
  178. rv.open = true
  179. // validate the mapping
  180. err = im.Validate()
  181. if err != nil {
  182. // note even if the mapping is invalid
  183. // we still return an open usable index
  184. return rv, err
  185. }
  186. rv.m = im
  187. indexStats.Register(rv)
  188. return rv, err
  189. }
  190. // Advanced returns implementation internals
  191. // necessary ONLY for advanced usage.
  192. func (i *indexImpl) Advanced() (index.Index, store.KVStore, error) {
  193. s, err := i.i.Advanced()
  194. if err != nil {
  195. return nil, nil, err
  196. }
  197. return i.i, s, nil
  198. }
  199. // Mapping returns the IndexMapping in use by this
  200. // Index.
  201. func (i *indexImpl) Mapping() mapping.IndexMapping {
  202. return i.m
  203. }
  204. // Index the object with the specified identifier.
  205. // The IndexMapping for this index will determine
  206. // how the object is indexed.
  207. func (i *indexImpl) Index(id string, data interface{}) (err error) {
  208. if id == "" {
  209. return ErrorEmptyID
  210. }
  211. i.mutex.RLock()
  212. defer i.mutex.RUnlock()
  213. if !i.open {
  214. return ErrorIndexClosed
  215. }
  216. doc := document.NewDocument(id)
  217. err = i.m.MapDocument(doc, data)
  218. if err != nil {
  219. return
  220. }
  221. err = i.i.Update(doc)
  222. return
  223. }
  224. // IndexAdvanced takes a document.Document object
  225. // skips the mapping and indexes it.
  226. func (i *indexImpl) IndexAdvanced(doc *document.Document) (err error) {
  227. if doc.ID == "" {
  228. return ErrorEmptyID
  229. }
  230. i.mutex.RLock()
  231. defer i.mutex.RUnlock()
  232. if !i.open {
  233. return ErrorIndexClosed
  234. }
  235. err = i.i.Update(doc)
  236. return
  237. }
  238. // Delete entries for the specified identifier from
  239. // the index.
  240. func (i *indexImpl) Delete(id string) (err error) {
  241. if id == "" {
  242. return ErrorEmptyID
  243. }
  244. i.mutex.RLock()
  245. defer i.mutex.RUnlock()
  246. if !i.open {
  247. return ErrorIndexClosed
  248. }
  249. err = i.i.Delete(id)
  250. return
  251. }
  252. // Batch executes multiple Index and Delete
  253. // operations at the same time. There are often
  254. // significant performance benefits when performing
  255. // operations in a batch.
  256. func (i *indexImpl) Batch(b *Batch) error {
  257. i.mutex.RLock()
  258. defer i.mutex.RUnlock()
  259. if !i.open {
  260. return ErrorIndexClosed
  261. }
  262. return i.i.Batch(b.internal)
  263. }
  264. // Document is used to find the values of all the
  265. // stored fields for a document in the index. These
  266. // stored fields are put back into a Document object
  267. // and returned.
  268. func (i *indexImpl) Document(id string) (doc *document.Document, err error) {
  269. i.mutex.RLock()
  270. defer i.mutex.RUnlock()
  271. if !i.open {
  272. return nil, ErrorIndexClosed
  273. }
  274. indexReader, err := i.i.Reader()
  275. if err != nil {
  276. return nil, err
  277. }
  278. defer func() {
  279. if cerr := indexReader.Close(); err == nil && cerr != nil {
  280. err = cerr
  281. }
  282. }()
  283. doc, err = indexReader.Document(id)
  284. if err != nil {
  285. return nil, err
  286. }
  287. return doc, nil
  288. }
  289. // DocCount returns the number of documents in the
  290. // index.
  291. func (i *indexImpl) DocCount() (count uint64, err error) {
  292. i.mutex.RLock()
  293. defer i.mutex.RUnlock()
  294. if !i.open {
  295. return 0, ErrorIndexClosed
  296. }
  297. // open a reader for this search
  298. indexReader, err := i.i.Reader()
  299. if err != nil {
  300. return 0, fmt.Errorf("error opening index reader %v", err)
  301. }
  302. defer func() {
  303. if cerr := indexReader.Close(); err == nil && cerr != nil {
  304. err = cerr
  305. }
  306. }()
  307. count, err = indexReader.DocCount()
  308. return
  309. }
  310. // Search executes a search request operation.
  311. // Returns a SearchResult object or an error.
  312. func (i *indexImpl) Search(req *SearchRequest) (sr *SearchResult, err error) {
  313. return i.SearchInContext(context.Background(), req)
  314. }
  315. var documentMatchEmptySize int
  316. var searchContextEmptySize int
  317. var facetResultEmptySize int
  318. var documentEmptySize int
  319. func init() {
  320. var dm search.DocumentMatch
  321. documentMatchEmptySize = dm.Size()
  322. var sc search.SearchContext
  323. searchContextEmptySize = sc.Size()
  324. var fr search.FacetResult
  325. facetResultEmptySize = fr.Size()
  326. var d document.Document
  327. documentEmptySize = d.Size()
  328. }
  329. // memNeededForSearch is a helper function that returns an estimate of RAM
  330. // needed to execute a search request.
  331. func memNeededForSearch(req *SearchRequest,
  332. searcher search.Searcher,
  333. topnCollector *collector.TopNCollector) uint64 {
  334. backingSize := req.Size + req.From + 1
  335. if req.Size+req.From > collector.PreAllocSizeSkipCap {
  336. backingSize = collector.PreAllocSizeSkipCap + 1
  337. }
  338. numDocMatches := backingSize + searcher.DocumentMatchPoolSize()
  339. estimate := 0
  340. // overhead, size in bytes from collector
  341. estimate += topnCollector.Size()
  342. // pre-allocing DocumentMatchPool
  343. estimate += searchContextEmptySize + numDocMatches*documentMatchEmptySize
  344. // searcher overhead
  345. estimate += searcher.Size()
  346. // overhead from results, lowestMatchOutsideResults
  347. estimate += (numDocMatches + 1) * documentMatchEmptySize
  348. // additional overhead from SearchResult
  349. estimate += reflectStaticSizeSearchResult + reflectStaticSizeSearchStatus
  350. // overhead from facet results
  351. if req.Facets != nil {
  352. estimate += len(req.Facets) * facetResultEmptySize
  353. }
  354. // highlighting, store
  355. if len(req.Fields) > 0 || req.Highlight != nil {
  356. // Size + From => number of hits
  357. estimate += (req.Size + req.From) * documentEmptySize
  358. }
  359. return uint64(estimate)
  360. }
  361. // SearchInContext executes a search request operation within the provided
  362. // Context. Returns a SearchResult object or an error.
  363. func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr *SearchResult, err error) {
  364. i.mutex.RLock()
  365. defer i.mutex.RUnlock()
  366. searchStart := time.Now()
  367. if !i.open {
  368. return nil, ErrorIndexClosed
  369. }
  370. collector := collector.NewTopNCollector(req.Size, req.From, req.Sort)
  371. // open a reader for this search
  372. indexReader, err := i.i.Reader()
  373. if err != nil {
  374. return nil, fmt.Errorf("error opening index reader %v", err)
  375. }
  376. defer func() {
  377. if cerr := indexReader.Close(); err == nil && cerr != nil {
  378. err = cerr
  379. }
  380. }()
  381. searcher, err := req.Query.Searcher(indexReader, i.m, search.SearcherOptions{
  382. Explain: req.Explain,
  383. IncludeTermVectors: req.IncludeLocations || req.Highlight != nil,
  384. Score: req.Score,
  385. })
  386. if err != nil {
  387. return nil, err
  388. }
  389. defer func() {
  390. if serr := searcher.Close(); err == nil && serr != nil {
  391. err = serr
  392. }
  393. }()
  394. if req.Facets != nil {
  395. facetsBuilder := search.NewFacetsBuilder(indexReader)
  396. for facetName, facetRequest := range req.Facets {
  397. if facetRequest.NumericRanges != nil {
  398. // build numeric range facet
  399. facetBuilder := facet.NewNumericFacetBuilder(facetRequest.Field, facetRequest.Size)
  400. for _, nr := range facetRequest.NumericRanges {
  401. facetBuilder.AddRange(nr.Name, nr.Min, nr.Max)
  402. }
  403. facetsBuilder.Add(facetName, facetBuilder)
  404. } else if facetRequest.DateTimeRanges != nil {
  405. // build date range facet
  406. facetBuilder := facet.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size)
  407. dateTimeParser := i.m.DateTimeParserNamed("")
  408. for _, dr := range facetRequest.DateTimeRanges {
  409. start, end := dr.ParseDates(dateTimeParser)
  410. facetBuilder.AddRange(dr.Name, start, end)
  411. }
  412. facetsBuilder.Add(facetName, facetBuilder)
  413. } else {
  414. // build terms facet
  415. facetBuilder := facet.NewTermsFacetBuilder(facetRequest.Field, facetRequest.Size)
  416. facetsBuilder.Add(facetName, facetBuilder)
  417. }
  418. }
  419. collector.SetFacetsBuilder(facetsBuilder)
  420. }
  421. memNeeded := memNeededForSearch(req, searcher, collector)
  422. if cb := ctx.Value(SearchQueryStartCallbackKey); cb != nil {
  423. if cbF, ok := cb.(SearchQueryStartCallbackFn); ok {
  424. err = cbF(memNeeded)
  425. }
  426. }
  427. if err != nil {
  428. return nil, err
  429. }
  430. if cb := ctx.Value(SearchQueryEndCallbackKey); cb != nil {
  431. if cbF, ok := cb.(SearchQueryEndCallbackFn); ok {
  432. defer func() {
  433. _ = cbF(memNeeded)
  434. }()
  435. }
  436. }
  437. err = collector.Collect(ctx, searcher, indexReader)
  438. if err != nil {
  439. return nil, err
  440. }
  441. hits := collector.Results()
  442. var highlighter highlight.Highlighter
  443. if req.Highlight != nil {
  444. // get the right highlighter
  445. highlighter, err = Config.Cache.HighlighterNamed(Config.DefaultHighlighter)
  446. if err != nil {
  447. return nil, err
  448. }
  449. if req.Highlight.Style != nil {
  450. highlighter, err = Config.Cache.HighlighterNamed(*req.Highlight.Style)
  451. if err != nil {
  452. return nil, err
  453. }
  454. }
  455. if highlighter == nil {
  456. return nil, fmt.Errorf("no highlighter named `%s` registered", *req.Highlight.Style)
  457. }
  458. }
  459. for _, hit := range hits {
  460. if i.name != "" {
  461. hit.Index = i.name
  462. }
  463. err = LoadAndHighlightFields(hit, req, i.name, indexReader, highlighter)
  464. if err != nil {
  465. return nil, err
  466. }
  467. }
  468. atomic.AddUint64(&i.stats.searches, 1)
  469. searchDuration := time.Since(searchStart)
  470. atomic.AddUint64(&i.stats.searchTime, uint64(searchDuration))
  471. if Config.SlowSearchLogThreshold > 0 &&
  472. searchDuration > Config.SlowSearchLogThreshold {
  473. logger.Printf("slow search took %s - %v", searchDuration, req)
  474. }
  475. return &SearchResult{
  476. Status: &SearchStatus{
  477. Total: 1,
  478. Successful: 1,
  479. },
  480. Request: req,
  481. Hits: hits,
  482. Total: collector.Total(),
  483. MaxScore: collector.MaxScore(),
  484. Took: searchDuration,
  485. Facets: collector.FacetResults(),
  486. }, nil
  487. }
  488. func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest,
  489. indexName string, r index.IndexReader,
  490. highlighter highlight.Highlighter) error {
  491. if len(req.Fields) > 0 || highlighter != nil {
  492. doc, err := r.Document(hit.ID)
  493. if err == nil && doc != nil {
  494. if len(req.Fields) > 0 {
  495. fieldsToLoad := deDuplicate(req.Fields)
  496. for _, f := range fieldsToLoad {
  497. for _, docF := range doc.Fields {
  498. if f == "*" || docF.Name() == f {
  499. var value interface{}
  500. switch docF := docF.(type) {
  501. case *document.TextField:
  502. value = string(docF.Value())
  503. case *document.NumericField:
  504. num, err := docF.Number()
  505. if err == nil {
  506. value = num
  507. }
  508. case *document.DateTimeField:
  509. datetime, err := docF.DateTime()
  510. if err == nil {
  511. value = datetime.Format(time.RFC3339)
  512. }
  513. case *document.BooleanField:
  514. boolean, err := docF.Boolean()
  515. if err == nil {
  516. value = boolean
  517. }
  518. case *document.GeoPointField:
  519. lon, err := docF.Lon()
  520. if err == nil {
  521. lat, err := docF.Lat()
  522. if err == nil {
  523. value = []float64{lon, lat}
  524. }
  525. }
  526. }
  527. if value != nil {
  528. hit.AddFieldValue(docF.Name(), value)
  529. }
  530. }
  531. }
  532. }
  533. }
  534. if highlighter != nil {
  535. highlightFields := req.Highlight.Fields
  536. if highlightFields == nil {
  537. // add all fields with matches
  538. highlightFields = make([]string, 0, len(hit.Locations))
  539. for k := range hit.Locations {
  540. highlightFields = append(highlightFields, k)
  541. }
  542. }
  543. for _, hf := range highlightFields {
  544. highlighter.BestFragmentsInField(hit, doc, hf, 1)
  545. }
  546. }
  547. } else if doc == nil {
  548. // unexpected case, a doc ID that was found as a search hit
  549. // was unable to be found during document lookup
  550. return ErrorIndexReadInconsistency
  551. }
  552. }
  553. return nil
  554. }
  555. // Fields returns the name of all the fields this
  556. // Index has operated on.
  557. func (i *indexImpl) Fields() (fields []string, err error) {
  558. i.mutex.RLock()
  559. defer i.mutex.RUnlock()
  560. if !i.open {
  561. return nil, ErrorIndexClosed
  562. }
  563. indexReader, err := i.i.Reader()
  564. if err != nil {
  565. return nil, err
  566. }
  567. defer func() {
  568. if cerr := indexReader.Close(); err == nil && cerr != nil {
  569. err = cerr
  570. }
  571. }()
  572. fields, err = indexReader.Fields()
  573. if err != nil {
  574. return nil, err
  575. }
  576. return fields, nil
  577. }
  578. func (i *indexImpl) FieldDict(field string) (index.FieldDict, error) {
  579. i.mutex.RLock()
  580. if !i.open {
  581. i.mutex.RUnlock()
  582. return nil, ErrorIndexClosed
  583. }
  584. indexReader, err := i.i.Reader()
  585. if err != nil {
  586. i.mutex.RUnlock()
  587. return nil, err
  588. }
  589. fieldDict, err := indexReader.FieldDict(field)
  590. if err != nil {
  591. i.mutex.RUnlock()
  592. return nil, err
  593. }
  594. return &indexImplFieldDict{
  595. index: i,
  596. indexReader: indexReader,
  597. fieldDict: fieldDict,
  598. }, nil
  599. }
  600. func (i *indexImpl) FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {
  601. i.mutex.RLock()
  602. if !i.open {
  603. i.mutex.RUnlock()
  604. return nil, ErrorIndexClosed
  605. }
  606. indexReader, err := i.i.Reader()
  607. if err != nil {
  608. i.mutex.RUnlock()
  609. return nil, err
  610. }
  611. fieldDict, err := indexReader.FieldDictRange(field, startTerm, endTerm)
  612. if err != nil {
  613. i.mutex.RUnlock()
  614. return nil, err
  615. }
  616. return &indexImplFieldDict{
  617. index: i,
  618. indexReader: indexReader,
  619. fieldDict: fieldDict,
  620. }, nil
  621. }
  622. func (i *indexImpl) FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error) {
  623. i.mutex.RLock()
  624. if !i.open {
  625. i.mutex.RUnlock()
  626. return nil, ErrorIndexClosed
  627. }
  628. indexReader, err := i.i.Reader()
  629. if err != nil {
  630. i.mutex.RUnlock()
  631. return nil, err
  632. }
  633. fieldDict, err := indexReader.FieldDictPrefix(field, termPrefix)
  634. if err != nil {
  635. i.mutex.RUnlock()
  636. return nil, err
  637. }
  638. return &indexImplFieldDict{
  639. index: i,
  640. indexReader: indexReader,
  641. fieldDict: fieldDict,
  642. }, nil
  643. }
  644. func (i *indexImpl) Close() error {
  645. i.mutex.Lock()
  646. defer i.mutex.Unlock()
  647. indexStats.UnRegister(i)
  648. i.open = false
  649. return i.i.Close()
  650. }
  651. func (i *indexImpl) Stats() *IndexStat {
  652. return i.stats
  653. }
  654. func (i *indexImpl) StatsMap() map[string]interface{} {
  655. return i.stats.statsMap()
  656. }
  657. func (i *indexImpl) GetInternal(key []byte) (val []byte, err error) {
  658. i.mutex.RLock()
  659. defer i.mutex.RUnlock()
  660. if !i.open {
  661. return nil, ErrorIndexClosed
  662. }
  663. reader, err := i.i.Reader()
  664. if err != nil {
  665. return nil, err
  666. }
  667. defer func() {
  668. if cerr := reader.Close(); err == nil && cerr != nil {
  669. err = cerr
  670. }
  671. }()
  672. val, err = reader.GetInternal(key)
  673. if err != nil {
  674. return nil, err
  675. }
  676. return val, nil
  677. }
  678. func (i *indexImpl) SetInternal(key, val []byte) error {
  679. i.mutex.RLock()
  680. defer i.mutex.RUnlock()
  681. if !i.open {
  682. return ErrorIndexClosed
  683. }
  684. return i.i.SetInternal(key, val)
  685. }
  686. func (i *indexImpl) DeleteInternal(key []byte) error {
  687. i.mutex.RLock()
  688. defer i.mutex.RUnlock()
  689. if !i.open {
  690. return ErrorIndexClosed
  691. }
  692. return i.i.DeleteInternal(key)
  693. }
  694. // NewBatch creates a new empty batch.
  695. func (i *indexImpl) NewBatch() *Batch {
  696. return &Batch{
  697. index: i,
  698. internal: index.NewBatch(),
  699. }
  700. }
  701. func (i *indexImpl) Name() string {
  702. return i.name
  703. }
  704. func (i *indexImpl) SetName(name string) {
  705. indexStats.UnRegister(i)
  706. i.name = name
  707. indexStats.Register(i)
  708. }
  709. type indexImplFieldDict struct {
  710. index *indexImpl
  711. indexReader index.IndexReader
  712. fieldDict index.FieldDict
  713. }
  714. func (f *indexImplFieldDict) Next() (*index.DictEntry, error) {
  715. return f.fieldDict.Next()
  716. }
  717. func (f *indexImplFieldDict) Close() error {
  718. defer f.index.mutex.RUnlock()
  719. err := f.fieldDict.Close()
  720. if err != nil {
  721. return err
  722. }
  723. return f.indexReader.Close()
  724. }
  725. // helper function to remove duplicate entries from slice of strings
  726. func deDuplicate(fields []string) []string {
  727. entries := make(map[string]struct{})
  728. ret := []string{}
  729. for _, entry := range fields {
  730. if _, exists := entries[entry]; !exists {
  731. entries[entry] = struct{}{}
  732. ret = append(ret, entry)
  733. }
  734. }
  735. return ret
  736. }