index.go 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293
  1. // Copyright (c) 2014 Couchbase, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bleve
  15. import (
  16. "context"
  17. "github.com/blevesearch/bleve/document"
  18. "github.com/blevesearch/bleve/index"
  19. "github.com/blevesearch/bleve/index/store"
  20. "github.com/blevesearch/bleve/mapping"
  21. "github.com/blevesearch/bleve/size"
  22. )
  23. // A Batch groups together multiple Index and Delete
  24. // operations you would like performed at the same
  25. // time. The Batch structure is NOT thread-safe.
  26. // You should only perform operations on a batch
  27. // from a single thread at a time. Once batch
  28. // execution has started, you may not modify it.
  29. type Batch struct {
  30. index Index
  31. internal *index.Batch
  32. lastDocSize uint64
  33. totalSize uint64
  34. }
  35. // Index adds the specified index operation to the
  36. // batch. NOTE: the bleve Index is not updated
  37. // until the batch is executed.
  38. func (b *Batch) Index(id string, data interface{}) error {
  39. if id == "" {
  40. return ErrorEmptyID
  41. }
  42. doc := document.NewDocument(id)
  43. err := b.index.Mapping().MapDocument(doc, data)
  44. if err != nil {
  45. return err
  46. }
  47. b.internal.Update(doc)
  48. b.lastDocSize = uint64(doc.Size() +
  49. len(id) + size.SizeOfString) // overhead from internal
  50. b.totalSize += b.lastDocSize
  51. return nil
  52. }
  53. func (b *Batch) LastDocSize() uint64 {
  54. return b.lastDocSize
  55. }
  56. func (b *Batch) TotalDocsSize() uint64 {
  57. return b.totalSize
  58. }
  59. // IndexAdvanced adds the specified index operation to the
  60. // batch which skips the mapping. NOTE: the bleve Index is not updated
  61. // until the batch is executed.
  62. func (b *Batch) IndexAdvanced(doc *document.Document) (err error) {
  63. if doc.ID == "" {
  64. return ErrorEmptyID
  65. }
  66. b.internal.Update(doc)
  67. return nil
  68. }
  69. // Delete adds the specified delete operation to the
  70. // batch. NOTE: the bleve Index is not updated until
  71. // the batch is executed.
  72. func (b *Batch) Delete(id string) {
  73. if id != "" {
  74. b.internal.Delete(id)
  75. }
  76. }
  77. // SetInternal adds the specified set internal
  78. // operation to the batch. NOTE: the bleve Index is
  79. // not updated until the batch is executed.
  80. func (b *Batch) SetInternal(key, val []byte) {
  81. b.internal.SetInternal(key, val)
  82. }
  83. // DeleteInternal adds the specified delete internal
  84. // operation to the batch. NOTE: the bleve Index is
  85. // not updated until the batch is executed.
  86. func (b *Batch) DeleteInternal(key []byte) {
  87. b.internal.DeleteInternal(key)
  88. }
  89. // Size returns the total number of operations inside the batch
  90. // including normal index operations and internal operations.
  91. func (b *Batch) Size() int {
  92. return len(b.internal.IndexOps) + len(b.internal.InternalOps)
  93. }
  94. // String prints a user friendly string representation of what
  95. // is inside this batch.
  96. func (b *Batch) String() string {
  97. return b.internal.String()
  98. }
  99. // Reset returns a Batch to the empty state so that it can
  100. // be re-used in the future.
  101. func (b *Batch) Reset() {
  102. b.internal.Reset()
  103. }
  104. func (b *Batch) Merge(o *Batch) {
  105. if o != nil && o.internal != nil {
  106. b.internal.Merge(o.internal)
  107. if o.LastDocSize() > 0 {
  108. b.lastDocSize = o.LastDocSize()
  109. }
  110. b.totalSize = uint64(b.internal.TotalDocSize())
  111. }
  112. }
  113. func (b *Batch) SetPersistedCallback(f index.BatchCallback) {
  114. b.internal.SetPersistedCallback(f)
  115. }
  116. func (b *Batch) PersistedCallback() index.BatchCallback {
  117. return b.internal.PersistedCallback()
  118. }
  119. // An Index implements all the indexing and searching
  120. // capabilities of bleve. An Index can be created
  121. // using the New() and Open() methods.
  122. //
  123. // Index() takes an input value, deduces a DocumentMapping for its type,
  124. // assigns string paths to its fields or values then applies field mappings on
  125. // them.
  126. //
  127. // The DocumentMapping used to index a value is deduced by the following rules:
  128. // 1) If value implements mapping.bleveClassifier interface, resolve the mapping
  129. // from BleveType().
  130. // 2) If value implements mapping.Classifier interface, resolve the mapping
  131. // from Type().
  132. // 3) If value has a string field or value at IndexMapping.TypeField.
  133. // (defaulting to "_type"), use it to resolve the mapping. Fields addressing
  134. // is described below.
  135. // 4) If IndexMapping.DefaultType is registered, return it.
  136. // 5) Return IndexMapping.DefaultMapping.
  137. //
  138. // Each field or nested field of the value is identified by a string path, then
  139. // mapped to one or several FieldMappings which extract the result for analysis.
  140. //
  141. // Struct values fields are identified by their "json:" tag, or by their name.
  142. // Nested fields are identified by prefixing with their parent identifier,
  143. // separated by a dot.
  144. //
  145. // Map values entries are identified by their string key. Entries not indexed
  146. // by strings are ignored. Entry values are identified recursively like struct
  147. // fields.
  148. //
  149. // Slice and array values are identified by their field name. Their elements
  150. // are processed sequentially with the same FieldMapping.
  151. //
  152. // String, float64 and time.Time values are identified by their field name.
  153. // Other types are ignored.
  154. //
  155. // Each value identifier is decomposed in its parts and recursively address
  156. // SubDocumentMappings in the tree starting at the root DocumentMapping. If a
  157. // mapping is found, all its FieldMappings are applied to the value. If no
  158. // mapping is found and the root DocumentMapping is dynamic, default mappings
  159. // are used based on value type and IndexMapping default configurations.
  160. //
  161. // Finally, mapped values are analyzed, indexed or stored. See
  162. // FieldMapping.Analyzer to know how an analyzer is resolved for a given field.
  163. //
  164. // Examples:
  165. //
  166. // type Date struct {
  167. // Day string `json:"day"`
  168. // Month string
  169. // Year string
  170. // }
  171. //
  172. // type Person struct {
  173. // FirstName string `json:"first_name"`
  174. // LastName string
  175. // BirthDate Date `json:"birth_date"`
  176. // }
  177. //
  178. // A Person value FirstName is mapped by the SubDocumentMapping at
  179. // "first_name". Its LastName is mapped by the one at "LastName". The day of
  180. // BirthDate is mapped to the SubDocumentMapping "day" of the root
  181. // SubDocumentMapping "birth_date". It will appear as the "birth_date.day"
  182. // field in the index. The month is mapped to "birth_date.Month".
  183. type Index interface {
  184. // Index analyzes, indexes or stores mapped data fields. Supplied
  185. // identifier is bound to analyzed data and will be retrieved by search
  186. // requests. See Index interface documentation for details about mapping
  187. // rules.
  188. Index(id string, data interface{}) error
  189. Delete(id string) error
  190. NewBatch() *Batch
  191. Batch(b *Batch) error
  192. // Document returns specified document or nil if the document is not
  193. // indexed or stored.
  194. Document(id string) (*document.Document, error)
  195. // DocCount returns the number of documents in the index.
  196. DocCount() (uint64, error)
  197. Search(req *SearchRequest) (*SearchResult, error)
  198. SearchInContext(ctx context.Context, req *SearchRequest) (*SearchResult, error)
  199. Fields() ([]string, error)
  200. FieldDict(field string) (index.FieldDict, error)
  201. FieldDictRange(field string, startTerm []byte, endTerm []byte) (index.FieldDict, error)
  202. FieldDictPrefix(field string, termPrefix []byte) (index.FieldDict, error)
  203. Close() error
  204. Mapping() mapping.IndexMapping
  205. Stats() *IndexStat
  206. StatsMap() map[string]interface{}
  207. GetInternal(key []byte) ([]byte, error)
  208. SetInternal(key, val []byte) error
  209. DeleteInternal(key []byte) error
  210. // Name returns the name of the index (by default this is the path)
  211. Name() string
  212. // SetName lets you assign your own logical name to this index
  213. SetName(string)
  214. // Advanced returns the indexer and data store, exposing lower level
  215. // methods to enumerate records and access data.
  216. Advanced() (index.Index, store.KVStore, error)
  217. }
  218. // New index at the specified path, must not exist.
  219. // The provided mapping will be used for all
  220. // Index/Search operations.
  221. func New(path string, mapping mapping.IndexMapping) (Index, error) {
  222. return newIndexUsing(path, mapping, Config.DefaultIndexType, Config.DefaultKVStore, nil)
  223. }
  224. // NewMemOnly creates a memory-only index.
  225. // The contents of the index is NOT persisted,
  226. // and will be lost once closed.
  227. // The provided mapping will be used for all
  228. // Index/Search operations.
  229. func NewMemOnly(mapping mapping.IndexMapping) (Index, error) {
  230. return newIndexUsing("", mapping, Config.DefaultIndexType, Config.DefaultMemKVStore, nil)
  231. }
  232. // NewUsing creates index at the specified path,
  233. // which must not already exist.
  234. // The provided mapping will be used for all
  235. // Index/Search operations.
  236. // The specified index type will be used.
  237. // The specified kvstore implementation will be used
  238. // and the provided kvconfig will be passed to its
  239. // constructor. Note that currently the values of kvconfig must
  240. // be able to be marshaled and unmarshaled using the encoding/json library (used
  241. // when reading/writing the index metadata file).
  242. func NewUsing(path string, mapping mapping.IndexMapping, indexType string, kvstore string, kvconfig map[string]interface{}) (Index, error) {
  243. return newIndexUsing(path, mapping, indexType, kvstore, kvconfig)
  244. }
  245. // Open index at the specified path, must exist.
  246. // The mapping used when it was created will be used for all Index/Search operations.
  247. func Open(path string) (Index, error) {
  248. return openIndexUsing(path, nil)
  249. }
  250. // OpenUsing opens index at the specified path, must exist.
  251. // The mapping used when it was created will be used for all Index/Search operations.
  252. // The provided runtimeConfig can override settings
  253. // persisted when the kvstore was created.
  254. func OpenUsing(path string, runtimeConfig map[string]interface{}) (Index, error) {
  255. return openIndexUsing(path, runtimeConfig)
  256. }