index_test.go 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218
  1. // Copyright (c) 2014 Couchbase, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bleve
  15. import (
  16. "context"
  17. "fmt"
  18. "io/ioutil"
  19. "log"
  20. "math"
  21. "os"
  22. "reflect"
  23. "sort"
  24. "strconv"
  25. "strings"
  26. "sync"
  27. "testing"
  28. "time"
  29. "github.com/blevesearch/bleve/analysis/analyzer/keyword"
  30. "github.com/blevesearch/bleve/document"
  31. "github.com/blevesearch/bleve/index"
  32. "github.com/blevesearch/bleve/index/store/boltdb"
  33. "github.com/blevesearch/bleve/index/store/null"
  34. "github.com/blevesearch/bleve/mapping"
  35. "github.com/blevesearch/bleve/search"
  36. "github.com/blevesearch/bleve/search/query"
  37. "github.com/blevesearch/bleve/index/scorch"
  38. "github.com/blevesearch/bleve/index/upsidedown"
  39. )
  40. func TestCrud(t *testing.T) {
  41. defer func() {
  42. err := os.RemoveAll("testidx")
  43. if err != nil {
  44. t.Fatal(err)
  45. }
  46. }()
  47. index, err := New("testidx", NewIndexMapping())
  48. if err != nil {
  49. t.Fatal(err)
  50. }
  51. doca := map[string]interface{}{
  52. "name": "marty",
  53. "desc": "gophercon india",
  54. }
  55. err = index.Index("a", doca)
  56. if err != nil {
  57. t.Error(err)
  58. }
  59. docy := map[string]interface{}{
  60. "name": "jasper",
  61. "desc": "clojure",
  62. }
  63. err = index.Index("y", docy)
  64. if err != nil {
  65. t.Error(err)
  66. }
  67. err = index.Delete("y")
  68. if err != nil {
  69. t.Error(err)
  70. }
  71. docx := map[string]interface{}{
  72. "name": "rose",
  73. "desc": "googler",
  74. }
  75. err = index.Index("x", docx)
  76. if err != nil {
  77. t.Error(err)
  78. }
  79. err = index.SetInternal([]byte("status"), []byte("pending"))
  80. if err != nil {
  81. t.Error(err)
  82. }
  83. docb := map[string]interface{}{
  84. "name": "steve",
  85. "desc": "cbft master",
  86. }
  87. batch := index.NewBatch()
  88. err = batch.Index("b", docb)
  89. if err != nil {
  90. t.Error(err)
  91. }
  92. batch.Delete("x")
  93. batch.SetInternal([]byte("batchi"), []byte("batchv"))
  94. batch.DeleteInternal([]byte("status"))
  95. err = index.Batch(batch)
  96. if err != nil {
  97. t.Error(err)
  98. }
  99. val, err := index.GetInternal([]byte("batchi"))
  100. if err != nil {
  101. t.Error(err)
  102. }
  103. if string(val) != "batchv" {
  104. t.Errorf("expected 'batchv', got '%s'", val)
  105. }
  106. val, err = index.GetInternal([]byte("status"))
  107. if err != nil {
  108. t.Error(err)
  109. }
  110. if val != nil {
  111. t.Errorf("expected nil, got '%s'", val)
  112. }
  113. err = index.SetInternal([]byte("seqno"), []byte("7"))
  114. if err != nil {
  115. t.Error(err)
  116. }
  117. err = index.SetInternal([]byte("status"), []byte("ready"))
  118. if err != nil {
  119. t.Error(err)
  120. }
  121. err = index.DeleteInternal([]byte("status"))
  122. if err != nil {
  123. t.Error(err)
  124. }
  125. val, err = index.GetInternal([]byte("status"))
  126. if err != nil {
  127. t.Error(err)
  128. }
  129. if val != nil {
  130. t.Errorf("expected nil, got '%s'", val)
  131. }
  132. val, err = index.GetInternal([]byte("seqno"))
  133. if err != nil {
  134. t.Error(err)
  135. }
  136. if string(val) != "7" {
  137. t.Errorf("expected '7', got '%s'", val)
  138. }
  139. // close the index, open it again, and try some more things
  140. err = index.Close()
  141. if err != nil {
  142. t.Fatal(err)
  143. }
  144. index, err = Open("testidx")
  145. if err != nil {
  146. t.Fatal(err)
  147. }
  148. defer func() {
  149. err := index.Close()
  150. if err != nil {
  151. t.Fatal(err)
  152. }
  153. }()
  154. count, err := index.DocCount()
  155. if err != nil {
  156. t.Fatal(err)
  157. }
  158. if count != 2 {
  159. t.Errorf("expected doc count 2, got %d", count)
  160. }
  161. doc, err := index.Document("a")
  162. if err != nil {
  163. t.Fatal(err)
  164. }
  165. if doc == nil {
  166. t.Errorf("expected doc not nil, got nil")
  167. }
  168. foundNameField := false
  169. for _, field := range doc.Fields {
  170. if field.Name() == "name" && string(field.Value()) == "marty" {
  171. foundNameField = true
  172. }
  173. }
  174. if !foundNameField {
  175. t.Errorf("expected to find field named 'name' with value 'marty'")
  176. }
  177. fields, err := index.Fields()
  178. if err != nil {
  179. t.Fatal(err)
  180. }
  181. expectedFields := map[string]bool{
  182. "_all": false,
  183. "name": false,
  184. "desc": false,
  185. }
  186. if len(fields) < len(expectedFields) {
  187. t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields))
  188. }
  189. for _, f := range fields {
  190. expectedFields[f] = true
  191. }
  192. for ef, efp := range expectedFields {
  193. if !efp {
  194. t.Errorf("field %s is missing", ef)
  195. }
  196. }
  197. }
  198. func TestIndexCreateNewOverExisting(t *testing.T) {
  199. defer func() {
  200. err := os.RemoveAll("testidx")
  201. if err != nil {
  202. t.Fatal(err)
  203. }
  204. }()
  205. index, err := New("testidx", NewIndexMapping())
  206. if err != nil {
  207. t.Fatal(err)
  208. }
  209. err = index.Close()
  210. if err != nil {
  211. t.Fatal(err)
  212. }
  213. index, err = New("testidx", NewIndexMapping())
  214. if err != ErrorIndexPathExists {
  215. t.Fatalf("expected error index path exists, got %v", err)
  216. }
  217. }
  218. func TestIndexOpenNonExisting(t *testing.T) {
  219. _, err := Open("doesnotexist")
  220. if err != ErrorIndexPathDoesNotExist {
  221. t.Fatalf("expected error index path does not exist, got %v", err)
  222. }
  223. }
  224. func TestIndexOpenMetaMissingOrCorrupt(t *testing.T) {
  225. defer func() {
  226. err := os.RemoveAll("testidx")
  227. if err != nil {
  228. t.Fatal(err)
  229. }
  230. }()
  231. index, err := New("testidx", NewIndexMapping())
  232. if err != nil {
  233. t.Fatal(err)
  234. }
  235. err = index.Close()
  236. if err != nil {
  237. t.Fatal(err)
  238. }
  239. // now intentionally change the storage type
  240. err = ioutil.WriteFile("testidx/index_meta.json", []byte(`{"storage":"mystery"}`), 0666)
  241. if err != nil {
  242. t.Fatal(err)
  243. }
  244. index, err = Open("testidx")
  245. if err != ErrorUnknownStorageType {
  246. t.Fatalf("expected error unknown storage type, got %v", err)
  247. }
  248. // now intentionally corrupt the metadata
  249. err = ioutil.WriteFile("testidx/index_meta.json", []byte("corrupted"), 0666)
  250. if err != nil {
  251. t.Fatal(err)
  252. }
  253. index, err = Open("testidx")
  254. if err != ErrorIndexMetaCorrupt {
  255. t.Fatalf("expected error index metadata corrupted, got %v", err)
  256. }
  257. // now intentionally remove the metadata
  258. err = os.Remove("testidx/index_meta.json")
  259. if err != nil {
  260. t.Fatal(err)
  261. }
  262. index, err = Open("testidx")
  263. if err != ErrorIndexMetaMissing {
  264. t.Fatalf("expected error index metadata missing, got %v", err)
  265. }
  266. }
  267. func TestInMemIndex(t *testing.T) {
  268. index, err := NewMemOnly(NewIndexMapping())
  269. if err != nil {
  270. t.Fatal(err)
  271. }
  272. err = index.Close()
  273. if err != nil {
  274. t.Fatal(err)
  275. }
  276. }
  277. func TestClosedIndex(t *testing.T) {
  278. index, err := NewMemOnly(NewIndexMapping())
  279. if err != nil {
  280. t.Fatal(err)
  281. }
  282. err = index.Close()
  283. if err != nil {
  284. t.Fatal(err)
  285. }
  286. err = index.Index("test", "test")
  287. if err != ErrorIndexClosed {
  288. t.Errorf("expected error index closed, got %v", err)
  289. }
  290. err = index.Delete("test")
  291. if err != ErrorIndexClosed {
  292. t.Errorf("expected error index closed, got %v", err)
  293. }
  294. b := index.NewBatch()
  295. err = index.Batch(b)
  296. if err != ErrorIndexClosed {
  297. t.Errorf("expected error index closed, got %v", err)
  298. }
  299. _, err = index.Document("test")
  300. if err != ErrorIndexClosed {
  301. t.Errorf("expected error index closed, got %v", err)
  302. }
  303. _, err = index.DocCount()
  304. if err != ErrorIndexClosed {
  305. t.Errorf("expected error index closed, got %v", err)
  306. }
  307. _, err = index.Search(NewSearchRequest(NewTermQuery("test")))
  308. if err != ErrorIndexClosed {
  309. t.Errorf("expected error index closed, got %v", err)
  310. }
  311. _, err = index.Fields()
  312. if err != ErrorIndexClosed {
  313. t.Errorf("expected error index closed, got %v", err)
  314. }
  315. }
  316. type slowQuery struct {
  317. actual query.Query
  318. delay time.Duration
  319. }
  320. func (s *slowQuery) Searcher(i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
  321. time.Sleep(s.delay)
  322. return s.actual.Searcher(i, m, options)
  323. }
  324. func TestSlowSearch(t *testing.T) {
  325. defer func() {
  326. err := os.RemoveAll("testidx")
  327. if err != nil {
  328. t.Fatal(err)
  329. }
  330. }()
  331. defer func() {
  332. // reset logger back to normal
  333. SetLog(log.New(ioutil.Discard, "bleve", log.LstdFlags))
  334. }()
  335. // set custom logger
  336. var sdw sawDataWriter
  337. SetLog(log.New(&sdw, "bleve", log.LstdFlags))
  338. index, err := New("testidx", NewIndexMapping())
  339. if err != nil {
  340. t.Fatal(err)
  341. }
  342. defer func() {
  343. err := index.Close()
  344. if err != nil {
  345. t.Fatal(err)
  346. }
  347. }()
  348. Config.SlowSearchLogThreshold = 1 * time.Minute
  349. query := NewTermQuery("water")
  350. req := NewSearchRequest(query)
  351. _, err = index.Search(req)
  352. if err != nil {
  353. t.Fatal(err)
  354. }
  355. if sdw.sawData {
  356. t.Errorf("expected to not see slow query logged, but did")
  357. }
  358. sq := &slowQuery{
  359. actual: query,
  360. delay: 50 * time.Millisecond, // on Windows timer resolution is 15ms
  361. }
  362. req.Query = sq
  363. Config.SlowSearchLogThreshold = 1 * time.Microsecond
  364. _, err = index.Search(req)
  365. if err != nil {
  366. t.Fatal(err)
  367. }
  368. if !sdw.sawData {
  369. t.Errorf("expected to see slow query logged, but didn't")
  370. }
  371. }
  372. type sawDataWriter struct {
  373. sawData bool
  374. }
  375. func (s *sawDataWriter) Write(p []byte) (n int, err error) {
  376. s.sawData = true
  377. return len(p), nil
  378. }
  379. func TestStoredFieldPreserved(t *testing.T) {
  380. defer func() {
  381. err := os.RemoveAll("testidx")
  382. if err != nil {
  383. t.Fatal(err)
  384. }
  385. }()
  386. index, err := New("testidx", NewIndexMapping())
  387. if err != nil {
  388. t.Fatal(err)
  389. }
  390. defer func() {
  391. err := index.Close()
  392. if err != nil {
  393. t.Fatal(err)
  394. }
  395. }()
  396. doca := map[string]interface{}{
  397. "name": "Marty",
  398. "desc": "GopherCON India",
  399. "bool": true,
  400. "num": float64(1),
  401. }
  402. err = index.Index("a", doca)
  403. if err != nil {
  404. t.Error(err)
  405. }
  406. q := NewTermQuery("marty")
  407. req := NewSearchRequest(q)
  408. req.Fields = []string{"name", "desc", "bool", "num"}
  409. res, err := index.Search(req)
  410. if err != nil {
  411. t.Error(err)
  412. }
  413. if len(res.Hits) != 1 {
  414. t.Fatalf("expected 1 hit, got %d", len(res.Hits))
  415. }
  416. if res.Hits[0].Fields["name"] != "Marty" {
  417. t.Errorf("expected 'Marty' got '%s'", res.Hits[0].Fields["name"])
  418. }
  419. if res.Hits[0].Fields["desc"] != "GopherCON India" {
  420. t.Errorf("expected 'GopherCON India' got '%s'", res.Hits[0].Fields["desc"])
  421. }
  422. if res.Hits[0].Fields["num"] != float64(1) {
  423. t.Errorf("expected '1' got '%v'", res.Hits[0].Fields["num"])
  424. }
  425. if res.Hits[0].Fields["bool"] != true {
  426. t.Errorf("expected 'true' got '%v'", res.Hits[0].Fields["bool"])
  427. }
  428. }
  429. func TestDict(t *testing.T) {
  430. defer func() {
  431. err := os.RemoveAll("testidx")
  432. if err != nil {
  433. t.Fatal(err)
  434. }
  435. }()
  436. index, err := New("testidx", NewIndexMapping())
  437. if err != nil {
  438. t.Fatal(err)
  439. }
  440. doca := map[string]interface{}{
  441. "name": "marty",
  442. "desc": "gophercon india",
  443. }
  444. err = index.Index("a", doca)
  445. if err != nil {
  446. t.Error(err)
  447. }
  448. docy := map[string]interface{}{
  449. "name": "jasper",
  450. "desc": "clojure",
  451. }
  452. err = index.Index("y", docy)
  453. if err != nil {
  454. t.Error(err)
  455. }
  456. docx := map[string]interface{}{
  457. "name": "rose",
  458. "desc": "googler",
  459. }
  460. err = index.Index("x", docx)
  461. if err != nil {
  462. t.Error(err)
  463. }
  464. dict, err := index.FieldDict("name")
  465. if err != nil {
  466. t.Error(err)
  467. }
  468. terms := []string{}
  469. de, err := dict.Next()
  470. for err == nil && de != nil {
  471. terms = append(terms, string(de.Term))
  472. de, err = dict.Next()
  473. }
  474. expectedTerms := []string{"jasper", "marty", "rose"}
  475. if !reflect.DeepEqual(terms, expectedTerms) {
  476. t.Errorf("expected %v, got %v", expectedTerms, terms)
  477. }
  478. err = dict.Close()
  479. if err != nil {
  480. t.Fatal(err)
  481. }
  482. // test start and end range
  483. dict, err = index.FieldDictRange("name", []byte("marty"), []byte("rose"))
  484. if err != nil {
  485. t.Error(err)
  486. }
  487. terms = []string{}
  488. de, err = dict.Next()
  489. for err == nil && de != nil {
  490. terms = append(terms, string(de.Term))
  491. de, err = dict.Next()
  492. }
  493. expectedTerms = []string{"marty", "rose"}
  494. if !reflect.DeepEqual(terms, expectedTerms) {
  495. t.Errorf("expected %v, got %v", expectedTerms, terms)
  496. }
  497. err = dict.Close()
  498. if err != nil {
  499. t.Fatal(err)
  500. }
  501. docz := map[string]interface{}{
  502. "name": "prefix",
  503. "desc": "bob cat cats catting dog doggy zoo",
  504. }
  505. err = index.Index("z", docz)
  506. if err != nil {
  507. t.Error(err)
  508. }
  509. dict, err = index.FieldDictPrefix("desc", []byte("cat"))
  510. if err != nil {
  511. t.Error(err)
  512. }
  513. terms = []string{}
  514. de, err = dict.Next()
  515. for err == nil && de != nil {
  516. terms = append(terms, string(de.Term))
  517. de, err = dict.Next()
  518. }
  519. expectedTerms = []string{"cat", "cats", "catting"}
  520. if !reflect.DeepEqual(terms, expectedTerms) {
  521. t.Errorf("expected %v, got %v", expectedTerms, terms)
  522. }
  523. stats := index.Stats()
  524. if stats == nil {
  525. t.Errorf("expected IndexStat, got nil")
  526. }
  527. err = dict.Close()
  528. if err != nil {
  529. t.Fatal(err)
  530. }
  531. err = index.Close()
  532. if err != nil {
  533. t.Fatal(err)
  534. }
  535. }
  536. func TestBatchString(t *testing.T) {
  537. defer func() {
  538. err := os.RemoveAll("testidx")
  539. if err != nil {
  540. t.Fatal(err)
  541. }
  542. }()
  543. index, err := New("testidx", NewIndexMapping())
  544. if err != nil {
  545. t.Fatal(err)
  546. }
  547. defer func() {
  548. err := index.Close()
  549. if err != nil {
  550. t.Fatal(err)
  551. }
  552. }()
  553. batch := index.NewBatch()
  554. err = batch.Index("a", []byte("{}"))
  555. if err != nil {
  556. t.Fatal(err)
  557. }
  558. batch.Delete("b")
  559. batch.SetInternal([]byte("c"), []byte{})
  560. batch.DeleteInternal([]byte("d"))
  561. batchStr := batch.String()
  562. if !strings.HasPrefix(batchStr, "Batch (2 ops, 2 internal ops)") {
  563. t.Errorf("expected to start with Batch (2 ops, 2 internal ops), did not")
  564. }
  565. if !strings.Contains(batchStr, "INDEX - 'a'") {
  566. t.Errorf("expected to contain INDEX - 'a', did not")
  567. }
  568. if !strings.Contains(batchStr, "DELETE - 'b'") {
  569. t.Errorf("expected to contain DELETE - 'b', did not")
  570. }
  571. if !strings.Contains(batchStr, "SET INTERNAL - 'c'") {
  572. t.Errorf("expected to contain SET INTERNAL - 'c', did not")
  573. }
  574. if !strings.Contains(batchStr, "DELETE INTERNAL - 'd'") {
  575. t.Errorf("expected to contain DELETE INTERNAL - 'd', did not")
  576. }
  577. }
  578. func TestIndexMetadataRaceBug198(t *testing.T) {
  579. defer func() {
  580. err := os.RemoveAll("testidx")
  581. if err != nil {
  582. t.Fatal(err)
  583. }
  584. }()
  585. index, err := New("testidx", NewIndexMapping())
  586. if err != nil {
  587. t.Fatal(err)
  588. }
  589. defer func() {
  590. err := index.Close()
  591. if err != nil {
  592. t.Fatal(err)
  593. }
  594. }()
  595. wg := sync.WaitGroup{}
  596. wg.Add(1)
  597. done := make(chan struct{})
  598. go func() {
  599. for {
  600. select {
  601. case <-done:
  602. wg.Done()
  603. return
  604. default:
  605. _, err2 := index.DocCount()
  606. if err2 != nil {
  607. t.Fatal(err2)
  608. }
  609. }
  610. }
  611. }()
  612. for i := 0; i < 100; i++ {
  613. batch := index.NewBatch()
  614. err = batch.Index("a", []byte("{}"))
  615. if err != nil {
  616. t.Fatal(err)
  617. }
  618. err = index.Batch(batch)
  619. if err != nil {
  620. t.Fatal(err)
  621. }
  622. }
  623. close(done)
  624. wg.Wait()
  625. }
  626. func TestSortMatchSearch(t *testing.T) {
  627. defer func() {
  628. err := os.RemoveAll("testidx")
  629. if err != nil {
  630. t.Fatal(err)
  631. }
  632. }()
  633. index, err := New("testidx", NewIndexMapping())
  634. if err != nil {
  635. t.Fatal(err)
  636. }
  637. names := []string{"Noam", "Uri", "David", "Yosef", "Eitan", "Itay", "Ariel", "Daniel", "Omer", "Yogev", "Yehonatan", "Moshe", "Mohammed", "Yusuf", "Omar"}
  638. days := []string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}
  639. numbers := []string{"One", "Two", "Three", "Four", "Five", "Six", "Seven", "Eight", "Nine", "Ten", "Eleven", "Twelve"}
  640. for i := 0; i < 200; i++ {
  641. doc := make(map[string]interface{})
  642. doc["Name"] = names[i%len(names)]
  643. doc["Day"] = days[i%len(days)]
  644. doc["Number"] = numbers[i%len(numbers)]
  645. err = index.Index(fmt.Sprintf("%d", i), doc)
  646. if err != nil {
  647. t.Fatal(err)
  648. }
  649. }
  650. req := NewSearchRequest(NewMatchQuery("One"))
  651. req.SortBy([]string{"Day", "Name"})
  652. req.Fields = []string{"*"}
  653. sr, err := index.Search(req)
  654. if err != nil {
  655. t.Fatal(err)
  656. }
  657. prev := ""
  658. for _, hit := range sr.Hits {
  659. val := hit.Fields["Day"].(string)
  660. if prev > val {
  661. t.Errorf("Hits must be sorted by 'Day'. Found '%s' before '%s'", prev, val)
  662. }
  663. prev = val
  664. }
  665. err = index.Close()
  666. if err != nil {
  667. t.Fatal(err)
  668. }
  669. }
  670. func TestIndexCountMatchSearch(t *testing.T) {
  671. defer func() {
  672. err := os.RemoveAll("testidx")
  673. if err != nil {
  674. t.Fatal(err)
  675. }
  676. }()
  677. index, err := New("testidx", NewIndexMapping())
  678. if err != nil {
  679. t.Fatal(err)
  680. }
  681. var wg sync.WaitGroup
  682. for i := 0; i < 10; i++ {
  683. wg.Add(1)
  684. go func(i int) {
  685. b := index.NewBatch()
  686. for j := 0; j < 200; j++ {
  687. id := fmt.Sprintf("%d", (i*200)+j)
  688. doc := struct {
  689. Body string
  690. }{
  691. Body: "match",
  692. }
  693. err := b.Index(id, doc)
  694. if err != nil {
  695. t.Fatal(err)
  696. }
  697. }
  698. err := index.Batch(b)
  699. if err != nil {
  700. t.Fatal(err)
  701. }
  702. wg.Done()
  703. }(i)
  704. }
  705. wg.Wait()
  706. // search for something that should match all documents
  707. sr, err := index.Search(NewSearchRequest(NewMatchQuery("match")))
  708. if err != nil {
  709. t.Fatal(err)
  710. }
  711. // get the index document count
  712. dc, err := index.DocCount()
  713. if err != nil {
  714. t.Fatal(err)
  715. }
  716. // make sure test is working correctly, doc count should 2000
  717. if dc != 2000 {
  718. t.Errorf("expected doc count 2000, got %d", dc)
  719. }
  720. // make sure our search found all the documents
  721. if dc != sr.Total {
  722. t.Errorf("expected search result total %d to match doc count %d", sr.Total, dc)
  723. }
  724. err = index.Close()
  725. if err != nil {
  726. t.Fatal(err)
  727. }
  728. }
  729. func TestBatchReset(t *testing.T) {
  730. defer func() {
  731. err := os.RemoveAll("testidx")
  732. if err != nil {
  733. t.Fatal(err)
  734. }
  735. }()
  736. index, err := New("testidx", NewIndexMapping())
  737. if err != nil {
  738. t.Fatal(err)
  739. }
  740. batch := index.NewBatch()
  741. err = batch.Index("k1", struct {
  742. Body string
  743. }{
  744. Body: "v1",
  745. })
  746. if err != nil {
  747. t.Error(err)
  748. }
  749. batch.Delete("k2")
  750. batch.SetInternal([]byte("k3"), []byte("v3"))
  751. batch.DeleteInternal([]byte("k4"))
  752. if batch.Size() != 4 {
  753. t.Logf("%v", batch)
  754. t.Errorf("expected batch size 4, got %d", batch.Size())
  755. }
  756. batch.Reset()
  757. if batch.Size() != 0 {
  758. t.Errorf("expected batch size 0 after reset, got %d", batch.Size())
  759. }
  760. err = index.Close()
  761. if err != nil {
  762. t.Fatal(err)
  763. }
  764. }
  765. func TestDocumentFieldArrayPositions(t *testing.T) {
  766. defer func() {
  767. err := os.RemoveAll("testidx")
  768. if err != nil {
  769. t.Fatal(err)
  770. }
  771. }()
  772. index, err := New("testidx", NewIndexMapping())
  773. if err != nil {
  774. t.Fatal(err)
  775. }
  776. // index a document with an array of strings
  777. err = index.Index("k", struct {
  778. Messages []string
  779. }{
  780. Messages: []string{
  781. "first",
  782. "second",
  783. "third",
  784. "last",
  785. },
  786. })
  787. if err != nil {
  788. t.Fatal(err)
  789. }
  790. // load the document
  791. doc, err := index.Document("k")
  792. if err != nil {
  793. t.Fatal(err)
  794. }
  795. for _, f := range doc.Fields {
  796. if reflect.DeepEqual(f.Value(), []byte("first")) {
  797. ap := f.ArrayPositions()
  798. if len(ap) < 1 {
  799. t.Errorf("expected an array position, got none")
  800. continue
  801. }
  802. if ap[0] != 0 {
  803. t.Errorf("expected 'first' in array position 0, got %d", ap[0])
  804. }
  805. }
  806. if reflect.DeepEqual(f.Value(), []byte("second")) {
  807. ap := f.ArrayPositions()
  808. if len(ap) < 1 {
  809. t.Errorf("expected an array position, got none")
  810. continue
  811. }
  812. if ap[0] != 1 {
  813. t.Errorf("expected 'second' in array position 1, got %d", ap[0])
  814. }
  815. }
  816. if reflect.DeepEqual(f.Value(), []byte("third")) {
  817. ap := f.ArrayPositions()
  818. if len(ap) < 1 {
  819. t.Errorf("expected an array position, got none")
  820. continue
  821. }
  822. if ap[0] != 2 {
  823. t.Errorf("expected 'third' in array position 2, got %d", ap[0])
  824. }
  825. }
  826. if reflect.DeepEqual(f.Value(), []byte("last")) {
  827. ap := f.ArrayPositions()
  828. if len(ap) < 1 {
  829. t.Errorf("expected an array position, got none")
  830. continue
  831. }
  832. if ap[0] != 3 {
  833. t.Errorf("expected 'last' in array position 3, got %d", ap[0])
  834. }
  835. }
  836. }
  837. // now index a document in the same field with a single string
  838. err = index.Index("k2", struct {
  839. Messages string
  840. }{
  841. Messages: "only",
  842. })
  843. if err != nil {
  844. t.Fatal(err)
  845. }
  846. // load the document
  847. doc, err = index.Document("k2")
  848. if err != nil {
  849. t.Fatal(err)
  850. }
  851. for _, f := range doc.Fields {
  852. if reflect.DeepEqual(f.Value(), []byte("only")) {
  853. ap := f.ArrayPositions()
  854. if len(ap) != 0 {
  855. t.Errorf("expected no array positions, got %d", len(ap))
  856. continue
  857. }
  858. }
  859. }
  860. err = index.Close()
  861. if err != nil {
  862. t.Fatal(err)
  863. }
  864. }
  865. func TestKeywordSearchBug207(t *testing.T) {
  866. defer func() {
  867. err := os.RemoveAll("testidx")
  868. if err != nil {
  869. t.Fatal(err)
  870. }
  871. }()
  872. f := NewTextFieldMapping()
  873. f.Analyzer = keyword.Name
  874. m := NewIndexMapping()
  875. m.DefaultMapping = NewDocumentMapping()
  876. m.DefaultMapping.AddFieldMappingsAt("Body", f)
  877. index, err := New("testidx", m)
  878. if err != nil {
  879. t.Fatal(err)
  880. }
  881. doc1 := struct {
  882. Body string
  883. }{
  884. Body: "a555c3bb06f7a127cda000005",
  885. }
  886. err = index.Index("a", doc1)
  887. if err != nil {
  888. t.Fatal(err)
  889. }
  890. doc2 := struct {
  891. Body string
  892. }{
  893. Body: "555c3bb06f7a127cda000005",
  894. }
  895. err = index.Index("b", doc2)
  896. if err != nil {
  897. t.Fatal(err)
  898. }
  899. // now search for these terms
  900. sreq := NewSearchRequest(NewTermQuery("a555c3bb06f7a127cda000005"))
  901. sres, err := index.Search(sreq)
  902. if err != nil {
  903. t.Fatal(err)
  904. }
  905. if sres.Total != 1 {
  906. t.Errorf("expected 1 result, got %d", sres.Total)
  907. }
  908. if sres.Hits[0].ID != "a" {
  909. t.Errorf("expecated id 'a', got '%s'", sres.Hits[0].ID)
  910. }
  911. sreq = NewSearchRequest(NewTermQuery("555c3bb06f7a127cda000005"))
  912. sres, err = index.Search(sreq)
  913. if err != nil {
  914. t.Fatal(err)
  915. }
  916. if sres.Total != 1 {
  917. t.Errorf("expected 1 result, got %d", sres.Total)
  918. }
  919. if sres.Hits[0].ID != "b" {
  920. t.Errorf("expecated id 'b', got '%s'", sres.Hits[0].ID)
  921. }
  922. // now do the same searches using query strings
  923. sreq = NewSearchRequest(NewQueryStringQuery("Body:a555c3bb06f7a127cda000005"))
  924. sres, err = index.Search(sreq)
  925. if err != nil {
  926. t.Fatal(err)
  927. }
  928. if sres.Total != 1 {
  929. t.Errorf("expected 1 result, got %d", sres.Total)
  930. }
  931. if sres.Hits[0].ID != "a" {
  932. t.Errorf("expecated id 'a', got '%s'", sres.Hits[0].ID)
  933. }
  934. sreq = NewSearchRequest(NewQueryStringQuery(`Body:555c3bb06f7a127cda000005`))
  935. sres, err = index.Search(sreq)
  936. if err != nil {
  937. t.Fatal(err)
  938. }
  939. if sres.Total != 1 {
  940. t.Errorf("expected 1 result, got %d", sres.Total)
  941. }
  942. if sres.Hits[0].ID != "b" {
  943. t.Errorf("expecated id 'b', got '%s'", sres.Hits[0].ID)
  944. }
  945. err = index.Close()
  946. if err != nil {
  947. t.Fatal(err)
  948. }
  949. }
  950. func TestTermVectorArrayPositions(t *testing.T) {
  951. defer func() {
  952. err := os.RemoveAll("testidx")
  953. if err != nil {
  954. t.Fatal(err)
  955. }
  956. }()
  957. index, err := New("testidx", NewIndexMapping())
  958. if err != nil {
  959. t.Fatal(err)
  960. }
  961. // index a document with an array of strings
  962. err = index.Index("k", struct {
  963. Messages []string
  964. }{
  965. Messages: []string{
  966. "first",
  967. "second",
  968. "third",
  969. "last",
  970. },
  971. })
  972. if err != nil {
  973. t.Fatal(err)
  974. }
  975. // search for this document in all field
  976. tq := NewTermQuery("second")
  977. tsr := NewSearchRequest(tq)
  978. tsr.IncludeLocations = true
  979. results, err := index.Search(tsr)
  980. if err != nil {
  981. t.Fatal(err)
  982. }
  983. if results.Total != 1 {
  984. t.Fatalf("expected 1 result, got %d", results.Total)
  985. }
  986. if len(results.Hits[0].Locations["Messages"]["second"]) < 1 {
  987. t.Fatalf("expected at least one location")
  988. }
  989. if len(results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions) < 1 {
  990. t.Fatalf("expected at least one location array position")
  991. }
  992. if results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0] != 1 {
  993. t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0])
  994. }
  995. // repeat search for this document in Messages field
  996. tq2 := NewTermQuery("third")
  997. tq2.SetField("Messages")
  998. tsr = NewSearchRequest(tq2)
  999. tsr.IncludeLocations = true
  1000. results, err = index.Search(tsr)
  1001. if err != nil {
  1002. t.Fatal(err)
  1003. }
  1004. if results.Total != 1 {
  1005. t.Fatalf("expected 1 result, got %d", results.Total)
  1006. }
  1007. if len(results.Hits[0].Locations["Messages"]["third"]) < 1 {
  1008. t.Fatalf("expected at least one location")
  1009. }
  1010. if len(results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions) < 1 {
  1011. t.Fatalf("expected at least one location array position")
  1012. }
  1013. if results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0] != 2 {
  1014. t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0])
  1015. }
  1016. err = index.Close()
  1017. if err != nil {
  1018. t.Fatal(err)
  1019. }
  1020. }
  1021. func TestDocumentStaticMapping(t *testing.T) {
  1022. defer func() {
  1023. err := os.RemoveAll("testidx")
  1024. if err != nil {
  1025. t.Fatal(err)
  1026. }
  1027. }()
  1028. m := NewIndexMapping()
  1029. m.DefaultMapping = NewDocumentStaticMapping()
  1030. m.DefaultMapping.AddFieldMappingsAt("Text", NewTextFieldMapping())
  1031. m.DefaultMapping.AddFieldMappingsAt("Date", NewDateTimeFieldMapping())
  1032. m.DefaultMapping.AddFieldMappingsAt("Numeric", NewNumericFieldMapping())
  1033. index, err := New("testidx", m)
  1034. if err != nil {
  1035. t.Fatal(err)
  1036. }
  1037. doc1 := struct {
  1038. Text string
  1039. IgnoredText string
  1040. Numeric float64
  1041. IgnoredNumeric float64
  1042. Date time.Time
  1043. IgnoredDate time.Time
  1044. }{
  1045. Text: "valid text",
  1046. IgnoredText: "ignored text",
  1047. Numeric: 10,
  1048. IgnoredNumeric: 20,
  1049. Date: time.Unix(1, 0),
  1050. IgnoredDate: time.Unix(2, 0),
  1051. }
  1052. err = index.Index("a", doc1)
  1053. if err != nil {
  1054. t.Fatal(err)
  1055. }
  1056. fields, err := index.Fields()
  1057. if err != nil {
  1058. t.Fatal(err)
  1059. }
  1060. sort.Strings(fields)
  1061. expectedFields := []string{"Date", "Numeric", "Text", "_all"}
  1062. if len(fields) < len(expectedFields) {
  1063. t.Fatalf("invalid field count: %d", len(fields))
  1064. }
  1065. for i, expected := range expectedFields {
  1066. if expected != fields[i] {
  1067. t.Fatalf("unexpected field[%d]: %s", i, fields[i])
  1068. }
  1069. }
  1070. err = index.Close()
  1071. if err != nil {
  1072. t.Fatal(err)
  1073. }
  1074. }
  1075. func TestIndexEmptyDocId(t *testing.T) {
  1076. defer func() {
  1077. err := os.RemoveAll("testidx")
  1078. if err != nil {
  1079. t.Fatal(err)
  1080. }
  1081. }()
  1082. index, err := New("testidx", NewIndexMapping())
  1083. if err != nil {
  1084. t.Fatal(err)
  1085. }
  1086. defer func() {
  1087. err := index.Close()
  1088. if err != nil {
  1089. t.Fatal(err)
  1090. }
  1091. }()
  1092. doc := map[string]interface{}{
  1093. "body": "nodocid",
  1094. }
  1095. err = index.Index("", doc)
  1096. if err != ErrorEmptyID {
  1097. t.Errorf("expect index empty doc id to fail")
  1098. }
  1099. err = index.Delete("")
  1100. if err != ErrorEmptyID {
  1101. t.Errorf("expect delete empty doc id to fail")
  1102. }
  1103. batch := index.NewBatch()
  1104. err = batch.Index("", doc)
  1105. if err != ErrorEmptyID {
  1106. t.Errorf("expect index empty doc id in batch to fail")
  1107. }
  1108. batch.Delete("")
  1109. if batch.Size() > 0 {
  1110. t.Errorf("expect delete empty doc id in batch to be ignored")
  1111. }
  1112. }
  1113. func TestDateTimeFieldMappingIssue287(t *testing.T) {
  1114. defer func() {
  1115. err := os.RemoveAll("testidx")
  1116. if err != nil {
  1117. t.Fatal(err)
  1118. }
  1119. }()
  1120. f := NewDateTimeFieldMapping()
  1121. m := NewIndexMapping()
  1122. m.DefaultMapping = NewDocumentMapping()
  1123. m.DefaultMapping.AddFieldMappingsAt("Date", f)
  1124. index, err := New("testidx", m)
  1125. if err != nil {
  1126. t.Fatal(err)
  1127. }
  1128. type doc struct {
  1129. Date time.Time
  1130. }
  1131. now := time.Now()
  1132. // 3hr ago to 1hr ago
  1133. for i := 0; i < 3; i++ {
  1134. d := doc{now.Add(time.Duration((i - 3)) * time.Hour)}
  1135. err = index.Index(strconv.FormatInt(int64(i), 10), d)
  1136. if err != nil {
  1137. t.Fatal(err)
  1138. }
  1139. }
  1140. // search range across all docs
  1141. start := now.Add(-4 * time.Hour)
  1142. end := now
  1143. sreq := NewSearchRequest(NewDateRangeQuery(start, end))
  1144. sres, err := index.Search(sreq)
  1145. if err != nil {
  1146. t.Fatal(err)
  1147. }
  1148. if sres.Total != 3 {
  1149. t.Errorf("expected 3 results, got %d", sres.Total)
  1150. }
  1151. // search range includes only oldest
  1152. start = now.Add(-4 * time.Hour)
  1153. end = now.Add(-121 * time.Minute)
  1154. sreq = NewSearchRequest(NewDateRangeQuery(start, end))
  1155. sres, err = index.Search(sreq)
  1156. if err != nil {
  1157. t.Fatal(err)
  1158. }
  1159. if sres.Total != 1 {
  1160. t.Errorf("expected 1 results, got %d", sres.Total)
  1161. }
  1162. if sres.Total > 0 && sres.Hits[0].ID != "0" {
  1163. t.Errorf("expecated id '0', got '%s'", sres.Hits[0].ID)
  1164. }
  1165. // search range includes only newest
  1166. start = now.Add(-61 * time.Minute)
  1167. end = now
  1168. sreq = NewSearchRequest(NewDateRangeQuery(start, end))
  1169. sres, err = index.Search(sreq)
  1170. if err != nil {
  1171. t.Fatal(err)
  1172. }
  1173. if sres.Total != 1 {
  1174. t.Errorf("expected 1 results, got %d", sres.Total)
  1175. }
  1176. if sres.Total > 0 && sres.Hits[0].ID != "2" {
  1177. t.Errorf("expecated id '2', got '%s'", sres.Hits[0].ID)
  1178. }
  1179. err = index.Close()
  1180. if err != nil {
  1181. t.Fatal(err)
  1182. }
  1183. }
  1184. func TestDocumentFieldArrayPositionsBug295(t *testing.T) {
  1185. defer func() {
  1186. err := os.RemoveAll("testidx")
  1187. if err != nil {
  1188. t.Fatal(err)
  1189. }
  1190. }()
  1191. index, err := New("testidx", NewIndexMapping())
  1192. if err != nil {
  1193. t.Fatal(err)
  1194. }
  1195. // index a document with an array of strings
  1196. err = index.Index("k", struct {
  1197. Messages []string
  1198. Another string
  1199. MoreData []string
  1200. }{
  1201. Messages: []string{
  1202. "bleve",
  1203. "bleve",
  1204. },
  1205. Another: "text",
  1206. MoreData: []string{
  1207. "a",
  1208. "b",
  1209. "c",
  1210. "bleve",
  1211. },
  1212. })
  1213. if err != nil {
  1214. t.Fatal(err)
  1215. }
  1216. // search for it in the messages field
  1217. tq := NewTermQuery("bleve")
  1218. tq.SetField("Messages")
  1219. tsr := NewSearchRequest(tq)
  1220. tsr.IncludeLocations = true
  1221. results, err := index.Search(tsr)
  1222. if err != nil {
  1223. t.Fatal(err)
  1224. }
  1225. if results.Total != 1 {
  1226. t.Fatalf("expected 1 result, got %d", results.Total)
  1227. }
  1228. if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 {
  1229. t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"]))
  1230. }
  1231. if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 {
  1232. t.Errorf("expected array position to be 0")
  1233. }
  1234. if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 {
  1235. t.Errorf("expected array position to be 1")
  1236. }
  1237. // search for it in all
  1238. tq = NewTermQuery("bleve")
  1239. tsr = NewSearchRequest(tq)
  1240. tsr.IncludeLocations = true
  1241. results, err = index.Search(tsr)
  1242. if err != nil {
  1243. t.Fatal(err)
  1244. }
  1245. if results.Total != 1 {
  1246. t.Fatalf("expected 1 result, got %d", results.Total)
  1247. }
  1248. if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 {
  1249. t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"]))
  1250. }
  1251. if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 {
  1252. t.Errorf("expected array position to be 0")
  1253. }
  1254. if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 {
  1255. t.Errorf("expected array position to be 1")
  1256. }
  1257. err = index.Close()
  1258. if err != nil {
  1259. t.Fatal(err)
  1260. }
  1261. }
  1262. func TestBooleanFieldMappingIssue109(t *testing.T) {
  1263. defer func() {
  1264. err := os.RemoveAll("testidx")
  1265. if err != nil {
  1266. t.Fatal(err)
  1267. }
  1268. }()
  1269. m := NewIndexMapping()
  1270. m.DefaultMapping = NewDocumentMapping()
  1271. m.DefaultMapping.AddFieldMappingsAt("Bool", NewBooleanFieldMapping())
  1272. index, err := New("testidx", m)
  1273. if err != nil {
  1274. t.Fatal(err)
  1275. }
  1276. type doc struct {
  1277. Bool bool
  1278. }
  1279. err = index.Index("true", &doc{Bool: true})
  1280. if err != nil {
  1281. t.Fatal(err)
  1282. }
  1283. err = index.Index("false", &doc{Bool: false})
  1284. if err != nil {
  1285. t.Fatal(err)
  1286. }
  1287. q := NewBoolFieldQuery(true)
  1288. q.SetField("Bool")
  1289. sreq := NewSearchRequest(q)
  1290. sres, err := index.Search(sreq)
  1291. if err != nil {
  1292. t.Fatal(err)
  1293. }
  1294. if sres.Total != 1 {
  1295. t.Errorf("expected 1 results, got %d", sres.Total)
  1296. }
  1297. q = NewBoolFieldQuery(false)
  1298. q.SetField("Bool")
  1299. sreq = NewSearchRequest(q)
  1300. sres, err = index.Search(sreq)
  1301. if err != nil {
  1302. t.Fatal(err)
  1303. }
  1304. if sres.Total != 1 {
  1305. t.Errorf("expected 1 results, got %d", sres.Total)
  1306. }
  1307. sreq = NewSearchRequest(NewBoolFieldQuery(true))
  1308. sres, err = index.Search(sreq)
  1309. if err != nil {
  1310. t.Fatal(err)
  1311. }
  1312. if sres.Total != 1 {
  1313. t.Errorf("expected 1 results, got %d", sres.Total)
  1314. }
  1315. err = index.Close()
  1316. if err != nil {
  1317. t.Fatal(err)
  1318. }
  1319. }
  1320. func TestSearchTimeout(t *testing.T) {
  1321. defer func() {
  1322. err := os.RemoveAll("testidx")
  1323. if err != nil {
  1324. t.Fatal(err)
  1325. }
  1326. }()
  1327. index, err := New("testidx", NewIndexMapping())
  1328. if err != nil {
  1329. t.Fatal(err)
  1330. }
  1331. defer func() {
  1332. err := index.Close()
  1333. if err != nil {
  1334. t.Fatal(err)
  1335. }
  1336. }()
  1337. // first run a search with an absurdly long timeout (should succeed)
  1338. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
  1339. defer cancel()
  1340. query := NewTermQuery("water")
  1341. req := NewSearchRequest(query)
  1342. _, err = index.SearchInContext(ctx, req)
  1343. if err != nil {
  1344. t.Fatal(err)
  1345. }
  1346. // now run a search again with an absurdly low timeout (should timeout)
  1347. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Microsecond)
  1348. defer cancel()
  1349. sq := &slowQuery{
  1350. actual: query,
  1351. delay: 50 * time.Millisecond, // on Windows timer resolution is 15ms
  1352. }
  1353. req.Query = sq
  1354. _, err = index.SearchInContext(ctx, req)
  1355. if err != context.DeadlineExceeded {
  1356. t.Fatalf("exected %v, got: %v", context.DeadlineExceeded, err)
  1357. }
  1358. // now run a search with a long timeout, but with a long query, and cancel it
  1359. ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
  1360. sq = &slowQuery{
  1361. actual: query,
  1362. delay: 100 * time.Millisecond, // on Windows timer resolution is 15ms
  1363. }
  1364. req = NewSearchRequest(sq)
  1365. cancel()
  1366. _, err = index.SearchInContext(ctx, req)
  1367. if err != context.Canceled {
  1368. t.Fatalf("exected %v, got: %v", context.Canceled, err)
  1369. }
  1370. }
  1371. // TestConfigCache exposes a concurrent map write with go 1.6
  1372. func TestConfigCache(t *testing.T) {
  1373. for i := 0; i < 100; i++ {
  1374. go func() {
  1375. _, err := Config.Cache.HighlighterNamed(Config.DefaultHighlighter)
  1376. if err != nil {
  1377. t.Error(err)
  1378. }
  1379. }()
  1380. }
  1381. }
  1382. func TestBatchRaceBug260(t *testing.T) {
  1383. defer func() {
  1384. err := os.RemoveAll("testidx")
  1385. if err != nil {
  1386. t.Fatal(err)
  1387. }
  1388. }()
  1389. i, err := New("testidx", NewIndexMapping())
  1390. if err != nil {
  1391. t.Fatal(err)
  1392. }
  1393. defer func() {
  1394. err := i.Close()
  1395. if err != nil {
  1396. t.Fatal(err)
  1397. }
  1398. }()
  1399. b := i.NewBatch()
  1400. err = b.Index("1", 1)
  1401. if err != nil {
  1402. t.Fatal(err)
  1403. }
  1404. err = i.Batch(b)
  1405. if err != nil {
  1406. t.Fatal(err)
  1407. }
  1408. b.Reset()
  1409. err = b.Index("2", 2)
  1410. if err != nil {
  1411. t.Fatal(err)
  1412. }
  1413. err = i.Batch(b)
  1414. if err != nil {
  1415. t.Fatal(err)
  1416. }
  1417. b.Reset()
  1418. }
  1419. func BenchmarkBatchOverhead(b *testing.B) {
  1420. defer func() {
  1421. err := os.RemoveAll("testidx")
  1422. if err != nil {
  1423. b.Fatal(err)
  1424. }
  1425. }()
  1426. m := NewIndexMapping()
  1427. i, err := NewUsing("testidx", m, Config.DefaultIndexType, null.Name, nil)
  1428. if err != nil {
  1429. b.Fatal(err)
  1430. }
  1431. for n := 0; n < b.N; n++ {
  1432. // put 1000 items in a batch
  1433. batch := i.NewBatch()
  1434. for i := 0; i < 1000; i++ {
  1435. err = batch.Index(fmt.Sprintf("%d", i), map[string]interface{}{"name": "bleve"})
  1436. if err != nil {
  1437. b.Fatal(err)
  1438. }
  1439. }
  1440. err = i.Batch(batch)
  1441. if err != nil {
  1442. b.Fatal(err)
  1443. }
  1444. batch.Reset()
  1445. }
  1446. }
  1447. func TestOpenReadonlyMultiple(t *testing.T) {
  1448. defer func() {
  1449. err := os.RemoveAll("testidx")
  1450. if err != nil {
  1451. t.Fatal(err)
  1452. }
  1453. }()
  1454. // build an index and close it
  1455. index, err := New("testidx", NewIndexMapping())
  1456. if err != nil {
  1457. t.Fatal(err)
  1458. }
  1459. doca := map[string]interface{}{
  1460. "name": "marty",
  1461. "desc": "gophercon india",
  1462. }
  1463. err = index.Index("a", doca)
  1464. if err != nil {
  1465. t.Fatal(err)
  1466. }
  1467. err = index.Close()
  1468. if err != nil {
  1469. t.Fatal(err)
  1470. }
  1471. // now open it read-only
  1472. index, err = OpenUsing("testidx", map[string]interface{}{
  1473. "read_only": true,
  1474. })
  1475. if err != nil {
  1476. t.Fatal(err)
  1477. }
  1478. // now open it again
  1479. index2, err := OpenUsing("testidx", map[string]interface{}{
  1480. "read_only": true,
  1481. })
  1482. if err != nil {
  1483. t.Fatal(err)
  1484. }
  1485. err = index.Close()
  1486. if err != nil {
  1487. t.Fatal(err)
  1488. }
  1489. err = index2.Close()
  1490. if err != nil {
  1491. t.Fatal(err)
  1492. }
  1493. }
  1494. // TestBug408 tests for VERY large values of size, even though actual result
  1495. // set may be reasonable size
  1496. func TestBug408(t *testing.T) {
  1497. type TestStruct struct {
  1498. ID string `json:"id"`
  1499. UserID *string `json:"user_id"`
  1500. }
  1501. docMapping := NewDocumentMapping()
  1502. docMapping.AddFieldMappingsAt("id", NewTextFieldMapping())
  1503. docMapping.AddFieldMappingsAt("user_id", NewTextFieldMapping())
  1504. indexMapping := NewIndexMapping()
  1505. indexMapping.DefaultMapping = docMapping
  1506. index, err := NewMemOnly(indexMapping)
  1507. if err != nil {
  1508. t.Fatal(err)
  1509. }
  1510. numToTest := 10
  1511. matchUserID := "match"
  1512. noMatchUserID := "no_match"
  1513. matchingDocIds := make(map[string]struct{})
  1514. for i := 0; i < numToTest; i++ {
  1515. ds := &TestStruct{"id_" + strconv.Itoa(i), nil}
  1516. if i%2 == 0 {
  1517. ds.UserID = &noMatchUserID
  1518. } else {
  1519. ds.UserID = &matchUserID
  1520. matchingDocIds[ds.ID] = struct{}{}
  1521. }
  1522. err = index.Index(ds.ID, ds)
  1523. if err != nil {
  1524. t.Fatal(err)
  1525. }
  1526. }
  1527. cnt, err := index.DocCount()
  1528. if err != nil {
  1529. t.Fatal(err)
  1530. }
  1531. if int(cnt) != numToTest {
  1532. t.Fatalf("expected %d documents in index, got %d", numToTest, cnt)
  1533. }
  1534. q := NewTermQuery(matchUserID)
  1535. q.SetField("user_id")
  1536. searchReq := NewSearchRequestOptions(q, math.MaxInt32, 0, false)
  1537. results, err := index.Search(searchReq)
  1538. if err != nil {
  1539. t.Fatal(err)
  1540. }
  1541. if int(results.Total) != numToTest/2 {
  1542. t.Fatalf("expected %d search hits, got %d", numToTest/2, results.Total)
  1543. }
  1544. for _, result := range results.Hits {
  1545. if _, found := matchingDocIds[result.ID]; !found {
  1546. t.Fatalf("document with ID %s not in results as expected", result.ID)
  1547. }
  1548. }
  1549. }
  1550. func TestIndexAdvancedCountMatchSearch(t *testing.T) {
  1551. defer func() {
  1552. err := os.RemoveAll("testidx")
  1553. if err != nil {
  1554. t.Fatal(err)
  1555. }
  1556. }()
  1557. index, err := New("testidx", NewIndexMapping())
  1558. if err != nil {
  1559. t.Fatal(err)
  1560. }
  1561. var wg sync.WaitGroup
  1562. for i := 0; i < 10; i++ {
  1563. wg.Add(1)
  1564. go func(i int) {
  1565. b := index.NewBatch()
  1566. for j := 0; j < 200; j++ {
  1567. id := fmt.Sprintf("%d", (i*200)+j)
  1568. doc := &document.Document{
  1569. ID: id,
  1570. Fields: []document.Field{
  1571. document.NewTextField("body", []uint64{}, []byte("match")),
  1572. },
  1573. CompositeFields: []*document.CompositeField{
  1574. document.NewCompositeField("_all", true, []string{}, []string{}),
  1575. },
  1576. }
  1577. err := b.IndexAdvanced(doc)
  1578. if err != nil {
  1579. t.Fatal(err)
  1580. }
  1581. }
  1582. err := index.Batch(b)
  1583. if err != nil {
  1584. t.Fatal(err)
  1585. }
  1586. wg.Done()
  1587. }(i)
  1588. }
  1589. wg.Wait()
  1590. // search for something that should match all documents
  1591. sr, err := index.Search(NewSearchRequest(NewMatchQuery("match")))
  1592. if err != nil {
  1593. t.Fatal(err)
  1594. }
  1595. // get the index document count
  1596. dc, err := index.DocCount()
  1597. if err != nil {
  1598. t.Fatal(err)
  1599. }
  1600. // make sure test is working correctly, doc count should 2000
  1601. if dc != 2000 {
  1602. t.Errorf("expected doc count 2000, got %d", dc)
  1603. }
  1604. // make sure our search found all the documents
  1605. if dc != sr.Total {
  1606. t.Errorf("expected search result total %d to match doc count %d", sr.Total, dc)
  1607. }
  1608. err = index.Close()
  1609. if err != nil {
  1610. t.Fatal(err)
  1611. }
  1612. }
  1613. func benchmarkSearchOverhead(indexType string, b *testing.B) {
  1614. defer func() {
  1615. err := os.RemoveAll("testidx")
  1616. if err != nil {
  1617. b.Fatal(err)
  1618. }
  1619. }()
  1620. index, err := NewUsing("testidx", NewIndexMapping(),
  1621. indexType, Config.DefaultKVStore, nil)
  1622. if err != nil {
  1623. b.Fatal(err)
  1624. }
  1625. defer func() {
  1626. err := index.Close()
  1627. if err != nil {
  1628. b.Fatal(err)
  1629. }
  1630. }()
  1631. elements := []string{"air", "water", "fire", "earth"}
  1632. for j := 0; j < 10000; j++ {
  1633. err = index.Index(fmt.Sprintf("%d", j),
  1634. map[string]interface{}{"name": elements[j%len(elements)]})
  1635. if err != nil {
  1636. b.Fatal(err)
  1637. }
  1638. }
  1639. query1 := NewTermQuery("water")
  1640. query2 := NewTermQuery("fire")
  1641. query := NewDisjunctionQuery(query1, query2)
  1642. req := NewSearchRequest(query)
  1643. b.ResetTimer()
  1644. for n := 0; n < b.N; n++ {
  1645. _, err = index.Search(req)
  1646. if err != nil {
  1647. b.Fatal(err)
  1648. }
  1649. }
  1650. }
  1651. func BenchmarkUpsidedownSearchOverhead(b *testing.B) {
  1652. benchmarkSearchOverhead(upsidedown.Name, b)
  1653. }
  1654. func BenchmarkScorchSearchOverhead(b *testing.B) {
  1655. benchmarkSearchOverhead(scorch.Name, b)
  1656. }
  1657. func TestSearchQueryCallback(t *testing.T) {
  1658. defer func() {
  1659. err := os.RemoveAll("testidx")
  1660. if err != nil {
  1661. t.Fatal(err)
  1662. }
  1663. }()
  1664. index, err := New("testidx", NewIndexMapping())
  1665. if err != nil {
  1666. t.Fatal(err)
  1667. }
  1668. defer func() {
  1669. err := index.Close()
  1670. if err != nil {
  1671. t.Fatal(err)
  1672. }
  1673. }()
  1674. elements := []string{"air", "water", "fire", "earth"}
  1675. for j := 0; j < 10000; j++ {
  1676. err = index.Index(fmt.Sprintf("%d", j),
  1677. map[string]interface{}{"name": elements[j%len(elements)]})
  1678. if err != nil {
  1679. t.Fatal(err)
  1680. }
  1681. }
  1682. query := NewTermQuery("water")
  1683. req := NewSearchRequest(query)
  1684. expErr := fmt.Errorf("MEM_LIMIT_EXCEEDED")
  1685. f := func(size uint64) error {
  1686. if size > 1000 {
  1687. return expErr
  1688. }
  1689. return nil
  1690. }
  1691. ctx := context.WithValue(context.Background(), SearchQueryStartCallbackKey,
  1692. SearchQueryStartCallbackFn(f))
  1693. _, err = index.SearchInContext(ctx, req)
  1694. if err != expErr {
  1695. t.Fatalf("Expected: %v, Got: %v", expErr, err)
  1696. }
  1697. }
  1698. func TestBatchMerge(t *testing.T) {
  1699. defer func() {
  1700. err := os.RemoveAll("testidx")
  1701. if err != nil {
  1702. t.Fatal(err)
  1703. }
  1704. }()
  1705. index, err := New("testidx", NewIndexMapping())
  1706. if err != nil {
  1707. t.Fatal(err)
  1708. }
  1709. doca := map[string]interface{}{
  1710. "name": "scorch",
  1711. "desc": "gophercon india",
  1712. "nation": "india",
  1713. }
  1714. batchA := index.NewBatch()
  1715. err = batchA.Index("a", doca)
  1716. if err != nil {
  1717. t.Error(err)
  1718. }
  1719. batchA.SetInternal([]byte("batchkA"), []byte("batchvA"))
  1720. docb := map[string]interface{}{
  1721. "name": "moss",
  1722. "desc": "gophercon MV",
  1723. }
  1724. batchB := index.NewBatch()
  1725. err = batchB.Index("b", docb)
  1726. if err != nil {
  1727. t.Error(err)
  1728. }
  1729. batchB.SetInternal([]byte("batchkB"), []byte("batchvB"))
  1730. docC := map[string]interface{}{
  1731. "name": "blahblah",
  1732. "desc": "inProgress",
  1733. "country": "usa",
  1734. }
  1735. batchC := index.NewBatch()
  1736. err = batchC.Index("c", docC)
  1737. if err != nil {
  1738. t.Error(err)
  1739. }
  1740. batchC.SetInternal([]byte("batchkC"), []byte("batchvC"))
  1741. batchC.SetInternal([]byte("batchkB"), []byte("batchvBNew"))
  1742. batchC.Delete("a")
  1743. batchC.DeleteInternal([]byte("batchkA"))
  1744. batchA.Merge(batchB)
  1745. if batchA.Size() != 4 {
  1746. t.Errorf("expected batch size 4, got %d", batchA.Size())
  1747. }
  1748. batchA.Merge(batchC)
  1749. if batchA.Size() != 6 {
  1750. t.Errorf("expected batch size 6, got %d", batchA.Size())
  1751. }
  1752. err = index.Batch(batchA)
  1753. if err != nil {
  1754. t.Fatal(err)
  1755. }
  1756. // close the index, open it again, and try some more things
  1757. err = index.Close()
  1758. if err != nil {
  1759. t.Fatal(err)
  1760. }
  1761. index, err = Open("testidx")
  1762. if err != nil {
  1763. t.Fatal(err)
  1764. }
  1765. defer func() {
  1766. err := index.Close()
  1767. if err != nil {
  1768. t.Fatal(err)
  1769. }
  1770. }()
  1771. count, err := index.DocCount()
  1772. if err != nil {
  1773. t.Fatal(err)
  1774. }
  1775. if count != 2 {
  1776. t.Errorf("expected doc count 2, got %d", count)
  1777. }
  1778. doc, err := index.Document("c")
  1779. if err != nil {
  1780. t.Fatal(err)
  1781. }
  1782. if doc == nil {
  1783. t.Errorf("expected doc not nil, got nil")
  1784. }
  1785. val, err := index.GetInternal([]byte("batchkB"))
  1786. if err != nil {
  1787. t.Fatal(err)
  1788. }
  1789. if val == nil || string(val) != "batchvBNew" {
  1790. t.Errorf("expected val: batchvBNew , got %s", val)
  1791. }
  1792. val, err = index.GetInternal([]byte("batchkA"))
  1793. if err != nil {
  1794. t.Fatal(err)
  1795. }
  1796. if val != nil {
  1797. t.Errorf("expected nil, got %s", val)
  1798. }
  1799. foundNameField := false
  1800. for _, field := range doc.Fields {
  1801. if field.Name() == "name" && string(field.Value()) == "blahblah" {
  1802. foundNameField = true
  1803. }
  1804. }
  1805. if !foundNameField {
  1806. t.Errorf("expected to find field named 'name' with value 'blahblah'")
  1807. }
  1808. fields, err := index.Fields()
  1809. if err != nil {
  1810. t.Fatal(err)
  1811. }
  1812. expectedFields := map[string]bool{
  1813. "_all": false,
  1814. "name": false,
  1815. "desc": false,
  1816. "country": false,
  1817. }
  1818. if len(fields) < len(expectedFields) {
  1819. t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields))
  1820. }
  1821. for _, f := range fields {
  1822. expectedFields[f] = true
  1823. }
  1824. for ef, efp := range expectedFields {
  1825. if !efp {
  1826. t.Errorf("field %s is missing", ef)
  1827. }
  1828. }
  1829. }
  1830. func TestBug1096(t *testing.T) {
  1831. defer func() {
  1832. err := os.RemoveAll("testidx")
  1833. if err != nil {
  1834. t.Fatal(err)
  1835. }
  1836. }()
  1837. // use default mapping
  1838. mapping := NewIndexMapping()
  1839. // create a scorch index with default SAFE batches
  1840. var idx Index
  1841. idx, err = NewUsing("testidx", mapping, "scorch", "scorch", nil)
  1842. if err != nil {
  1843. log.Fatal(err)
  1844. }
  1845. defer func() {
  1846. err := idx.Close()
  1847. if err != nil {
  1848. t.Fatal(err)
  1849. }
  1850. }()
  1851. // create a single batch instance that we will reuse
  1852. // this should be safe because we have single goroutine
  1853. // and we always wait for batch execution to finish
  1854. batch := idx.NewBatch()
  1855. // number of batches to execute
  1856. for i := 0; i < 10; i++ {
  1857. // number of documents to put into the batch
  1858. for j := 0; j < 91; j++ {
  1859. // create a doc id 0-90 (important so that we get id's 9 and 90)
  1860. // this could duplicate something already in the index
  1861. // this too should be OK and update the item in the index
  1862. id := fmt.Sprintf("%d", j)
  1863. err = batch.Index(id, map[string]interface{}{
  1864. "name": id,
  1865. "batch": fmt.Sprintf("%d", i),
  1866. })
  1867. if err != nil {
  1868. log.Fatal(err)
  1869. }
  1870. }
  1871. // execute the batch
  1872. err = idx.Batch(batch)
  1873. if err != nil {
  1874. log.Fatal(err)
  1875. }
  1876. // reset the batch before reusing it
  1877. batch.Reset()
  1878. }
  1879. // search for docs having name starting with the number 9
  1880. q := NewWildcardQuery("9*")
  1881. q.SetField("name")
  1882. req := NewSearchRequestOptions(q, 1000, 0, false)
  1883. req.Fields = []string{"*"}
  1884. var res *SearchResult
  1885. res, err = idx.Search(req)
  1886. if err != nil {
  1887. log.Fatal(err)
  1888. }
  1889. // we expect only 2 hits, for docs 9 and 90
  1890. if res.Total > 2 {
  1891. t.Fatalf("expected only 2 hits '9' and '90', got %v", res)
  1892. }
  1893. }
  1894. func TestDataRaceBug1092(t *testing.T) {
  1895. defer func() {
  1896. rerr := os.RemoveAll("testidx")
  1897. if rerr != nil {
  1898. t.Fatal(rerr)
  1899. }
  1900. }()
  1901. // use default mapping
  1902. mapping := NewIndexMapping()
  1903. var idx Index
  1904. idx, err = NewUsing("testidx", mapping, upsidedown.Name, boltdb.Name, nil)
  1905. if err != nil {
  1906. log.Fatal(err)
  1907. }
  1908. defer func() {
  1909. cerr := idx.Close()
  1910. if cerr != nil {
  1911. t.Fatal(cerr)
  1912. }
  1913. }()
  1914. batch := idx.NewBatch()
  1915. for i := 0; i < 10; i++ {
  1916. err = idx.Batch(batch)
  1917. if err != nil {
  1918. t.Error(err)
  1919. }
  1920. batch.Reset()
  1921. }
  1922. }
  1923. func TestBatchRaceBug1149(t *testing.T) {
  1924. defer func() {
  1925. err := os.RemoveAll("testidx")
  1926. if err != nil {
  1927. t.Fatal(err)
  1928. }
  1929. }()
  1930. i, err := New("testidx", NewIndexMapping())
  1931. //i, err := NewUsing("testidx", NewIndexMapping(), "scorch", "scorch", nil)
  1932. if err != nil {
  1933. t.Fatal(err)
  1934. }
  1935. defer func() {
  1936. err := i.Close()
  1937. if err != nil {
  1938. t.Fatal(err)
  1939. }
  1940. }()
  1941. b := i.NewBatch()
  1942. b.Delete("1")
  1943. err = i.Batch(b)
  1944. if err != nil {
  1945. t.Fatal(err)
  1946. }
  1947. b.Reset()
  1948. err = i.Batch(b)
  1949. if err != nil {
  1950. t.Fatal(err)
  1951. }
  1952. b.Reset()
  1953. }