search_test.go 26 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225
  1. // Copyright (c) 2014 Couchbase, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package bleve
  15. import (
  16. "encoding/json"
  17. "fmt"
  18. "os"
  19. "reflect"
  20. "strconv"
  21. "strings"
  22. "testing"
  23. "time"
  24. "github.com/blevesearch/bleve/analysis"
  25. "github.com/blevesearch/bleve/analysis/analyzer/custom"
  26. "github.com/blevesearch/bleve/analysis/analyzer/keyword"
  27. "github.com/blevesearch/bleve/analysis/analyzer/standard"
  28. "github.com/blevesearch/bleve/analysis/token/lowercase"
  29. "github.com/blevesearch/bleve/analysis/tokenizer/single"
  30. "github.com/blevesearch/bleve/analysis/tokenizer/whitespace"
  31. "github.com/blevesearch/bleve/document"
  32. "github.com/blevesearch/bleve/index/scorch"
  33. "github.com/blevesearch/bleve/index/upsidedown"
  34. "github.com/blevesearch/bleve/mapping"
  35. "github.com/blevesearch/bleve/search"
  36. "github.com/blevesearch/bleve/search/query"
  37. )
  38. func TestSearchResultString(t *testing.T) {
  39. tests := []struct {
  40. result *SearchResult
  41. str string
  42. }{
  43. {
  44. result: &SearchResult{
  45. Request: &SearchRequest{
  46. Size: 10,
  47. },
  48. Total: 5,
  49. Took: 1 * time.Second,
  50. Hits: search.DocumentMatchCollection{
  51. &search.DocumentMatch{},
  52. &search.DocumentMatch{},
  53. &search.DocumentMatch{},
  54. &search.DocumentMatch{},
  55. &search.DocumentMatch{},
  56. },
  57. },
  58. str: "5 matches, showing 1 through 5, took 1s",
  59. },
  60. {
  61. result: &SearchResult{
  62. Request: &SearchRequest{
  63. Size: 0,
  64. },
  65. Total: 5,
  66. Hits: search.DocumentMatchCollection{},
  67. },
  68. str: "5 matches",
  69. },
  70. {
  71. result: &SearchResult{
  72. Request: &SearchRequest{
  73. Size: 10,
  74. },
  75. Total: 0,
  76. Hits: search.DocumentMatchCollection{},
  77. },
  78. str: "No matches",
  79. },
  80. }
  81. for _, test := range tests {
  82. srstring := test.result.String()
  83. if !strings.HasPrefix(srstring, test.str) {
  84. t.Errorf("expected to start %s, got %s", test.str, srstring)
  85. }
  86. }
  87. }
  88. func TestSearchResultMerge(t *testing.T) {
  89. l := &SearchResult{
  90. Status: &SearchStatus{
  91. Total: 1,
  92. Successful: 1,
  93. Errors: make(map[string]error),
  94. },
  95. Total: 1,
  96. MaxScore: 1,
  97. Hits: search.DocumentMatchCollection{
  98. &search.DocumentMatch{
  99. ID: "a",
  100. Score: 1,
  101. },
  102. },
  103. }
  104. r := &SearchResult{
  105. Status: &SearchStatus{
  106. Total: 1,
  107. Successful: 1,
  108. Errors: make(map[string]error),
  109. },
  110. Total: 1,
  111. MaxScore: 2,
  112. Hits: search.DocumentMatchCollection{
  113. &search.DocumentMatch{
  114. ID: "b",
  115. Score: 2,
  116. },
  117. },
  118. }
  119. expected := &SearchResult{
  120. Status: &SearchStatus{
  121. Total: 2,
  122. Successful: 2,
  123. Errors: make(map[string]error),
  124. },
  125. Total: 2,
  126. MaxScore: 2,
  127. Hits: search.DocumentMatchCollection{
  128. &search.DocumentMatch{
  129. ID: "a",
  130. Score: 1,
  131. },
  132. &search.DocumentMatch{
  133. ID: "b",
  134. Score: 2,
  135. },
  136. },
  137. }
  138. l.Merge(r)
  139. if !reflect.DeepEqual(l, expected) {
  140. t.Errorf("expected %#v, got %#v", expected, l)
  141. }
  142. }
  143. func TestUnmarshalingSearchResult(t *testing.T) {
  144. searchResponse := []byte(`{
  145. "status":{
  146. "total":1,
  147. "failed":1,
  148. "successful":0,
  149. "errors":{
  150. "default_index_362ce020b3d62b13_348f5c3c":"context deadline exceeded"
  151. }
  152. },
  153. "request":{
  154. "query":{
  155. "match":"emp",
  156. "field":"type",
  157. "boost":1,
  158. "prefix_length":0,
  159. "fuzziness":0
  160. },
  161. "size":10000000,
  162. "from":0,
  163. "highlight":null,
  164. "fields":[],
  165. "facets":null,
  166. "explain":false
  167. },
  168. "hits":null,
  169. "total_hits":0,
  170. "max_score":0,
  171. "took":0,
  172. "facets":null
  173. }`)
  174. rv := &SearchResult{
  175. Status: &SearchStatus{
  176. Errors: make(map[string]error),
  177. },
  178. }
  179. err = json.Unmarshal(searchResponse, rv)
  180. if err != nil {
  181. t.Error(err)
  182. }
  183. if len(rv.Status.Errors) != 1 {
  184. t.Errorf("expected 1 error, got %d", len(rv.Status.Errors))
  185. }
  186. }
  187. func TestFacetNumericDateRangeRequests(t *testing.T) {
  188. var drMissingErr = fmt.Errorf("date range query must specify either start, end or both for range name 'testName'")
  189. var nrMissingErr = fmt.Errorf("numeric range query must specify either min, max or both for range name 'testName'")
  190. var drNrErr = fmt.Errorf("facet can only conain numeric ranges or date ranges, not both")
  191. var drNameDupErr = fmt.Errorf("date ranges contains duplicate name 'testName'")
  192. var nrNameDupErr = fmt.Errorf("numeric ranges contains duplicate name 'testName'")
  193. value := float64(5)
  194. tests := []struct {
  195. facet *FacetRequest
  196. result error
  197. }{
  198. {
  199. facet: &FacetRequest{
  200. Field: "Date_Range_Success_With_StartEnd",
  201. Size: 1,
  202. DateTimeRanges: []*dateTimeRange{
  203. &dateTimeRange{Name: "testName", Start: time.Unix(0, 0), End: time.Now()},
  204. },
  205. },
  206. result: nil,
  207. },
  208. {
  209. facet: &FacetRequest{
  210. Field: "Date_Range_Success_With_Start",
  211. Size: 1,
  212. DateTimeRanges: []*dateTimeRange{
  213. &dateTimeRange{Name: "testName", Start: time.Unix(0, 0)},
  214. },
  215. },
  216. result: nil,
  217. },
  218. {
  219. facet: &FacetRequest{
  220. Field: "Date_Range_Success_With_End",
  221. Size: 1,
  222. DateTimeRanges: []*dateTimeRange{
  223. &dateTimeRange{Name: "testName", End: time.Now()},
  224. },
  225. },
  226. result: nil,
  227. },
  228. {
  229. facet: &FacetRequest{
  230. Field: "Numeric_Range_Success_With_MinMax",
  231. Size: 1,
  232. NumericRanges: []*numericRange{
  233. &numericRange{Name: "testName", Min: &value, Max: &value},
  234. },
  235. },
  236. result: nil,
  237. },
  238. {
  239. facet: &FacetRequest{
  240. Field: "Numeric_Range_Success_With_Min",
  241. Size: 1,
  242. NumericRanges: []*numericRange{
  243. &numericRange{Name: "testName", Min: &value},
  244. },
  245. },
  246. result: nil,
  247. },
  248. {
  249. facet: &FacetRequest{
  250. Field: "Numeric_Range_Success_With_Max",
  251. Size: 1,
  252. NumericRanges: []*numericRange{
  253. &numericRange{Name: "testName", Max: &value},
  254. },
  255. },
  256. result: nil,
  257. },
  258. {
  259. facet: &FacetRequest{
  260. Field: "Date_Range_Missing_Failure",
  261. Size: 1,
  262. DateTimeRanges: []*dateTimeRange{
  263. &dateTimeRange{Name: "testName2", Start: time.Unix(0, 0)},
  264. &dateTimeRange{Name: "testName1", End: time.Now()},
  265. &dateTimeRange{Name: "testName"},
  266. },
  267. },
  268. result: drMissingErr,
  269. },
  270. {
  271. facet: &FacetRequest{
  272. Field: "Numeric_Range_Missing_Failure",
  273. Size: 1,
  274. NumericRanges: []*numericRange{
  275. &numericRange{Name: "testName2", Min: &value},
  276. &numericRange{Name: "testName1", Max: &value},
  277. &numericRange{Name: "testName"},
  278. },
  279. },
  280. result: nrMissingErr,
  281. },
  282. {
  283. facet: &FacetRequest{
  284. Field: "Numeric_And_DateRanges_Failure",
  285. Size: 1,
  286. NumericRanges: []*numericRange{
  287. &numericRange{Name: "testName", Max: &value},
  288. },
  289. DateTimeRanges: []*dateTimeRange{
  290. &dateTimeRange{Name: "testName", End: time.Now()},
  291. },
  292. },
  293. result: drNrErr,
  294. },
  295. {
  296. facet: &FacetRequest{
  297. Field: "Numeric_Range_Name_Repeat_Failure",
  298. Size: 1,
  299. NumericRanges: []*numericRange{
  300. &numericRange{Name: "testName", Min: &value},
  301. &numericRange{Name: "testName", Max: &value},
  302. },
  303. },
  304. result: nrNameDupErr,
  305. },
  306. {
  307. facet: &FacetRequest{
  308. Field: "Date_Range_Name_Repeat_Failure",
  309. Size: 1,
  310. DateTimeRanges: []*dateTimeRange{
  311. &dateTimeRange{Name: "testName", Start: time.Unix(0, 0)},
  312. &dateTimeRange{Name: "testName", End: time.Now()},
  313. },
  314. },
  315. result: drNameDupErr,
  316. },
  317. }
  318. for _, test := range tests {
  319. result := test.facet.Validate()
  320. if !reflect.DeepEqual(result, test.result) {
  321. t.Errorf("expected %#v, got %#v", test.result, result)
  322. }
  323. }
  324. }
  325. func TestSearchResultFacetsMerge(t *testing.T) {
  326. lowmed := "2010-01-01"
  327. medhi := "2011-01-01"
  328. hihigher := "2012-01-01"
  329. fr := &search.FacetResult{
  330. Field: "birthday",
  331. Total: 100,
  332. Missing: 25,
  333. Other: 25,
  334. DateRanges: []*search.DateRangeFacet{
  335. {
  336. Name: "low",
  337. End: &lowmed,
  338. Count: 25,
  339. },
  340. {
  341. Name: "med",
  342. Count: 24,
  343. Start: &lowmed,
  344. End: &medhi,
  345. },
  346. {
  347. Name: "hi",
  348. Count: 1,
  349. Start: &medhi,
  350. End: &hihigher,
  351. },
  352. },
  353. }
  354. frs := search.FacetResults{
  355. "birthdays": fr,
  356. }
  357. l := &SearchResult{
  358. Status: &SearchStatus{
  359. Total: 10,
  360. Successful: 1,
  361. Errors: make(map[string]error),
  362. },
  363. Total: 10,
  364. MaxScore: 1,
  365. }
  366. r := &SearchResult{
  367. Status: &SearchStatus{
  368. Total: 1,
  369. Successful: 1,
  370. Errors: make(map[string]error),
  371. },
  372. Total: 1,
  373. MaxScore: 2,
  374. Facets: frs,
  375. }
  376. expected := &SearchResult{
  377. Status: &SearchStatus{
  378. Total: 11,
  379. Successful: 2,
  380. Errors: make(map[string]error),
  381. },
  382. Total: 11,
  383. MaxScore: 2,
  384. Facets: frs,
  385. }
  386. l.Merge(r)
  387. if !reflect.DeepEqual(l, expected) {
  388. t.Errorf("expected %#v, got %#v", expected, l)
  389. }
  390. }
  391. func TestMemoryNeededForSearchResult(t *testing.T) {
  392. query := NewTermQuery("blah")
  393. req := NewSearchRequest(query)
  394. var sr SearchResult
  395. expect := sr.Size()
  396. var dm search.DocumentMatch
  397. expect += 10 * dm.Size()
  398. estimate := MemoryNeededForSearchResult(req)
  399. if estimate != uint64(expect) {
  400. t.Errorf("estimate not what is expected: %v != %v", estimate, expect)
  401. }
  402. }
  403. // https://github.com/blevesearch/bleve/issues/954
  404. func TestNestedBooleanSearchers(t *testing.T) {
  405. // create an index with a custom analyzer
  406. idxMapping := NewIndexMapping()
  407. if err := idxMapping.AddCustomAnalyzer("3xbla", map[string]interface{}{
  408. "type": custom.Name,
  409. "tokenizer": whitespace.Name,
  410. "token_filters": []interface{}{lowercase.Name, "stop_en"},
  411. }); err != nil {
  412. t.Fatal(err)
  413. }
  414. idxMapping.DefaultAnalyzer = "3xbla"
  415. idx, err := New("testidx", idxMapping)
  416. if err != nil {
  417. t.Fatal(err)
  418. }
  419. defer func() {
  420. err = idx.Close()
  421. if err != nil {
  422. t.Fatal(err)
  423. }
  424. err = os.RemoveAll("testidx")
  425. if err != nil {
  426. t.Fatal(err)
  427. }
  428. }()
  429. // create and insert documents as a batch
  430. batch := idx.NewBatch()
  431. matches := 0
  432. for i := 0; i < 100; i++ {
  433. hostname := fmt.Sprintf("planner_hostname_%d", i%5)
  434. metadata := map[string]string{"region": fmt.Sprintf("planner_us-east-%d", i%5)}
  435. // Expected matches
  436. if (hostname == "planner_hostname_1" || hostname == "planner_hostname_2") &&
  437. metadata["region"] == "planner_us-east-1" {
  438. matches++
  439. }
  440. doc := document.NewDocument(strconv.Itoa(i))
  441. doc.Fields = []document.Field{
  442. document.NewTextFieldCustom("hostname", []uint64{}, []byte(hostname),
  443. document.IndexField,
  444. &analysis.Analyzer{
  445. Tokenizer: single.NewSingleTokenTokenizer(),
  446. TokenFilters: []analysis.TokenFilter{
  447. lowercase.NewLowerCaseFilter(),
  448. },
  449. },
  450. ),
  451. }
  452. for k, v := range metadata {
  453. doc.AddField(document.NewTextFieldWithIndexingOptions(
  454. fmt.Sprintf("metadata.%s", k), []uint64{}, []byte(v), document.IndexField))
  455. }
  456. doc.CompositeFields = []*document.CompositeField{
  457. document.NewCompositeFieldWithIndexingOptions(
  458. "_all", true, []string{"text"}, []string{},
  459. document.IndexField|document.IncludeTermVectors),
  460. }
  461. if err = batch.IndexAdvanced(doc); err != nil {
  462. t.Fatal(err)
  463. }
  464. }
  465. if err = idx.Batch(batch); err != nil {
  466. t.Fatal(err)
  467. }
  468. que, err := query.ParseQuery([]byte(
  469. `{
  470. "conjuncts": [
  471. {
  472. "must": {
  473. "conjuncts": [
  474. {
  475. "disjuncts": [
  476. {
  477. "match": "planner_hostname_1",
  478. "field": "hostname"
  479. },
  480. {
  481. "match": "planner_hostname_2",
  482. "field": "hostname"
  483. }
  484. ]
  485. }
  486. ]
  487. }
  488. },
  489. {
  490. "must": {
  491. "conjuncts": [
  492. {
  493. "match": "planner_us-east-1",
  494. "field": "metadata.region"
  495. }
  496. ]
  497. }
  498. }
  499. ]
  500. }`,
  501. ))
  502. if err != nil {
  503. t.Fatal(err)
  504. }
  505. req := NewSearchRequest(que)
  506. req.Size = 100
  507. req.Fields = []string{"hostname", "metadata.region"}
  508. searchResults, err := idx.Search(req)
  509. if err != nil {
  510. t.Fatal(err)
  511. }
  512. if matches != len(searchResults.Hits) {
  513. t.Fatalf("Unexpected result set, %v != %v", matches, len(searchResults.Hits))
  514. }
  515. }
  516. func TestNestedBooleanMustNotSearcherUpsidedown(t *testing.T) {
  517. // create an index with default settings
  518. idxMapping := NewIndexMapping()
  519. idx, err := New("testidx", idxMapping)
  520. if err != nil {
  521. t.Fatal(err)
  522. }
  523. defer func() {
  524. err = idx.Close()
  525. if err != nil {
  526. t.Fatal(err)
  527. }
  528. err = os.RemoveAll("testidx")
  529. if err != nil {
  530. t.Fatal(err)
  531. }
  532. }()
  533. // create and insert documents as a batch
  534. batch := idx.NewBatch()
  535. docs := []struct {
  536. id string
  537. hasRole bool
  538. investigationId string
  539. }{
  540. {
  541. id: "1@1",
  542. hasRole: true,
  543. investigationId: "1",
  544. },
  545. {
  546. id: "1@2",
  547. hasRole: false,
  548. investigationId: "2",
  549. },
  550. {
  551. id: "2@1",
  552. hasRole: true,
  553. investigationId: "1",
  554. },
  555. {
  556. id: "2@2",
  557. hasRole: false,
  558. investigationId: "2",
  559. },
  560. {
  561. id: "3@1",
  562. hasRole: true,
  563. investigationId: "1",
  564. },
  565. {
  566. id: "3@2",
  567. hasRole: false,
  568. investigationId: "2",
  569. },
  570. {
  571. id: "4@1",
  572. hasRole: true,
  573. investigationId: "1",
  574. },
  575. {
  576. id: "5@1",
  577. hasRole: true,
  578. investigationId: "1",
  579. },
  580. {
  581. id: "6@1",
  582. hasRole: true,
  583. investigationId: "1",
  584. },
  585. {
  586. id: "7@1",
  587. hasRole: true,
  588. investigationId: "1",
  589. },
  590. }
  591. for i := 0; i < len(docs); i++ {
  592. doc := document.NewDocument(docs[i].id)
  593. doc.Fields = []document.Field{
  594. document.NewTextField("id", []uint64{}, []byte(docs[i].id)),
  595. document.NewBooleanField("hasRole", []uint64{}, docs[i].hasRole),
  596. document.NewTextField("investigationId", []uint64{}, []byte(docs[i].investigationId)),
  597. }
  598. doc.CompositeFields = []*document.CompositeField{
  599. document.NewCompositeFieldWithIndexingOptions(
  600. "_all", true, []string{"text"}, []string{},
  601. document.IndexField|document.IncludeTermVectors),
  602. }
  603. if err = batch.IndexAdvanced(doc); err != nil {
  604. t.Fatal(err)
  605. }
  606. }
  607. if err = idx.Batch(batch); err != nil {
  608. t.Fatal(err)
  609. }
  610. tq := NewTermQuery("1")
  611. tq.SetField("investigationId")
  612. // using must not, for cases that the field did not exists at all
  613. hasRole := NewBoolFieldQuery(true)
  614. hasRole.SetField("hasRole")
  615. noRole := NewBooleanQuery()
  616. noRole.AddMustNot(hasRole)
  617. oneRolesOrNoRoles := NewBooleanQuery()
  618. oneRolesOrNoRoles.AddShould(noRole)
  619. oneRolesOrNoRoles.SetMinShould(1)
  620. q := NewConjunctionQuery(tq, oneRolesOrNoRoles)
  621. sr := NewSearchRequestOptions(q, 100, 0, false)
  622. sr.Fields = []string{"hasRole"}
  623. sr.Highlight = NewHighlight()
  624. res, err := idx.Search(sr)
  625. if err != nil {
  626. t.Fatal(err)
  627. }
  628. if res.Total != 0 {
  629. t.Fatalf("Unexpected result, %v != 0", res.Total)
  630. }
  631. }
  632. func TestSearchScorchOverEmptyKeyword(t *testing.T) {
  633. defaultIndexType := Config.DefaultIndexType
  634. Config.DefaultIndexType = scorch.Name
  635. dmap := mapping.NewDocumentMapping()
  636. dmap.DefaultAnalyzer = standard.Name
  637. fm := mapping.NewTextFieldMapping()
  638. fm.Analyzer = keyword.Name
  639. fm1 := mapping.NewTextFieldMapping()
  640. fm1.Analyzer = standard.Name
  641. dmap.AddFieldMappingsAt("id", fm)
  642. dmap.AddFieldMappingsAt("name", fm1)
  643. imap := mapping.NewIndexMapping()
  644. imap.DefaultMapping = dmap
  645. imap.DefaultAnalyzer = standard.Name
  646. idx, err := New("testidx", imap)
  647. if err != nil {
  648. t.Fatal(err)
  649. }
  650. defer func() {
  651. err = idx.Close()
  652. if err != nil {
  653. t.Fatal(err)
  654. }
  655. err = os.RemoveAll("testidx")
  656. if err != nil {
  657. t.Fatal(err)
  658. }
  659. Config.DefaultIndexType = defaultIndexType
  660. }()
  661. for i := 0; i < 10; i++ {
  662. err = idx.Index(fmt.Sprint(i), map[string]string{"name": fmt.Sprintf("test%d", i), "id": ""})
  663. if err != nil {
  664. t.Fatal(err)
  665. }
  666. }
  667. count, err := idx.DocCount()
  668. if err != nil {
  669. t.Fatal(err)
  670. }
  671. if count != 10 {
  672. t.Fatalf("Unexpected doc count: %v, expected 10", count)
  673. }
  674. q := query.NewWildcardQuery("test*")
  675. sr := NewSearchRequestOptions(q, 40, 0, false)
  676. res, err := idx.Search(sr)
  677. if err != nil {
  678. t.Fatal(err)
  679. }
  680. if res.Total != 10 {
  681. t.Fatalf("Unexpected search hits: %v, expected 10", res.Total)
  682. }
  683. }
  684. func TestMultipleNestedBooleanMustNotSearchersOnScorch(t *testing.T) {
  685. defaultIndexType := Config.DefaultIndexType
  686. Config.DefaultIndexType = scorch.Name
  687. // create an index with default settings
  688. idxMapping := NewIndexMapping()
  689. idx, err := New("testidx", idxMapping)
  690. if err != nil {
  691. t.Fatal(err)
  692. }
  693. defer func() {
  694. err = idx.Close()
  695. if err != nil {
  696. t.Fatal(err)
  697. }
  698. err = os.RemoveAll("testidx")
  699. if err != nil {
  700. t.Fatal(err)
  701. }
  702. Config.DefaultIndexType = defaultIndexType
  703. }()
  704. // create and insert documents as a batch
  705. batch := idx.NewBatch()
  706. doc := document.NewDocument("1-child-0")
  707. doc.Fields = []document.Field{
  708. document.NewTextField("id", []uint64{}, []byte("1-child-0")),
  709. document.NewBooleanField("hasRole", []uint64{}, false),
  710. document.NewTextField("roles", []uint64{}, []byte("R1")),
  711. document.NewNumericField("type", []uint64{}, 0),
  712. }
  713. doc.CompositeFields = []*document.CompositeField{
  714. document.NewCompositeFieldWithIndexingOptions(
  715. "_all", true, []string{"text"}, []string{},
  716. document.IndexField|document.IncludeTermVectors),
  717. }
  718. if err = batch.IndexAdvanced(doc); err != nil {
  719. t.Fatal(err)
  720. }
  721. docs := []struct {
  722. id string
  723. hasRole bool
  724. typ int
  725. }{
  726. {
  727. id: "16d6fa37-48fd-4dea-8b3d-a52bddf73951",
  728. hasRole: false,
  729. typ: 9,
  730. },
  731. {
  732. id: "18fa9eb2-8b1f-46f0-8b56-b4c551213f78",
  733. hasRole: false,
  734. typ: 9,
  735. },
  736. {
  737. id: "3085855b-d74b-474a-86c3-9bf3e4504382",
  738. hasRole: false,
  739. typ: 9,
  740. },
  741. {
  742. id: "38ef5d28-0f85-4fb0-8a94-dd20751c3364",
  743. hasRole: false,
  744. typ: 9,
  745. },
  746. }
  747. for i := 0; i < len(docs); i++ {
  748. doc := document.NewDocument(docs[i].id)
  749. doc.Fields = []document.Field{
  750. document.NewTextField("id", []uint64{}, []byte(docs[i].id)),
  751. document.NewBooleanField("hasRole", []uint64{}, docs[i].hasRole),
  752. document.NewNumericField("type", []uint64{}, float64(docs[i].typ)),
  753. }
  754. doc.CompositeFields = []*document.CompositeField{
  755. document.NewCompositeFieldWithIndexingOptions(
  756. "_all", true, []string{"text"}, []string{},
  757. document.IndexField|document.IncludeTermVectors),
  758. }
  759. if err = batch.IndexAdvanced(doc); err != nil {
  760. t.Fatal(err)
  761. }
  762. }
  763. if err = idx.Batch(batch); err != nil {
  764. t.Fatal(err)
  765. }
  766. batch = idx.NewBatch()
  767. // Update 1st doc
  768. doc = document.NewDocument("1-child-0")
  769. doc.Fields = []document.Field{
  770. document.NewTextField("id", []uint64{}, []byte("1-child-0")),
  771. document.NewBooleanField("hasRole", []uint64{}, false),
  772. document.NewNumericField("type", []uint64{}, 0),
  773. }
  774. doc.CompositeFields = []*document.CompositeField{
  775. document.NewCompositeFieldWithIndexingOptions(
  776. "_all", true, []string{"text"}, []string{},
  777. document.IndexField|document.IncludeTermVectors),
  778. }
  779. if err = batch.IndexAdvanced(doc); err != nil {
  780. t.Fatal(err)
  781. }
  782. if err = idx.Batch(batch); err != nil {
  783. t.Fatal(err)
  784. }
  785. inclusive := true
  786. val := float64(9)
  787. q := query.NewNumericRangeInclusiveQuery(&val, &val, &inclusive, &inclusive)
  788. q.SetField("type")
  789. initialQuery := query.NewBooleanQuery(nil, nil, []query.Query{q})
  790. // using must not, for cases that the field did not exists at all
  791. hasRole := NewBoolFieldQuery(true)
  792. hasRole.SetField("hasRole")
  793. noRole := NewBooleanQuery()
  794. noRole.AddMustNot(hasRole)
  795. rq := query.NewBooleanQuery([]query.Query{initialQuery, noRole}, nil, nil)
  796. sr := NewSearchRequestOptions(rq, 100, 0, false)
  797. sr.Fields = []string{"id", "hasRole", "type"}
  798. sr.Highlight = NewHighlight()
  799. res, err := idx.Search(sr)
  800. if err != nil {
  801. t.Fatal(err)
  802. }
  803. if res.Total != 1 {
  804. t.Fatalf("Unexpected result, %v != 1", res.Total)
  805. }
  806. }
  807. func testBooleanMustNotSearcher(t *testing.T, indexName string) {
  808. im := NewIndexMapping()
  809. idx, err := NewUsing("testidx", im, indexName, Config.DefaultKVStore, nil)
  810. if err != nil {
  811. t.Fatal(err)
  812. }
  813. defer func() {
  814. err = idx.Close()
  815. if err != nil {
  816. t.Fatal(err)
  817. }
  818. err := os.RemoveAll("testidx")
  819. if err != nil {
  820. t.Fatal(err)
  821. }
  822. }()
  823. docs := []struct {
  824. Name string
  825. HasRole bool
  826. }{
  827. {
  828. Name: "13900",
  829. },
  830. {
  831. Name: "13901",
  832. },
  833. {
  834. Name: "13965",
  835. },
  836. {
  837. Name: "13966",
  838. HasRole: true,
  839. },
  840. {
  841. Name: "13967",
  842. HasRole: true,
  843. },
  844. }
  845. for _, doc := range docs {
  846. err := idx.Index(doc.Name, doc)
  847. if err != nil {
  848. t.Fatal(err)
  849. }
  850. }
  851. lhs := NewDocIDQuery([]string{"13965", "13966", "13967"})
  852. hasRole := NewBoolFieldQuery(true)
  853. hasRole.SetField("HasRole")
  854. rhs := NewBooleanQuery()
  855. rhs.AddMustNot(hasRole)
  856. var compareLeftRightAndConjunction = func(idx Index, left, right query.Query) error {
  857. // left
  858. lr := NewSearchRequestOptions(left, 100, 0, false)
  859. lres, err := idx.Search(lr)
  860. if err != nil {
  861. return fmt.Errorf("error left: %v", err)
  862. }
  863. lresIds := map[string]struct{}{}
  864. for i := range lres.Hits {
  865. lresIds[lres.Hits[i].ID] = struct{}{}
  866. }
  867. // right
  868. rr := NewSearchRequestOptions(right, 100, 0, false)
  869. rres, err := idx.Search(rr)
  870. if err != nil {
  871. return fmt.Errorf("error right: %v", err)
  872. }
  873. rresIds := map[string]struct{}{}
  874. for i := range rres.Hits {
  875. rresIds[rres.Hits[i].ID] = struct{}{}
  876. }
  877. // conjunction
  878. cr := NewSearchRequestOptions(NewConjunctionQuery(left, right), 100, 0, false)
  879. cres, err := idx.Search(cr)
  880. if err != nil {
  881. return fmt.Errorf("error conjunction: %v", err)
  882. }
  883. for i := range cres.Hits {
  884. if _, ok := lresIds[cres.Hits[i].ID]; ok {
  885. if _, ok := rresIds[cres.Hits[i].ID]; !ok {
  886. return fmt.Errorf("error id %s missing from right", cres.Hits[i].ID)
  887. }
  888. } else {
  889. return fmt.Errorf("error id %s missing from left", cres.Hits[i].ID)
  890. }
  891. }
  892. return nil
  893. }
  894. err = compareLeftRightAndConjunction(idx, lhs, rhs)
  895. if err != nil {
  896. t.Fatal(err)
  897. }
  898. }
  899. func TestBooleanMustNotSearcherUpsidedown(t *testing.T) {
  900. testBooleanMustNotSearcher(t, upsidedown.Name)
  901. }
  902. func TestBooleanMustNotSearcherScorch(t *testing.T) {
  903. testBooleanMustNotSearcher(t, scorch.Name)
  904. }
  905. func TestQueryStringEmptyConjunctionSearcher(t *testing.T) {
  906. mapping := NewIndexMapping()
  907. mapping.DefaultAnalyzer = keyword.Name
  908. index, err := NewMemOnly(mapping)
  909. if err != nil {
  910. t.Fatal(err)
  911. }
  912. defer func() {
  913. _ = index.Close()
  914. }()
  915. query := NewQueryStringQuery("foo:bar +baz:\"\"")
  916. searchReq := NewSearchRequest(query)
  917. _, _ = index.Search(searchReq)
  918. }
  919. func TestDisjunctionQueryIncorrectMin(t *testing.T) {
  920. // create an index with default settings
  921. idxMapping := NewIndexMapping()
  922. idx, err := New("testidx", idxMapping)
  923. if err != nil {
  924. t.Fatal(err)
  925. }
  926. defer func() {
  927. err = idx.Close()
  928. if err != nil {
  929. t.Fatal(err)
  930. }
  931. err = os.RemoveAll("testidx")
  932. if err != nil {
  933. t.Fatal(err)
  934. }
  935. }()
  936. // create and insert documents as a batch
  937. batch := idx.NewBatch()
  938. docs := []struct {
  939. field1 string
  940. field2 int
  941. }{
  942. {
  943. field1: "one",
  944. field2: 1,
  945. },
  946. {
  947. field1: "two",
  948. field2: 2,
  949. },
  950. }
  951. for i := 0; i < len(docs); i++ {
  952. doc := document.NewDocument(strconv.Itoa(docs[i].field2))
  953. doc.Fields = []document.Field{
  954. document.NewTextField("field1", []uint64{}, []byte(docs[i].field1)),
  955. document.NewNumericField("field2", []uint64{}, float64(docs[i].field2)),
  956. }
  957. doc.CompositeFields = []*document.CompositeField{
  958. document.NewCompositeFieldWithIndexingOptions(
  959. "_all", true, []string{"text"}, []string{},
  960. document.IndexField|document.IncludeTermVectors),
  961. }
  962. if err = batch.IndexAdvanced(doc); err != nil {
  963. t.Fatal(err)
  964. }
  965. }
  966. if err = idx.Batch(batch); err != nil {
  967. t.Fatal(err)
  968. }
  969. tq := NewTermQuery("one")
  970. dq := NewDisjunctionQuery(tq)
  971. dq.SetMin(2)
  972. sr := NewSearchRequestOptions(dq, 1, 0, false)
  973. res, err := idx.Search(sr)
  974. if err != nil {
  975. t.Fatal(err)
  976. }
  977. if res.Total > 0 {
  978. t.Fatalf("Expected 0 matches as disjunction query contains a single clause"+
  979. " but got: %v", res.Total)
  980. }
  981. }
  982. func TestBooleanShouldMinPropagation(t *testing.T) {
  983. idx, err := New("testidx", NewIndexMapping())
  984. if err != nil {
  985. t.Fatal(err)
  986. }
  987. defer func() {
  988. err = idx.Close()
  989. if err != nil {
  990. t.Fatal(err)
  991. }
  992. err := os.RemoveAll("testidx")
  993. if err != nil {
  994. t.Fatal(err)
  995. }
  996. }()
  997. doc1 := map[string]interface{}{
  998. "dept": "queen",
  999. "name": "cersei lannister",
  1000. }
  1001. doc2 := map[string]interface{}{
  1002. "dept": "kings guard",
  1003. "name": "jaime lannister",
  1004. }
  1005. batch := idx.NewBatch()
  1006. if err = batch.Index("doc1", doc1); err != nil {
  1007. t.Fatal(err)
  1008. }
  1009. if err = batch.Index("doc2", doc2); err != nil {
  1010. t.Fatal(err)
  1011. }
  1012. if err = idx.Batch(batch); err != nil {
  1013. t.Fatal(err)
  1014. }
  1015. // term dictionaries in the index for field..
  1016. // dept: queen kings guard
  1017. // name: cersei jaime lannister
  1018. // the following match query would match doc2
  1019. mq1 := NewMatchQuery("kings guard")
  1020. mq1.SetField("dept")
  1021. // the following match query would match both doc1 and doc2,
  1022. // as both docs share common lastname
  1023. mq2 := NewMatchQuery("jaime lannister")
  1024. mq2.SetField("name")
  1025. bq := NewBooleanQuery()
  1026. bq.AddShould(mq1)
  1027. bq.AddMust(mq2)
  1028. sr := NewSearchRequest(bq)
  1029. res, err := idx.Search(sr)
  1030. if err != nil {
  1031. t.Fatal(err)
  1032. }
  1033. if res.Total != 2 {
  1034. t.Errorf("Expected 2 results, but got: %v", res.Total)
  1035. }
  1036. }
  1037. func TestDisjunctionMinPropagation(t *testing.T) {
  1038. idx, err := New("testidx", NewIndexMapping())
  1039. if err != nil {
  1040. t.Fatal(err)
  1041. }
  1042. defer func() {
  1043. err = idx.Close()
  1044. if err != nil {
  1045. t.Fatal(err)
  1046. }
  1047. err := os.RemoveAll("testidx")
  1048. if err != nil {
  1049. t.Fatal(err)
  1050. }
  1051. }()
  1052. doc1 := map[string]interface{}{
  1053. "dept": "finance",
  1054. "name": "xyz",
  1055. }
  1056. doc2 := map[string]interface{}{
  1057. "dept": "marketing",
  1058. "name": "xyz",
  1059. }
  1060. doc3 := map[string]interface{}{
  1061. "dept": "engineering",
  1062. "name": "abc",
  1063. }
  1064. batch := idx.NewBatch()
  1065. if err = batch.Index("doc1", doc1); err != nil {
  1066. t.Fatal(err)
  1067. }
  1068. if err = batch.Index("doc2", doc2); err != nil {
  1069. t.Fatal(err)
  1070. }
  1071. if err = batch.Index("doc3", doc3); err != nil {
  1072. t.Fatal(err)
  1073. }
  1074. if err = idx.Batch(batch); err != nil {
  1075. t.Fatal(err)
  1076. }
  1077. mq1 := NewMatchQuery("finance")
  1078. mq2 := NewMatchQuery("marketing")
  1079. dq := NewDisjunctionQuery(mq1, mq2)
  1080. dq.SetMin(3)
  1081. dq2 := NewDisjunctionQuery(dq)
  1082. dq2.SetMin(1)
  1083. sr := NewSearchRequest(dq2)
  1084. res, err := idx.Search(sr)
  1085. if err != nil {
  1086. t.Fatal(err)
  1087. }
  1088. if res.Total != 0 {
  1089. t.Fatalf("Expect 0 results, but got: %v", res.Total)
  1090. }
  1091. }