freq_test.go 3.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173
  1. // Copyright (c) 2014 Couchbase, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package analysis
  15. import (
  16. "reflect"
  17. "testing"
  18. )
  19. func TestTokenFrequency(t *testing.T) {
  20. tokens := TokenStream{
  21. &Token{
  22. Term: []byte("water"),
  23. Position: 1,
  24. Start: 0,
  25. End: 5,
  26. },
  27. &Token{
  28. Term: []byte("water"),
  29. Position: 2,
  30. Start: 6,
  31. End: 11,
  32. },
  33. }
  34. expectedResult := TokenFrequencies{
  35. "water": &TokenFreq{
  36. Term: []byte("water"),
  37. Locations: []*TokenLocation{
  38. {
  39. Position: 1,
  40. Start: 0,
  41. End: 5,
  42. },
  43. {
  44. Position: 2,
  45. Start: 6,
  46. End: 11,
  47. },
  48. },
  49. frequency: 2,
  50. },
  51. }
  52. result := TokenFrequency(tokens, nil, true)
  53. if !reflect.DeepEqual(result, expectedResult) {
  54. t.Errorf("expected %#v, got %#v", expectedResult, result)
  55. }
  56. }
  57. func TestTokenFrequenciesMergeAll(t *testing.T) {
  58. tf1 := TokenFrequencies{
  59. "water": &TokenFreq{
  60. Term: []byte("water"),
  61. Locations: []*TokenLocation{
  62. {
  63. Position: 1,
  64. Start: 0,
  65. End: 5,
  66. },
  67. {
  68. Position: 2,
  69. Start: 6,
  70. End: 11,
  71. },
  72. },
  73. },
  74. }
  75. tf2 := TokenFrequencies{
  76. "water": &TokenFreq{
  77. Term: []byte("water"),
  78. Locations: []*TokenLocation{
  79. {
  80. Position: 1,
  81. Start: 0,
  82. End: 5,
  83. },
  84. {
  85. Position: 2,
  86. Start: 6,
  87. End: 11,
  88. },
  89. },
  90. },
  91. }
  92. expectedResult := TokenFrequencies{
  93. "water": &TokenFreq{
  94. Term: []byte("water"),
  95. Locations: []*TokenLocation{
  96. {
  97. Position: 1,
  98. Start: 0,
  99. End: 5,
  100. },
  101. {
  102. Position: 2,
  103. Start: 6,
  104. End: 11,
  105. },
  106. {
  107. Field: "tf2",
  108. Position: 1,
  109. Start: 0,
  110. End: 5,
  111. },
  112. {
  113. Field: "tf2",
  114. Position: 2,
  115. Start: 6,
  116. End: 11,
  117. },
  118. },
  119. },
  120. }
  121. tf1.MergeAll("tf2", tf2)
  122. if !reflect.DeepEqual(tf1, expectedResult) {
  123. t.Errorf("expected %#v, got %#v", expectedResult, tf1)
  124. }
  125. }
  126. func TestTokenFrequenciesMergeAllLeftEmpty(t *testing.T) {
  127. tf1 := TokenFrequencies{}
  128. tf2 := TokenFrequencies{
  129. "water": &TokenFreq{
  130. Term: []byte("water"),
  131. Locations: []*TokenLocation{
  132. {
  133. Position: 1,
  134. Start: 0,
  135. End: 5,
  136. },
  137. {
  138. Position: 2,
  139. Start: 6,
  140. End: 11,
  141. },
  142. },
  143. },
  144. }
  145. expectedResult := TokenFrequencies{
  146. "water": &TokenFreq{
  147. Term: []byte("water"),
  148. Locations: []*TokenLocation{
  149. {
  150. Field: "tf2",
  151. Position: 1,
  152. Start: 0,
  153. End: 5,
  154. },
  155. {
  156. Field: "tf2",
  157. Position: 2,
  158. Start: 6,
  159. End: 11,
  160. },
  161. },
  162. },
  163. }
  164. tf1.MergeAll("tf2", tf2)
  165. if !reflect.DeepEqual(tf1, expectedResult) {
  166. t.Errorf("expected %#v, got %#v", expectedResult, tf1)
  167. }
  168. }