api0_test.go 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220
  1. //
  2. // Copyright (C) 2017-2018 Marcus Rohrmoser, http://purl.mro.name/ShaarliGo
  3. //
  4. // This program is free software: you can redistribute it and/or modify
  5. // it under the terms of the GNU General Public License as published by
  6. // the Free Software Foundation, either version 3 of the License, or
  7. // (at your option) any later version.
  8. //
  9. // This program is distributed in the hope that it will be useful,
  10. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12. // GNU General Public License for more details.
  13. //
  14. // You should have received a copy of the GNU General Public License
  15. // along with this program. If not, see <http://www.gnu.org/licenses/>.
  16. //
  17. package main
  18. import (
  19. "crypto/rand"
  20. "encoding/base64"
  21. "encoding/binary"
  22. "encoding/hex"
  23. "fmt"
  24. "hash/crc32"
  25. "io"
  26. "net/url"
  27. "strings"
  28. "time"
  29. "github.com/stretchr/testify/assert"
  30. "testing"
  31. )
  32. func TestURLQuery(t *testing.T) {
  33. t.Parallel()
  34. par := mustParseURL("a/b/c?post=foo").Query()
  35. assert.Equal(t, 1, len(par["post"]), "Na klar")
  36. assert.Equal(t, "foo", par["post"][0], "Na klar")
  37. nul := mustParseURL("")
  38. assert.False(t, nul.IsAbs(), "Na klar")
  39. http := mustParseURL("http://")
  40. assert.True(t, http.IsAbs(), "Na klar")
  41. assert.Equal(t, "", http.Hostname(), "Na klar")
  42. purl := fmt.Sprintf("?post=%s&title=%s&source=%s", url.QueryEscape("http://example.com/foo?bar=baz#grr"), url.QueryEscape("A first post"), url.QueryEscape("me"))
  43. par = mustParseURL(purl).Query()
  44. assert.Equal(t, 1, len(par["post"]), "Na klar")
  45. assert.Equal(t, "http://example.com/foo?bar=baz#grr", par["post"][0], "Na klar")
  46. assert.Equal(t, "A first post", par["title"][0], "Na klar")
  47. assert.Equal(t, "me", par["source"][0], "Na klar")
  48. assert.Equal(t, "foo", strings.SplitN("pub/posts/foo/", "/", 4)[2], "Na klar")
  49. }
  50. func TestLfTimeFmt(t *testing.T) {
  51. t.Parallel()
  52. loc, err := time.LoadLocation("Europe/Berlin")
  53. assert.Nil(t, err, "aua")
  54. // loc = time.Local
  55. t0, err := time.ParseInLocation(fmtTimeLfTime, "20171106_223225", loc)
  56. assert.Nil(t, err, "aua")
  57. assert.Equal(t, "2017-11-06T22:32:25+01:00", t0.Format(time.RFC3339), "Na klar")
  58. assert.Equal(t, "20171106_223225", t0.Format(fmtTimeLfTime), "Na klar")
  59. }
  60. func TestParseLinkUrl(t *testing.T) {
  61. t.Parallel()
  62. // vorher schon geklärt:
  63. // - ist's die rechte Seite von :// eines bekannten Links
  64. // - ist's eine Id eines bekannten Links
  65. u, e := url.Parse("www.heise.de")
  66. assert.Nil(t, e, "aua")
  67. assert.Equal(t, "www.heise.de", u.String(), "aua")
  68. assert.False(t, u.IsAbs(), "aua")
  69. u, e = url.Parse("http://www.heise.de")
  70. assert.Equal(t, "http://www.heise.de", u.String(), "aua")
  71. assert.True(t, u.IsAbs(), "aua")
  72. u, e = url.Parse("https://www.heise.de")
  73. assert.Equal(t, "https://www.heise.de", u.String(), "aua")
  74. assert.True(t, u.IsAbs(), "aua")
  75. u, e = url.Parse("voo8Uo")
  76. assert.Equal(t, "voo8Uo", u.String(), "aua")
  77. assert.False(t, u.IsAbs(), "aua")
  78. assert.Nil(t, e, "aua")
  79. assert.NotNil(t, u, "aua")
  80. assert.Equal(t, "http://heise.de", parseLinkUrl("heise.de").String(), "aua")
  81. assert.Equal(t, "http://heise.de", parseLinkUrl("http://heise.de").String(), "aua")
  82. assert.Equal(t, "https://heise.de", parseLinkUrl("https://heise.de").String(), "aua")
  83. assert.Nil(t, parseLinkUrl("Eine Notiz"), "aua")
  84. assert.Nil(t, parseLinkUrl("genau www.heise.de"), "aua")
  85. }
  86. func TestToken(t *testing.T) {
  87. t.Parallel()
  88. src := []byte("foo\x00bar8901234567890")
  89. assert.Equal(t, 20, len(src), "sicher")
  90. hx := hex.EncodeToString(src)
  91. assert.Equal(t, 40, len(hx), "sicher")
  92. assert.Equal(t, "666f6f0062617238393031323334353637383930", hx, "Na klar")
  93. src = make([]byte, 20)
  94. _, err := io.ReadFull(rand.Reader, src)
  95. assert.Nil(t, err, "aua")
  96. assert.NotNil(t, hex.EncodeToString(src), "aua")
  97. }
  98. /* Returns the small hash of a string, using RFC 4648 base64url format
  99. eg. smallHash('20111006_131924') --> yZH23w
  100. Small hashes:
  101. - are unique (well, as unique as crc32, at last)
  102. - are always 6 characters long.
  103. - only use the following characters: a-z A-Z 0-9 - _ @
  104. - are NOT cryptographically secure (they CAN be forged)
  105. In Shaarli, they are used as a tinyurl-like link to individual entries.
  106. https://github.com/sebsauvage/Shaarli/blob/master/index.php#L228
  107. */
  108. func smallHash(text string) string {
  109. // ret:= rtrim(base64_encode(hash('crc32',$text,true)),'=');
  110. crc := crc32.ChecksumIEEE([]byte(text))
  111. bs := make([]byte, 4) // https://stackoverflow.com/a/16889357
  112. binary.LittleEndian.PutUint32(bs, crc)
  113. return base64.RawURLEncoding.EncodeToString(bs)
  114. }
  115. func smallDateHash(tt time.Time) string {
  116. bs := make([]byte, 4) // https://stackoverflow.com/a/16889357
  117. // unix time in seconds as uint32
  118. binary.LittleEndian.PutUint32(bs, uint32(tt.Unix()&0xFFFFFFFF))
  119. return base64.RawURLEncoding.EncodeToString(bs)
  120. }
  121. /*
  122. func smallHashRandom() string {
  123. bs := make([]byte, 4)
  124. io.ReadFull(rand.Reader, bs)
  125. return base64.RawURLEncoding.EncodeToString(bs)
  126. }
  127. */
  128. func TestSmallHash(t *testing.T) {
  129. t.Parallel()
  130. // assert.Equal(t, uint32(0xc2d07353), crc32.Checksum([]byte("20171108_231054"), crc32.MakeTable((0xC96C5795<<32)|0xD7870F42)), "hm")
  131. // assert.Equal(t, "wtBzUw", smallHash("20171108_231054"), "aha")
  132. // assert.Equal(t, "yZH23w", smallHash("20111006_131924"), "the original from https://github.com/sebsauvage/Shaarli/blob/master/index.php#L228")
  133. assert.Equal(t, "AfsQ8g", smallHash("20111006_131924"), "strange - that's what GO produces.")
  134. tt, _ := time.Parse(fmtTimeLfTime, "20171108_231054")
  135. assert.Equal(t, "_o4DWg", smallDateHash(tt), "aha")
  136. }
  137. func TestApi0LinkFormMap(t *testing.T) {
  138. t.Parallel()
  139. e := Entry{}
  140. assert.Equal(t, map[string]interface{}{"lf_tags": "", "lf_linkdate": "00010101_000000", "lf_title": ""}, e.api0LinkFormMap(), "oha")
  141. e = Entry{
  142. Title: HumanText{Body: "My #Post"},
  143. }
  144. assert.Equal(t, map[string]interface{}{"lf_linkdate": "00010101_000000", "lf_title": "My #Post", "lf_tags": ""}, e.api0LinkFormMap(), "oha")
  145. e = Entry{
  146. Title: HumanText{Body: "My #Post"},
  147. Categories: []Category{Category{Term: "Post"}, Category{Term: "tag1"}},
  148. }
  149. assert.Equal(t, map[string]interface{}{"lf_tags": "tag1", "lf_linkdate": "00010101_000000", "lf_title": "My #Post"}, e.api0LinkFormMap(), "oha")
  150. // assert.Equal(t, map[string]string{"lf_linkdate": "00010101_000000", "lf_title": "My #Post", "lf_tags": "tag1"}, e.api0LinkFormMap(), "oha")
  151. }
  152. func TestUrlFromPostParam(t *testing.T) {
  153. t.Parallel()
  154. assert.Equal(t, "http://foo", urlFromPostParam("http://foo").String(), "oha")
  155. assert.Equal(t, "http://foo.de", urlFromPostParam("foo.de").String(), "oha")
  156. assert.Equal(t, "http://example.com/und?noch#was", urlFromPostParam("example.com/und?noch#was").String(), "oha")
  157. assert.Nil(t, urlFromPostParam("/sub/dir#frag"), "oha")
  158. assert.Nil(t, urlFromPostParam("two parts"), "oha")
  159. assert.Nil(t, urlFromPostParam("#two"), "oha")
  160. assert.Nil(t, urlFromPostParam("example.com und noch was"), "oha")
  161. assert.Nil(t, urlFromPostParam("http://example.com und noch was"), "oha")
  162. assert.Nil(t, urlFromPostParam("foohoo"), "oha")
  163. }
  164. func TestSanitiseURLString(t *testing.T) {
  165. // sanitizeUrl
  166. t.Parallel()
  167. sanitizers := []RegexpReplaceAllString{
  168. {Regexp: "[\\?&]utm_source=.*$", ReplaceAllString: ""}, // We remove the annoying parameters added by FeedBurner and GoogleFeedProxy (?utm_source=...)
  169. {Regexp: "#xtor=RSS-.*$", ReplaceAllString: ""},
  170. {Regexp: "^(?i)(?:https?://)?(?:(?:www|m)\\.)?heise\\.de/.*?(-\\d+)(?:\\.html)?(?:[\\?#].*)?$", ReplaceAllString: "https://heise.de/${1}"},
  171. {Regexp: "^(?i)(?:https?://)?(?:(?:www|m)\\.)?spiegel\\.de/.*?-(\\d+)(?:\\.html.*)?", ReplaceAllString: "https://spiegel.de/article.do?id=${1}"},
  172. {Regexp: "^(?i)(?:https?://)?(?:(?:www|m)\\.)?sueddeutsche\\.de/.*?-(\\d+\\.\\d+)(?:\\.html.*)?$", ReplaceAllString: "https://sz.de/${1}"},
  173. {Regexp: "^(?i)(?:https?://)?(?:(?:www|m)\\.)?youtube.com/watch\\?v=([^&]+)(?:.*&(t=[^&]+))?(?:.*)$", ReplaceAllString: "https://youtu.be/${1}?${2}"},
  174. }
  175. assert.Equal(t, "https://heise.de/-3905244", sanitiseURLString("www.heise.de/newsticker/meldung/Finanzaufsicht-warnt-vor-Bitcoin-Investment-Kurs-sackt-wieder-unter-10-000-US-Dollar-3905244.html", sanitizers), "oha")
  176. assert.Equal(t, "https://spiegel.de/article.do?id=1181168", sanitiseURLString("https://www.spiegel.de/politik/ausland/glyphosat-barbara-hendricks-und-christian-schmidt-sprechen-sich-aus-a-1181168.html", sanitizers), "oha")
  177. assert.Equal(t, "https://sz.de/1.3772485", sanitiseURLString("http://www.sueddeutsche.de/politik/exklusiv-spd-ministerpraesident-weil-warnt-vor-instabilitaet-in-deutschland-1.3772485", sanitizers), "oha")
  178. assert.Equal(t, "https://youtu.be/hzf3hTUKk8U?", sanitiseURLString("https://www.youtube.com/watch?v=hzf3hTUKk8U&feature=youtu.be", sanitizers), "oha")
  179. assert.Equal(t, "https://youtu.be/hzf3hTUKk8U?t=14m4s", sanitiseURLString("youtube.com/watch?v=hzf3hTUKk8U&feature=youtu.be&t=14m4s", sanitizers), "oha")
  180. assert.Equal(t, "https://youtu.be/e-5obm1G_FY?t=14m4s", sanitiseURLString("https://youtu.be/e-5obm1G_FY?t=14m4s", sanitizers), "oha")
  181. }