From 9382717f7dbd00aa4e3cd01f53ba6f9a282ebe67 Mon Sep 17 00:00:00 2001 From: Luigi Date: Sun, 8 Oct 2023 21:02:46 +0200 Subject: [PATCH] chore: use search service --- api/api.gen.go | 40 ++++-- cmd/serve.go | 2 - go.mod | 3 +- go.sum | 6 + internal/app/feed.go | 11 -- .../wiki => client/wikipedia}/cache.go | 2 +- .../wiki => client/wikipedia}/client.go | 93 +++++++++++- .../wikipedia/client_test.go} | 20 +-- .../wiki => client/wikipedia}/page.go | 20 +-- .../wiki => client/wikipedia}/request.go | 2 +- internal/container/container.go | 15 +- internal/informer/wiki.go | 91 ------------ internal/repository/bleve/bleve_test.go | 134 ------------------ internal/search/search.go | 18 +++ internal/webserver/server.go | 23 +-- 15 files changed, 188 insertions(+), 292 deletions(-) rename internal/{informer/wiki => client/wikipedia}/cache.go (99%) rename internal/{informer/wiki => client/wikipedia}/client.go (50%) rename internal/{informer/wiki_test.go => client/wikipedia/client_test.go} (71%) rename internal/{informer/wiki => client/wikipedia}/page.go (97%) rename internal/{informer/wiki => client/wikipedia}/request.go (99%) delete mode 100644 internal/informer/wiki.go delete mode 100644 internal/repository/bleve/bleve_test.go create mode 100644 internal/search/search.go diff --git a/api/api.gen.go b/api/api.gen.go index 28bec07..3080aab 100644 --- a/api/api.gen.go +++ b/api/api.gen.go @@ -60,6 +60,15 @@ type SourceReleaseVersion struct { Version string `json:"version"` } +// WikiResult defines model for WikiResult. +type WikiResult struct { + Language string `json:"language"` + Link string `json:"link"` + Summary string `json:"summary"` + Thumbnail string `json:"thumbnail"` + Title string `json:"title"` +} + // GetV1VersionJSONBody defines parameters for GetV1Version. type GetV1VersionJSONBody map[string]interface{} @@ -162,21 +171,22 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/6RWXW/iOBT9K+juPqYkoVBo3mak2RHah/2Y2ZFWoz6Y5EIMjm3smwyhyn9fOUlJWkJL", - "d56K4ut7js85vu4jxCrTSqIkC9Ej2DjFjNU/PxmjjPuhjdJoiGP9OVYJur9UaoQIuCTcoIHKgwytZZv+", - "oiXD5QaqygOD+5wbTCD6fir0mmYP3lO9Wm0xJvDgcGNJacE3Kbl2PIEINsl2MTV4OGSHNdU9f0NMllkL", - "+ZwlcRJDTDzIjXiboSvy2ibX0pvpeRoLridlsuvoEWbn7BJGNbm1MhkjiOoPN8Qzp8kZY/50xF8NriGC", - "X/zOM781zO+0qDwQTG7yYSs8EFzuBhesyk08vMfmWcZMObh2SeoXkjZlXauWyAm2x9prBLpSeBOm+XEz", - "E/eq3Oka9guaAs3HnIvkGxrLlRyKcZZxOqc9jLFbxfvJ4jZf5EEa1BhF1/iaBpaz253e7tkhDH9A9VKc", - "lk3X9uGk69BprhNmbmYrKtj0MAvL254wFzVZOYy3gjZA51l2Xt1bV/2NApnF0+4XWpzy0NA5U+J9Ihzv", - "7KIM87u9KII2HUMkzrQQzNI/2uUw+XB1TA7biRRyWoRzfUz+T0xwtVciZbPj+nAvz2NSnI7+nF5fo6HT", - "XSfV/XZj9SqmnSikceBu9si16s1TWFolHOjoa8rlbnQzavBGH/5c9tIbQTgOXCyURsk0hwhux8F4Ah5o", - "RmktsF+EvkVm4tRfIyb+4z5HU1ZuaYM1KWcHI67k0rH7jPQt/FJvcLPuL1dd9zMsQ0JjIfr+CNyBOwzw", - "QLLMUd63lZ2QZHL02qduaHA9uGKrlbRNFiZB0IwMSShrakxrweOanL+1jb9dP06Y2asGtnsdqpM7zBhW", - "Nq4naGPDNTVq/vG7q5q9k8Zr6M3jPgC1lIRGMjFqLtvoqXA4MprF2fHeFHMW7sKmnfO1l/rLXnbhdM6g", - "pY8qKd91wOehrtrr8hPGvT33uqE1aFL/qYR/VW5Gnz99HaFMtOLy0r1bC7OapvhjcVeEtj2GrdGaTNf/", - "sUBKpG3k+814HPP2IpK7h2NCS34RQuX1yyPfFypmIlWWokWwCHrr4WQ+DsbBOGwXHi6MTxXOpuEqvpuq", - "g4Kq+i8AAP//ZNYHKjAKAAA=", + "H4sIAAAAAAAC/8xW32/bNhD+V4LbHlVLcuzY0dsKdEWwh2Ht2mEo8kBLZ4s2f5mkFMuB/veBlCIpsZy6", + "RQf0yYZ4vPvu+44f+Qip5EoKFNZA8ggmzZET//ed1lK7P0pLhdpS9J9TmaH7tZVCSIAKixvUUAfA0Riy", + "GS4aq6nYQF0HoHFfUI0ZJF+6wKBJdh88xcvVFlMLARzeGCsVo5vcunQ0gwQ22XY503g48MPa+py/I2Z3", + "vC35HKWllo0hCaDQ7OsIXVDQJrkU3lwt8pRRNa2yXQ/PIj9FlxHrwa2l5sRC4j+8sZQ7Tk4Q06cWf9W4", + "hgR+CXvNwlawsOeiDoARsSnGpQiAUbEbXTCy0On4HlNwTnQ1unaO6heUNmF9qhZIV3aAOmgIupB4HefF", + "cTNnt7LaKV/2I+oS9duCsuwzakOlGBtjzqk9hT1eY7dK99PldbEsojzyNco+8SUJDCXXO7Xdk0McP0D9", + "kpwWTZ/2vuN1rJvLiFno+cqWZHaYx9X1gJiznKxcja8N2gicZ7Pz6l4f9QEZEoPd7hdcdPPQwDlh4ttI", + "ON6YZRUXN3tWRu10jIE44YIRYz8pN4fZbxePyWE7FUzMynihjtn3jAmu9pLlZH5cH27F6ZiUXevP4Q05", + "GuvuMqputxujVqndsVJoj/4fuqMf0BTMjhH0XR7zmpHkBV8JQtmPsJknc+ncps8+cJoTi/GUU7GWg0sE", + "PolUioxaKgVhg0OaQDyJHDapUBBFIYHrSTSZQgCK2NzTFJZx6PJpjjp8oDsaPu4L1FXtFjfoiXW0Epf9", + "zsnwHu3n+K7d4hT4y8X7nJpwtKgNJF8egToArg4EIAh3QPdtZE+I1QUG7a0+Rt69CzZKCtOoOo2ixh2F", + "ReHBEaUYTT28cGuaUe7zvXbaB9Pjac3QpJoq21D35x+OufkPrNc8WEZK3QmLWhB21RjIVRdYB14fg0Sn", + "ebhGzC5T56Pf4K7cn1kbapGbi94N7pFSd0eBaE2qn1SycedSJOXHW10uSLyLodN1YL7ntew90imDxr6V", + "WfVNDZ44SP0/Hqrnd+AZkYZGC//KQl+9f/f3FYpMSSrO2f+a6dUsx4flTRmbtg3jqzUz7R/OkFurTBKG", + "RNFJMfTFSUaqsIyhDoahSRgymRKWS2OTZbSMBuvxdDGJJtEkbhfuz9zgMp7P4lV6M5MHCXX9XwAAAP//", + "MmHoN7MMAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/cmd/serve.go b/cmd/serve.go index 87afe12..a1ce61e 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -77,8 +77,6 @@ func NewServeCommand(version version.Build) *cobra.Command { go serve.UpdateResources(&source, sourceService, c) - w := informer.NewWiki() - return c.GetAPIServer().Start() }, } diff --git a/go.mod b/go.mod index 8a10013..51a2495 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/unconditionalday/server go 1.21 require ( + github.com/anaskhan96/soup v1.2.5 github.com/deepmap/oapi-codegen v1.15.0 github.com/getkin/kin-openapi v0.120.0 github.com/labstack/echo/v4 v4.11.1 @@ -10,6 +11,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.5 go.uber.org/zap v1.26.0 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) require ( @@ -111,7 +113,6 @@ require ( go.uber.org/multierr v1.10.0 // indirect golang.org/x/arch v0.4.0 // indirect golang.org/x/crypto v0.13.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.15.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index a311dd0..94dd3b7 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/SlyMarbo/rss v1.0.5 h1:DPcZ4aOXXHJ5yNLXY1q/57frIixMmAvTtLxDE3fsMEI= github.com/SlyMarbo/rss v1.0.5/go.mod h1:w6Bhn1BZs91q4OlEnJVZEUNRJmlbFmV7BkAlgCN8ofM= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM= +github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= @@ -525,6 +527,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= @@ -547,6 +550,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -585,6 +589,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -758,6 +763,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/internal/app/feed.go b/internal/app/feed.go index 012d327..a3ca353 100644 --- a/internal/app/feed.go +++ b/internal/app/feed.go @@ -17,17 +17,6 @@ type FeedRepository interface { Delete(doc Feed) error } -type FeedRepository interface { - // Search returns the results of a search query. - Find(query string) ([]Feed, error) - // Index indexes a document. - Save(doc Feed) error - // Delete deletes a document. - Delete(doc Feed) error - // Close closes the database. - Close() error -} - type Feed struct { Title string `json:"title"` Link string `json:"link"` diff --git a/internal/informer/wiki/cache.go b/internal/client/wikipedia/cache.go similarity index 99% rename from internal/informer/wiki/cache.go rename to internal/client/wikipedia/cache.go index c9893f3..f1d9a55 100644 --- a/internal/informer/wiki/cache.go +++ b/internal/client/wikipedia/cache.go @@ -1,4 +1,4 @@ -package wiki +package wikipedia import ( "crypto/sha256" diff --git a/internal/informer/wiki/client.go b/internal/client/wikipedia/client.go similarity index 50% rename from internal/informer/wiki/client.go rename to internal/client/wikipedia/client.go index 61dfc83..d89cdb9 100644 --- a/internal/informer/wiki/client.go +++ b/internal/client/wikipedia/client.go @@ -1,4 +1,4 @@ -package wiki +package wikipedia import ( "encoding/json" @@ -7,6 +7,8 @@ import ( "io" "net/http" "time" + + "github.com/unconditionalday/server/internal/search" ) type Client struct { @@ -21,8 +23,13 @@ const ( maxCacheMemory = 500 ) +var ( + ErrEmptyQuery = errors.New("query string must not be empty") + ErrEmptyLanguage = errors.New("language string must not be empty") +) + // Create a new WikiClient -func New() *Client { +func NewClient() *Client { return &Client{ userAgent: "unconditional.day", URL: "https://%v.wikipedia.org/w/api.php", @@ -34,9 +41,9 @@ func New() *Client { /* Make a request to the Wikipedia API using the given search parameters. -Returns a RequestResult (You can see the model in the models.go file) +Returns a RequestResult */ -func (c *Client) DoRequest(args map[string]string, wikiLang string) (RequestResult, error) { +func (c *Client) doRequest(args map[string]string, wikiLang string) (RequestResult, error) { const ReqPerSec = 199 const ApiGap = time.Second / ReqPerSec @@ -105,3 +112,81 @@ Update the last time we call the API (API should) func (c *Client) updateLastCall(now time.Time) { c.lastCall = now } + +func (w *Client) FetchEntityDetails(query string, lang string) (search.EntityDetails, error) { + if query == "" { + return search.EntityDetails{}, ErrEmptyQuery + } + + if lang == "" { + return search.EntityDetails{}, ErrEmptyLanguage + } + + args := map[string]string{ + "action": "query", + "list": "search", + "srprop": "", + "srlimit": "1", + "srsearch": query, + } + + res, err := w.doRequest(args, lang) + if err != nil { + return search.EntityDetails{}, err + } + + if len(res.Query.Search) == 0 { + return search.EntityDetails{}, nil + } + + title := res.Query.Search[0].Title + + wikiPage, err := MakeWikipediaPage(-1, title, "", false, w, lang) + if len(wikiPage.Disambiguation) != 0 { + title = wikiPage.Disambiguation[0] + wikiPage, err = MakeWikipediaPage(-1, title, "", false, w, lang) + } + + if err != nil { + return search.EntityDetails{}, err + } + + summary, err := wikiPage.GetSummary(w, lang) + if err != nil { + return search.EntityDetails{}, err + } + + thumbnail, err := wikiPage.GetThumbURL(w, lang) + if err != nil { + return search.EntityDetails{}, err + } + + return search.EntityDetails{ + Title: wikiPage.Title, + Language: wikiPage.Language, + Link: wikiPage.URL, + Source: "Wikipedia", + Summary: summary, + Thumbnail: thumbnail, + }, nil +} + +func (c *Client) Suggest(_input, lang string) (string, error) { + args := map[string]string{ + "action": "query", + "list": "search", + "srlimit": "1", + "srprop": "", + "srinfo": "suggestion", + "srsearch": _input, + } + + res, err := c.doRequest(args, lang) + if err != nil { + return "", err + } + if res.Error.Code != "" { + return "", errors.New(res.Error.Info) + } + return res.Query.SearchInfo.Suggestion, nil +} diff --git a/internal/informer/wiki_test.go b/internal/client/wikipedia/client_test.go similarity index 71% rename from internal/informer/wiki_test.go rename to internal/client/wikipedia/client_test.go index 8b45281..ff97fe6 100644 --- a/internal/informer/wiki_test.go +++ b/internal/client/wikipedia/client_test.go @@ -1,11 +1,11 @@ -package informer_test +package wikipedia_test import ( "errors" "testing" "github.com/stretchr/testify/assert" - "github.com/unconditionalday/server/internal/informer" + "github.com/unconditionalday/server/internal/client/wikipedia" ) type TestInput struct { @@ -15,15 +15,15 @@ type TestInput struct { type TestExpect struct { validRes bool - err error + err error } func TestWikiSearch(t *testing.T) { t.Parallel() testCases := []struct { - name string - input TestInput + name string + input TestInput output TestExpect }{ { @@ -31,7 +31,7 @@ func TestWikiSearch(t *testing.T) { input: TestInput{query: "", lang: "en"}, output: TestExpect{ validRes: false, - err: errors.New("query string must not be empty"), + err: errors.New("query string must not be empty"), }, }, { @@ -39,7 +39,7 @@ func TestWikiSearch(t *testing.T) { input: TestInput{query: "Lorem ipsum", lang: ""}, output: TestExpect{ validRes: false, - err: errors.New("language string must not be empty"), + err: errors.New("language string must not be empty"), }, }, { @@ -47,7 +47,7 @@ func TestWikiSearch(t *testing.T) { input: TestInput{query: "Salvini", lang: "en"}, output: TestExpect{ validRes: true, - err: nil, + err: nil, }, }, } @@ -57,8 +57,8 @@ func TestWikiSearch(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - w := informer.NewWiki() - actual, err := w.Search(tc.input.query, tc.input.lang) + w := wikipedia.NewClient() + actual, err := w.FetchEntityDetails(tc.input.query, tc.input.lang) if tc.output.err != nil { assert.Equal(t, tc.output.err.Error(), err.Error()) diff --git a/internal/informer/wiki/page.go b/internal/client/wikipedia/page.go similarity index 97% rename from internal/informer/wiki/page.go rename to internal/client/wikipedia/page.go index 6ff3416..9b9ffca 100644 --- a/internal/informer/wiki/page.go +++ b/internal/client/wikipedia/page.go @@ -1,4 +1,4 @@ -package wiki +package wikipedia import ( "errors" @@ -62,7 +62,7 @@ func (page *WikipediaPage) GetContent(client *Client, lang string) (string, erro "rvprop": "ids", "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return "", err } @@ -91,7 +91,7 @@ func (page *WikipediaPage) GetHTML(client *Client, lang string) (string, error) "rvparse": "", "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return "", err } @@ -154,7 +154,7 @@ func (page *WikipediaPage) GetSummary(client *Client, lang string) (string, erro "exlimit": "1", "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return "", err } @@ -176,7 +176,7 @@ func (page *WikipediaPage) ContinuedQuery(args map[string]string, client *Client new_args := maps.Clone(args) mapsx.Update(new_args, last) - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return result, err } @@ -269,7 +269,7 @@ func (page *WikipediaPage) GetThumbURL(client *Client, lang string) (string, err "pithumbsize": "500", } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return "", err } @@ -294,7 +294,7 @@ func (page *WikipediaPage) GetCoordinate(client *Client, lang string) ([]float64 "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return []float64{}, err } @@ -396,7 +396,7 @@ func (page *WikipediaPage) GetSectionList(client *Client, lang string) ([]string if page.Title != "" { args["page"] = page.Title } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return []string{}, err } @@ -466,7 +466,7 @@ func MakeWikipediaPage(pageid int, title string, originaltitle string, redirect if originaltitle != "" { page.OriginalTitle = originaltitle } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return page, err } @@ -522,7 +522,7 @@ func MakeWikipediaPage(pageid int, title string, originaltitle string, redirect "rvlimit": "1", "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return page, err } diff --git a/internal/informer/wiki/request.go b/internal/client/wikipedia/request.go similarity index 99% rename from internal/informer/wiki/request.go rename to internal/client/wikipedia/request.go index 571bbda..350dc11 100644 --- a/internal/informer/wiki/request.go +++ b/internal/client/wikipedia/request.go @@ -1,4 +1,4 @@ -package wiki +package wikipedia type InnerBasic struct { Aster string `json:"*"` diff --git a/internal/container/container.go b/internal/container/container.go index 62306cd..63fceb2 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -10,8 +10,10 @@ import ( "github.com/unconditionalday/server/internal/app" "github.com/unconditionalday/server/internal/client/github" + "github.com/unconditionalday/server/internal/client/wikipedia" "github.com/unconditionalday/server/internal/parser" bleveRepo "github.com/unconditionalday/server/internal/repository/bleve" + "github.com/unconditionalday/server/internal/search" "github.com/unconditionalday/server/internal/version" "github.com/unconditionalday/server/internal/webserver" blevex "github.com/unconditionalday/server/internal/x/bleve" @@ -68,6 +70,7 @@ type Services struct { apiServer *webserver.Server feedRepository *bleveRepo.FeedRepository sourceClient *github.Client + searchClient *wikipedia.Client httpClient *netx.HttpClient logger *zap.Logger parser *parser.Parser @@ -91,7 +94,7 @@ func (c *Container) GetAPIServer() *webserver.Server { AllowedOrigins: c.Parameters.ServerAllowedOrigins, } - c.apiServer = webserver.NewServer(config, c.GetFeedRepository(), c.SourceRelease, c.BuildVersion, c.GetLogger()) + c.apiServer = webserver.NewServer(config, c.GetFeedRepository(), c.SourceRelease, c.GetSearchClient(), c.BuildVersion, c.GetLogger()) return c.apiServer } @@ -128,6 +131,16 @@ func (c *Container) GetSourceClient() app.SourceClient { return c.sourceClient } +func (c *Container) GetSearchClient() search.SearchClient { + if c.searchClient != nil { + return c.searchClient + } + + c.searchClient = wikipedia.NewClient() + + return c.searchClient +} + func (c *Container) GetVersioning() version.Versioning { if c.versioning != nil { return c.versioning diff --git a/internal/informer/wiki.go b/internal/informer/wiki.go deleted file mode 100644 index cca79b1..0000000 --- a/internal/informer/wiki.go +++ /dev/null @@ -1,91 +0,0 @@ -package informer - -import ( - "errors" - - "github.com/unconditionalday/server/internal/informer/wiki" -) - -var ( - ErrEmptyQuery = errors.New("query string must not be empty") - ErrEmptyLanguage = errors.New("language string must not be empty") -) - -type Wiki struct { - client *wiki.Client -} - -func NewWiki() *Wiki { - return &Wiki{ - client: wiki.New(), - } -} - -func (w *Wiki) Search(query string, lang string) (WikiInfo, error) { - if query == "" { - return WikiInfo{}, ErrEmptyQuery - } - - if lang == "" { - return WikiInfo{}, ErrEmptyLanguage - } - - args := map[string]string{ - "action": "query", - "list": "search", - "srprop": "", - "srlimit": "1", - "srsearch": query, - } - - res, err := w.client.DoRequest(args, lang) - if err != nil { - return WikiInfo{}, err - } - - if len(res.Query.Search) == 0 { - return WikiInfo{}, nil - } - - title := res.Query.Search[0].Title - - wikiPage, err := wiki.MakeWikipediaPage(-1, title, "", false, w.client, lang) - if wikiPage.Disambiguation != nil { - title = wikiPage.Disambiguation[0] - wikiPage, err = wiki.MakeWikipediaPage(-1, title, "", false, w.client, lang) - } - - if err != nil { - return WikiInfo{}, err - } - - summary, err := wikiPage.GetSummary(w.client, lang) - if err != nil { - return WikiInfo{}, err - } - - thumbnail, err := wikiPage.GetThumbURL(w.client, lang) - if err != nil { - return WikiInfo{}, err - } - - return WikiInfo{ - Title: wikiPage.Title, - Language: wikiPage.Language, - Link: wikiPage.URL, - Summary: summary, - Thumbnail: thumbnail, - }, nil -} - -type WikiInfo struct { - Title string - Link string - Summary string - Thumbnail string - Language string -} - -func (r WikiInfo) IsValid() bool { - return r.Title != "" && r.Link != "" && r.Summary != "" && r.Thumbnail != "" && r.Language != "" -} diff --git a/internal/repository/bleve/bleve_test.go b/internal/repository/bleve/bleve_test.go deleted file mode 100644 index c1ad058..0000000 --- a/internal/repository/bleve/bleve_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package bleve_test - -import ( - "os" - "testing" - "time" - - "github.com/blevesearch/bleve/v2/mapping" - "github.com/unconditionalday/server/internal/app" - "github.com/unconditionalday/server/internal/repository/bleve" -) - -func TestBleveIndex(t *testing.T) { - testCases := []struct { - name string - b *bleve.Bleve - wantErr bool - }{ - { - name: "bleve is created", - wantErr: false, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var err error - - tc.b, err = bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) - - defer os.RemoveAll("test.bleve") - - if tc.b == nil { - t.Fatalf("expected bleve to be created") - } - - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - }) - } -} - -func TestSave(t *testing.T) { - testCases := []struct { - name string - document app.Feed - wantErr bool - }{ - { - name: "document is saved", - wantErr: false, - document: app.Feed{ - Title: "test", - Link: "link", - Language: "it", - Image: &app.Image{}, - Summary: "Lorem Ipsum", - Source: "Unconditional Day", - Date: time.Time{}, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - b, err := bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) - if b == nil { - t.Fatalf("expected bleve to be created") - } - - defer os.RemoveAll("test.bleve") - - err = b.Save(tc.document) - - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - }) - } -} - -func TestFind(t *testing.T) { - testCases := []struct { - name string - document app.Feed - wantErr bool - }{ - { - name: "document is found", - wantErr: false, - document: app.Feed{ - Title: "test", - Link: "link", - Language: "it", - Image: &app.Image{}, - Summary: "Lorem Ipsum", - Source: "Unconditional Day", - Date: time.Now(), - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - b, err := bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) - if b == nil { - t.Fatalf("expected bleve to be created") - } - - defer os.RemoveAll("test.bleve") - - err = b.Save(tc.document) - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - - found, err := b.Find(tc.document.Title) - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - - if found == nil { - t.Errorf("expected document to be found") - } - - if found[0].Title != tc.document.Title { - t.Errorf("expected document to be found") - } - - }) - } -} diff --git a/internal/search/search.go b/internal/search/search.go new file mode 100644 index 0000000..521a7ae --- /dev/null +++ b/internal/search/search.go @@ -0,0 +1,18 @@ +package search + +type EntityDetails struct { + Title string + Link string + Summary string + Thumbnail string + Language string + Source string +} + +func (i EntityDetails) IsValid() bool { + return i.Title != "" && i.Link != "" && i.Summary != "" && i.Thumbnail != "" && i.Language != "" && i.Source != "" +} + +type SearchClient interface { + FetchEntityDetails(query, locale string) (EntityDetails, error) +} diff --git a/internal/webserver/server.go b/internal/webserver/server.go index a5c422d..018362c 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -10,6 +10,7 @@ import ( api "github.com/unconditionalday/server/api" "github.com/unconditionalday/server/internal/app" + "github.com/unconditionalday/server/internal/search" "github.com/unconditionalday/server/internal/version" ) @@ -18,7 +19,7 @@ type Server struct { feedRepo app.FeedRepository source *app.SourceRelease buildVersion version.Build - wiki *informer.Wiki + search search.SearchClient logger *zap.Logger client *echo.Echo } @@ -29,13 +30,13 @@ type Config struct { AllowedOrigins []string } -func NewServer(config Config, repo app.FeedRepository, source *app.SourceRelease, version version.Build, logger *zap.Logger) *Server { +func NewServer(config Config, repo app.FeedRepository, source *app.SourceRelease, search search.SearchClient, version version.Build, logger *zap.Logger) *Server { return &Server{ config: config, feedRepo: repo, source: source, buildVersion: version, - wiki: wiki, + search: search, logger: logger, client: echo.New(), } @@ -116,24 +117,24 @@ func (s *Server) GetV1Version(ctx echo.Context) error { func (s *Server) GetV1InformerWikiQuery(ctx echo.Context, query string) error { // TODO: add language support - wikiRes, err := s.wiki.Search(query, "en") + searchRes, err := s.search.FetchEntityDetails(query, "en") if err != nil { e := api.Error{ - Code: 500, + Code: http.StatusInternalServerError, Message: "Internal Server Error", } s.logger.Error("wiki search", zap.Error(err)) - return ctx.JSON(500, e) + return ctx.JSON(http.StatusInternalServerError, e) } res := api.WikiResult{ - Language: wikiRes.Language, - Link: wikiRes.Link, - Summary: wikiRes.Summary, - Thumbnail: wikiRes.Thumbnail, - Title: wikiRes.Title, + Language: searchRes.Language, + Link: searchRes.Link, + Summary: searchRes.Summary, + Thumbnail: searchRes.Thumbnail, + Title: searchRes.Title, } return ctx.JSON(200, res)