Skip to content

Commit

Permalink
chore: use search service
Browse files Browse the repository at this point in the history
  • Loading branch information
@luigibarbato
luigibarbato committed Oct 8, 2023
1 parent f1aafa5 commit 9382717
Show file tree
Hide file tree
Showing 15 changed files with 188 additions and 292 deletions.
40 changes: 25 additions & 15 deletions api/api.gen.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 0 additions & 2 deletions cmd/serve.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,6 @@ func NewServeCommand(version version.Build) *cobra.Command {

go serve.UpdateResources(&source, sourceService, c)

w := informer.NewWiki()

return c.GetAPIServer().Start()
},
}
Expand Down
3 changes: 2 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ module github.com/unconditionalday/server
go 1.21

require (
github.com/anaskhan96/soup v1.2.5
github.com/deepmap/oapi-codegen v1.15.0
github.com/getkin/kin-openapi v0.120.0
github.com/labstack/echo/v4 v4.11.1
github.com/labstack/gommon v0.4.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/pflag v1.0.5
go.uber.org/zap v1.26.0
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
)

require (
Expand Down Expand Up @@ -111,7 +113,6 @@ require (
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/arch v0.4.0 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/net v0.15.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
Expand Down
6 changes: 6 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ github.com/SlyMarbo/rss v1.0.5 h1:DPcZ4aOXXHJ5yNLXY1q/57frIixMmAvTtLxDE3fsMEI=
github.com/SlyMarbo/rss v1.0.5/go.mod h1:w6Bhn1BZs91q4OlEnJVZEUNRJmlbFmV7BkAlgCN8ofM=
github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM=
github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
Expand Down Expand Up @@ -525,6 +527,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
Expand All @@ -547,6 +550,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down Expand Up @@ -585,6 +589,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down Expand Up @@ -758,6 +763,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
Expand Down
11 changes: 0 additions & 11 deletions internal/app/feed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,6 @@ type FeedRepository interface {
Delete(doc Feed) error
}

type FeedRepository interface {
// Search returns the results of a search query.
Find(query string) ([]Feed, error)
// Index indexes a document.
Save(doc Feed) error
// Delete deletes a document.
Delete(doc Feed) error
// Close closes the database.
Close() error
}

type Feed struct {
Title string `json:"title"`
Link string `json:"link"`
Expand Down
2 changes: 1 addition & 1 deletion internal/informer/wiki/cache.go → internal/client/wikipedia/cache.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package wiki
package wikipedia

import (
"crypto/sha256"
Expand Down
93 changes: 89 additions & 4 deletions internal/informer/wiki/client.go → internal/client/wikipedia/client.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package wiki
package wikipedia

import (
"encoding/json"
Expand All @@ -7,6 +7,8 @@ import (
"io"
"net/http"
"time"

"github.com/unconditionalday/server/internal/search"
)

type Client struct {
Expand All @@ -21,8 +23,13 @@ const (
maxCacheMemory = 500
)

var (
ErrEmptyQuery = errors.New("query string must not be empty")
ErrEmptyLanguage = errors.New("language string must not be empty")
)

// Create a new WikiClient
func New() *Client {
func NewClient() *Client {
return &Client{
userAgent: "unconditional.day",
URL: "https://%v.wikipedia.org/w/api.php",
Expand All @@ -34,9 +41,9 @@ func New() *Client {
/*
Make a request to the Wikipedia API using the given search parameters.
Returns a RequestResult (You can see the model in the models.go file)
Returns a RequestResult
*/
func (c *Client) DoRequest(args map[string]string, wikiLang string) (RequestResult, error) {
func (c *Client) doRequest(args map[string]string, wikiLang string) (RequestResult, error) {
const ReqPerSec = 199
const ApiGap = time.Second / ReqPerSec

Expand Down Expand Up @@ -105,3 +112,81 @@ Update the last time we call the API (API should)
func (c *Client) updateLastCall(now time.Time) {
c.lastCall = now
}

func (w *Client) FetchEntityDetails(query string, lang string) (search.EntityDetails, error) {
if query == "" {
return search.EntityDetails{}, ErrEmptyQuery
}

if lang == "" {
return search.EntityDetails{}, ErrEmptyLanguage
}

args := map[string]string{
"action": "query",
"list": "search",
"srprop": "",
"srlimit": "1",
"srsearch": query,
}

res, err := w.doRequest(args, lang)
if err != nil {
return search.EntityDetails{}, err
}

if len(res.Query.Search) == 0 {
return search.EntityDetails{}, nil
}

title := res.Query.Search[0].Title

wikiPage, err := MakeWikipediaPage(-1, title, "", false, w, lang)
if len(wikiPage.Disambiguation) != 0 {
title = wikiPage.Disambiguation[0]
wikiPage, err = MakeWikipediaPage(-1, title, "", false, w, lang)
}

if err != nil {
return search.EntityDetails{}, err
}

summary, err := wikiPage.GetSummary(w, lang)
if err != nil {
return search.EntityDetails{}, err
}

thumbnail, err := wikiPage.GetThumbURL(w, lang)
if err != nil {
return search.EntityDetails{}, err
}

return search.EntityDetails{
Title: wikiPage.Title,
Language: wikiPage.Language,
Link: wikiPage.URL,
Source: "Wikipedia",
Summary: summary,
Thumbnail: thumbnail,
}, nil
}

func (c *Client) Suggest(_input, lang string) (string, error) {
args := map[string]string{
"action": "query",
"list": "search",
"srlimit": "1",
"srprop": "",
"srinfo": "suggestion",
"srsearch": _input,
}

res, err := c.doRequest(args, lang)
if err != nil {
return "", err
}
if res.Error.Code != "" {
return "", errors.New(res.Error.Info)
}
return res.Query.SearchInfo.Suggestion, nil
}
20 changes: 10 additions & 10 deletions internal/informer/wiki_test.go → internal/client/wikipedia/client_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package informer_test
package wikipedia_test

import (
"errors"
"testing"

"github.com/stretchr/testify/assert"
"github.com/unconditionalday/server/internal/informer"
"github.com/unconditionalday/server/internal/client/wikipedia"
)

type TestInput struct {
Expand All @@ -15,39 +15,39 @@ type TestInput struct {

type TestExpect struct {
validRes bool
err error
err error
}

func TestWikiSearch(t *testing.T) {
t.Parallel()

testCases := []struct {
name string
input TestInput
name string
input TestInput
output TestExpect
}{
{
name: "empty query string",
input: TestInput{query: "", lang: "en"},
output: TestExpect{
validRes: false,
err: errors.New("query string must not be empty"),
err: errors.New("query string must not be empty"),
},
},
{
name: "empty language",
input: TestInput{query: "Lorem ipsum", lang: ""},
output: TestExpect{
validRes: false,
err: errors.New("language string must not be empty"),
err: errors.New("language string must not be empty"),
},
},
{
name: "valid query",
input: TestInput{query: "Salvini", lang: "en"},
output: TestExpect{
validRes: true,
err: nil,
err: nil,
},
},
}
Expand All @@ -57,8 +57,8 @@ func TestWikiSearch(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

w := informer.NewWiki()
actual, err := w.Search(tc.input.query, tc.input.lang)
w := wikipedia.NewClient()
actual, err := w.FetchEntityDetails(tc.input.query, tc.input.lang)

if tc.output.err != nil {
assert.Equal(t, tc.output.err.Error(), err.Error())
Expand Down
Loading

0 comments on commit 9382717

Please sign in to comment.