From 757a1342a7700eddfc4b7a9db0627aae935dc70c Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Sun, 11 Dec 2022 20:24:10 +0100 Subject: [PATCH 01/25] chore: refactor --- cmd/source/download.go | 7 +++++++ internal/app/feed.go | 15 +++++++++++++++ internal/webserver/server.go | 3 +++ 3 files changed, 25 insertions(+) diff --git a/cmd/source/download.go b/cmd/source/download.go index 09d8cb6..40ad8a4 100644 --- a/cmd/source/download.go +++ b/cmd/source/download.go @@ -1,7 +1,14 @@ package source import ( +<<<<<<< HEAD "errors" +======= + "github.com/unconditionalday/server/internal/cobrax" + "github.com/unconditionalday/server/internal/iox" + "github.com/unconditionalday/server/internal/netx" + "github.com/unconditionalday/server/internal/service" +>>>>>>> ed637fb (chore: refactor) "github.com/spf13/cobra" diff --git a/internal/app/feed.go b/internal/app/feed.go index a3ca353..c7b0428 100644 --- a/internal/app/feed.go +++ b/internal/app/feed.go @@ -17,6 +17,21 @@ type FeedRepository interface { Delete(doc Feed) error } +type FeedRepository interface { + // Search returns the results of a search query. + Find(query string) ([]Feed, error) + // Index indexes a document. + Save(doc Feed) error + // Delete deletes a document. + Delete(doc Feed) error + // Update updates a document. + Update(doc Feed) error + // Index indexes a document. + Index(id string, doc Feed) error + // Close closes the database. + Close() error +} + type Feed struct { Title string `json:"title"` Link string `json:"link"` diff --git a/internal/webserver/server.go b/internal/webserver/server.go index beb8af1..b54cc2f 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -4,6 +4,9 @@ import ( "fmt" "net/http" + api "github.com/unconditionalday/server/api" + "github.com/unconditionalday/server/internal/app" + "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "go.uber.org/zap" From f53f2bcb33032b68bc646b287e7471519fb08418 Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Mon, 12 Dec 2022 22:28:17 +0100 Subject: [PATCH 02/25] chore: format code --- cmd/source/download.go | 7 ------- internal/webserver/server.go | 3 --- 2 files changed, 10 deletions(-) diff --git a/cmd/source/download.go b/cmd/source/download.go index 40ad8a4..09d8cb6 100644 --- a/cmd/source/download.go +++ b/cmd/source/download.go @@ -1,14 +1,7 @@ package source import ( -<<<<<<< HEAD "errors" -======= - "github.com/unconditionalday/server/internal/cobrax" - "github.com/unconditionalday/server/internal/iox" - "github.com/unconditionalday/server/internal/netx" - "github.com/unconditionalday/server/internal/service" ->>>>>>> ed637fb (chore: refactor) "github.com/spf13/cobra" diff --git a/internal/webserver/server.go b/internal/webserver/server.go index b54cc2f..beb8af1 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -4,9 +4,6 @@ import ( "fmt" "net/http" - api "github.com/unconditionalday/server/api" - "github.com/unconditionalday/server/internal/app" - "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "go.uber.org/zap" From 495d45baadb1aef4294aeaac6e4ab2ebd850ed65 Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Tue, 13 Dec 2022 18:37:07 +0100 Subject: [PATCH 03/25] feat: add special chars capability,deploy workflow --- .github/workflows/deploy.yaml | 39 ++++++++++++++++++++++++++++++++++ internal/parser/parser_test.go | 5 +++++ 2 files changed, 44 insertions(+) create mode 100644 .github/workflows/deploy.yaml diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 0000000..8fd68bb --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,39 @@ +name: Deploy +on: + push: + branches: + - main + +env: + FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version-file: go.mod + cache: true + - name: Prepare tools + run: make prepare-ci + - name: Get dependencies + run: go get -v -t -d ./... + - name: Install dependencies + run: go mod download + - name: Unit test + run: make test-unit + - name: Integration test + run: make test-integration + - name: Build + run: make build + deploy: + name: Deploy + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: superfly/flyctl-actions/setup-flyctl@master + - run: flyctl deploy diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index adfdafd..24fa8d8 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -46,6 +46,11 @@ func TestParse(t *testing.T) { input: "

hello

\n[Continue reading...]", expected: "hello", }, + { + name: "single word with html tags and new line and bloated text and special characters", + input: "

hello

\n[Continue reading...]\n–", + expected: "hello", + }, } for _, tc := range testCases { tc := tc From c20963889f84ba3542f93cf743b1153b4e840eb9 Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Tue, 13 Dec 2022 22:48:17 +0100 Subject: [PATCH 04/25] chore: pin fly action version, optimize workflow logic, optimize fly config --- .github/workflows/deploy.yaml | 2 +- fly.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 8fd68bb..cc59992 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -34,6 +34,6 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: superfly/flyctl-actions/setup-flyctl@master - run: flyctl deploy diff --git a/fly.toml b/fly.toml index 5e5742b..0976624 100644 --- a/fly.toml +++ b/fly.toml @@ -42,4 +42,4 @@ processes = [] interval = "15s" restart_limit = 2 timeout = "10s" - \ No newline at end of file + From cb13c4b4180ed396e810b27342e5d8e16b8816ae Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Tue, 27 Dec 2022 20:10:39 +0100 Subject: [PATCH 05/25] chore: small refactor,add some bleve test --- internal/app/feed.go | 2 - internal/repository/bleve/bleve_test.go | 195 ++++++++++++++++++++++++ 2 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 internal/repository/bleve/bleve_test.go diff --git a/internal/app/feed.go b/internal/app/feed.go index c7b0428..cc45dde 100644 --- a/internal/app/feed.go +++ b/internal/app/feed.go @@ -26,8 +26,6 @@ type FeedRepository interface { Delete(doc Feed) error // Update updates a document. Update(doc Feed) error - // Index indexes a document. - Index(id string, doc Feed) error // Close closes the database. Close() error } diff --git a/internal/repository/bleve/bleve_test.go b/internal/repository/bleve/bleve_test.go new file mode 100644 index 0000000..f04d3ff --- /dev/null +++ b/internal/repository/bleve/bleve_test.go @@ -0,0 +1,195 @@ +package bleve_test + +import ( + "os" + "testing" + "time" + + "github.com/blevesearch/bleve/mapping" + "github.com/unconditionalday/server/internal/app" + "github.com/unconditionalday/server/internal/repository/bleve" +) + +func TestBleveIndex(t *testing.T) { + testCases := []struct { + name string + b *bleve.Bleve + wantErr bool + }{ + { + name: "bleve is created", + wantErr: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var err error + + tc.b, err = bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) + + defer os.RemoveAll("test.bleve") + + if tc.b == nil { + t.Fatalf("expected bleve to be created") + } + + if (err != nil) != tc.wantErr { + t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) + return + } + }) + } +} + +func TestSave(t *testing.T) { + testCases := []struct { + name string + document app.Feed + wantErr bool + }{ + { + name: "document is saved", + wantErr: false, + document: app.Feed{ + Title: "test", + Link: "link", + Language: "it", + Image: &app.Image{}, + Summary: "Lorem Ipsum", + Source: "Unconditional Day", + Date: time.Time{}, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + b, err := bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) + if b == nil { + t.Fatalf("expected bleve to be created") + } + + defer os.RemoveAll("test.bleve") + + err = b.Save(tc.document) + + if (err != nil) != tc.wantErr { + t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) + return + } + }) + } +} + +func TestFind(t *testing.T) { + testCases := []struct { + name string + document app.Feed + wantErr bool + }{ + { + name: "document is found", + wantErr: false, + document: app.Feed{ + Title: "test", + Link: "link", + Language: "it", + Image: &app.Image{}, + Summary: "Lorem Ipsum", + Source: "Unconditional Day", + Date: time.Now(), + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + b, err := bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) + if b == nil { + t.Fatalf("expected bleve to be created") + } + + defer os.RemoveAll("test.bleve") + + err = b.Save(tc.document) + if (err != nil) != tc.wantErr { + t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) + return + } + + found, err := b.Find(tc.document.Title) + if (err != nil) != tc.wantErr { + t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) + return + } + + if found == nil { + t.Errorf("expected document to be found") + } + + if found[0].Title != tc.document.Title { + t.Errorf("expected document to be found") + } + + }) + } +} + +func TestUpdate(t *testing.T) { + testCases := []struct { + name string + document app.Feed + wantErr bool + }{ + { + name: "document is updated", + wantErr: false, + document: app.Feed{ + Title: "test", + Link: "link", + Language: "it", + Image: &app.Image{}, + Summary: "Lorem Ipsum", + Source: "Unconditional Day", + Date: time.Now(), + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + b, err := bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) + if b == nil { + t.Fatalf("expected bleve to be created") + } + + defer os.RemoveAll("test.bleve") + + err = b.Save(tc.document) + if (err != nil) != tc.wantErr { + t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) + return + } + + tc.document.Summary = "New Summary" + + err = b.Update(tc.document) + if (err != nil) != tc.wantErr { + t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) + return + } + + found, err := b.Find(tc.document.Title) + if (err != nil) != tc.wantErr { + t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) + return + } + + if found == nil { + t.Errorf("expected document to be found") + } + + if found[0].Summary == tc.document.Summary { + t.Errorf("expected document to be found") + } + + }) + } +} From d17be922d8fd700779c876bf3f2cb3f2a40e9989 Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Thu, 29 Jun 2023 11:03:51 +0200 Subject: [PATCH 06/25] chore: packages refactor --- internal/app/feed.go | 2 - internal/repository/bleve/bleve_test.go | 61 ------------------------- 2 files changed, 63 deletions(-) diff --git a/internal/app/feed.go b/internal/app/feed.go index cc45dde..012d327 100644 --- a/internal/app/feed.go +++ b/internal/app/feed.go @@ -24,8 +24,6 @@ type FeedRepository interface { Save(doc Feed) error // Delete deletes a document. Delete(doc Feed) error - // Update updates a document. - Update(doc Feed) error // Close closes the database. Close() error } diff --git a/internal/repository/bleve/bleve_test.go b/internal/repository/bleve/bleve_test.go index f04d3ff..35d179b 100644 --- a/internal/repository/bleve/bleve_test.go +++ b/internal/repository/bleve/bleve_test.go @@ -132,64 +132,3 @@ func TestFind(t *testing.T) { }) } } - -func TestUpdate(t *testing.T) { - testCases := []struct { - name string - document app.Feed - wantErr bool - }{ - { - name: "document is updated", - wantErr: false, - document: app.Feed{ - Title: "test", - Link: "link", - Language: "it", - Image: &app.Image{}, - Summary: "Lorem Ipsum", - Source: "Unconditional Day", - Date: time.Now(), - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - b, err := bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) - if b == nil { - t.Fatalf("expected bleve to be created") - } - - defer os.RemoveAll("test.bleve") - - err = b.Save(tc.document) - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - - tc.document.Summary = "New Summary" - - err = b.Update(tc.document) - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - - found, err := b.Find(tc.document.Title) - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - - if found == nil { - t.Errorf("expected document to be found") - } - - if found[0].Summary == tc.document.Summary { - t.Errorf("expected document to be found") - } - - }) - } -} From 1e0a17972217cfc434740197f98c04c036491c34 Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Thu, 29 Jun 2023 13:50:30 +0200 Subject: [PATCH 07/25] chore: disable auto deploy for now --- .github/workflows/deploy.yaml | 39 ----------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 .github/workflows/deploy.yaml diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml deleted file mode 100644 index cc59992..0000000 --- a/.github/workflows/deploy.yaml +++ /dev/null @@ -1,39 +0,0 @@ -name: Deploy -on: - push: - branches: - - main - -env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version-file: go.mod - cache: true - - name: Prepare tools - run: make prepare-ci - - name: Get dependencies - run: go get -v -t -d ./... - - name: Install dependencies - run: go mod download - - name: Unit test - run: make test-unit - - name: Integration test - run: make test-integration - - name: Build - run: make build - deploy: - name: Deploy - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: superfly/flyctl-actions/setup-flyctl@master - - run: flyctl deploy From 8cfa54aed8c121ef2b48dbc9bc3ee850147e8b8b Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Thu, 29 Jun 2023 16:03:49 +0200 Subject: [PATCH 08/25] fix: remove useless test --- internal/parser/parser_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index 24fa8d8..adfdafd 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -46,11 +46,6 @@ func TestParse(t *testing.T) { input: "

hello

\n[Continue reading...]", expected: "hello", }, - { - name: "single word with html tags and new line and bloated text and special characters", - input: "

hello

\n[Continue reading...]\n–", - expected: "hello", - }, } for _, tc := range testCases { tc := tc From cfb5eb8a663b6b73e844b89b49c4bb19497b0e47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 13:27:30 +0000 Subject: [PATCH 09/25] chore(deps): bump github.com/stretchr/testify from 1.8.1 to 1.8.4 Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.1 to 1.8.4. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.1...v1.8.4) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index b5758e7..ec5f192 100644 --- a/go.sum +++ b/go.sum @@ -378,7 +378,6 @@ github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= From 136a3765310f442b09f520440713e5c36eeeb37e Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Thu, 29 Jun 2023 16:09:56 +0200 Subject: [PATCH 10/25] feat: add dependabot auto-merge --- .github/workflows/db-auto-merge.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/db-auto-merge.yaml b/.github/workflows/db-auto-merge.yaml index 7b6b044..4929971 100644 --- a/.github/workflows/db-auto-merge.yaml +++ b/.github/workflows/db-auto-merge.yaml @@ -1,3 +1,4 @@ +<<<<<<< HEAD name: Dependabot auto-merge on: pull_request @@ -20,3 +21,19 @@ jobs: env: PR_URL: ${{github.event.pull_request.html_url}} GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} +======= +name: auto-merge + +on: + pull_request: + +jobs: + auto-merge: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: ahmadnassri/action-dependabot-auto-merge@v2 + with: + target: minor + github-token: ${{ github.token }} +>>>>>>> 0e7e5ce (feat: add dependabot auto-merge) From 999772e347181df5ef9b3e9b31b99b98cbcc07cc Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Thu, 29 Jun 2023 16:22:22 +0200 Subject: [PATCH 11/25] chore: use custom private secret for dependabot --- .github/workflows/db-auto-merge.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/db-auto-merge.yaml b/.github/workflows/db-auto-merge.yaml index 4929971..dc30341 100644 --- a/.github/workflows/db-auto-merge.yaml +++ b/.github/workflows/db-auto-merge.yaml @@ -35,5 +35,9 @@ jobs: - uses: ahmadnassri/action-dependabot-auto-merge@v2 with: target: minor +<<<<<<< HEAD github-token: ${{ github.token }} >>>>>>> 0e7e5ce (feat: add dependabot auto-merge) +======= + github-token: ${{ secrets.ACCESS_TOKEN }} +>>>>>>> 4b36d7b (chore: use custom private secret for dependabot) From 27feb67aa708506d3e20a46d534233231d17f493 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:14:52 +0000 Subject: [PATCH 12/25] chore(deps): bump github.com/spf13/viper from 1.13.0 to 1.16.0 Bumps [github.com/spf13/viper](https://github.com/spf13/viper) from 1.13.0 to 1.16.0. - [Release notes](https://github.com/spf13/viper/releases) - [Commits](https://github.com/spf13/viper/compare/v1.13.0...v1.16.0) --- updated-dependencies: - dependency-name: github.com/spf13/viper dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.sum | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go.sum b/go.sum index ec5f192..ca0f3ea 100644 --- a/go.sum +++ b/go.sum @@ -378,6 +378,7 @@ github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0 github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -429,6 +430,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= @@ -524,6 +526,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= @@ -581,8 +584,10 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From f9eec0ecd55de6f9abf9a2cda0d0219a1f3a0f79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Jun 2023 17:32:26 +0000 Subject: [PATCH 13/25] chore(deps): bump github.com/deepmap/oapi-codegen from 1.11.0 to 1.13.0 Bumps [github.com/deepmap/oapi-codegen](https://github.com/deepmap/oapi-codegen) from 1.11.0 to 1.13.0. - [Release notes](https://github.com/deepmap/oapi-codegen/releases) - [Commits](https://github.com/deepmap/oapi-codegen/compare/v1.11.0...v1.13.0) --- updated-dependencies: - dependency-name: github.com/deepmap/oapi-codegen dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.sum | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index ca0f3ea..5210438 100644 --- a/go.sum +++ b/go.sum @@ -334,6 +334,7 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= @@ -527,7 +528,6 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= @@ -551,6 +551,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -591,7 +593,6 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From ffb63affa85b358d01dd472a1e7167c0c4db95c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jul 2023 17:54:56 +0000 Subject: [PATCH 14/25] chore(deps): bump go.uber.org/zap from 1.17.0 to 1.24.0 Bumps [go.uber.org/zap](https://github.com/uber-go/zap) from 1.17.0 to 1.24.0. - [Release notes](https://github.com/uber-go/zap/releases) - [Changelog](https://github.com/uber-go/zap/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber-go/zap/compare/v1.17.0...v1.24.0) --- updated-dependencies: - dependency-name: go.uber.org/zap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.sum | 4 ---- 1 file changed, 4 deletions(-) diff --git a/go.sum b/go.sum index 5210438..9df109f 100644 --- a/go.sum +++ b/go.sum @@ -527,7 +527,6 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= @@ -550,7 +549,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -586,10 +584,8 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 0b18e49ad56045e630c4b5a60be0ca59563f37bd Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Mon, 24 Jul 2023 20:25:51 +0200 Subject: [PATCH 15/25] chore: update to bleve v2 and go 1.20 --- go.sum | 2 -- internal/repository/bleve/bleve_test.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go.sum b/go.sum index 9df109f..b9253d0 100644 --- a/go.sum +++ b/go.sum @@ -549,8 +549,6 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/repository/bleve/bleve_test.go b/internal/repository/bleve/bleve_test.go index 35d179b..c1ad058 100644 --- a/internal/repository/bleve/bleve_test.go +++ b/internal/repository/bleve/bleve_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/v2/mapping" "github.com/unconditionalday/server/internal/app" "github.com/unconditionalday/server/internal/repository/bleve" ) From 9f25282932b2db7dbe2258d3ad81729f4741044b Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Mon, 24 Jul 2023 20:39:58 +0200 Subject: [PATCH 16/25] fix: use auto-merge from gh docs --- .github/workflows/db-auto-merge.yaml | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/db-auto-merge.yaml b/.github/workflows/db-auto-merge.yaml index dc30341..4a70d9b 100644 --- a/.github/workflows/db-auto-merge.yaml +++ b/.github/workflows/db-auto-merge.yaml @@ -1,4 +1,5 @@ <<<<<<< HEAD +<<<<<<< HEAD name: Dependabot auto-merge on: pull_request @@ -23,17 +24,25 @@ jobs: GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} ======= name: auto-merge +======= +name: Dependabot auto-merge +on: pull_request +>>>>>>> b8ec7e1 (fix: use auto-merge from gh docs) -on: - pull_request: +permissions: + contents: write + pull-requests: write jobs: - auto-merge: + dependabot: runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} steps: - - uses: actions/checkout@v2 - - uses: ahmadnassri/action-dependabot-auto-merge@v2 + - name: Dependabot metadata + id: metadata + uses: dependabot/fetch-metadata@v1 with: +<<<<<<< HEAD target: minor <<<<<<< HEAD github-token: ${{ github.token }} @@ -41,3 +50,11 @@ jobs: ======= github-token: ${{ secrets.ACCESS_TOKEN }} >>>>>>> 4b36d7b (chore: use custom private secret for dependabot) +======= + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Enable auto-merge for Dependabot PRs + run: gh pr merge --auto --merge "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} +>>>>>>> b8ec7e1 (fix: use auto-merge from gh docs) From 53398b7fac45b1785e19074fdfb8ac2f7a3e00e1 Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Mon, 24 Jul 2023 20:46:46 +0200 Subject: [PATCH 17/25] chore: dependabot auto-rebase instead of auto-merge --- .github/workflows/db-auto-merge.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/db-auto-merge.yaml b/.github/workflows/db-auto-merge.yaml index 4a70d9b..4626309 100644 --- a/.github/workflows/db-auto-merge.yaml +++ b/.github/workflows/db-auto-merge.yaml @@ -19,6 +19,7 @@ jobs: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs run: gh pr merge --auto --rebase "$PR_URL" +<<<<<<< HEAD env: PR_URL: ${{github.event.pull_request.html_url}} GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} @@ -54,6 +55,8 @@ jobs: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs run: gh pr merge --auto --merge "$PR_URL" +======= +>>>>>>> b345346 (chore: dependabot auto-rebase instead of auto-merge) env: PR_URL: ${{github.event.pull_request.html_url}} GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} From 06ede3b5d282b61b816c8c91ad49cbbf2abfb457 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:00:54 +0000 Subject: [PATCH 18/25] chore(deps): bump github.com/deepmap/oapi-codegen from 1.13.2 to 1.13.3 Bumps [github.com/deepmap/oapi-codegen](https://github.com/deepmap/oapi-codegen) from 1.13.2 to 1.13.3. - [Release notes](https://github.com/deepmap/oapi-codegen/releases) - [Commits](https://github.com/deepmap/oapi-codegen/compare/v1.13.2...v1.13.3) --- updated-dependencies: - dependency-name: github.com/deepmap/oapi-codegen dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.sum | 3 --- 1 file changed, 3 deletions(-) diff --git a/go.sum b/go.sum index b9253d0..aa35317 100644 --- a/go.sum +++ b/go.sum @@ -297,7 +297,6 @@ github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3x github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.11.1 h1:dEpLU2FLg4UVmvCGPuk/APjlH6GDpbEPti61srUUUs4= github.com/labstack/echo/v4 v4.11.1/go.mod h1:YuYRTSM3CHs2ybfrL8Px48bO6BAnYIN4l8wSTMP6BDQ= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= @@ -384,7 +383,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -765,7 +763,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 1b76211716fa4d530a2ab01be4702e031be66bc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Aug 2023 23:02:12 +0000 Subject: [PATCH 19/25] chore(deps): bump github.com/deepmap/oapi-codegen from 1.13.3 to 1.13.4 Bumps [github.com/deepmap/oapi-codegen](https://github.com/deepmap/oapi-codegen) from 1.13.3 to 1.13.4. - [Release notes](https://github.com/deepmap/oapi-codegen/releases) - [Commits](https://github.com/deepmap/oapi-codegen/compare/v1.13.3...v1.13.4) --- updated-dependencies: - dependency-name: github.com/deepmap/oapi-codegen dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- internal/parser/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/parser/parser.go b/internal/parser/parser.go index c034bc0..9adbbdd 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -60,7 +60,7 @@ func removeHTML(content string) string { // function to remove new lines from the content using regex func removeNewLines(content string) string { // regex to match the text to remove - re := regexp.MustCompile(`\r?\n`) + re := regexp.MustCompile(` ?\n`) return re.ReplaceAllString(content, "") } From 4d3051043906d098e179480fff2030064474b1ea Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Wed, 30 Aug 2023 21:24:56 +0200 Subject: [PATCH 20/25] feat: add wiki informer --- api/api.gen.go | 29 ++ api/swagger.yaml | 46 +++ go.sum | 1 + internal/informer/wiki.go | 79 +++++ internal/informer/wiki/cache.go | 109 ++++++ internal/informer/wiki/http.go | 102 ++++++ internal/informer/wiki/models.go | 99 ++++++ internal/informer/wiki/page.go | 554 +++++++++++++++++++++++++++++++ internal/informer/wiki_test.go | 78 +++++ internal/webserver/server.go | 24 ++ internal/x/maps/maps.go | 17 + internal/x/net/http.go | 7 + 12 files changed, 1145 insertions(+) create mode 100644 internal/informer/wiki.go create mode 100644 internal/informer/wiki/cache.go create mode 100644 internal/informer/wiki/http.go create mode 100644 internal/informer/wiki/models.go create mode 100644 internal/informer/wiki/page.go create mode 100644 internal/informer/wiki_test.go create mode 100644 internal/x/maps/maps.go diff --git a/api/api.gen.go b/api/api.gen.go index 11cc0dc..4a2d278 100644 --- a/api/api.gen.go +++ b/api/api.gen.go @@ -66,9 +66,21 @@ type GetV1VersionJSONBody map[string]interface{} // GetV1VersionJSONRequestBody defines body for GetV1Version for application/json ContentType. type GetV1VersionJSONRequestBody GetV1VersionJSONBody +// WikiResult defines model for WikiResult. +type WikiResult struct { + Language string `json:"language"` + Link string `json:"link"` + Summary string `json:"summary"` + Thumbnail string `json:"thumbnail"` + Title string `json:"title"` +} + // ServerInterface represents all server handlers. type ServerInterface interface { + // (GET /v1/informer/wiki/{query}) + GetV1InformerWikiQuery(ctx echo.Context, query string) error + // (GET /v1/search/feed/{query}) GetV1SearchFeedQuery(ctx echo.Context, query string) error // Your GET endpoint @@ -81,6 +93,22 @@ type ServerInterfaceWrapper struct { Handler ServerInterface } +// GetV1InformerWikiQuery converts echo context to params. +func (w *ServerInterfaceWrapper) GetV1InformerWikiQuery(ctx echo.Context) error { + var err error + // ------------- Path parameter "query" ------------- + var query string + + err = runtime.BindStyledParameterWithLocation("simple", false, "query", runtime.ParamLocationPath, ctx.Param("query"), &query) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter query: %s", err)) + } + + // Invoke the callback with all the unmarshalled arguments + err = w.Handler.GetV1InformerWikiQuery(ctx, query) + return err +} + // GetV1SearchFeedQuery converts echo context to params. func (w *ServerInterfaceWrapper) GetV1SearchFeedQuery(ctx echo.Context) error { var err error @@ -134,6 +162,7 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL Handler: si, } + router.GET(baseURL+"/v1/informer/wiki/:query", wrapper.GetV1InformerWikiQuery) router.GET(baseURL+"/v1/search/feed/:query", wrapper.GetV1SearchFeedQuery) router.GET(baseURL+"/v1/version", wrapper.GetV1Version) diff --git a/api/swagger.yaml b/api/swagger.yaml index 0bc53e8..d5a8acc 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -45,6 +45,7 @@ paths: content: application/json: schema: + type: object $ref: "#/components/schemas/ServerVersion" operationId: get-v1-version x-stoplight: @@ -55,6 +56,30 @@ paths: schema: type: object properties: {} + /v1/informer/wiki/{query}: + get: + responses: + "200": + description: OK + content: + application/json: + schema: + type: object + $ref: "#/components/schemas/WikiResult" + "500": + description: Internal Server Error + content: + application/json: + schema: + type: object + $ref: "#/components/schemas/Error" + parameters: + - name: query + in: path + required: true + schema: + type: string + components: schemas: FeedItem: @@ -97,6 +122,27 @@ components: - title x-stoplight: id: g5p7hclip2ydk + + WikiResult: + type: object + properties: + title: + type: string + link: + type: string + summary: + type: string + thumbnail: + type: string + language: + type: string + + required: + - title + - link + - summary + - thumbnail + - language Error: type: object properties: diff --git a/go.sum b/go.sum index aa35317..ba0027f 100644 --- a/go.sum +++ b/go.sum @@ -383,6 +383,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= diff --git a/internal/informer/wiki.go b/internal/informer/wiki.go new file mode 100644 index 0000000..f7ebb16 --- /dev/null +++ b/internal/informer/wiki.go @@ -0,0 +1,79 @@ +package informer + +import ( + "errors" + + "github.com/unconditionalday/server/internal/informer/wiki" +) + +type Wiki struct { + client wiki.Client +} + +func NewWiki() *Wiki { + return &Wiki{ + client: wiki.New(), + } +} + +func (w *Wiki) Search(query string, lang string) (Result, error) { + if query == "" { + return Result{}, errors.New("query string must not be empty") + } + + if lang == "" { + return Result{}, errors.New("language string must not be empty") + } + + args := map[string]string{ + "action": "query", + "list": "search", + "srprop": "", + "srlimit": "1", + "srsearch": query, + } + + res, err := w.client.RequestWikiApi(args, lang) + if err != nil { + return Result{}, err + } + + title := res.Query.Search[0].Title + + wikiPage, err := wiki.MakeWikipediaPage(-1, title, "", false, w.client, lang) + if err != nil { + if err.Error() == "disambiguation" { + return Result{}, errors.New("ambiguous result") + } + } + + summary, err := wikiPage.GetSummary(w.client, lang) + if err != nil { + return Result{}, err + } + + thumbnail, err := wikiPage.GetThumbURL(w.client, lang) + if err != nil { + return Result{}, err + } + + return Result{ + Title: wikiPage.Title, + Language: wikiPage.Language, + Link: wikiPage.URL, + Summary: summary, + Thumbnail: thumbnail, + }, nil +} + +type Result struct { + Title string + Link string + Summary string + Thumbnail string + Language string +} + +func (r Result) IsValid() bool { + return r.Title != "" && r.Link != "" && r.Summary != "" && r.Thumbnail != "" && r.Language != "" +} diff --git a/internal/informer/wiki/cache.go b/internal/informer/wiki/cache.go new file mode 100644 index 0000000..f0ef6c9 --- /dev/null +++ b/internal/informer/wiki/cache.go @@ -0,0 +1,109 @@ +package wiki + +import ( + "crypto/sha256" + "errors" + "time" +) + +var ( + CacheExpiration time.Duration = 12 * time.Hour // Max time that a request can exist in the cache + MaxCacheMemory int = 500 // Max request can exist in the cache +) + +// Find and delete string s in string slice +func FindAndDel(arr []string, s string) []string { + index := 0 + for i, v := range arr { + if v == s { + index = i + break + } + } + return append(arr[:index], arr[index+1:]...) +} + +/* +Create a cache that store: + +- Key: API request URL + +- Value: RequestResponse +*/ +func MakeWikiCache() WikiCache { + res := WikiCache{} + res.Memory = map[string]RequestResult{} + res.HashedKeyQueue = make([]string, 0, MaxCacheMemory) + res.CreatedTime = map[string]time.Time{} + return res +} + +// Cache to store Wikipedia request result +type WikiCache struct { + Memory map[string]RequestResult // Map store request result + HashedKeyQueue []string // Key queue. Delete the first item if reach max cache + CreatedTime map[string]time.Time // Map store created time +} + +// Hash a string into SHA256 +func HashCacheKey(s string) string { + hasher := sha256.New() + hasher.Write([]byte(s)) + return string(hasher.Sum(nil)) +} + +// Get WikiCache current number of cache +func (cache WikiCache) GetLen() int { + return len(cache.HashedKeyQueue) +} + +// Add cache into the WikiCache +func (cache *WikiCache) Add(s string, res RequestResult) { + if len(cache.Memory) >= MaxCacheMemory { + cache.Pop() + } + key := HashCacheKey(s) + if cache.Memory == nil { + cache.Memory = map[string]RequestResult{} + cache.CreatedTime = map[string]time.Time{} + cache.HashedKeyQueue = make([]string, 0, MaxCacheMemory) + } + if _, ok := cache.Memory[key]; !ok { + cache.Memory[key] = res + cache.CreatedTime[key] = time.Now() + cache.HashedKeyQueue = append(cache.HashedKeyQueue, key) + } +} + +// Get response from the Cache +func (cache *WikiCache) Get(s string) (RequestResult, error) { + key := HashCacheKey(s) + if value, ok := cache.Memory[key]; ok { + if time.Since(cache.CreatedTime[key]) <= CacheExpiration { + cache.HashedKeyQueue = FindAndDel(cache.HashedKeyQueue, key) + cache.HashedKeyQueue = append(cache.HashedKeyQueue, key) + return value, nil + } else { + cache.HashedKeyQueue = FindAndDel(cache.HashedKeyQueue, key) + delete(cache.Memory, key) + return RequestResult{}, errors.New("the data is outdated") + } + } + return RequestResult{}, errors.New("cache key not exist") +} + +// Delete the first key in the Cache +func (cache *WikiCache) Pop() { + if len(cache.HashedKeyQueue) == 0 { + return + } + delete(cache.Memory, cache.HashedKeyQueue[0]) + cache.HashedKeyQueue = cache.HashedKeyQueue[1:] +} + +// Clear the whole Cache +func (cache *WikiCache) Clear() { + *cache = WikiCache{} + // This line to avoid declare but not used error + _ = cache +} diff --git a/internal/informer/wiki/http.go b/internal/informer/wiki/http.go new file mode 100644 index 0000000..bc3884e --- /dev/null +++ b/internal/informer/wiki/http.go @@ -0,0 +1,102 @@ +package wiki + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" +) + +type Client struct { + userAgent string + wikiURL string + lastCall time.Time + cache WikiCache +} + +// Create a new WikiClient +func New() Client { + return Client{ + userAgent: "unconditionalday", + wikiURL: "https://%v.wikipedia.org/w/api.php", + lastCall: time.Now(), + cache: MakeWikiCache(), + } +} + +/* +Make a request to the Wikipedia API using the given search parameters. + +Returns a RequestResult (You can see the model in the models.go file) +*/ +func (c *Client) RequestWikiApi(args map[string]string, wikiLang string) (RequestResult, error) { + const ReqPerSec = 199 + const ApiGap = time.Second / ReqPerSec + + url := fmt.Sprintf(c.wikiURL, wikiLang) + // Make new request object + request, err := http.NewRequest("GET", url, nil) + if err != nil { + return RequestResult{}, err + } + // Add header + request.Header.Set("User-Agent", c.userAgent) + q := request.URL.Query() + // Add parameters + if args["format"] == "" { + args["format"] = "json" + } + if args["action"] == "" { + args["action"] = "query" + } + for k, v := range args { + q.Add(k, v) + } + request.URL.RawQuery = q.Encode() + now := time.Now() + if now.Sub(c.lastCall) < ApiGap { + wait := c.lastCall.Add(ApiGap).Sub(now) + time.Sleep(wait) + now = time.Now() + } + // Check in cache + full_url := request.URL.String() + r, err := c.cache.Get(full_url) + if err == nil { + return r, nil + } + + // Make GET request + client := http.Client{Timeout: 10 * time.Second} + res, err := client.Do(request) + defer c.updateLastCall(now) + if err != nil { + return RequestResult{}, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return RequestResult{}, errors.New("unable to fetch the results") + } + // Read body + body, err := io.ReadAll(res.Body) + if err != nil { + return RequestResult{}, err + } + // Parse + var result RequestResult + err = json.Unmarshal([]byte(body), &result) + if err != nil { + return RequestResult{}, err + } + c.cache.Add(full_url, result) + return result, nil +} + +/* +Update the last time we call the API (API should) +*/ +func (c *Client) updateLastCall(now time.Time) { + c.lastCall = now +} diff --git a/internal/informer/wiki/models.go b/internal/informer/wiki/models.go new file mode 100644 index 0000000..ef1114b --- /dev/null +++ b/internal/informer/wiki/models.go @@ -0,0 +1,99 @@ +package wiki + +type RequestError struct { + Code string `json:"code"` + Info string `json:"info"` + Aster string `json:"*"` +} + +type InnerBasic struct { + Aster string `json:"*"` +} + +type InnerSearchInfo struct { + TotalHits int `json:"totalhits"` + Suggestion string `json:"suggestion"` + SuggestionSnippet string `json:"suggestionsnippet"` +} + +type InnerSearch struct { + Ns int `json:"ns"` + Title string `json:"title"` + PageID int `json:"pageid"` + Size int `json:"size"` + Wordcount int `json:"wordcount"` + Snippet string `json:"snippet"` + Timestamp string `json:"timestamp"` +} + +type InnerPage struct { + Ns int `json:"ns"` + Title string `json:"title"` + PageID int `json:"pageid"` + ContentModel string `json:"contentmodel"` + PageLanguage string `json:"pagelanguage"` + PageLanguageTmlCode string `json:"pagelanguagetmlcode"` + PageLanguageDir string `json:"pagelanguagedir"` + Touched string `json:"touched"` + LastRevid int `json:"lastrevid"` + Length int `json:"length"` + FullURL string `json:"fullurl"` + EditURL string `json:"editurl"` + CanonicalURL string `json:"canonicalurl"` + PageProps map[string]string `json:"pageprops"` + Missing string `json:"missing"` + Extract string `json:"extract"` + Revision []map[string]interface{} `json:"revisions"` + Extlink []map[string]string `json:"extlinks"` + Link []map[string]interface{} `json:"links"` + Category []map[string]interface{} `json:"categories"` + Thumbnail Thumbnail `json:"thumbnail"` + PageImage string `json:"pageimage"` + ImageInfo []map[string]string `json:"imageinfo"` + Coordinate []map[string]interface{} `json:"coordinates"` +} + +type Thumbnail struct { + Source string `json:"source"` + Width int `json:"width"` + Height int `json:"height"` +} + +type InnerGeoSearch struct { + PageID int `json:"pageid"` + Ns int `json:"ns"` + Title string `json:"title"` + Latitude float32 `json:"lat"` + Longitude float32 `json:"lom"` + Distance float32 `json:"dist"` + Primary string `json:"primary"` +} + +type InnerNormalize struct { + From string `json:"from"` + To string `json:"to"` +} + +type RequestQuery struct { + SearchInfo InnerSearchInfo `json:"searchinfo"` + Normalize []InnerNormalize `json:"normalized"` + Redirect []InnerNormalize `json:"redirects"` + Search []InnerSearch `json:"search"` + GeoSearch []InnerGeoSearch `json:"geosearch"` + Page map[string]InnerPage `json:"pages"` + Random []InnerSearch `json:"random"` + Language []map[string]string `json:"languages"` +} + +/* +The result of calling the Wikipedia API +*/ +type RequestResult struct { + Error *RequestError `json:"error"` + Warning map[string]InnerBasic `json:"warnings"` + Batchcomplete string `json:"batchcomplete"` + Query *RequestQuery `json:"query"` + Servedby string `json:"servedby"` + Continue map[string]interface{} `json:"continue"` + Parse map[string]interface{} `json:"parse"` +} diff --git a/internal/informer/wiki/page.go b/internal/informer/wiki/page.go new file mode 100644 index 0000000..e30c195 --- /dev/null +++ b/internal/informer/wiki/page.go @@ -0,0 +1,554 @@ +package wiki + +import ( + "errors" + "maps" + "slices" + + mapsx "github.com/unconditionalday/server/internal/x/maps" + netx "github.com/unconditionalday/server/internal/x/net" + + "reflect" + "strconv" + "strings" +) + +// Result after we parse the response of Wikipedia API. +// Some attributes must be get manually using the WikipediaPage methods +type WikipediaPage struct { + PageID int `json:"pageid"` + Title string `json:"title"` + OriginalTitle string `json:"originaltitle"` + Content string `json:"content"` + HTML string `json:"html"` + URL string `json:"fullurl"` + RevisionID float64 `json:"revid"` + ParentID float64 `json:"parentid"` + Summary string `json:"summary"` + CheckedImage bool `json:"checkedimage"` + CheckedThumb bool `json:"checkedthumb"` + Thumbnail string `json:"thumbnail"` + Images []string `json:"images"` + Coordinate []float64 `json:"coordinates"` + Language string `json:"lang"` + Reference []string `json:"references"` + Link []string `json:"links"` + Category []string `json:"categories"` + Section []string `json:"sections"` + SectionOffset map[string][]int `json:"sectionoffset"` + Disambiguation []string `json:"disambiguation"` +} + +/* +Return true if the 2 pages are the same +*/ +func (page WikipediaPage) Equal(other WikipediaPage) bool { + return page.PageID == other.PageID +} + +/* +Get the string content of the page. Save it into the page.Content for later use +*/ +func (page *WikipediaPage) GetContent(client Client, lang string) (string, error) { + if page.Content != "" { + return page.Content, nil + } + pageid := strconv.Itoa(page.PageID) + args := map[string]string{ + "action": "query", + "prop": "extracts|revisions", + "explaintext": "", + "rvprop": "ids", + "titles": page.Title, + } + res, err := client.RequestWikiApi(args, lang) + if err != nil { + return "", err + } + if res.Error.Code != "" { + return "", errors.New(res.Error.Info) + } + page.Content = res.Query.Page[pageid].Extract + page.RevisionID = res.Query.Page[pageid].Revision[0]["revid"].(float64) + page.ParentID = res.Query.Page[pageid].Revision[0]["parentid"].(float64) + + return page.Content, nil +} + +/* +Get the html of the page. Save it into the page.HTML for later use\ + +**Warning:: This can get pretty slow on long pages. +*/ +func (page *WikipediaPage) GetHTML(client Client, lang string) (string, error) { + if page.HTML != "" { + return page.HTML, nil + } + args := map[string]string{ + "action": "query", + "prop": "revisions", + "rvprop": "content", + "rvlimit": strconv.Itoa(1), + "rvparse": "", + "titles": page.Title, + } + res, err := client.RequestWikiApi(args, lang) + if err != nil { + return "", err + } + if res.Error.Code != "" { + return "", errors.New(res.Error.Info) + } + page.HTML = res.Query.Page[strconv.Itoa(page.PageID)].Revision[0]["*"].(string) + return page.HTML, nil +} + +/* +Get the revid of the page. Save it into the page.HTML for later use + +The revision ID is a number that uniquely identifies the current version of the page. +It can be used to create the permalink or for other direct API calls. See Help:Page history +for more information. +*/ +func (page *WikipediaPage) GetRevisionID(client Client, lang string) (float64, error) { + if page.RevisionID != 0 { + return page.RevisionID, nil + } + _, err := page.GetContent(client, lang) + if err != nil { + return -1, err + } + return page.RevisionID, nil +} + +/* +Revision ID of the parent version of the current revision of this page. + +See “revision_id“ for more information. +*/ +func (page *WikipediaPage) GetParentID(client Client, lang string) (float64, error) { + if page.RevisionID != 0 { + return page.ParentID, nil + } + _, err := page.GetContent(client, lang) + if err != nil { + return -1, err + } + return page.ParentID, nil +} + +/* +String summary of a page +*/ +func (page *WikipediaPage) GetSummary(client Client, lang string) (string, error) { + if page.Summary != "" { + return page.Summary, nil + } + + pageid := strconv.Itoa(page.PageID) + args := map[string]string{ + "action": "query", + "prop": "extracts", + "explaintext": "", + "exintro": "", + "exsentences": "3", + "exlimit": "1", + "titles": page.Title, + } + res, err := client.RequestWikiApi(args, lang) + if err != nil { + return "", err + } + + page.Summary = res.Query.Page[pageid].Extract + return page.Summary, nil +} + +/* +Based on +*/ +func (page *WikipediaPage) ContinuedQuery(args map[string]string, client Client, lang string) ([]interface{}, error) { + // args["pageids"] = strconv.Itoa(page.PageID) + args["titles"] = page.Title + last := map[string]interface{}{} + prop := args["prop"] + result := make([]interface{}, 0, 7) + for { + new_args := maps.Clone(args) + mapsx.Update(new_args, last) + + res, err := client.RequestWikiApi(args, lang) + if err != nil { + return result, err + } + if res.Error.Code != "" { + return result, errors.New(res.Error.Info) + } + + if reflect.DeepEqual(RequestQuery{}, res.Query) { + break + } + + if _, ok := args["generator"]; ok { + for _, v := range res.Query.Page { + result = append(result, v) + } + } else { + if prop == "extlinks" { + temp := res.Query.Page[strconv.Itoa(page.PageID)].Extlink + for _, v := range temp { + result = append(result, v["*"]) + } + } else { + temp := []map[string]interface{}{} + switch prop { + case "links": + temp = res.Query.Page[strconv.Itoa(page.PageID)].Link + case "categories": + temp = res.Query.Page[strconv.Itoa(page.PageID)].Category + } + for _, v := range temp { + result = append(result, v["title"].(string)) + } + } + + } + + if len(res.Continue) == 0 { + break + } + + last = res.Continue + } + return result, nil +} + +/* +List of URLs of images on the page. +*/ +func (page *WikipediaPage) GetImagesURL(client Client, lang string) ([]string, error) { + if page.CheckedImage { + return page.Images, nil + } + args := map[string]string{ + "action": "query", + "generator": "images", + "gimlimit": "max", + "prop": "imageinfo", + "iiprop": "url", + } + + res, err := page.ContinuedQuery(args, client, lang) + if err != nil && len(res) == 0 { + return []string{}, err + } + result := make([]string, 0, 7) + for _, v := range res { + temp := v.(InnerPage).ImageInfo + if len(temp) > 0 { + result = append(result, temp[0]["url"]) + } + } + page.CheckedImage = true + page.Images = result + return page.Images, nil +} + +/* +Get Thumbnail URL of the page +*/ +func (page *WikipediaPage) GetThumbURL(client Client, lang string) (string, error) { + if page.CheckedThumb { + return page.Thumbnail, nil + } + + args := map[string]string{ + "action": "query", + "prop": "pageimages", + "titles": page.Title, + "piprop": "thumbnail", + "pithumbsize": "500", + } + + res, err := client.RequestWikiApi(args, lang) + if err != nil { + return "", err + } + + page.CheckedThumb = true + page.Thumbnail = res.Query.Page[strconv.Itoa(page.PageID)].Thumbnail.Source + + return page.Thumbnail, nil +} + +/* +Slice of float64 in the form of (lat, lon) +*/ +func (page *WikipediaPage) GetCoordinate(client Client, lang string) ([]float64, error) { + if len(page.Coordinate) == 2 { + return page.Coordinate, nil + } + args := map[string]string{ + "action": "query", + "prop": "coordinates", + "colimit": "max", + "titles": page.Title, + } + + res, err := client.RequestWikiApi(args, lang) + if err != nil { + return []float64{}, err + } + if res.Error.Code != "" { + return []float64{}, errors.New(res.Error.Info) + } + + if reflect.DeepEqual(RequestQuery{}, res.Query) { + page.Coordinate = []float64{-1, -1} + return page.Coordinate, nil + } else { + temp := res.Query.Page[strconv.Itoa(page.PageID)].Coordinate[0] + page.Coordinate = []float64{temp["lat"].(float64), temp["lon"].(float64)} + } + return page.Coordinate, nil +} + +/* + List of URLs of external links on a page. + May include external links within page that aren't technically cited anywhere. +*/ +func (page *WikipediaPage) GetReference(client Client, lang string) ([]string, error) { + if len(page.Reference) > 0 { + return page.Reference, nil + } + args := map[string]string{ + "action": "query", + "prop": "extlinks", + "ellimit": "max", + } + res, err := page.ContinuedQuery(args, client, lang) + if err != nil && len(res) == 0 { + return []string{}, err + } + for _, v := range res { + page.Reference = append(page.Reference, netx.HelpAddURL(v.(string))) + } + + return page.Reference, nil +} + +/* + List of titles of Wikipedia page links on a page. + **Note:: Only includes articles from namespace 0, meaning no Category, User talk, or other meta-Wikipedia pages. +*/ +func (page *WikipediaPage) GetLink(client Client, lang string) ([]string, error) { + if len(page.Link) > 0 { + return page.Link, nil + } + args := map[string]string{ + "action": "query", + "prop": "links", + "plnamespace": "0", + "pllimit": "max", + } + res, err := page.ContinuedQuery(args, client, lang) + if err != nil && len(res) == 0 { + return []string{}, err + } + for _, v := range res { + page.Link = append(page.Link, v.(string)) + } + return page.Link, nil +} + +/* +List of categories of a page. +*/ +func (page *WikipediaPage) GetCategory(client Client, lang string) ([]string, error) { + if len(page.Category) > 0 { + return page.Category, nil + } + args := map[string]string{ + "action": "query", + "prop": "categories", + "cllimit": "max", + } + res, err := page.ContinuedQuery(args, client, lang) + if err != nil && len(res) == 0 { + return []string{}, err + } + for _, v := range res { + page.Category = append(page.Category, strings.Replace(v.(string), "Category:", "", 1)) + } + return page.Category, nil +} + +/* +List of section titles from the table of contents on the page. +*/ +func (page *WikipediaPage) GetSectionList(client Client, lang string) ([]string, error) { + if len(page.Section) > 0 { + return page.Section, nil + } + args := map[string]string{ + "action": "parse", + "prop": "sections", + } + if page.Title != "" { + args["page"] = page.Title + } + res, err := client.RequestWikiApi(args, lang) + if err != nil { + return []string{}, err + } + if res.Error.Code != "" { + return []string{}, errors.New(res.Error.Info) + } + for _, v := range res.Parse["sections"].([]interface{}) { + page.Section = append(page.Section, v.(map[string]interface{})["line"].(string)) + } + return page.Section, nil +} + +func (page *WikipediaPage) GetSection(section string, client Client, lang string) (string, error) { + sections, err := page.GetSectionList(client, lang) + if err != nil { + return "", err + } + if !slices.Contains(sections, section) { + return "", errors.New("section not exist") + } + content, err := page.GetContent(client, lang) + if err != nil { + return "", err + } + if page.SectionOffset == nil { + page.SectionOffset = map[string][]int{} + } + if value, ok := page.SectionOffset[section]; ok { + return content[value[0]:value[1]], nil + } + sectiontitle := "== " + section + " ==" + start := strings.Index(content, sectiontitle) + len(sectiontitle) + // If you cannot find the section in the content (but it's there in the API for some reason) + if start < len(sectiontitle) { + page.SectionOffset[section] = []int{0, 0} + return "", nil + } + end := start + strings.Index(content[start:], "==") + if end == -1 { + page.SectionOffset[section] = []int{start, len(content)} + return content[start:], nil + } + page.SectionOffset[section] = []int{start, end} + return strings.TrimSpace(strings.TrimLeft(content[start:end], "=")), nil +} + +/* + Load basic information from Wikipedia. + + Confirm that page exists. If it's a disambiguation page, get a list of suggesting +*/ +func MakeWikipediaPage(pageid int, title string, originaltitle string, redirect bool, client Client, lang string) (WikipediaPage, error) { + page := WikipediaPage{} + args := map[string]string{ + "action": "query", + "prop": "info|pageprops", + "inprop": "url", + "ppprop": "disambiguation", + "redirects": "", + } + page.Title = title + page.OriginalTitle = title + if pageid != -1 { + args["pageids"] = strconv.Itoa(pageid) + page.PageID = pageid + } else { + args["titles"] = title + } + if originaltitle != "" { + page.OriginalTitle = originaltitle + } + res, err := client.RequestWikiApi(args, lang) + if err != nil { + return page, err + } + + target := InnerPage{} + target.Missing = "false" + var index string + for i, v := range res.Query.Page { + index = i + target = v + break + } + + if pageid == -1 { + page.PageID = target.PageID + } + if title == "" { + page.Title = target.Title + page.OriginalTitle = target.Title + } + + page.Language = target.PageLanguage + + if target.Missing == "" && index == "-1" { + return page, errors.New("missing") + } + // if field redirects exist + if len(res.Query.Redirect) > 0 { + if !redirect { + return page, errors.New("set the redirect argument to true to allow automatic redirects") + } + tempstr := page.Title + if len(res.Query.Normalize) > 0 { + if res.Query.Normalize[0].From != page.Title { + return page, errors.New("an unexpected weird error, report me if it happened") + } + tempstr = res.Query.Normalize[0].To + } + if tempstr != res.Query.Redirect[0].From { + return page, errors.New("an unexpected weird error, report me if it happened") + } + return MakeWikipediaPage(-1, res.Query.Redirect[0].To, "", redirect, client, lang) + } + + // If the page is a disambiguation page + if _, ok := target.PageProps["disambiguation"]; ok { + return WikipediaPage{}, errors.New("disambiguation") + // args = map[string]string{ + // "action": "query", + // "prop": "revisions", + // "rvprop": "content", + // "rvparse": "", + // "rvlimit": strconv.Itoa(1), + // "titles": page.Title, + // } + // res, err := client.RequestWikiApi(args, lang) + // if err != nil { + // return page, err + // } + + // html := res.Query.Page[strconv.Itoa(page.PageID)].Revision[0]["*"].(string) + // doc := soup.HTMLParse(html) + // links := doc.FindAll("li") + // disa := make([]string, 0, 10) + // for _, link := range links { + // li := link.FindAll("a") + // for _, l := range li { + // if ref, ok := l.Attrs()["title"]; ok { + // if len(ref) >= 1 && !slices.Contains(disa, ref) { + // disa = append(disa, ref) + // } + // } + // } + // } + // page.Disambiguation = disa + // return page, nil + } + + page.URL = target.FullURL + + return page, nil +} diff --git a/internal/informer/wiki_test.go b/internal/informer/wiki_test.go new file mode 100644 index 0000000..f95f1f6 --- /dev/null +++ b/internal/informer/wiki_test.go @@ -0,0 +1,78 @@ +package informer_test + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/unconditionalday/server/internal/informer" +) + +type TestInput struct { + query string + lang string +} + +type TestExpect struct { + validRes bool + err error +} + +func TestWikiSearch(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + input TestInput + output TestExpect + }{ + { + name: "empty query string", + input: TestInput{query: "", lang: "en"}, + output: TestExpect{ + validRes: false, + err: errors.New("query string must not be empty"), + }, + }, + { + name: "empty language", + input: TestInput{query: "Lorem ipsum", lang: ""}, + output: TestExpect{ + validRes: false, + err: errors.New("language string must not be empty"), + }, + }, + { + name: "ambiguous result", + input: TestInput{query: "Rossi", lang: "en"}, + output: TestExpect{ + validRes: false, + err: errors.New("ambiguous result"), + }, + }, + { + name: "valid query", + input: TestInput{query: "Salvini", lang: "en"}, + output: TestExpect{ + validRes: true, + err: nil, + }, + }, + } + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + w := informer.NewWiki() + actual, err := w.Search(tc.input.query, tc.input.lang) + + if tc.output.err != nil { + assert.Equal(t, tc.output.err.Error(), err.Error()) + } else { + assert.Equal(t, tc.output.validRes, actual.IsValid()) + } + }) + } +} diff --git a/internal/webserver/server.go b/internal/webserver/server.go index beb8af1..2af685b 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -111,3 +111,27 @@ func (s *Server) GetV1Version(ctx echo.Context) error { return ctx.JSON(http.StatusOK, v) } + +func (s *Server) GetV1InformerWikiQuery(ctx echo.Context, query string) error { + w := informer.NewWiki() + + wikiRes, err := w.Search(query, "en") + if err != nil { + e := api.Error{ + Code: 500, + Message: "Internal Server Error", + } + + return ctx.JSON(500, e) + } + + res := api.WikiResult{ + Language: wikiRes.Language, + Link: wikiRes.Link, + Summary: wikiRes.Summary, + Thumbnail: wikiRes.Thumbnail, + Title: wikiRes.Title, + } + + return ctx.JSON(200, res) +} diff --git a/internal/x/maps/maps.go b/internal/x/maps/maps.go new file mode 100644 index 0000000..a03742d --- /dev/null +++ b/internal/x/maps/maps.go @@ -0,0 +1,17 @@ +package maps + +import "strconv" + +/* +Update map a using map b +*/ +func Update(a map[string]string, b map[string]interface{}) { + for k, v := range b { + switch t := v.(type) { + case int: + a[k] = strconv.Itoa(t) + case string: + a[k] = t + } + } +} diff --git a/internal/x/net/http.go b/internal/x/net/http.go index b153b8b..f830545 100644 --- a/internal/x/net/http.go +++ b/internal/x/net/http.go @@ -21,3 +21,10 @@ func (h *HttpClient) Download(src string) ([]byte, error) { return ioutil.ReadAll(resp.Body) } + +func HelpAddURL(s string) string { + if s[0:4] == "http" { + return s + } + return "http:" + s +} From 321cf05c18c64b0dc65f3d7fcdeb164b770edb6c Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Wed, 6 Sep 2023 22:28:17 +0200 Subject: [PATCH 21/25] chore: refactor --- cmd/serve.go | 1 + internal/informer/wiki.go | 40 +++--- internal/informer/wiki/cache.go | 57 +++++---- internal/informer/wiki/{http.go => client.go} | 25 ++-- internal/informer/wiki/page.go | 119 +++++++++--------- .../informer/wiki/{models.go => request.go} | 35 +++--- internal/webserver/server.go | 7 +- 7 files changed, 154 insertions(+), 130 deletions(-) rename internal/informer/wiki/{http.go => client.go} (81%) rename internal/informer/wiki/{models.go => request.go} (95%) diff --git a/cmd/serve.go b/cmd/serve.go index a1ce61e..64ef21f 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -10,6 +10,7 @@ import ( "github.com/unconditionalday/server/internal/service" "github.com/unconditionalday/server/internal/version" cobrax "github.com/unconditionalday/server/internal/x/cobra" + "go.uber.org/zap" ) var ( diff --git a/internal/informer/wiki.go b/internal/informer/wiki.go index f7ebb16..cca79b1 100644 --- a/internal/informer/wiki.go +++ b/internal/informer/wiki.go @@ -6,8 +6,13 @@ import ( "github.com/unconditionalday/server/internal/informer/wiki" ) +var ( + ErrEmptyQuery = errors.New("query string must not be empty") + ErrEmptyLanguage = errors.New("language string must not be empty") +) + type Wiki struct { - client wiki.Client + client *wiki.Client } func NewWiki() *Wiki { @@ -16,13 +21,13 @@ func NewWiki() *Wiki { } } -func (w *Wiki) Search(query string, lang string) (Result, error) { +func (w *Wiki) Search(query string, lang string) (WikiInfo, error) { if query == "" { - return Result{}, errors.New("query string must not be empty") + return WikiInfo{}, ErrEmptyQuery } if lang == "" { - return Result{}, errors.New("language string must not be empty") + return WikiInfo{}, ErrEmptyLanguage } args := map[string]string{ @@ -33,31 +38,38 @@ func (w *Wiki) Search(query string, lang string) (Result, error) { "srsearch": query, } - res, err := w.client.RequestWikiApi(args, lang) + res, err := w.client.DoRequest(args, lang) if err != nil { - return Result{}, err + return WikiInfo{}, err + } + + if len(res.Query.Search) == 0 { + return WikiInfo{}, nil } title := res.Query.Search[0].Title wikiPage, err := wiki.MakeWikipediaPage(-1, title, "", false, w.client, lang) + if wikiPage.Disambiguation != nil { + title = wikiPage.Disambiguation[0] + wikiPage, err = wiki.MakeWikipediaPage(-1, title, "", false, w.client, lang) + } + if err != nil { - if err.Error() == "disambiguation" { - return Result{}, errors.New("ambiguous result") - } + return WikiInfo{}, err } summary, err := wikiPage.GetSummary(w.client, lang) if err != nil { - return Result{}, err + return WikiInfo{}, err } thumbnail, err := wikiPage.GetThumbURL(w.client, lang) if err != nil { - return Result{}, err + return WikiInfo{}, err } - return Result{ + return WikiInfo{ Title: wikiPage.Title, Language: wikiPage.Language, Link: wikiPage.URL, @@ -66,7 +78,7 @@ func (w *Wiki) Search(query string, lang string) (Result, error) { }, nil } -type Result struct { +type WikiInfo struct { Title string Link string Summary string @@ -74,6 +86,6 @@ type Result struct { Language string } -func (r Result) IsValid() bool { +func (r WikiInfo) IsValid() bool { return r.Title != "" && r.Link != "" && r.Summary != "" && r.Thumbnail != "" && r.Language != "" } diff --git a/internal/informer/wiki/cache.go b/internal/informer/wiki/cache.go index f0ef6c9..c9893f3 100644 --- a/internal/informer/wiki/cache.go +++ b/internal/informer/wiki/cache.go @@ -6,11 +6,6 @@ import ( "time" ) -var ( - CacheExpiration time.Duration = 12 * time.Hour // Max time that a request can exist in the cache - MaxCacheMemory int = 500 // Max request can exist in the cache -) - // Find and delete string s in string slice func FindAndDel(arr []string, s string) []string { index := 0 @@ -30,43 +25,59 @@ Create a cache that store: - Value: RequestResponse */ -func MakeWikiCache() WikiCache { - res := WikiCache{} - res.Memory = map[string]RequestResult{} - res.HashedKeyQueue = make([]string, 0, MaxCacheMemory) - res.CreatedTime = map[string]time.Time{} - return res +func MakeWikiCache(expiration time.Duration, maxMemory int) *Cache { + if expiration != 0 { + expiration = (12 * time.Hour) + } + + if maxMemory != 0 { + maxMemory = 500 + } + + c := &Cache{ + Memory: map[string]RequestResult{}, + MaxMemory: maxMemory, + Expiration: expiration, + HashedKeyQueue: make([]string, 0, maxMemory), + CreatedTime: map[string]time.Time{}, + } + + return c } // Cache to store Wikipedia request result -type WikiCache struct { +type Cache struct { Memory map[string]RequestResult // Map store request result - HashedKeyQueue []string // Key queue. Delete the first item if reach max cache - CreatedTime map[string]time.Time // Map store created time + HashedKeyQueue []string // Key queue. Delete the first item if reach max cache + CreatedTime map[string]time.Time // Map store created time + Expiration time.Duration // Cache expiration + MaxMemory int // Max cache memory } // Hash a string into SHA256 func HashCacheKey(s string) string { hasher := sha256.New() hasher.Write([]byte(s)) + return string(hasher.Sum(nil)) } // Get WikiCache current number of cache -func (cache WikiCache) GetLen() int { +func (cache Cache) GetLen() int { return len(cache.HashedKeyQueue) } // Add cache into the WikiCache -func (cache *WikiCache) Add(s string, res RequestResult) { - if len(cache.Memory) >= MaxCacheMemory { +func (cache *Cache) Add(s string, res RequestResult) { + if len(cache.Memory) >= cache.MaxMemory { cache.Pop() } + key := HashCacheKey(s) if cache.Memory == nil { cache.Memory = map[string]RequestResult{} cache.CreatedTime = map[string]time.Time{} - cache.HashedKeyQueue = make([]string, 0, MaxCacheMemory) + cache.HashedKeyQueue = make([]string, 0, cache.MaxMemory) } if _, ok := cache.Memory[key]; !ok { cache.Memory[key] = res @@ -76,10 +87,10 @@ func (cache *WikiCache) Add(s string, res RequestResult) { } // Get response from the Cache -func (cache *WikiCache) Get(s string) (RequestResult, error) { +func (cache *Cache) Get(s string) (RequestResult, error) { key := HashCacheKey(s) if value, ok := cache.Memory[key]; ok { - if time.Since(cache.CreatedTime[key]) <= CacheExpiration { + if time.Since(cache.CreatedTime[key]) <= cache.Expiration { cache.HashedKeyQueue = FindAndDel(cache.HashedKeyQueue, key) cache.HashedKeyQueue = append(cache.HashedKeyQueue, key) return value, nil @@ -93,7 +104,7 @@ func (cache *WikiCache) Get(s string) (RequestResult, error) { } // Delete the first key in the Cache -func (cache *WikiCache) Pop() { +func (cache *Cache) Pop() { if len(cache.HashedKeyQueue) == 0 { return } @@ -102,8 +113,8 @@ func (cache *WikiCache) Pop() { } // Clear the whole Cache -func (cache *WikiCache) Clear() { - *cache = WikiCache{} +func (cache *Cache) Clear() { + *cache = Cache{} // This line to avoid declare but not used error _ = cache } diff --git a/internal/informer/wiki/http.go b/internal/informer/wiki/client.go similarity index 81% rename from internal/informer/wiki/http.go rename to internal/informer/wiki/client.go index bc3884e..61dfc83 100644 --- a/internal/informer/wiki/http.go +++ b/internal/informer/wiki/client.go @@ -11,18 +11,23 @@ import ( type Client struct { userAgent string - wikiURL string + URL string lastCall time.Time - cache WikiCache + cache *Cache } +const ( + cacheExpiration = 12 * time.Hour + maxCacheMemory = 500 +) + // Create a new WikiClient -func New() Client { - return Client{ - userAgent: "unconditionalday", - wikiURL: "https://%v.wikipedia.org/w/api.php", +func New() *Client { + return &Client{ + userAgent: "unconditional.day", + URL: "https://%v.wikipedia.org/w/api.php", lastCall: time.Now(), - cache: MakeWikiCache(), + cache: MakeWikiCache(cacheExpiration, maxCacheMemory), } } @@ -31,11 +36,11 @@ Make a request to the Wikipedia API using the given search parameters. Returns a RequestResult (You can see the model in the models.go file) */ -func (c *Client) RequestWikiApi(args map[string]string, wikiLang string) (RequestResult, error) { +func (c *Client) DoRequest(args map[string]string, wikiLang string) (RequestResult, error) { const ReqPerSec = 199 const ApiGap = time.Second / ReqPerSec - - url := fmt.Sprintf(c.wikiURL, wikiLang) + + url := fmt.Sprintf(c.URL, wikiLang) // Make new request object request, err := http.NewRequest("GET", url, nil) if err != nil { diff --git a/internal/informer/wiki/page.go b/internal/informer/wiki/page.go index e30c195..6ff3416 100644 --- a/internal/informer/wiki/page.go +++ b/internal/informer/wiki/page.go @@ -2,11 +2,12 @@ package wiki import ( "errors" - "maps" - "slices" + "github.com/anaskhan96/soup" mapsx "github.com/unconditionalday/server/internal/x/maps" netx "github.com/unconditionalday/server/internal/x/net" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" "reflect" "strconv" @@ -49,7 +50,7 @@ func (page WikipediaPage) Equal(other WikipediaPage) bool { /* Get the string content of the page. Save it into the page.Content for later use */ -func (page *WikipediaPage) GetContent(client Client, lang string) (string, error) { +func (page *WikipediaPage) GetContent(client *Client, lang string) (string, error) { if page.Content != "" { return page.Content, nil } @@ -61,13 +62,11 @@ func (page *WikipediaPage) GetContent(client Client, lang string) (string, error "rvprop": "ids", "titles": page.Title, } - res, err := client.RequestWikiApi(args, lang) + res, err := client.DoRequest(args, lang) if err != nil { return "", err } - if res.Error.Code != "" { - return "", errors.New(res.Error.Info) - } + page.Content = res.Query.Page[pageid].Extract page.RevisionID = res.Query.Page[pageid].Revision[0]["revid"].(float64) page.ParentID = res.Query.Page[pageid].Revision[0]["parentid"].(float64) @@ -80,7 +79,7 @@ Get the html of the page. Save it into the page.HTML for later use\ **Warning:: This can get pretty slow on long pages. */ -func (page *WikipediaPage) GetHTML(client Client, lang string) (string, error) { +func (page *WikipediaPage) GetHTML(client *Client, lang string) (string, error) { if page.HTML != "" { return page.HTML, nil } @@ -92,7 +91,7 @@ func (page *WikipediaPage) GetHTML(client Client, lang string) (string, error) { "rvparse": "", "titles": page.Title, } - res, err := client.RequestWikiApi(args, lang) + res, err := client.DoRequest(args, lang) if err != nil { return "", err } @@ -110,7 +109,7 @@ The revision ID is a number that uniquely identifies the current version of the It can be used to create the permalink or for other direct API calls. See Help:Page history for more information. */ -func (page *WikipediaPage) GetRevisionID(client Client, lang string) (float64, error) { +func (page *WikipediaPage) GetRevisionID(client *Client, lang string) (float64, error) { if page.RevisionID != 0 { return page.RevisionID, nil } @@ -126,7 +125,7 @@ Revision ID of the parent version of the current revision of this page. See “revision_id“ for more information. */ -func (page *WikipediaPage) GetParentID(client Client, lang string) (float64, error) { +func (page *WikipediaPage) GetParentID(client *Client, lang string) (float64, error) { if page.RevisionID != 0 { return page.ParentID, nil } @@ -140,7 +139,7 @@ func (page *WikipediaPage) GetParentID(client Client, lang string) (float64, err /* String summary of a page */ -func (page *WikipediaPage) GetSummary(client Client, lang string) (string, error) { +func (page *WikipediaPage) GetSummary(client *Client, lang string) (string, error) { if page.Summary != "" { return page.Summary, nil } @@ -155,7 +154,7 @@ func (page *WikipediaPage) GetSummary(client Client, lang string) (string, error "exlimit": "1", "titles": page.Title, } - res, err := client.RequestWikiApi(args, lang) + res, err := client.DoRequest(args, lang) if err != nil { return "", err } @@ -167,7 +166,7 @@ func (page *WikipediaPage) GetSummary(client Client, lang string) (string, error /* Based on */ -func (page *WikipediaPage) ContinuedQuery(args map[string]string, client Client, lang string) ([]interface{}, error) { +func (page *WikipediaPage) ContinuedQuery(args map[string]string, client *Client, lang string) ([]interface{}, error) { // args["pageids"] = strconv.Itoa(page.PageID) args["titles"] = page.Title last := map[string]interface{}{} @@ -177,7 +176,7 @@ func (page *WikipediaPage) ContinuedQuery(args map[string]string, client Client, new_args := maps.Clone(args) mapsx.Update(new_args, last) - res, err := client.RequestWikiApi(args, lang) + res, err := client.DoRequest(args, lang) if err != nil { return result, err } @@ -226,7 +225,7 @@ func (page *WikipediaPage) ContinuedQuery(args map[string]string, client Client, /* List of URLs of images on the page. */ -func (page *WikipediaPage) GetImagesURL(client Client, lang string) ([]string, error) { +func (page *WikipediaPage) GetImagesURL(client *Client, lang string) ([]string, error) { if page.CheckedImage { return page.Images, nil } @@ -257,7 +256,7 @@ func (page *WikipediaPage) GetImagesURL(client Client, lang string) ([]string, e /* Get Thumbnail URL of the page */ -func (page *WikipediaPage) GetThumbURL(client Client, lang string) (string, error) { +func (page *WikipediaPage) GetThumbURL(client *Client, lang string) (string, error) { if page.CheckedThumb { return page.Thumbnail, nil } @@ -270,7 +269,7 @@ func (page *WikipediaPage) GetThumbURL(client Client, lang string) (string, erro "pithumbsize": "500", } - res, err := client.RequestWikiApi(args, lang) + res, err := client.DoRequest(args, lang) if err != nil { return "", err } @@ -284,7 +283,7 @@ func (page *WikipediaPage) GetThumbURL(client Client, lang string) (string, erro /* Slice of float64 in the form of (lat, lon) */ -func (page *WikipediaPage) GetCoordinate(client Client, lang string) ([]float64, error) { +func (page *WikipediaPage) GetCoordinate(client *Client, lang string) ([]float64, error) { if len(page.Coordinate) == 2 { return page.Coordinate, nil } @@ -295,7 +294,7 @@ func (page *WikipediaPage) GetCoordinate(client Client, lang string) ([]float64, "titles": page.Title, } - res, err := client.RequestWikiApi(args, lang) + res, err := client.DoRequest(args, lang) if err != nil { return []float64{}, err } @@ -317,7 +316,7 @@ func (page *WikipediaPage) GetCoordinate(client Client, lang string) ([]float64, List of URLs of external links on a page. May include external links within page that aren't technically cited anywhere. */ -func (page *WikipediaPage) GetReference(client Client, lang string) ([]string, error) { +func (page *WikipediaPage) GetReference(client *Client, lang string) ([]string, error) { if len(page.Reference) > 0 { return page.Reference, nil } @@ -341,7 +340,7 @@ func (page *WikipediaPage) GetReference(client Client, lang string) ([]string, e List of titles of Wikipedia page links on a page. **Note:: Only includes articles from namespace 0, meaning no Category, User talk, or other meta-Wikipedia pages. */ -func (page *WikipediaPage) GetLink(client Client, lang string) ([]string, error) { +func (page *WikipediaPage) GetLink(client *Client, lang string) ([]string, error) { if len(page.Link) > 0 { return page.Link, nil } @@ -364,7 +363,7 @@ func (page *WikipediaPage) GetLink(client Client, lang string) ([]string, error) /* List of categories of a page. */ -func (page *WikipediaPage) GetCategory(client Client, lang string) ([]string, error) { +func (page *WikipediaPage) GetCategory(client *Client, lang string) ([]string, error) { if len(page.Category) > 0 { return page.Category, nil } @@ -386,7 +385,7 @@ func (page *WikipediaPage) GetCategory(client Client, lang string) ([]string, er /* List of section titles from the table of contents on the page. */ -func (page *WikipediaPage) GetSectionList(client Client, lang string) ([]string, error) { +func (page *WikipediaPage) GetSectionList(client *Client, lang string) ([]string, error) { if len(page.Section) > 0 { return page.Section, nil } @@ -397,20 +396,18 @@ func (page *WikipediaPage) GetSectionList(client Client, lang string) ([]string, if page.Title != "" { args["page"] = page.Title } - res, err := client.RequestWikiApi(args, lang) + res, err := client.DoRequest(args, lang) if err != nil { return []string{}, err } - if res.Error.Code != "" { - return []string{}, errors.New(res.Error.Info) - } + for _, v := range res.Parse["sections"].([]interface{}) { page.Section = append(page.Section, v.(map[string]interface{})["line"].(string)) } return page.Section, nil } -func (page *WikipediaPage) GetSection(section string, client Client, lang string) (string, error) { +func (page *WikipediaPage) GetSection(section string, client *Client, lang string) (string, error) { sections, err := page.GetSectionList(client, lang) if err != nil { return "", err @@ -449,7 +446,7 @@ func (page *WikipediaPage) GetSection(section string, client Client, lang string Confirm that page exists. If it's a disambiguation page, get a list of suggesting */ -func MakeWikipediaPage(pageid int, title string, originaltitle string, redirect bool, client Client, lang string) (WikipediaPage, error) { +func MakeWikipediaPage(pageid int, title string, originaltitle string, redirect bool, client *Client, lang string) (WikipediaPage, error) { page := WikipediaPage{} args := map[string]string{ "action": "query", @@ -469,7 +466,7 @@ func MakeWikipediaPage(pageid int, title string, originaltitle string, redirect if originaltitle != "" { page.OriginalTitle = originaltitle } - res, err := client.RequestWikiApi(args, lang) + res, err := client.DoRequest(args, lang) if err != nil { return page, err } @@ -515,37 +512,37 @@ func MakeWikipediaPage(pageid int, title string, originaltitle string, redirect } // If the page is a disambiguation page + // TODO: Needs more love here to get the disambiguation list of pages in the right way if _, ok := target.PageProps["disambiguation"]; ok { - return WikipediaPage{}, errors.New("disambiguation") - // args = map[string]string{ - // "action": "query", - // "prop": "revisions", - // "rvprop": "content", - // "rvparse": "", - // "rvlimit": strconv.Itoa(1), - // "titles": page.Title, - // } - // res, err := client.RequestWikiApi(args, lang) - // if err != nil { - // return page, err - // } - - // html := res.Query.Page[strconv.Itoa(page.PageID)].Revision[0]["*"].(string) - // doc := soup.HTMLParse(html) - // links := doc.FindAll("li") - // disa := make([]string, 0, 10) - // for _, link := range links { - // li := link.FindAll("a") - // for _, l := range li { - // if ref, ok := l.Attrs()["title"]; ok { - // if len(ref) >= 1 && !slices.Contains(disa, ref) { - // disa = append(disa, ref) - // } - // } - // } - // } - // page.Disambiguation = disa - // return page, nil + args = map[string]string{ + "action": "query", + "prop": "revisions", + "rvprop": "content", + "rvparse": "", + "rvlimit": "1", + "titles": page.Title, + } + res, err := client.DoRequest(args, lang) + if err != nil { + return page, err + } + + html := res.Query.Page[strconv.Itoa(page.PageID)].Revision[0]["*"].(string) + doc := soup.HTMLParse(html) + links := doc.FindAll("li") + disa := make([]string, 0, 10) + for _, link := range links { + li := link.FindAll("a") + for _, l := range li { + if ref, ok := l.Attrs()["title"]; ok { + if len(ref) >= 1 && !slices.Contains(disa, ref) { + disa = append(disa, ref) + } + } + } + } + page.Disambiguation = disa + return page, nil } page.URL = target.FullURL diff --git a/internal/informer/wiki/models.go b/internal/informer/wiki/request.go similarity index 95% rename from internal/informer/wiki/models.go rename to internal/informer/wiki/request.go index ef1114b..571bbda 100644 --- a/internal/informer/wiki/models.go +++ b/internal/informer/wiki/request.go @@ -1,11 +1,5 @@ package wiki -type RequestError struct { - Code string `json:"code"` - Info string `json:"info"` - Aster string `json:"*"` -} - type InnerBasic struct { Aster string `json:"*"` } @@ -74,6 +68,22 @@ type InnerNormalize struct { To string `json:"to"` } +type RequestResult struct { + Error *RequestError `json:"error"` + Warning map[string]InnerBasic `json:"warnings"` + Batchcomplete string `json:"batchcomplete"` + Query *RequestQuery `json:"query"` + Servedby string `json:"servedby"` + Continue map[string]interface{} `json:"continue"` + Parse map[string]interface{} `json:"parse"` +} + +type RequestError struct { + Code string `json:"code"` + Info string `json:"info"` + Aster string `json:"*"` +} + type RequestQuery struct { SearchInfo InnerSearchInfo `json:"searchinfo"` Normalize []InnerNormalize `json:"normalized"` @@ -84,16 +94,3 @@ type RequestQuery struct { Random []InnerSearch `json:"random"` Language []map[string]string `json:"languages"` } - -/* -The result of calling the Wikipedia API -*/ -type RequestResult struct { - Error *RequestError `json:"error"` - Warning map[string]InnerBasic `json:"warnings"` - Batchcomplete string `json:"batchcomplete"` - Query *RequestQuery `json:"query"` - Servedby string `json:"servedby"` - Continue map[string]interface{} `json:"continue"` - Parse map[string]interface{} `json:"parse"` -} diff --git a/internal/webserver/server.go b/internal/webserver/server.go index 2af685b..b9bf9e4 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -113,15 +113,16 @@ func (s *Server) GetV1Version(ctx echo.Context) error { } func (s *Server) GetV1InformerWikiQuery(ctx echo.Context, query string) error { - w := informer.NewWiki() - - wikiRes, err := w.Search(query, "en") + // TODO: add language support + wikiRes, err := s.wiki.Search(query, "en") if err != nil { e := api.Error{ Code: 500, Message: "Internal Server Error", } + s.logger.Error("wiki search", zap.Error(err)) + return ctx.JSON(500, e) } From 6c53fbc56032c73f194c6e8b16e6ed4159f81d0a Mon Sep 17 00:00:00 2001 From: Luigi Barbato Date: Wed, 6 Sep 2023 22:32:32 +0200 Subject: [PATCH 22/25] chore: remove old test case --- internal/informer/wiki_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/informer/wiki_test.go b/internal/informer/wiki_test.go index f95f1f6..8b45281 100644 --- a/internal/informer/wiki_test.go +++ b/internal/informer/wiki_test.go @@ -42,14 +42,6 @@ func TestWikiSearch(t *testing.T) { err: errors.New("language string must not be empty"), }, }, - { - name: "ambiguous result", - input: TestInput{query: "Rossi", lang: "en"}, - output: TestExpect{ - validRes: false, - err: errors.New("ambiguous result"), - }, - }, { name: "valid query", input: TestInput{query: "Salvini", lang: "en"}, From 90404a0b36a97c81e3e0521a516ad528b72d7f0d Mon Sep 17 00:00:00 2001 From: Luigi Date: Sun, 8 Oct 2023 21:02:46 +0200 Subject: [PATCH 23/25] chore: use search service --- api/api.gen.go | 40 ++++-- go.mod | 3 +- go.sum | 5 + internal/app/feed.go | 11 -- .../wiki => client/wikipedia}/cache.go | 2 +- .../wiki => client/wikipedia}/client.go | 93 +++++++++++- .../wikipedia/client_test.go} | 20 +-- .../wiki => client/wikipedia}/page.go | 20 +-- .../wiki => client/wikipedia}/request.go | 2 +- internal/container/container.go | 15 +- internal/informer/wiki.go | 91 ------------ internal/repository/bleve/bleve_test.go | 134 ------------------ internal/search/search.go | 18 +++ internal/webserver/server.go | 21 +-- 14 files changed, 187 insertions(+), 288 deletions(-) rename internal/{informer/wiki => client/wikipedia}/cache.go (99%) rename internal/{informer/wiki => client/wikipedia}/client.go (50%) rename internal/{informer/wiki_test.go => client/wikipedia/client_test.go} (71%) rename internal/{informer/wiki => client/wikipedia}/page.go (97%) rename internal/{informer/wiki => client/wikipedia}/request.go (99%) delete mode 100644 internal/informer/wiki.go delete mode 100644 internal/repository/bleve/bleve_test.go create mode 100644 internal/search/search.go diff --git a/api/api.gen.go b/api/api.gen.go index 4a2d278..0dbb7da 100644 --- a/api/api.gen.go +++ b/api/api.gen.go @@ -60,6 +60,15 @@ type SourceReleaseVersion struct { Version string `json:"version"` } +// WikiResult defines model for WikiResult. +type WikiResult struct { + Language string `json:"language"` + Link string `json:"link"` + Summary string `json:"summary"` + Thumbnail string `json:"thumbnail"` + Title string `json:"title"` +} + // GetV1VersionJSONBody defines parameters for GetV1Version. type GetV1VersionJSONBody map[string]interface{} @@ -171,21 +180,22 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/6RWXW/iOBT9K+juPqYkoVBo3mak2RHah/2Y2ZFWoz6Y5EIMjm3smwyhyn9fOUlJWkJL", - "d56K4ut7js85vu4jxCrTSqIkC9Ej2DjFjNU/PxmjjPuhjdJoiGP9OVYJur9UaoQIuCTcoIHKgwytZZv+", - "oiXD5QaqygOD+5wbTCD6fir0mmYP3lO9Wm0xJvDgcGNJacE3Kbl2PIEINsl2MTV4OGSHNdU9f0NMllkL", - "+ZwlcRJDTDzIjXiboSvy2ibX0pvpeRoLridlsuvoEWbn7BJGNbm1MhkjiOoPN8Qzp8kZY/50xF8NriGC", - "X/zOM781zO+0qDwQTG7yYSs8EFzuBhesyk08vMfmWcZMObh2SeoXkjZlXauWyAm2x9prBLpSeBOm+XEz", - "E/eq3Oka9guaAs3HnIvkGxrLlRyKcZZxOqc9jLFbxfvJ4jZf5EEa1BhF1/iaBpaz253e7tkhDH9A9VKc", - "lk3X9uGk69BprhNmbmYrKtj0MAvL254wFzVZOYy3gjZA51l2Xt1bV/2NApnF0+4XWpzy0NA5U+J9Ihzv", - "7KIM87u9KII2HUMkzrQQzNI/2uUw+XB1TA7biRRyWoRzfUz+T0xwtVciZbPj+nAvz2NSnI7+nF5fo6HT", - "XSfV/XZj9SqmnSikceBu9si16s1TWFolHOjoa8rlbnQzavBGH/5c9tIbQTgOXCyURsk0hwhux8F4Ah5o", - "RmktsF+EvkVm4tRfIyb+4z5HU1ZuaYM1KWcHI67k0rH7jPQt/FJvcLPuL1dd9zMsQ0JjIfr+CNyBOwzw", - "QLLMUd63lZ2QZHL02qduaHA9uGKrlbRNFiZB0IwMSShrakxrweOanL+1jb9dP06Y2asGtnsdqpM7zBhW", - "Nq4naGPDNTVq/vG7q5q9k8Zr6M3jPgC1lIRGMjFqLtvoqXA4MprF2fHeFHMW7sKmnfO1l/rLXnbhdM6g", - "pY8qKd91wOehrtrr8hPGvT33uqE1aFL/qYR/VW5Gnz99HaFMtOLy0r1bC7OapvhjcVeEtj2GrdGaTNf/", - "sUBKpG3k+814HPP2IpK7h2NCS34RQuX1yyPfFypmIlWWokWwCHrr4WQ+DsbBOGwXHi6MTxXOpuEqvpuq", - "g4Kq+i8AAP//ZNYHKjAKAAA=", + "H4sIAAAAAAAC/8xW32/bNhD+V4LbHlVLcuzY0dsKdEWwh2Ht2mEo8kBLZ4s2f5mkFMuB/veBlCIpsZy6", + "RQf0yYZ4vPvu+44f+Qip5EoKFNZA8ggmzZET//ed1lK7P0pLhdpS9J9TmaH7tZVCSIAKixvUUAfA0Riy", + "GS4aq6nYQF0HoHFfUI0ZJF+6wKBJdh88xcvVFlMLARzeGCsVo5vcunQ0gwQ22XY503g48MPa+py/I2Z3", + "vC35HKWllo0hCaDQ7OsIXVDQJrkU3lwt8pRRNa2yXQ/PIj9FlxHrwa2l5sRC4j+8sZQ7Tk4Q06cWf9W4", + "hgR+CXvNwlawsOeiDoARsSnGpQiAUbEbXTCy0On4HlNwTnQ1unaO6heUNmF9qhZIV3aAOmgIupB4HefF", + "cTNnt7LaKV/2I+oS9duCsuwzakOlGBtjzqk9hT1eY7dK99PldbEsojzyNco+8SUJDCXXO7Xdk0McP0D9", + "kpwWTZ/2vuN1rJvLiFno+cqWZHaYx9X1gJiznKxcja8N2gicZ7Pz6l4f9QEZEoPd7hdcdPPQwDlh4ttI", + "ON6YZRUXN3tWRu10jIE44YIRYz8pN4fZbxePyWE7FUzMynihjtn3jAmu9pLlZH5cH27F6ZiUXevP4Q05", + "GuvuMqputxujVqndsVJoj/4fuqMf0BTMjhH0XR7zmpHkBV8JQtmPsJknc+ncps8+cJoTi/GUU7GWg0sE", + "PolUioxaKgVhg0OaQDyJHDapUBBFIYHrSTSZQgCK2NzTFJZx6PJpjjp8oDsaPu4L1FXtFjfoiXW0Epf9", + "zsnwHu3n+K7d4hT4y8X7nJpwtKgNJF8egToArg4EIAh3QPdtZE+I1QUG7a0+Rt69CzZKCtOoOo2ixh2F", + "ReHBEaUYTT28cGuaUe7zvXbaB9Pjac3QpJoq21D35x+OufkPrNc8WEZK3QmLWhB21RjIVRdYB14fg0Sn", + "ebhGzC5T56Pf4K7cn1kbapGbi94N7pFSd0eBaE2qn1SycedSJOXHW10uSLyLodN1YL7ntew90imDxr6V", + "WfVNDZ44SP0/Hqrnd+AZkYZGC//KQl+9f/f3FYpMSSrO2f+a6dUsx4flTRmbtg3jqzUz7R/OkFurTBKG", + "RNFJMfTFSUaqsIyhDoahSRgymRKWS2OTZbSMBuvxdDGJJtEkbhfuz9zgMp7P4lV6M5MHCXX9XwAAAP//", + "MmHoN7MMAAA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/go.mod b/go.mod index 8a10013..51a2495 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/unconditionalday/server go 1.21 require ( + github.com/anaskhan96/soup v1.2.5 github.com/deepmap/oapi-codegen v1.15.0 github.com/getkin/kin-openapi v0.120.0 github.com/labstack/echo/v4 v4.11.1 @@ -10,6 +11,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/pflag v1.0.5 go.uber.org/zap v1.26.0 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) require ( @@ -111,7 +113,6 @@ require ( go.uber.org/multierr v1.10.0 // indirect golang.org/x/arch v0.4.0 // indirect golang.org/x/crypto v0.13.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.15.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index ba0027f..8dd7b9b 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/SlyMarbo/rss v1.0.5 h1:DPcZ4aOXXHJ5yNLXY1q/57frIixMmAvTtLxDE3fsMEI= github.com/SlyMarbo/rss v1.0.5/go.mod h1:w6Bhn1BZs91q4OlEnJVZEUNRJmlbFmV7BkAlgCN8ofM= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/anaskhan96/soup v1.2.5 h1:V/FHiusdTrPrdF4iA1YkVxsOpdNcgvqT1hG+YtcZ5hM= +github.com/anaskhan96/soup v1.2.5/go.mod h1:6YnEp9A2yywlYdM4EgDz9NEHclocMepEtku7wg6Cq3s= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= @@ -526,6 +528,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= @@ -548,6 +551,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -586,6 +590,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/app/feed.go b/internal/app/feed.go index 012d327..a3ca353 100644 --- a/internal/app/feed.go +++ b/internal/app/feed.go @@ -17,17 +17,6 @@ type FeedRepository interface { Delete(doc Feed) error } -type FeedRepository interface { - // Search returns the results of a search query. - Find(query string) ([]Feed, error) - // Index indexes a document. - Save(doc Feed) error - // Delete deletes a document. - Delete(doc Feed) error - // Close closes the database. - Close() error -} - type Feed struct { Title string `json:"title"` Link string `json:"link"` diff --git a/internal/informer/wiki/cache.go b/internal/client/wikipedia/cache.go similarity index 99% rename from internal/informer/wiki/cache.go rename to internal/client/wikipedia/cache.go index c9893f3..f1d9a55 100644 --- a/internal/informer/wiki/cache.go +++ b/internal/client/wikipedia/cache.go @@ -1,4 +1,4 @@ -package wiki +package wikipedia import ( "crypto/sha256" diff --git a/internal/informer/wiki/client.go b/internal/client/wikipedia/client.go similarity index 50% rename from internal/informer/wiki/client.go rename to internal/client/wikipedia/client.go index 61dfc83..d89cdb9 100644 --- a/internal/informer/wiki/client.go +++ b/internal/client/wikipedia/client.go @@ -1,4 +1,4 @@ -package wiki +package wikipedia import ( "encoding/json" @@ -7,6 +7,8 @@ import ( "io" "net/http" "time" + + "github.com/unconditionalday/server/internal/search" ) type Client struct { @@ -21,8 +23,13 @@ const ( maxCacheMemory = 500 ) +var ( + ErrEmptyQuery = errors.New("query string must not be empty") + ErrEmptyLanguage = errors.New("language string must not be empty") +) + // Create a new WikiClient -func New() *Client { +func NewClient() *Client { return &Client{ userAgent: "unconditional.day", URL: "https://%v.wikipedia.org/w/api.php", @@ -34,9 +41,9 @@ func New() *Client { /* Make a request to the Wikipedia API using the given search parameters. -Returns a RequestResult (You can see the model in the models.go file) +Returns a RequestResult */ -func (c *Client) DoRequest(args map[string]string, wikiLang string) (RequestResult, error) { +func (c *Client) doRequest(args map[string]string, wikiLang string) (RequestResult, error) { const ReqPerSec = 199 const ApiGap = time.Second / ReqPerSec @@ -105,3 +112,81 @@ Update the last time we call the API (API should) func (c *Client) updateLastCall(now time.Time) { c.lastCall = now } + +func (w *Client) FetchEntityDetails(query string, lang string) (search.EntityDetails, error) { + if query == "" { + return search.EntityDetails{}, ErrEmptyQuery + } + + if lang == "" { + return search.EntityDetails{}, ErrEmptyLanguage + } + + args := map[string]string{ + "action": "query", + "list": "search", + "srprop": "", + "srlimit": "1", + "srsearch": query, + } + + res, err := w.doRequest(args, lang) + if err != nil { + return search.EntityDetails{}, err + } + + if len(res.Query.Search) == 0 { + return search.EntityDetails{}, nil + } + + title := res.Query.Search[0].Title + + wikiPage, err := MakeWikipediaPage(-1, title, "", false, w, lang) + if len(wikiPage.Disambiguation) != 0 { + title = wikiPage.Disambiguation[0] + wikiPage, err = MakeWikipediaPage(-1, title, "", false, w, lang) + } + + if err != nil { + return search.EntityDetails{}, err + } + + summary, err := wikiPage.GetSummary(w, lang) + if err != nil { + return search.EntityDetails{}, err + } + + thumbnail, err := wikiPage.GetThumbURL(w, lang) + if err != nil { + return search.EntityDetails{}, err + } + + return search.EntityDetails{ + Title: wikiPage.Title, + Language: wikiPage.Language, + Link: wikiPage.URL, + Source: "Wikipedia", + Summary: summary, + Thumbnail: thumbnail, + }, nil +} + +func (c *Client) Suggest(_input, lang string) (string, error) { + args := map[string]string{ + "action": "query", + "list": "search", + "srlimit": "1", + "srprop": "", + "srinfo": "suggestion", + "srsearch": _input, + } + + res, err := c.doRequest(args, lang) + if err != nil { + return "", err + } + if res.Error.Code != "" { + return "", errors.New(res.Error.Info) + } + return res.Query.SearchInfo.Suggestion, nil +} diff --git a/internal/informer/wiki_test.go b/internal/client/wikipedia/client_test.go similarity index 71% rename from internal/informer/wiki_test.go rename to internal/client/wikipedia/client_test.go index 8b45281..ff97fe6 100644 --- a/internal/informer/wiki_test.go +++ b/internal/client/wikipedia/client_test.go @@ -1,11 +1,11 @@ -package informer_test +package wikipedia_test import ( "errors" "testing" "github.com/stretchr/testify/assert" - "github.com/unconditionalday/server/internal/informer" + "github.com/unconditionalday/server/internal/client/wikipedia" ) type TestInput struct { @@ -15,15 +15,15 @@ type TestInput struct { type TestExpect struct { validRes bool - err error + err error } func TestWikiSearch(t *testing.T) { t.Parallel() testCases := []struct { - name string - input TestInput + name string + input TestInput output TestExpect }{ { @@ -31,7 +31,7 @@ func TestWikiSearch(t *testing.T) { input: TestInput{query: "", lang: "en"}, output: TestExpect{ validRes: false, - err: errors.New("query string must not be empty"), + err: errors.New("query string must not be empty"), }, }, { @@ -39,7 +39,7 @@ func TestWikiSearch(t *testing.T) { input: TestInput{query: "Lorem ipsum", lang: ""}, output: TestExpect{ validRes: false, - err: errors.New("language string must not be empty"), + err: errors.New("language string must not be empty"), }, }, { @@ -47,7 +47,7 @@ func TestWikiSearch(t *testing.T) { input: TestInput{query: "Salvini", lang: "en"}, output: TestExpect{ validRes: true, - err: nil, + err: nil, }, }, } @@ -57,8 +57,8 @@ func TestWikiSearch(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - w := informer.NewWiki() - actual, err := w.Search(tc.input.query, tc.input.lang) + w := wikipedia.NewClient() + actual, err := w.FetchEntityDetails(tc.input.query, tc.input.lang) if tc.output.err != nil { assert.Equal(t, tc.output.err.Error(), err.Error()) diff --git a/internal/informer/wiki/page.go b/internal/client/wikipedia/page.go similarity index 97% rename from internal/informer/wiki/page.go rename to internal/client/wikipedia/page.go index 6ff3416..9b9ffca 100644 --- a/internal/informer/wiki/page.go +++ b/internal/client/wikipedia/page.go @@ -1,4 +1,4 @@ -package wiki +package wikipedia import ( "errors" @@ -62,7 +62,7 @@ func (page *WikipediaPage) GetContent(client *Client, lang string) (string, erro "rvprop": "ids", "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return "", err } @@ -91,7 +91,7 @@ func (page *WikipediaPage) GetHTML(client *Client, lang string) (string, error) "rvparse": "", "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return "", err } @@ -154,7 +154,7 @@ func (page *WikipediaPage) GetSummary(client *Client, lang string) (string, erro "exlimit": "1", "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return "", err } @@ -176,7 +176,7 @@ func (page *WikipediaPage) ContinuedQuery(args map[string]string, client *Client new_args := maps.Clone(args) mapsx.Update(new_args, last) - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return result, err } @@ -269,7 +269,7 @@ func (page *WikipediaPage) GetThumbURL(client *Client, lang string) (string, err "pithumbsize": "500", } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return "", err } @@ -294,7 +294,7 @@ func (page *WikipediaPage) GetCoordinate(client *Client, lang string) ([]float64 "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return []float64{}, err } @@ -396,7 +396,7 @@ func (page *WikipediaPage) GetSectionList(client *Client, lang string) ([]string if page.Title != "" { args["page"] = page.Title } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return []string{}, err } @@ -466,7 +466,7 @@ func MakeWikipediaPage(pageid int, title string, originaltitle string, redirect if originaltitle != "" { page.OriginalTitle = originaltitle } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return page, err } @@ -522,7 +522,7 @@ func MakeWikipediaPage(pageid int, title string, originaltitle string, redirect "rvlimit": "1", "titles": page.Title, } - res, err := client.DoRequest(args, lang) + res, err := client.doRequest(args, lang) if err != nil { return page, err } diff --git a/internal/informer/wiki/request.go b/internal/client/wikipedia/request.go similarity index 99% rename from internal/informer/wiki/request.go rename to internal/client/wikipedia/request.go index 571bbda..350dc11 100644 --- a/internal/informer/wiki/request.go +++ b/internal/client/wikipedia/request.go @@ -1,4 +1,4 @@ -package wiki +package wikipedia type InnerBasic struct { Aster string `json:"*"` diff --git a/internal/container/container.go b/internal/container/container.go index 62306cd..63fceb2 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -10,8 +10,10 @@ import ( "github.com/unconditionalday/server/internal/app" "github.com/unconditionalday/server/internal/client/github" + "github.com/unconditionalday/server/internal/client/wikipedia" "github.com/unconditionalday/server/internal/parser" bleveRepo "github.com/unconditionalday/server/internal/repository/bleve" + "github.com/unconditionalday/server/internal/search" "github.com/unconditionalday/server/internal/version" "github.com/unconditionalday/server/internal/webserver" blevex "github.com/unconditionalday/server/internal/x/bleve" @@ -68,6 +70,7 @@ type Services struct { apiServer *webserver.Server feedRepository *bleveRepo.FeedRepository sourceClient *github.Client + searchClient *wikipedia.Client httpClient *netx.HttpClient logger *zap.Logger parser *parser.Parser @@ -91,7 +94,7 @@ func (c *Container) GetAPIServer() *webserver.Server { AllowedOrigins: c.Parameters.ServerAllowedOrigins, } - c.apiServer = webserver.NewServer(config, c.GetFeedRepository(), c.SourceRelease, c.BuildVersion, c.GetLogger()) + c.apiServer = webserver.NewServer(config, c.GetFeedRepository(), c.SourceRelease, c.GetSearchClient(), c.BuildVersion, c.GetLogger()) return c.apiServer } @@ -128,6 +131,16 @@ func (c *Container) GetSourceClient() app.SourceClient { return c.sourceClient } +func (c *Container) GetSearchClient() search.SearchClient { + if c.searchClient != nil { + return c.searchClient + } + + c.searchClient = wikipedia.NewClient() + + return c.searchClient +} + func (c *Container) GetVersioning() version.Versioning { if c.versioning != nil { return c.versioning diff --git a/internal/informer/wiki.go b/internal/informer/wiki.go deleted file mode 100644 index cca79b1..0000000 --- a/internal/informer/wiki.go +++ /dev/null @@ -1,91 +0,0 @@ -package informer - -import ( - "errors" - - "github.com/unconditionalday/server/internal/informer/wiki" -) - -var ( - ErrEmptyQuery = errors.New("query string must not be empty") - ErrEmptyLanguage = errors.New("language string must not be empty") -) - -type Wiki struct { - client *wiki.Client -} - -func NewWiki() *Wiki { - return &Wiki{ - client: wiki.New(), - } -} - -func (w *Wiki) Search(query string, lang string) (WikiInfo, error) { - if query == "" { - return WikiInfo{}, ErrEmptyQuery - } - - if lang == "" { - return WikiInfo{}, ErrEmptyLanguage - } - - args := map[string]string{ - "action": "query", - "list": "search", - "srprop": "", - "srlimit": "1", - "srsearch": query, - } - - res, err := w.client.DoRequest(args, lang) - if err != nil { - return WikiInfo{}, err - } - - if len(res.Query.Search) == 0 { - return WikiInfo{}, nil - } - - title := res.Query.Search[0].Title - - wikiPage, err := wiki.MakeWikipediaPage(-1, title, "", false, w.client, lang) - if wikiPage.Disambiguation != nil { - title = wikiPage.Disambiguation[0] - wikiPage, err = wiki.MakeWikipediaPage(-1, title, "", false, w.client, lang) - } - - if err != nil { - return WikiInfo{}, err - } - - summary, err := wikiPage.GetSummary(w.client, lang) - if err != nil { - return WikiInfo{}, err - } - - thumbnail, err := wikiPage.GetThumbURL(w.client, lang) - if err != nil { - return WikiInfo{}, err - } - - return WikiInfo{ - Title: wikiPage.Title, - Language: wikiPage.Language, - Link: wikiPage.URL, - Summary: summary, - Thumbnail: thumbnail, - }, nil -} - -type WikiInfo struct { - Title string - Link string - Summary string - Thumbnail string - Language string -} - -func (r WikiInfo) IsValid() bool { - return r.Title != "" && r.Link != "" && r.Summary != "" && r.Thumbnail != "" && r.Language != "" -} diff --git a/internal/repository/bleve/bleve_test.go b/internal/repository/bleve/bleve_test.go deleted file mode 100644 index c1ad058..0000000 --- a/internal/repository/bleve/bleve_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package bleve_test - -import ( - "os" - "testing" - "time" - - "github.com/blevesearch/bleve/v2/mapping" - "github.com/unconditionalday/server/internal/app" - "github.com/unconditionalday/server/internal/repository/bleve" -) - -func TestBleveIndex(t *testing.T) { - testCases := []struct { - name string - b *bleve.Bleve - wantErr bool - }{ - { - name: "bleve is created", - wantErr: false, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - var err error - - tc.b, err = bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) - - defer os.RemoveAll("test.bleve") - - if tc.b == nil { - t.Fatalf("expected bleve to be created") - } - - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - }) - } -} - -func TestSave(t *testing.T) { - testCases := []struct { - name string - document app.Feed - wantErr bool - }{ - { - name: "document is saved", - wantErr: false, - document: app.Feed{ - Title: "test", - Link: "link", - Language: "it", - Image: &app.Image{}, - Summary: "Lorem Ipsum", - Source: "Unconditional Day", - Date: time.Time{}, - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - b, err := bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) - if b == nil { - t.Fatalf("expected bleve to be created") - } - - defer os.RemoveAll("test.bleve") - - err = b.Save(tc.document) - - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - }) - } -} - -func TestFind(t *testing.T) { - testCases := []struct { - name string - document app.Feed - wantErr bool - }{ - { - name: "document is found", - wantErr: false, - document: app.Feed{ - Title: "test", - Link: "link", - Language: "it", - Image: &app.Image{}, - Summary: "Lorem Ipsum", - Source: "Unconditional Day", - Date: time.Now(), - }, - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - b, err := bleve.NewBleveIndex("test.bleve", mapping.NewIndexMapping()) - if b == nil { - t.Fatalf("expected bleve to be created") - } - - defer os.RemoveAll("test.bleve") - - err = b.Save(tc.document) - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - - found, err := b.Find(tc.document.Title) - if (err != nil) != tc.wantErr { - t.Errorf("NewBleve() error = %v, wantErr %v", err, tc.wantErr) - return - } - - if found == nil { - t.Errorf("expected document to be found") - } - - if found[0].Title != tc.document.Title { - t.Errorf("expected document to be found") - } - - }) - } -} diff --git a/internal/search/search.go b/internal/search/search.go new file mode 100644 index 0000000..521a7ae --- /dev/null +++ b/internal/search/search.go @@ -0,0 +1,18 @@ +package search + +type EntityDetails struct { + Title string + Link string + Summary string + Thumbnail string + Language string + Source string +} + +func (i EntityDetails) IsValid() bool { + return i.Title != "" && i.Link != "" && i.Summary != "" && i.Thumbnail != "" && i.Language != "" && i.Source != "" +} + +type SearchClient interface { + FetchEntityDetails(query, locale string) (EntityDetails, error) +} diff --git a/internal/webserver/server.go b/internal/webserver/server.go index b9bf9e4..018362c 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -10,6 +10,7 @@ import ( api "github.com/unconditionalday/server/api" "github.com/unconditionalday/server/internal/app" + "github.com/unconditionalday/server/internal/search" "github.com/unconditionalday/server/internal/version" ) @@ -18,6 +19,7 @@ type Server struct { feedRepo app.FeedRepository source *app.SourceRelease buildVersion version.Build + search search.SearchClient logger *zap.Logger client *echo.Echo } @@ -28,12 +30,13 @@ type Config struct { AllowedOrigins []string } -func NewServer(config Config, repo app.FeedRepository, source *app.SourceRelease, version version.Build, logger *zap.Logger) *Server { +func NewServer(config Config, repo app.FeedRepository, source *app.SourceRelease, search search.SearchClient, version version.Build, logger *zap.Logger) *Server { return &Server{ config: config, feedRepo: repo, source: source, buildVersion: version, + search: search, logger: logger, client: echo.New(), } @@ -114,24 +117,24 @@ func (s *Server) GetV1Version(ctx echo.Context) error { func (s *Server) GetV1InformerWikiQuery(ctx echo.Context, query string) error { // TODO: add language support - wikiRes, err := s.wiki.Search(query, "en") + searchRes, err := s.search.FetchEntityDetails(query, "en") if err != nil { e := api.Error{ - Code: 500, + Code: http.StatusInternalServerError, Message: "Internal Server Error", } s.logger.Error("wiki search", zap.Error(err)) - return ctx.JSON(500, e) + return ctx.JSON(http.StatusInternalServerError, e) } res := api.WikiResult{ - Language: wikiRes.Language, - Link: wikiRes.Link, - Summary: wikiRes.Summary, - Thumbnail: wikiRes.Thumbnail, - Title: wikiRes.Title, + Language: searchRes.Language, + Link: searchRes.Link, + Summary: searchRes.Summary, + Thumbnail: searchRes.Thumbnail, + Title: searchRes.Title, } return ctx.JSON(200, res) From ec133e78287ca1536eab62142dd8a4182123f2ff Mon Sep 17 00:00:00 2001 From: Luigi Date: Sun, 8 Oct 2023 21:48:13 +0200 Subject: [PATCH 24/25] chore: refactor --- .github/workflows/db-auto-merge.yaml | 41 ---------------------------- api/api.gen.go | 9 ------ cmd/serve.go | 1 - 3 files changed, 51 deletions(-) diff --git a/.github/workflows/db-auto-merge.yaml b/.github/workflows/db-auto-merge.yaml index 4626309..7b6b044 100644 --- a/.github/workflows/db-auto-merge.yaml +++ b/.github/workflows/db-auto-merge.yaml @@ -1,5 +1,3 @@ -<<<<<<< HEAD -<<<<<<< HEAD name: Dependabot auto-merge on: pull_request @@ -19,45 +17,6 @@ jobs: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Enable auto-merge for Dependabot PRs run: gh pr merge --auto --rebase "$PR_URL" -<<<<<<< HEAD env: PR_URL: ${{github.event.pull_request.html_url}} GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} -======= -name: auto-merge -======= -name: Dependabot auto-merge -on: pull_request ->>>>>>> b8ec7e1 (fix: use auto-merge from gh docs) - -permissions: - contents: write - pull-requests: write - -jobs: - dependabot: - runs-on: ubuntu-latest - if: ${{ github.actor == 'dependabot[bot]' }} - steps: - - name: Dependabot metadata - id: metadata - uses: dependabot/fetch-metadata@v1 - with: -<<<<<<< HEAD - target: minor -<<<<<<< HEAD - github-token: ${{ github.token }} ->>>>>>> 0e7e5ce (feat: add dependabot auto-merge) -======= - github-token: ${{ secrets.ACCESS_TOKEN }} ->>>>>>> 4b36d7b (chore: use custom private secret for dependabot) -======= - github-token: "${{ secrets.GITHUB_TOKEN }}" - - name: Enable auto-merge for Dependabot PRs - run: gh pr merge --auto --merge "$PR_URL" -======= ->>>>>>> b345346 (chore: dependabot auto-rebase instead of auto-merge) - env: - PR_URL: ${{github.event.pull_request.html_url}} - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} ->>>>>>> b8ec7e1 (fix: use auto-merge from gh docs) diff --git a/api/api.gen.go b/api/api.gen.go index 0dbb7da..3080aab 100644 --- a/api/api.gen.go +++ b/api/api.gen.go @@ -75,15 +75,6 @@ type GetV1VersionJSONBody map[string]interface{} // GetV1VersionJSONRequestBody defines body for GetV1Version for application/json ContentType. type GetV1VersionJSONRequestBody GetV1VersionJSONBody -// WikiResult defines model for WikiResult. -type WikiResult struct { - Language string `json:"language"` - Link string `json:"link"` - Summary string `json:"summary"` - Thumbnail string `json:"thumbnail"` - Title string `json:"title"` -} - // ServerInterface represents all server handlers. type ServerInterface interface { diff --git a/cmd/serve.go b/cmd/serve.go index 64ef21f..a1ce61e 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -10,7 +10,6 @@ import ( "github.com/unconditionalday/server/internal/service" "github.com/unconditionalday/server/internal/version" cobrax "github.com/unconditionalday/server/internal/x/cobra" - "go.uber.org/zap" ) var ( From f0aee1e6bdd763bab4834c1bcf0034ac0d417c0e Mon Sep 17 00:00:00 2001 From: Luigi Date: Tue, 10 Oct 2023 00:36:03 +0200 Subject: [PATCH 25/25] chore: rename search fetch method --- internal/client/wikipedia/client.go | 42 +++++++----------------- internal/client/wikipedia/client_test.go | 2 +- internal/search/search.go | 6 ++-- internal/webserver/server.go | 2 +- 4 files changed, 16 insertions(+), 36 deletions(-) diff --git a/internal/client/wikipedia/client.go b/internal/client/wikipedia/client.go index d89cdb9..41e4780 100644 --- a/internal/client/wikipedia/client.go +++ b/internal/client/wikipedia/client.go @@ -26,6 +26,7 @@ const ( var ( ErrEmptyQuery = errors.New("query string must not be empty") ErrEmptyLanguage = errors.New("language string must not be empty") + ErrDisambiguationResult = errors.New("disambiguation result") ) // Create a new WikiClient @@ -113,13 +114,13 @@ func (c *Client) updateLastCall(now time.Time) { c.lastCall = now } -func (w *Client) FetchEntityDetails(query string, lang string) (search.EntityDetails, error) { +func (w *Client) FetchContextDetails(query string, lang string) (search.ContextDetails, error) { if query == "" { - return search.EntityDetails{}, ErrEmptyQuery + return search.ContextDetails{}, ErrEmptyQuery } if lang == "" { - return search.EntityDetails{}, ErrEmptyLanguage + return search.ContextDetails{}, ErrEmptyLanguage } args := map[string]string{ @@ -132,36 +133,35 @@ func (w *Client) FetchEntityDetails(query string, lang string) (search.EntityDet res, err := w.doRequest(args, lang) if err != nil { - return search.EntityDetails{}, err + return search.ContextDetails{}, err } if len(res.Query.Search) == 0 { - return search.EntityDetails{}, nil + return search.ContextDetails{}, nil } title := res.Query.Search[0].Title wikiPage, err := MakeWikipediaPage(-1, title, "", false, w, lang) if len(wikiPage.Disambiguation) != 0 { - title = wikiPage.Disambiguation[0] - wikiPage, err = MakeWikipediaPage(-1, title, "", false, w, lang) + return search.ContextDetails{}, nil } if err != nil { - return search.EntityDetails{}, err + return search.ContextDetails{}, err } summary, err := wikiPage.GetSummary(w, lang) if err != nil { - return search.EntityDetails{}, err + return search.ContextDetails{}, err } thumbnail, err := wikiPage.GetThumbURL(w, lang) if err != nil { - return search.EntityDetails{}, err + return search.ContextDetails{}, err } - return search.EntityDetails{ + return search.ContextDetails{ Title: wikiPage.Title, Language: wikiPage.Language, Link: wikiPage.URL, @@ -170,23 +170,3 @@ func (w *Client) FetchEntityDetails(query string, lang string) (search.EntityDet Thumbnail: thumbnail, }, nil } - -func (c *Client) Suggest(_input, lang string) (string, error) { - args := map[string]string{ - "action": "query", - "list": "search", - "srlimit": "1", - "srprop": "", - "srinfo": "suggestion", - "srsearch": _input, - } - - res, err := c.doRequest(args, lang) - if err != nil { - return "", err - } - if res.Error.Code != "" { - return "", errors.New(res.Error.Info) - } - return res.Query.SearchInfo.Suggestion, nil -} diff --git a/internal/client/wikipedia/client_test.go b/internal/client/wikipedia/client_test.go index ff97fe6..a968d88 100644 --- a/internal/client/wikipedia/client_test.go +++ b/internal/client/wikipedia/client_test.go @@ -58,7 +58,7 @@ func TestWikiSearch(t *testing.T) { t.Parallel() w := wikipedia.NewClient() - actual, err := w.FetchEntityDetails(tc.input.query, tc.input.lang) + actual, err := w.FetchContextDetails(tc.input.query, tc.input.lang) if tc.output.err != nil { assert.Equal(t, tc.output.err.Error(), err.Error()) diff --git a/internal/search/search.go b/internal/search/search.go index 521a7ae..5884047 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -1,6 +1,6 @@ package search -type EntityDetails struct { +type ContextDetails struct { Title string Link string Summary string @@ -9,10 +9,10 @@ type EntityDetails struct { Source string } -func (i EntityDetails) IsValid() bool { +func (i ContextDetails) IsValid() bool { return i.Title != "" && i.Link != "" && i.Summary != "" && i.Thumbnail != "" && i.Language != "" && i.Source != "" } type SearchClient interface { - FetchEntityDetails(query, locale string) (EntityDetails, error) + FetchContextDetails(query, locale string) (ContextDetails, error) } diff --git a/internal/webserver/server.go b/internal/webserver/server.go index 018362c..18addd5 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -117,7 +117,7 @@ func (s *Server) GetV1Version(ctx echo.Context) error { func (s *Server) GetV1InformerWikiQuery(ctx echo.Context, query string) error { // TODO: add language support - searchRes, err := s.search.FetchEntityDetails(query, "en") + searchRes, err := s.search.FetchContextDetails(query, "en") if err != nil { e := api.Error{ Code: http.StatusInternalServerError,