From a9fe0d8e580b2c1167960e73befdeb48a316c1b8 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 14 Oct 2023 22:28:14 +1300 Subject: [PATCH 01/10] Chore: Replace html2text modules with simplified internal function The module microcosm-cc/bluemonday now requires Go v1.21 and is quite frankly an overkill as Mailpit only needs to convert HTML to a single line (no formatting). --- go.mod | 3 -- go.sum | 16 ------ internal/storage/utils.go | 10 ++-- internal/tools/html.go | 11 ---- internal/tools/html2text/html2text.go | 72 +++++++++++++++++++++++++++ internal/tools/snippets.go | 8 ++- 6 files changed, 78 insertions(+), 42 deletions(-) create mode 100644 internal/tools/html2text/html2text.go diff --git a/go.mod b/go.mod index 7276b1a47..de3d85d1d 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,9 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/jhillyerd/enmime v1.0.1 - github.com/k3a/html2text v1.2.1 github.com/klauspost/compress v1.17.0 github.com/leporo/sqlf v1.4.0 github.com/mhale/smtpd v0.8.0 - github.com/microcosm-cc/bluemonday v1.0.25 github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.9.3 @@ -33,7 +31,6 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/aymerick/douceur v0.2.0 // indirect github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect github.com/cznic/ql v1.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 87fc6de3e..c5a4f444f 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,6 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/axllent/semver v0.0.1 h1:QqF+KSGxgj8QZzSXAvKFqjGWE5792ksOnQhludToK8E= github.com/axllent/semver v0.0.1/go.mod h1:2xSPzvG8n9mRfdtxSvWvfTfQGWfHsMsHO1iZnKATMSc= -github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= -github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI= github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -59,8 +57,6 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= @@ -73,10 +69,6 @@ github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQykt github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jhillyerd/enmime v1.0.1 h1:y6RyqIgBOI2hIinOXIzmeB+ITRVls0zTJIm5GwgXnjE= github.com/jhillyerd/enmime v1.0.1/go.mod h1:LMMbm6oTlzWHghPavqHtOrP/NosVv3l42CUrZjn03/Q= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/k3a/html2text v1.2.1 h1:nvnKgBvBR/myqrwfLuiqecUtaK1lB9hGziIJKatNFVY= -github.com/k3a/html2text v1.2.1/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= @@ -99,8 +91,6 @@ github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwp github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0= github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4= -github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= -github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -123,10 +113,6 @@ github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -166,7 +152,6 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -210,7 +195,6 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= diff --git a/internal/storage/utils.go b/internal/storage/utils.go index 58f91b0ab..877b993fa 100644 --- a/internal/storage/utils.go +++ b/internal/storage/utils.go @@ -11,9 +11,9 @@ import ( "github.com/axllent/mailpit/config" "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/tools/html2text" "github.com/axllent/mailpit/server/websockets" "github.com/jhillyerd/enmime" - "github.com/k3a/html2text" "github.com/leporo/sqlf" ) @@ -39,12 +39,8 @@ func createSearchText(env *enmime.Envelope) string { b.WriteString(env.GetHeader("Bcc") + " ") b.WriteString(env.GetHeader("Reply-To") + " ") b.WriteString(env.GetHeader("Return-Path") + " ") - h := strings.TrimSpace( - html2text.HTML2TextWithOptions( - env.HTML, - html2text.WithLinksInnerText(), - ), - ) + + h := html2text.Strip(env.HTML, true) if h != "" { b.WriteString(h + " ") } else { diff --git a/internal/tools/html.go b/internal/tools/html.go index 06de636b7..6e5e8832b 100644 --- a/internal/tools/html.go +++ b/internal/tools/html.go @@ -2,9 +2,7 @@ package tools import ( "fmt" - "strings" - "github.com/microcosm-cc/bluemonday" "golang.org/x/net/html" ) @@ -19,12 +17,3 @@ func GetHTMLAttributeVal(e *html.Node, key string) (string, error) { return "", fmt.Errorf("%s not found", key) } - -// StripHTML returns text from an HTML string -func stripHTML(h string) string { - p := bluemonday.StrictPolicy() - // // ensure joining html elements are spaced apart, eg table cells etc - h = strings.ReplaceAll(h, "><", "> <") - // return p.Sanitize(h) - return html.UnescapeString(p.Sanitize(h)) -} diff --git a/internal/tools/html2text/html2text.go b/internal/tools/html2text/html2text.go new file mode 100644 index 000000000..0940d067a --- /dev/null +++ b/internal/tools/html2text/html2text.go @@ -0,0 +1,72 @@ +// Package html2text is a simple library to convert HTML to plain text +package html2text + +import ( + "bytes" + "log" + "regexp" + "strings" + + "golang.org/x/net/html" +) + +var ( + re = regexp.MustCompile(`\s+`) + spaceRe = regexp.MustCompile(`(?mi)<\/(div|p|td|th|h[1-6]|ul|ol|li|address|article|aside|blockquote|dl|dt|footer|header|hr|main|nav|pre|table|thead|tfoot|video)><`) + brRe = regexp.MustCompile(`(?mi)<(br /|br)>`) + imgRe = regexp.MustCompile(`(?mi)<(img)`) + skip = make(map[string]bool) +) + +func init() { + skip["script"] = true + skip["title"] = true + skip["head"] = true + skip["link"] = true + skip["meta"] = true + skip["style"] = true + skip["noscript"] = true +} + +// Strip will convert a HTML string to plain text +func Strip(h string, includeLinks bool) string { + h = spaceRe.ReplaceAllString(h, " <") + h = brRe.ReplaceAllString(h, " ") + h = imgRe.ReplaceAllString(h, " <$1") + var buffer bytes.Buffer + doc, err := html.Parse(strings.NewReader(h)) + if err != nil { + log.Fatal(err) + } + + extract(doc, &buffer, includeLinks) + return clean(buffer.String()) +} + +func extract(node *html.Node, buff *bytes.Buffer, includeLinks bool) { + if node.Type == html.TextNode { + data := node.Data + if data != "" { + buff.WriteString(data) + } + } + for c := node.FirstChild; c != nil; c = c.NextSibling { + if _, skip := skip[c.Data]; !skip { + if includeLinks && c.Data == "a" { + for _, a := range c.Attr { + if a.Key == "href" && strings.HasPrefix(strings.ToLower(a.Val), "http") { + buff.WriteString(" " + a.Val + " ") + } + } + } + extract(c, buff, includeLinks) + } + } +} + +func clean(text string) string { + // replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184 + text = strings.ReplaceAll(text, string('\uFEFF'), " ") + text = re.ReplaceAllString(text, " ") + return strings.TrimSpace(text) +} diff --git a/internal/tools/snippets.go b/internal/tools/snippets.go index 5b8510f28..1322caf17 100644 --- a/internal/tools/snippets.go +++ b/internal/tools/snippets.go @@ -3,6 +3,8 @@ package tools import ( "regexp" "strings" + + "github.com/axllent/mailpit/internal/tools/html2text" ) // CreateSnippet returns a message snippet. It will use the HTML version (if it exists) @@ -12,17 +14,13 @@ func CreateSnippet(text, html string) string { html = strings.TrimSpace(html) limit := 200 spaceRe := regexp.MustCompile(`\s+`) - nlRe := regexp.MustCompile(`\r?\n`) if text == "" && html == "" { return "" } if html != "" { - data := nlRe.ReplaceAllString(stripHTML(html), " ") - // replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184 - data = strings.ReplaceAll(data, string('\uFEFF'), " ") - data = strings.TrimSpace(spaceRe.ReplaceAllString(data, " ")) + data := html2text.Strip(html, false) if len(data) <= limit { return data From e9d44c55a1fe87ec45071065c7b9b1625d828d2c Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 14 Oct 2023 22:28:52 +1300 Subject: [PATCH 02/10] Tests: Add html2text tests --- .github/workflows/tests.yml | 2 +- internal/tools/html2text/html2text_test.go | 56 ++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 internal/tools/html2text/html2text_test.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e26ce529f..c26e25830 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- - - run: go test ./internal/storage ./server ./internal/tools -v + - run: go test ./internal/storage ./server ./internal/tools ./internal/tools/html2text -v - run: go test ./internal/storage -bench=. # build the assets diff --git a/internal/tools/html2text/html2text_test.go b/internal/tools/html2text/html2text_test.go new file mode 100644 index 000000000..dd1763918 --- /dev/null +++ b/internal/tools/html2text/html2text_test.go @@ -0,0 +1,56 @@ +package html2text + +import "testing" + +func TestPlain(t *testing.T) { + tests := map[string]string{} + tests["this is a test"] = "this is a test" + tests["thiS IS a Test"] = "thiS IS a Test" + tests["thiS IS a Test :-)"] = "thiS IS a Test :-)" + tests["

This is a test.

"] = "This is a test." + tests["

Paragraph 1

Paragraph 2

"] = "Paragraph 1 Paragraph 2" + tests["

Heading

Paragraph

"] = "Heading Paragraph" + tests["Alphabet chars"] = "Alphabet chars" + tests["Alphabet chars."] = "Alphabet chars." + tests["
FirstSecond
"] = "First Second" + tests[`

Heading

+

Paragraph

`] = "Heading Paragraph" + tests[`

Heading

linked text

`] = "Heading linked text" + // broken html + tests[`

Heading

linked text.`] = "Heading linked text." + + for str, expected := range tests { + res := Strip(str, false) + if res != expected { + t.Log("error:", res, "!=", expected) + t.Fail() + } + } +} + +func TestWithLinks(t *testing.T) { + tests := map[string]string{} + tests["this is a test"] = "this is a test" + tests["thiS IS a Test"] = "thiS IS a Test" + tests["thiS IS a Test :-)"] = "thiS IS a Test :-)" + tests["

This is a test.

"] = "This is a test." + tests["

Paragraph 1

Paragraph 2

"] = "Paragraph 1 Paragraph 2" + tests["

Heading

Paragraph

"] = "Heading Paragraph" + tests["Alphabet chars"] = "Alphabet chars" + tests["Alphabet chars."] = "Alphabet chars." + tests["
FirstSecond
"] = "First Second" + tests["

Heading

Paragraph

"] = "Heading Paragraph" + tests[`

Heading

+

Paragraph

`] = "Heading Paragraph" + tests[`

Heading

linked text

`] = "Heading https://github.com linked text" + // broken html + tests[`

Heading

linked text.`] = "Heading https://github.com linked text." + + for str, expected := range tests { + res := Strip(str, true) + if res != expected { + t.Log("error:", res, "!=", expected) + t.Fail() + } + } +} From cded4d25fc778d6b116a50b7bb0ca665e94768b7 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 15 Oct 2023 19:24:06 +1300 Subject: [PATCH 03/10] Swagger: Update swagger documentation See #188 --- server/apiv1/api.go | 81 ++---------- server/apiv1/structs.go | 5 +- server/apiv1/swagger.go | 138 ++++++++++++++------ server/ui/api/v1/swagger.json | 234 +++++++++++++++++++--------------- 4 files changed, 242 insertions(+), 216 deletions(-) diff --git a/server/apiv1/api.go b/server/apiv1/api.go index c1f64a74d..4fb20e10c 100644 --- a/server/apiv1/api.go +++ b/server/apiv1/api.go @@ -144,11 +144,11 @@ func Search(w http.ResponseWriter, r *http.Request) { // DeleteSearch will delete all messages matching a search func DeleteSearch(w http.ResponseWriter, r *http.Request) { - // swagger:route DELETE /api/v1/search messages MessagesSummary + // swagger:route DELETE /api/v1/search messages DeleteSearch // // # Delete messages by search // - // Deletes messages matching a search. + // Delete all messages matching a search. // // Produces: // - application/json @@ -196,7 +196,7 @@ func GetMessage(w http.ResponseWriter, r *http.Request) { // Parameters: // + name: ID // in: path - // description: Database ID + // description: Message database ID // required: true // type: string // @@ -237,7 +237,7 @@ func DownloadAttachment(w http.ResponseWriter, r *http.Request) { // Parameters: // + name: ID // in: path - // description: Database ID + // description: Message database ID // required: true // type: string // + name: PartID @@ -362,11 +362,11 @@ func DownloadRaw(w http.ResponseWriter, r *http.Request) { // DeleteMessages (method: DELETE) deletes all messages matching IDS. func DeleteMessages(w http.ResponseWriter, r *http.Request) { - // swagger:route DELETE /api/v1/messages messages Delete + // swagger:route DELETE /api/v1/messages messages DeleteMessages // // # Delete messages // - // If no IDs are provided then all messages are deleted. + // Delete individual or all messages. If no IDs are provided then all messages are deleted. // // Consumes: // - application/json @@ -376,13 +376,6 @@ func DeleteMessages(w http.ResponseWriter, r *http.Request) { // // Schemes: http, https // - // Parameters: - // + name: ids - // in: body - // description: Database IDs to delete - // required: false - // type: DeleteRequest - // // Responses: // 200: OKResponse // default: ErrorResponse @@ -406,7 +399,7 @@ func DeleteMessages(w http.ResponseWriter, r *http.Request) { } } - w.Header().Add("Content-Type", "application/json") + w.Header().Add("Content-Type", "application/plain") _, _ = w.Write([]byte("ok")) } @@ -427,13 +420,6 @@ func SetReadStatus(w http.ResponseWriter, r *http.Request) { // // Schemes: http, https // - // Parameters: - // + name: ids - // in: body - // description: Database IDs to update - // required: false - // type: SetReadStatusRequest - // // Responses: // 200: OKResponse // default: ErrorResponse @@ -491,7 +477,7 @@ func SetReadStatus(w http.ResponseWriter, r *http.Request) { // GetTags (method: GET) will get all tags currently in use func GetTags(w http.ResponseWriter, _ *http.Request) { - // swagger:route GET /api/v1/tags tags SetTags + // swagger:route GET /api/v1/tags tags GetTags // // # Get all current tags // @@ -524,7 +510,7 @@ func SetTags(w http.ResponseWriter, r *http.Request) { // // # Set message tags // - // To remove all tags from a message, pass an empty tags array. + // This will overwrite any existing tags for selected message database IDs. To remove all tags from a message, pass an empty tags array. // // Consumes: // - application/json @@ -534,13 +520,6 @@ func SetTags(w http.ResponseWriter, r *http.Request) { // // Schemes: http, https // - // Parameters: - // + name: ids - // in: body - // description: Database IDs to update - // required: true - // type: SetTagsRequest - // // Responses: // 200: OKResponse // default: ErrorResponse @@ -576,11 +555,11 @@ func SetTags(w http.ResponseWriter, r *http.Request) { // ReleaseMessage (method: POST) will release a message via a pre-configured external SMTP server. // If no IDs are provided then all messages are updated. func ReleaseMessage(w http.ResponseWriter, r *http.Request) { - // swagger:route POST /api/v1/message/{ID}/release message Release + // swagger:route POST /api/v1/message/{ID}/release message ReleaseMessage // // # Release message // - // Release a message via a pre-configured external SMTP server.. + // Release a message via a pre-configured external SMTP server. This is only enabled if message relaying has been configured. // // Consumes: // - application/json @@ -590,18 +569,6 @@ func ReleaseMessage(w http.ResponseWriter, r *http.Request) { // // Schemes: http, https // - // Parameters: - // + name: ID - // in: path - // description: Database ID - // required: true - // type: string - // + name: to - // in: body - // description: Array of email addresses to release message to - // required: true - // type: ReleaseMessageRequest - // // Responses: // 200: OKResponse // default: ErrorResponse @@ -618,7 +585,7 @@ func ReleaseMessage(w http.ResponseWriter, r *http.Request) { decoder := json.NewDecoder(r.Body) - data := releaseMessageRequest{} + data := releaseMessageRequestBody{} if err := decoder.Decode(&data); err != nil { httpError(w, err.Error()) @@ -702,7 +669,7 @@ func ReleaseMessage(w http.ResponseWriter, r *http.Request) { // HTMLCheck returns a summary of the HTML client support func HTMLCheck(w http.ResponseWriter, r *http.Request) { - // swagger:route GET /api/v1/message/{ID}/html-check Other HTMLCheckResponse + // swagger:route GET /api/v1/message/{ID}/html-check Other HTMLCheck // // # HTML check (beta) // @@ -716,13 +683,6 @@ func HTMLCheck(w http.ResponseWriter, r *http.Request) { // // Schemes: http, https // - // Parameters: - // + name: ID - // in: path - // description: Database ID - // required: true - // type: string - // // Responses: // 200: HTMLCheckResponse // default: ErrorResponse @@ -754,7 +714,7 @@ func HTMLCheck(w http.ResponseWriter, r *http.Request) { // LinkCheck returns a summary of links in the email func LinkCheck(w http.ResponseWriter, r *http.Request) { - // swagger:route GET /api/v1/message/{ID}/link-check Other LinkCheckResponse + // swagger:route GET /api/v1/message/{ID}/link-check Other LinkCheck // // # Link check (beta) // @@ -768,19 +728,6 @@ func LinkCheck(w http.ResponseWriter, r *http.Request) { // // Schemes: http, https // - // Parameters: - // + name: ID - // in: path - // description: Database ID - // required: true - // type: string - // + name: follow - // in: query - // description: Follow redirects - // required: false - // type: boolean - // default: false - // // Responses: // 200: LinkCheckResponse // default: ErrorResponse diff --git a/server/apiv1/structs.go b/server/apiv1/structs.go index 318ef6944..8bf1290b4 100644 --- a/server/apiv1/structs.go +++ b/server/apiv1/structs.go @@ -22,9 +22,6 @@ type MessagesSummary struct { // Total number of messages matching current query MessagesCount int `json:"messages_count"` - // // Number of results returned on current page - // Count int `json:"count"` - // Pagination offset Start int `json:"start"` @@ -32,7 +29,7 @@ type MessagesSummary struct { Tags []string `json:"tags"` // Messages summary - // in:body + // in: body Messages []storage.MessageSummary `json:"messages"` } diff --git a/server/apiv1/swagger.go b/server/apiv1/swagger.go index 43e06469c..583714fed 100644 --- a/server/apiv1/swagger.go +++ b/server/apiv1/swagger.go @@ -6,6 +6,8 @@ package apiv1 // swagger:response InfoResponse type infoResponse struct { // Application information + // + // in: body Body appInformation } @@ -13,6 +15,8 @@ type infoResponse struct { // swagger:response WebUIConfigurationResponse type webUIConfigurationResponse struct { // Web UI configuration settings + // + // in: body Body webUIConfiguration } @@ -28,81 +32,137 @@ type messagesSummaryResponse struct { // swagger:model MessageHeaders type messageHeaders map[string][]string +// swagger:parameters DeleteMessages +type deleteMessagesParams struct { + // in: body + Body *deleteMessagesRequestBody +} + // Delete request // swagger:model DeleteRequest -type deleteRequest struct { - // ids - // in:body +type deleteMessagesRequestBody struct { + // Array of message database IDs + // + // required: false + // example: ["5dec4247-812e-4b77-9101-e25ad406e9ea", "8ac66bbc-2d9a-4c41-ad99-00aa75fa674e"] IDs []string `json:"ids"` } +// swagger:parameters SetReadStatus +type setReadStatusParams struct { + // in: body + Body *setReadStatusRequestBody +} + // Set read status request -// swagger:model SetReadStatusRequest -type setReadStatusRequest struct { +// swagger:model setReadStatusRequestBody +type setReadStatusRequestBody struct { // Read status + // + // required: false + // default: false + // example: true Read bool `json:"read"` - // ids - // in:body + // Array of message database IDs + // + // required: false + // example: ["5dec4247-812e-4b77-9101-e25ad406e9ea", "8ac66bbc-2d9a-4c41-ad99-00aa75fa674e"] IDs []string `json:"ids"` } +// swagger:parameters SetTags +type setTagsParams struct { + // in: body + Body *setTagsRequestBody +} + // Set tags request -// swagger:model SetTagsRequest -type setTagsRequest struct { - // Tags - // in:body +// swagger:model setTagsRequestBody +type setTagsRequestBody struct { + // Array of tag names to set + // + // required: true + // example: ["Tag 1", "Tag 2"] Tags []string `json:"tags"` - // IDs - // in:body + // Array of message database IDs + // + // required: true + // example: ["5dec4247-812e-4b77-9101-e25ad406e9ea", "8ac66bbc-2d9a-4c41-ad99-00aa75fa674e"] IDs []string `json:"ids"` } +// swagger:parameters ReleaseMessage +type releaseMessageParams struct { + // Message database ID + // + // in: path + // description: Message database ID + // required: true + ID string + + // in: body + Body *releaseMessageRequestBody +} + // Release request -// swagger:model ReleaseMessageRequest -type releaseMessageRequest struct { - // To - // in:body +// swagger:model releaseMessageRequestBody +type releaseMessageRequestBody struct { + // Array of email addresses to relay the message to + // + // required: true + // example: ["user1@example.com", "user2@example.com"] To []string `json:"to"` } +// swagger:parameters HTMLCheck +type htmlCheckParams struct { + // Message database ID + // + // in: path + // description: Message database ID + // required: true + ID string +} + +// swagger:parameters LinkCheck +type linkCheckParams struct { + // Message database ID + // + // in: path + // description: Message database ID + // required: true + ID string + + // Follow redirects + // + // in: query + // description: Follow redirects + // required: false + // default: false + Follow string `json:"follow"` +} + // Binary data response inherits the attachment's content type // swagger:response BinaryResponse -type binaryResponse struct { - // in: body - Body string -} +type binaryResponse string // Plain text response // swagger:response TextResponse -type textResponse struct { - // in: body - Body string -} +type textResponse string // HTML response // swagger:response HTMLResponse -type htmlResponse struct { - // in: body - Body string -} +type htmlResponse string // Error response // swagger:response ErrorResponse -type errorResponse struct { - // The error message - // in: body - Body string -} +type errorResponse string // Plain text "ok" response // swagger:response OKResponse -type okResponse struct { - // Default response - // in: body - Body string -} +type okResponse string // Plain JSON array response // swagger:response ArrayResponse diff --git a/server/ui/api/v1/swagger.json b/server/ui/api/v1/swagger.json index d0a85a5f4..c85fff451 100644 --- a/server/ui/api/v1/swagger.json +++ b/server/ui/api/v1/swagger.json @@ -66,7 +66,7 @@ "parameters": [ { "type": "string", - "description": "Database ID", + "description": "Message database ID", "name": "ID", "in": "path", "required": true @@ -136,11 +136,11 @@ "Other" ], "summary": "HTML check (beta)", - "operationId": "HTMLCheckResponse", + "operationId": "HTMLCheck", "parameters": [ { "type": "string", - "description": "Database ID", + "description": "Message database ID", "name": "ID", "in": "path", "required": true @@ -173,18 +173,19 @@ "Other" ], "summary": "Link check (beta)", - "operationId": "LinkCheckResponse", + "operationId": "LinkCheck", "parameters": [ { "type": "string", - "description": "Database ID", + "description": "Message database ID", "name": "ID", "in": "path", "required": true }, { - "type": "boolean", - "default": false, + "type": "string", + "default": "false", + "x-go-name": "Follow", "description": "Follow redirects", "name": "follow", "in": "query" @@ -223,7 +224,7 @@ "parameters": [ { "type": "string", - "description": "Database ID", + "description": "Message database ID", "name": "ID", "in": "path", "required": true @@ -323,7 +324,7 @@ }, "/api/v1/message/{ID}/release": { "post": { - "description": "Release a message via a pre-configured external SMTP server..", + "description": "Release a message via a pre-configured external SMTP server. This is only enabled if message relaying has been configured.", "consumes": [ "application/json" ], @@ -338,24 +339,20 @@ "message" ], "summary": "Release message", - "operationId": "Release", + "operationId": "ReleaseMessage", "parameters": [ { "type": "string", - "description": "Database ID", + "description": "Message database ID", "name": "ID", "in": "path", "required": true }, { - "description": "Array of email addresses to release message to", - "name": "to", + "name": "Body", "in": "body", - "required": true, "schema": { - "description": "Array of email addresses to release message to", - "type": "object", - "$ref": "#/definitions/ReleaseMessageRequest" + "$ref": "#/definitions/releaseMessageRequestBody" } } ], @@ -428,13 +425,10 @@ "operationId": "SetReadStatus", "parameters": [ { - "description": "Database IDs to update", - "name": "ids", + "name": "Body", "in": "body", "schema": { - "description": "Database IDs to update", - "type": "object", - "$ref": "#/definitions/SetReadStatusRequest" + "$ref": "#/definitions/setReadStatusRequestBody" } } ], @@ -448,7 +442,7 @@ } }, "delete": { - "description": "If no IDs are provided then all messages are deleted.", + "description": "Delete individual or all messages. If no IDs are provided then all messages are deleted.", "consumes": [ "application/json" ], @@ -463,15 +457,12 @@ "messages" ], "summary": "Delete messages", - "operationId": "Delete", + "operationId": "DeleteMessages", "parameters": [ { - "description": "Database IDs to delete", - "name": "ids", + "name": "Body", "in": "body", "schema": { - "description": "Database IDs to delete", - "type": "object", "$ref": "#/definitions/DeleteRequest" } } @@ -534,7 +525,7 @@ } }, "delete": { - "description": "Deletes messages matching a search.", + "description": "Delete all messages matching a search.", "produces": [ "application/json" ], @@ -546,7 +537,7 @@ "messages" ], "summary": "Delete messages by search", - "operationId": "MessagesSummary", + "operationId": "DeleteSearch", "parameters": [ { "type": "string", @@ -580,7 +571,7 @@ "tags" ], "summary": "Get all current tags", - "operationId": "SetTags", + "operationId": "GetTags", "responses": { "200": { "$ref": "#/responses/ArrayResponse" @@ -591,7 +582,7 @@ } }, "put": { - "description": "To remove all tags from a message, pass an empty tags array.", + "description": "This will overwrite any existing tags for selected message database IDs. To remove all tags from a message, pass an empty tags array.", "consumes": [ "application/json" ], @@ -609,14 +600,10 @@ "operationId": "SetTags", "parameters": [ { - "description": "Database IDs to update", - "name": "ids", + "name": "Body", "in": "body", - "required": true, "schema": { - "description": "Database IDs to update", - "type": "object", - "$ref": "#/definitions/SetTagsRequest" + "$ref": "#/definitions/setTagsRequestBody" } } ], @@ -807,15 +794,19 @@ "type": "object", "properties": { "ids": { - "description": "ids\nin:body", + "description": "Array of message database IDs", "type": "array", "items": { "type": "string" }, - "x-go-name": "IDs" + "x-go-name": "IDs", + "example": [ + "5dec4247-812e-4b77-9101-e25ad406e9ea", + "8ac66bbc-2d9a-4c41-ad99-00aa75fa674e" + ] } }, - "x-go-name": "deleteRequest", + "x-go-name": "deleteMessagesRequestBody", "x-go-package": "github.com/axllent/mailpit/server/apiv1" }, "HTMLCheckResponse": { @@ -1188,6 +1179,10 @@ "type": "integer", "format": "int64" }, + "Snippet": { + "description": "Message snippet includes up to 250 characters", + "type": "string" + }, "Subject": { "description": "Email subject", "type": "string" @@ -1214,7 +1209,7 @@ "type": "object", "properties": { "messages": { - "description": "Messages summary\nin:body", + "description": "Messages summary\nin: body", "type": "array", "items": { "$ref": "#/definitions/MessageSummary" @@ -1256,99 +1251,121 @@ }, "x-go-package": "github.com/axllent/mailpit/server/apiv1" }, - "ReleaseMessageRequest": { + "WebUIConfiguration": { + "description": "Response includes global web UI settings", + "type": "object", + "properties": { + "DisableHTMLCheck": { + "description": "Whether the HTML check has been globally disabled", + "type": "boolean" + }, + "MessageRelay": { + "description": "Message Relay information", + "type": "object", + "properties": { + "Enabled": { + "description": "Whether message relaying (release) is enabled", + "type": "boolean" + }, + "RecipientAllowlist": { + "description": "Allowlist of accepted recipients", + "type": "string" + }, + "ReturnPath": { + "description": "Enforced Return-Path (if set) for relay bounces", + "type": "string" + }, + "SMTPServer": { + "description": "The configured SMTP server address", + "type": "string" + } + } + } + }, + "x-go-name": "webUIConfiguration", + "x-go-package": "github.com/axllent/mailpit/server/apiv1" + }, + "releaseMessageRequestBody": { "description": "Release request", "type": "object", + "required": [ + "to" + ], "properties": { "to": { - "description": "To\nin:body", + "description": "Array of email addresses to relay the message to", "type": "array", "items": { "type": "string" }, - "x-go-name": "To" + "x-go-name": "To", + "example": [ + "user1@example.com", + "user2@example.com" + ] } }, - "x-go-name": "releaseMessageRequest", "x-go-package": "github.com/axllent/mailpit/server/apiv1" }, - "SetReadStatusRequest": { + "setReadStatusRequestBody": { "description": "Set read status request", "type": "object", "properties": { "ids": { - "description": "ids\nin:body", + "description": "Array of message database IDs", "type": "array", "items": { "type": "string" }, - "x-go-name": "IDs" + "x-go-name": "IDs", + "example": [ + "5dec4247-812e-4b77-9101-e25ad406e9ea", + "8ac66bbc-2d9a-4c41-ad99-00aa75fa674e" + ] }, "read": { "description": "Read status", "type": "boolean", - "x-go-name": "Read" + "default": false, + "x-go-name": "Read", + "example": true } }, - "x-go-name": "setReadStatusRequest", "x-go-package": "github.com/axllent/mailpit/server/apiv1" }, - "SetTagsRequest": { + "setTagsRequestBody": { "description": "Set tags request", "type": "object", + "required": [ + "tags", + "ids" + ], "properties": { "ids": { - "description": "IDs\nin:body", + "description": "Array of message database IDs", "type": "array", "items": { "type": "string" }, - "x-go-name": "IDs" + "x-go-name": "IDs", + "example": [ + "5dec4247-812e-4b77-9101-e25ad406e9ea", + "8ac66bbc-2d9a-4c41-ad99-00aa75fa674e" + ] }, "tags": { - "description": "Tags\nin:body", + "description": "Array of tag names to set", "type": "array", "items": { "type": "string" }, - "x-go-name": "Tags" + "x-go-name": "Tags", + "example": [ + "Tag 1", + "Tag 2" + ] } }, - "x-go-name": "setTagsRequest", - "x-go-package": "github.com/axllent/mailpit/server/apiv1" - }, - "WebUIConfiguration": { - "description": "Response includes global web UI settings", - "type": "object", - "properties": { - "DisableHTMLCheck": { - "description": "Whether the HTML check has been globally disabled", - "type": "boolean" - }, - "MessageRelay": { - "description": "Message Relay information", - "type": "object", - "properties": { - "Enabled": { - "description": "Whether message relaying (release) is enabled", - "type": "boolean" - }, - "RecipientAllowlist": { - "description": "Allowlist of accepted recipients", - "type": "string" - }, - "ReturnPath": { - "description": "Enforced Return-Path (if set) for relay bounces", - "type": "string" - }, - "SMTPServer": { - "description": "The configured SMTP server address", - "type": "string" - } - } - } - }, - "x-go-name": "webUIConfiguration", "x-go-package": "github.com/axllent/mailpit/server/apiv1" } }, @@ -1363,23 +1380,27 @@ } }, "BinaryResponse": { - "description": "Binary data response inherits the attachment's content type" + "description": "Binary data response inherits the attachment's content type", + "schema": { + "type": "string" + } }, "ErrorResponse": { - "description": "Error response" + "description": "Error response", + "schema": { + "type": "string" + } }, "HTMLResponse": { - "description": "HTML response" + "description": "HTML response", + "schema": { + "type": "string" + } }, "InfoResponse": { "description": "Application information", "schema": { "$ref": "#/definitions/AppInformation" - }, - "headers": { - "Body": { - "description": "Application information" - } } }, "MessagesSummaryResponse": { @@ -1389,20 +1410,21 @@ } }, "OKResponse": { - "description": "Plain text \"ok\" response" + "description": "Plain text \"ok\" response", + "schema": { + "type": "string" + } }, "TextResponse": { - "description": "Plain text response" + "description": "Plain text response", + "schema": { + "type": "string" + } }, "WebUIConfigurationResponse": { "description": "Web UI configuration", "schema": { "$ref": "#/definitions/WebUIConfiguration" - }, - "headers": { - "Body": { - "description": "Web UI configuration settings" - } } } } From e5646372030a8affb967ab4ee7064f541c6c020f Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 15 Oct 2023 20:55:43 +1300 Subject: [PATCH 04/10] Tests: Add test to validate swagger.json --- .github/workflows/tests.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c26e25830..deea3d74e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,8 @@ jobs: with: go-version: ${{ matrix.go-version }} - uses: actions/checkout@v4 - - uses: actions/cache@v3 + - name: Run Go tests + uses: actions/cache@v3 with: path: | ~/.cache/go-build @@ -28,9 +29,16 @@ jobs: - run: go test ./internal/storage -bench=. # build the assets - - uses: actions/setup-node@v3 + - name: Build web UI + uses: actions/setup-node@v3 with: node-version: 18 cache: 'npm' - run: npm install - run: npm run package + + # validate the swagger file + - name: Validate OpenAPI definition + uses: char0n/swagger-editor-validate@v1 + with: + definition-file: server/ui/api/v1/swagger.json From e72dd8d9b63f80bafc92779b448f757fb4747760 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 15 Oct 2023 22:02:57 +1300 Subject: [PATCH 05/10] Replace unprintable characters with space in html2text --- .github/workflows/tests.yml | 2 +- internal/tools/html2text/html2text.go | 10 ++ internal/tools/html2text/html2text_test.go | 194 +++++++++++++++++++++ 3 files changed, 205 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index deea3d74e..ce99ba53f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: restore-keys: | ${{ runner.os }}-go- - run: go test ./internal/storage ./server ./internal/tools ./internal/tools/html2text -v - - run: go test ./internal/storage -bench=. + - run: go test ./internal/storage ./internal/tools/html2text -bench=. # build the assets - name: Build web UI diff --git a/internal/tools/html2text/html2text.go b/internal/tools/html2text/html2text.go index 0940d067a..9f3f6f6f9 100644 --- a/internal/tools/html2text/html2text.go +++ b/internal/tools/html2text/html2text.go @@ -6,6 +6,7 @@ import ( "log" "regexp" "strings" + "unicode" "golang.org/x/net/html" ) @@ -67,6 +68,15 @@ func extract(node *html.Node, buff *bytes.Buffer, includeLinks bool) { func clean(text string) string { // replace \uFEFF with space, see https://github.com/golang/go/issues/42274#issuecomment-1017258184 text = strings.ReplaceAll(text, string('\uFEFF'), " ") + + // remove non-printable characters + text = strings.Map(func(r rune) rune { + if unicode.IsPrint(r) { + return r + } + return []rune(" ")[0] + }, text) + text = re.ReplaceAllString(text, " ") return strings.TrimSpace(text) } diff --git a/internal/tools/html2text/html2text_test.go b/internal/tools/html2text/html2text_test.go index dd1763918..96ca9a13d 100644 --- a/internal/tools/html2text/html2text_test.go +++ b/internal/tools/html2text/html2text_test.go @@ -54,3 +54,197 @@ func TestWithLinks(t *testing.T) { } } } + +func BenchmarkPlain(b *testing.B) { + for i := 0; i < b.N; i++ { + Strip(htmlTestData, false) + } +} + +func BenchmarkLinks(b *testing.B) { + for i := 0; i < b.N; i++ { + Strip(htmlTestData, true) + } +} + +var htmlTestData = ` + + + + + [axllent/mailpit] Run failed: .github/workflows/tests.yml - feature/swagger (284335a) + + + + + + + +
+
+ + + + +
+ + + + + + +
 
+ + + + + +
+ GitHub +

+ [axllent/mailpit] .github/workflows/tests.yml workflow run + +

+
+ + + + + + +
 
+ +
+ + + + +
+ + + + +
+ + + + +
+ + + + + +
+ + + + +
+ + + + + + + + +
 
+ +

.github/workflows/tests.yml: No jobs were run

+ + + + + + +
 
+ + + + + + + +
+ + + + +
+ + + + +
+ + View workflow run + +
+
+ +
+ + + + + + +
 
+ + +
+
+ + + + +
+
+
+ + + + +
+ + + + + + +
 
+ + + + + + + +
 
+ +

—
You are receiving this because you are subscribed to this thread.
Manage your GitHub Actions notifications

+ +
+ + + + +
+ + + + + + +
 
+ +

GitHub, Inc. ・88 Colin P Kelly Jr Street ・San Francisco, CA 94107

+
+ +
+
+ +

                                                           
+ +` From 4c3e073b0c1a825cd687fe575f7cb07157909bef Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Mon, 16 Oct 2023 17:34:10 +1300 Subject: [PATCH 06/10] Change swagger BinaryResponse to os.File --- server/apiv1/swagger.go | 7 ++++++- server/ui/api/v1/swagger.json | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/server/apiv1/swagger.go b/server/apiv1/swagger.go index 583714fed..b5ed980d6 100644 --- a/server/apiv1/swagger.go +++ b/server/apiv1/swagger.go @@ -1,5 +1,7 @@ package apiv1 +import "os" + // These structs are for the purpose of defining swagger HTTP responses // Application information @@ -146,7 +148,10 @@ type linkCheckParams struct { // Binary data response inherits the attachment's content type // swagger:response BinaryResponse -type binaryResponse string +type binaryResponse struct { + // in: body + File os.File +} // Plain text response // swagger:response TextResponse diff --git a/server/ui/api/v1/swagger.json b/server/ui/api/v1/swagger.json index c85fff451..29c18da5d 100644 --- a/server/ui/api/v1/swagger.json +++ b/server/ui/api/v1/swagger.json @@ -809,6 +809,11 @@ "x-go-name": "deleteMessagesRequestBody", "x-go-package": "github.com/axllent/mailpit/server/apiv1" }, + "File": { + "type": "object", + "title": "File represents an open file descriptor.", + "x-go-package": "os" + }, "HTMLCheckResponse": { "description": "Response represents the HTML check response struct", "type": "object", @@ -1382,7 +1387,7 @@ "BinaryResponse": { "description": "Binary data response inherits the attachment's content type", "schema": { - "type": "string" + "$ref": "#/definitions/File" } }, "ErrorResponse": { From 0e01b9ff73c79e7b86e9e2ba9298f7111b8223ee Mon Sep 17 00:00:00 2001 From: Cyril Jouve Date: Mon, 16 Oct 2023 05:53:21 +0200 Subject: [PATCH 07/10] Chore: Replace satori/go.uuid with github.com/google/uuid (#190) Fixes #189 --- go.mod | 3 +-- go.sum | 2 -- internal/storage/database.go | 4 ++-- server/apiv1/api.go | 4 ++-- server/smtpd/smtpd.go | 4 ++-- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index de3d85d1d..845da12e2 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/axllent/semver v0.0.1 github.com/disintegration/imaging v1.6.2 github.com/gomarkdown/markdown v0.0.0-20230922112808-5421fefb8386 + github.com/google/uuid v1.3.1 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/jhillyerd/enmime v1.0.1 @@ -15,7 +16,6 @@ require ( github.com/leporo/sqlf v1.4.0 github.com/mhale/smtpd v0.8.0 github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e - github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 @@ -35,7 +35,6 @@ require ( github.com/cznic/ql v1.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect - github.com/google/uuid v1.3.1 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 // indirect diff --git a/go.sum b/go.sum index c5a4f444f..0d1501565 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,6 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= diff --git a/internal/storage/database.go b/internal/storage/database.go index c6d55edcf..9e8e0f7f2 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -22,10 +22,10 @@ import ( "github.com/axllent/mailpit/internal/logger" "github.com/axllent/mailpit/internal/tools" "github.com/axllent/mailpit/server/websockets" + "github.com/google/uuid" "github.com/jhillyerd/enmime" "github.com/klauspost/compress/zstd" "github.com/leporo/sqlf" - uuid "github.com/satori/go.uuid" // sqlite (native) - https://gitlab.com/cznic/sqlite _ "modernc.org/sqlite" @@ -161,7 +161,7 @@ func Store(body []byte) (string, error) { searchText := createSearchText(env) // generate unique ID - id := uuid.NewV4().String() + id := uuid.New().String() summaryJSON, err := json.Marshal(obj) if err != nil { diff --git a/server/apiv1/api.go b/server/apiv1/api.go index c1f64a74d..d201672d8 100644 --- a/server/apiv1/api.go +++ b/server/apiv1/api.go @@ -17,8 +17,8 @@ import ( "github.com/axllent/mailpit/internal/storage" "github.com/axllent/mailpit/internal/tools" "github.com/axllent/mailpit/server/smtpd" + "github.com/google/uuid" "github.com/gorilla/mux" - uuid "github.com/satori/go.uuid" ) // GetMessages returns a paginated list of messages as JSON @@ -686,7 +686,7 @@ func ReleaseMessage(w http.ResponseWriter, r *http.Request) { } // generate unique ID - uid := uuid.NewV4().String() + "@mailpit" + uid := uuid.New().String() + "@mailpit" // add unique ID msg = append([]byte("Message-Id: <"+uid+">\r\n"), msg...) diff --git a/server/smtpd/smtpd.go b/server/smtpd/smtpd.go index e95310667..d0e310e51 100644 --- a/server/smtpd/smtpd.go +++ b/server/smtpd/smtpd.go @@ -13,8 +13,8 @@ import ( "github.com/axllent/mailpit/internal/auth" "github.com/axllent/mailpit/internal/logger" "github.com/axllent/mailpit/internal/storage" + "github.com/google/uuid" "github.com/mhale/smtpd" - uuid "github.com/satori/go.uuid" ) func mailHandler(origin net.Addr, from string, to []string, data []byte) error { @@ -57,7 +57,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error { // add a message ID if not set if messageID == "" { // generate unique ID - messageID = uuid.NewV4().String() + "@mailpit" + messageID = uuid.New().String() + "@mailpit" // add unique ID data = append([]byte("Message-Id: <"+messageID+">\r\n"), data...) } else if config.IgnoreDuplicateIDs { From 7185649bbd8c39c7a1bf9bc6033e6d22510d4bce Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Mon, 16 Oct 2023 21:15:56 +1300 Subject: [PATCH 08/10] Libs: Update Go modules --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 845da12e2..a678cb787 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/jhillyerd/enmime v1.0.1 - github.com/klauspost/compress v1.17.0 + github.com/klauspost/compress v1.17.1 github.com/leporo/sqlf v1.4.0 github.com/mhale/smtpd v0.8.0 github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e diff --git a/go.sum b/go.sum index 0d1501565..29659da3d 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,8 @@ github.com/jhillyerd/enmime v1.0.1 h1:y6RyqIgBOI2hIinOXIzmeB+ITRVls0zTJIm5GwgXnj github.com/jhillyerd/enmime v1.0.1/go.mod h1:LMMbm6oTlzWHghPavqHtOrP/NosVv3l42CUrZjn03/Q= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= -github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= From 82d54d354c63b4121d0bba4f7d89478d885e271c Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Mon, 16 Oct 2023 21:17:35 +1300 Subject: [PATCH 09/10] Libs: Update node modules --- package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index ebd758064..7a0694086 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1847,9 +1847,9 @@ } }, "node_modules/object-inspect": { - "version": "1.12.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", - "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.0.tgz", + "integrity": "sha512-HQ4J+ic8hKrgIt3mqk6cVOVrW2ozL4KdvHlqpBv9vDYWx9ysAgENAdvy4FoGF+KFdhR7nQTNm5J0ctAeOwn+3g==", "funding": { "url": "https://github.com/sponsors/ljharb" } From b4f5aa3640ba70f95278dea286af9e0dc5a09720 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Mon, 16 Oct 2023 21:18:10 +1300 Subject: [PATCH 10/10] Release v1.9.8 --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f572a24a5..d7135143f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ Notable changes to Mailpit will be documented in this file. +## [v1.9.8] + +### Chore +- Replace satori/go.uuid with github.com/google/uuid ([#190](https://github.com/axllent/mailpit/issues/190)) +- Replace html2text modules with simplified internal function + +### Libs +- Update node modules +- Update Go modules + +### Swagger +- Update swagger documentation + +### Tests +- Add test to validate swagger.json +- Add html2text tests + + ## [v1.9.7] ### Fix