From fd2c06241f78fcf0cd0cf2d945d749332f8b1c21 Mon Sep 17 00:00:00 2001 From: Jeremy Jay Date: Thu, 16 Feb 2023 23:48:07 -0500 Subject: [PATCH] update http handling and add pfam warning --- data/data.go | 2 +- data/http.go | 19 ++++++++++++++----- data/http_wasm.go | 6 ++++++ data/pfam.go | 8 ++++---- data/uniprot.go | 38 ++++++++++++++++++++++++-------------- 5 files changed, 49 insertions(+), 24 deletions(-) diff --git a/data/data.go b/data/data.go index a886316..6395ba2 100644 --- a/data/data.go +++ b/data/data.go @@ -24,7 +24,7 @@ import ( ) // MotifNames has human-readable names -// - mostly from http://pfam-legacy.xfam.org/help#tabview=tab9 +// - mostly from http://pfam-legacy.xfam.org/help#tabview=tab9 var MotifNames = map[string]string{ "disorder": "Disordered region", "low_complexity": "Low complexity region", diff --git a/data/http.go b/data/http.go index d85ad13..372c50c 100644 --- a/data/http.go +++ b/data/http.go @@ -1,19 +1,28 @@ +//go:build !wasm // +build !wasm package data import ( + "crypto/tls" + "fmt" "net/http" "net/url" - "crypto/tls" + "os" ) func httpGet(url string) (*http.Response, error) { + return http.Get(url) +} + +func httpGetInsecure(url string) (*http.Response, error) { tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - client := &http.Client{Transport: tr} - return client.Get(url) + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + client := &http.Client{Transport: tr} + fmt.Fprintln(os.Stderr, "WARNING: making insecure request to ", url) + fmt.Fprintln(os.Stderr, " eventually this will no longer work correctly!") + return client.Get(url) } func httpPostForm(url string, vals url.Values) (*http.Response, error) { diff --git a/data/http_wasm.go b/data/http_wasm.go index d7ae78f..2c8f58b 100644 --- a/data/http_wasm.go +++ b/data/http_wasm.go @@ -1,3 +1,4 @@ +//go:build wasm // +build wasm package data @@ -24,6 +25,11 @@ func httpGet(url string) (*http.Response, error) { return resp, err } +func httpGetInsecure(url string) (*http.Response, error) { + // I am not willing to test if this can be configured in WASM. + return httpGet(url) +} + // implements http.PostForm but makes wasm's fetch work with CORS func httpPostForm(wurl string, vals url.Values) (*http.Response, error) { body := strings.NewReader(vals.Encode()) diff --git a/data/pfam.go b/data/pfam.go index 628c611..ffc462b 100644 --- a/data/pfam.go +++ b/data/pfam.go @@ -29,7 +29,7 @@ const PfamGraphicURL = "https://pfam-legacy.xfam.org/protein/%s/graphic" func GetPfamGraphicData(accession string) (*GraphicResponse, error) { queryURL := fmt.Sprintf(PfamGraphicURL, accession) - resp, err := httpGet(queryURL) + resp, err := httpGetInsecure(queryURL) if err != nil { if err, ok := err.(net.Error); ok && err.Timeout() { fmt.Fprintf(os.Stderr, "Unable to connect to Pfam. Check your internet connection or try again later.") @@ -47,9 +47,9 @@ func GetPfamGraphicData(accession string) (*GraphicResponse, error) { data := []GraphicResponse{} err = json.Unmarshal(respBytes, &data) - //if err != nil { - // return nil, err - //} + if err != nil { + return nil, err + } if len(data) != 1 { return nil, fmt.Errorf("pfam returned invalid result") } diff --git a/data/uniprot.go b/data/uniprot.go index a6d8ccf..68a1c42 100644 --- a/data/uniprot.go +++ b/data/uniprot.go @@ -19,6 +19,7 @@ package data import ( "bytes" + "compress/gzip" "encoding/json" "fmt" "io" @@ -60,13 +61,27 @@ func getValueForKey(line, key string) string { return "" } +func uniprotDecompress(respBytes []byte) []byte { + // uniprot's REST implementation doesn't set a valid Content-Encoding header when + // gzipping the response, so Go's automatic gzip decompression doesn't work. + // since they'll probably fix it after put in this workaround, we'll just try + // to un-gzip and replace the content if it doesn't fail. + + buf := bytes.NewReader(respBytes) + zrdr, err := gzip.NewReader(buf) + if err != nil { + return respBytes + } + data, err := io.ReadAll(zrdr) + if err == nil { + return data + } + return respBytes +} + func GetUniprotGraphicData(accession string) (*GraphicResponse, error) { queryURL := fmt.Sprintf(UniprotDataURL, accession) - client := &http.Client{} - req, err := http.NewRequest("GET", queryURL, nil) - req.Header.Add("Accept-Encoding", "UTF-8") - resp, err := client.Do(req) - + resp, err := httpGet(queryURL) if err != nil { if err, ok := err.(net.Error); ok && err.Timeout() { fmt.Fprintf(os.Stderr, "Unable to connect to Uniprot. Check your internet connection or try again later.") @@ -79,6 +94,7 @@ func GetUniprotGraphicData(accession string) (*GraphicResponse, error) { if err != nil { return nil, err } + respBytes = uniprotDecompress(respBytes) if resp.StatusCode != 200 { return nil, fmt.Errorf("pfam error: %s", resp.Status) } @@ -172,14 +188,7 @@ const UNIPROTRESTURL = "https://rest.uniprot.org/uniprotkb/search?query=%s+AND+r func GetProtID(symbol string) (string, error) { apiURL := fmt.Sprintf(UNIPROTRESTURL, symbol) - client := &http.Client{} - req, err := http.NewRequest("GET", apiURL, nil) - req.Header.Add("Accept-Encoding", "UTF-8") - resp, err := client.Do(req) - if err != nil { - log.Fatal(err) - } - defer resp.Body.Close() + resp, err := http.Get(apiURL) if err != nil { if err, ok := err.(net.Error); ok && err.Timeout() { fmt.Fprintf(os.Stderr, "Unable to connect to Uniprot. Check your internet connection or try again later.") @@ -187,11 +196,12 @@ func GetProtID(symbol string) (string, error) { } return "", err } - + defer resp.Body.Close() respBytes, err := io.ReadAll(resp.Body) if err != nil { return "", err } + respBytes = uniprotDecompress(respBytes) if resp.StatusCode != 200 { return "", fmt.Errorf("uniprot error: %s", resp.Status) }