From 7d6c1fe88e6d7971f134dcf75e1cceb0028ade0a Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Wed, 6 Sep 2023 18:23:45 +0100 Subject: [PATCH 01/23] GetWithQueryParams function to create requests with retry flag --- internal/wrappers/azure-http.go | 3 --- internal/wrappers/bitbucket-http.go | 32 ++++------------------------- internal/wrappers/client.go | 30 +++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/internal/wrappers/azure-http.go b/internal/wrappers/azure-http.go index 07d90acde..89f55277c 100644 --- a/internal/wrappers/azure-http.go +++ b/internal/wrappers/azure-http.go @@ -30,7 +30,6 @@ const ( azureTop = "$top" azurePage = "$skip" azureLayoutTime = "2006-01-02" - basicFormat = "Basic %s" failedAuth = "failed Azure Authentication" unauthorized = "unauthorized: verify if the organization you provided is correct" azurePageLenValue = 100 @@ -112,8 +111,6 @@ func (g *AzureHTTPWrapper) get( queryParams map[string]string, authFormat string, ) (bool, error) { - var err error - req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return false, err diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index 6b890995a..6f1db6036 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -277,52 +277,28 @@ func collectPageBitBucket( } func getBitBucket(client *http.Client, token, url string, target interface{}, queryParams map[string]string) error { - var err error - - req, err := http.NewRequest(http.MethodGet, url, nil) + resp, err := GetWithQueryParams(client, url, token, basicFormat, queryParams) if err != nil { return err } - if len(token) > 0 { - req.Header.Add(AuthorizationHeader, fmt.Sprintf(basicFormat, token)) - } - - q := req.URL.Query() - for k, v := range queryParams { - q.Add(k, v) - } - - req.URL.RawQuery = q.Encode() - resp, err := client.Do(req) - if err != nil { - return err - } - logger.PrintRequest(req) - - defer func() { - _ = resp.Body.Close() - }() - - logger.PrintResponse(resp, true) - switch resp.StatusCode { case http.StatusOK: err = json.NewDecoder(resp.Body).Decode(target) if err != nil { return err } - // State sent when expired token + // State sent when expired token case http.StatusUnauthorized: err = errors.New(failedBitbucketAuth) return err - // State sent when no token is provided + // State sent when no token is provided case http.StatusForbidden: err = errors.New(failedBitbucketAuth) return err case http.StatusNotFound: err = errors.New(failedBitbucketNotFound) return err - // Case the commit/project does not exist in the organization + // Case the commit/project does not exist in the organization default: body, err := io.ReadAll(resp.Body) if err != nil { diff --git a/internal/wrappers/client.go b/internal/wrappers/client.go index 448fb4386..6b4350d7e 100644 --- a/internal/wrappers/client.go +++ b/internal/wrappers/client.go @@ -35,6 +35,8 @@ const ( MissingURI = "When using client-id and client-secret please provide base-uri or base-auth-uri" MissingTenant = "Failed to authenticate - please provide tenant" jwtError = "Error retrieving %s from jwt token" + basicFormat = "Basic %s" + bearearFormat = "Bearer %s" ) type ClientCredentialsInfo struct { @@ -356,6 +358,34 @@ func SendHTTPRequestWithJSONContentType(method, path string, body io.Reader, aut return resp, nil } +func GetWithQueryParams(client *http.Client, url, token, authFormat string, queryParams map[string]string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + if len(token) > 0 { + req.Header.Add(AuthorizationHeader, fmt.Sprintf(authFormat, token)) + } + + q := req.URL.Query() + for k, v := range queryParams { + q.Add(k, v) + } + + req.URL.RawQuery = q.Encode() + resp, err := request(client, req, true) + if err != nil { + return nil, err + } + logger.PrintRequest(req) + + defer func() { + _ = resp.Body.Close() + }() + + logger.PrintResponse(resp, true) + return resp, nil +} func GetAccessToken() (string, error) { authURI, err := getAuthURI() if err != nil { From a5fb73449077c348f161069fc1af61a1f497d369 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 14:52:39 +0100 Subject: [PATCH 02/23] refactoring client.go --- internal/wrappers/bitbucket-http.go | 2 + internal/wrappers/client.go | 71 ++++++++++------------------- 2 files changed, 26 insertions(+), 47 deletions(-) diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index 6f1db6036..c351d2a30 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -281,6 +281,8 @@ func getBitBucket(client *http.Client, token, url string, target interface{}, qu if err != nil { return err } + defer resp.Body.Close() + switch resp.StatusCode { case http.StatusOK: err = json.NewDecoder(resp.Body).Decode(target) diff --git a/internal/wrappers/client.go b/internal/wrappers/client.go index c01b94715..ee610300d 100644 --- a/internal/wrappers/client.go +++ b/internal/wrappers/client.go @@ -29,6 +29,7 @@ const ( NoTimeout = 0 ntlmProxyToken = "ntlm" checkmarxURLError = "Could not reach provided Checkmarx server" + invalidCredentialsError = "Provided credentials are invalid" APIKeyDecodeErrorFormat = "Token decoding error: %s" tryPrintOffset = 2 retryLimitPrintOffset = 1 @@ -37,6 +38,9 @@ const ( jwtError = "Error retrieving %s from jwt token" basicFormat = "Basic %s" bearearFormat = "Bearer %s" + contentTypeHeader = "Content-Type" + formURLContentType = "application/x-www-form-urlencoded" + jsonContentType = "application/json" ) type ClientCredentialsInfo struct { @@ -198,16 +202,12 @@ func SendHTTPRequestByFullURLContentLength( client := GetClient(timeout) setAgentName(req) if auth { - enrichWithOath2Credentials(req, accessToken) + enrichWithOath2Credentials(req, accessToken, bearearFormat) } req = addReqMonitor(req) - var resp *http.Response - resp, err = request(client, req, bodyPrint) - if err != nil { - return nil, err - } - return resp, nil + + return request(client, req, bodyPrint) } func addReqMonitor(req *http.Request) *http.Request { @@ -256,19 +256,13 @@ func SendHTTPRequestPasswordAuth(method string, body io.Reader, timeout uint, us if err != nil { return nil, err } - req.Header.Add("content-type", "application/json") + req.Header.Add(contentTypeHeader, jsonContentType) err = enrichWithPasswordCredentials(req, username, password, adminClientID, adminClientSecret) if err != nil { return nil, err } - var resp *http.Response - req = addReqMonitor(req) - resp, err = doRequest(client, req) - if err != nil { - return nil, err - } - return resp, nil + return doRequest(client, req) } func SendPrivateHTTPRequestWithQueryParams( @@ -304,7 +298,7 @@ func HTTPRequestWithQueryParams( q.Add(k, v) } req.URL.RawQuery = q.Encode() - enrichWithOath2Credentials(req, accessToken) + enrichWithOath2Credentials(req, accessToken, bearearFormat) var resp *http.Response resp, err = request(client, req, printBody) if err != nil { @@ -329,8 +323,8 @@ func addTenantAuthURI(baseAuthURI string) (string, error) { return fmt.Sprintf("%s/%s", strings.Trim(baseAuthURI, "/"), authPath), nil } -func enrichWithOath2Credentials(request *http.Request, accessToken string) { - request.Header.Add("Authorization", "Bearer "+accessToken) +func enrichWithOath2Credentials(request *http.Request, accessToken, authFormat string) { + request.Header.Add(AuthorizationHeader, fmt.Sprintf(authFormat, accessToken)) } func SendHTTPRequestWithJSONContentType(method, path string, body io.Reader, auth bool, timeout uint) ( @@ -344,21 +338,16 @@ func SendHTTPRequestWithJSONContentType(method, path string, body io.Reader, aut req, err := http.NewRequest(method, fullURL, body) client := GetClient(timeout) setAgentName(req) - req.Header.Add("Content-Type", "application/json") + req.Header.Add("Content-Type", jsonContentType) if err != nil { return nil, err } if auth { - enrichWithOath2Credentials(req, accessToken) + enrichWithOath2Credentials(req, accessToken, bearearFormat) } req = addReqMonitor(req) - var resp *http.Response - resp, err = doRequest(client, req) - if err != nil { - return nil, err - } - return resp, nil + return doRequest(client, req) } func GetWithQueryParams(client *http.Client, url, token, authFormat string, queryParams map[string]string) (*http.Response, error) { @@ -367,27 +356,16 @@ func GetWithQueryParams(client *http.Client, url, token, authFormat string, quer return nil, err } if len(token) > 0 { - req.Header.Add(AuthorizationHeader, fmt.Sprintf(authFormat, token)) + enrichWithOath2Credentials(req, token, authFormat) } q := req.URL.Query() for k, v := range queryParams { q.Add(k, v) } - req.URL.RawQuery = q.Encode() - resp, err := request(client, req, true) - if err != nil { - return nil, err - } - logger.PrintRequest(req) - - defer func() { - _ = resp.Body.Close() - }() - - logger.PrintResponse(resp, true) - return resp, nil + req = addReqMonitor(req) + return request(client, req, true) } func GetAccessToken() (string, error) { authURI, err := getAuthURI() @@ -432,8 +410,7 @@ func enrichWithPasswordCredentials( "failed to authenticate", ) } - - request.Header.Add("Authorization", "Bearer "+accessToken) + enrichWithOath2Credentials(request, accessToken, bearearFormat) return nil } @@ -488,7 +465,7 @@ func getNewToken(credentialsPayload, authServerURI string) (string, error) { return "", err } req = addReqMonitor(req) - req.Header.Add("content-type", "application/x-www-form-urlencoded") + req.Header.Add(contentTypeHeader, formURLContentType) clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) client := GetClient(clientTimeout) @@ -498,13 +475,13 @@ func getNewToken(credentialsPayload, authServerURI string) (string, error) { return "", errors.Errorf("%s %s", checkmarxURLError, authURL) } if res.StatusCode == http.StatusBadRequest { - return "", errors.Errorf("%v %s \n", res.StatusCode, "Provided credentials are invalid") + return "", errors.Errorf("%d %s \n", res.StatusCode, invalidCredentialsError) } if res.StatusCode == http.StatusNotFound { - return "", errors.Errorf("%v %s \n", res.StatusCode, "Provided Tenant Name is invalid") + return "", errors.Errorf("%d %s \n", res.StatusCode, "Provided Tenant Name is invalid") } if res.StatusCode == http.StatusUnauthorized { - return "", errors.Errorf("%v %s \n", res.StatusCode, "Provided credentials are invalid") + return "", errors.Errorf("%d %s \n", res.StatusCode, invalidCredentialsError) } body, _ := ioutil.ReadAll(res.Body) @@ -516,7 +493,7 @@ func getNewToken(credentialsPayload, authServerURI string) (string, error) { return "", err } - return "", errors.Errorf("%v %s %s", res.StatusCode, credentialsErr.Error, credentialsErr.Description) + return "", errors.Errorf("%d %s %s", res.StatusCode, credentialsErr.Error, credentialsErr.Description) } defer func() { From 0751a2a7bd38c098945d67f89b70f86f00103116 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 15:05:34 +0100 Subject: [PATCH 03/23] refactoring client.go - linter --- internal/wrappers/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/wrappers/client.go b/internal/wrappers/client.go index ee610300d..e6fc4002f 100644 --- a/internal/wrappers/client.go +++ b/internal/wrappers/client.go @@ -350,8 +350,8 @@ func SendHTTPRequestWithJSONContentType(method, path string, body io.Reader, aut return doRequest(client, req) } -func GetWithQueryParams(client *http.Client, url, token, authFormat string, queryParams map[string]string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) +func GetWithQueryParams(client *http.Client, urlAddress, token, authFormat string, queryParams map[string]string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodGet, urlAddress, http.NoBody) if err != nil { return nil, err } From 3ca4605abef63e657630d829252dc28891020310 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 15:15:15 +0100 Subject: [PATCH 04/23] more refactoring --- internal/commands/scarealtime/sca-realtime-utils.go | 2 +- internal/wrappers/logs-http.go | 2 +- internal/wrappers/predicates-http.go | 2 +- internal/wrappers/projects-http.go | 6 +++--- internal/wrappers/results-pdf-http.go | 4 ++-- internal/wrappers/results-sbom-http.go | 4 ++-- internal/wrappers/risks-overview-http.go | 2 +- internal/wrappers/scans-http.go | 8 ++++---- internal/wrappers/tenant-http.go | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/internal/commands/scarealtime/sca-realtime-utils.go b/internal/commands/scarealtime/sca-realtime-utils.go index 005cf7b04..02d119c66 100644 --- a/internal/commands/scarealtime/sca-realtime-utils.go +++ b/internal/commands/scarealtime/sca-realtime-utils.go @@ -162,7 +162,7 @@ func downloadSCAResolverHashFile(scaResolverHashURL, scaResolverZipFileNameHash func downloadFile(downloadURLPath, fileName string) error { logger.PrintIfVerbose("Downloading " + fileName + " from: " + downloadURLPath) - response, err := wrappers.SendHTTPRequestByFullURL(http.MethodGet, downloadURLPath, nil, false, 0, "", true) + response, err := wrappers.SendHTTPRequestByFullURL(http.MethodGet, downloadURLPath, http.NoBody, false, 0, "", true) if err != nil { return errors.Errorf("Invoking HTTP request to upload file failed - %s", err.Error()) } diff --git a/internal/wrappers/logs-http.go b/internal/wrappers/logs-http.go index 39fd8f5f6..b94d2b474 100644 --- a/internal/wrappers/logs-http.go +++ b/internal/wrappers/logs-http.go @@ -28,7 +28,7 @@ func NewLogsWrapper(path string) LogsWrapper { func (l *LogsHTTPWrapper) GetLog(scanID, scanType string) (string, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) reportPath := fmt.Sprintf("%s/%s/%s", l.path, scanID, scanType) - resp, err := SendHTTPRequest(http.MethodGet, reportPath, nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodGet, reportPath, http.NoBody, true, clientTimeout) if err != nil { return "", err } diff --git a/internal/wrappers/predicates-http.go b/internal/wrappers/predicates-http.go index 757c89f18..d853c1188 100644 --- a/internal/wrappers/predicates-http.go +++ b/internal/wrappers/predicates-http.go @@ -48,7 +48,7 @@ func (r *ResultsPredicatesHTTPWrapper) GetAllPredicatesForSimilarityID(similarit var request = "/" + similarityID + "?project-ids=" + projectID logger.PrintIfVerbose(fmt.Sprintf("Sending GET request to %s", r.path+request)) - return handleResponseWithBody(SendHTTPRequest(http.MethodGet, r.path+request, nil, true, clientTimeout)) + return handleResponseWithBody(SendHTTPRequest(http.MethodGet, r.path+request, http.NoBody, true, clientTimeout)) } func (r *ResultsPredicatesHTTPWrapper) SetPath(newPath string) { diff --git a/internal/wrappers/projects-http.go b/internal/wrappers/projects-http.go index 64de7b35b..e1d8eec54 100644 --- a/internal/wrappers/projects-http.go +++ b/internal/wrappers/projects-http.go @@ -117,7 +117,7 @@ func (p *ProjectsHTTPWrapper) GetByID(projectID string) ( *ErrorModel, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - resp, err := SendHTTPRequest(http.MethodGet, p.path+"/"+projectID, nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodGet, p.path+"/"+projectID, http.NoBody, true, clientTimeout) if err != nil { return nil, nil, err } @@ -162,7 +162,7 @@ func (p *ProjectsHTTPWrapper) GetBranchesByID(projectID string, params map[strin func (p *ProjectsHTTPWrapper) Delete(projectID string) (*ErrorModel, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - resp, err := SendHTTPRequest(http.MethodDelete, p.path+"/"+projectID, nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodDelete, p.path+"/"+projectID, http.NoBody, true, clientTimeout) if err != nil { return nil, err } @@ -174,7 +174,7 @@ func (p *ProjectsHTTPWrapper) Tags() ( *ErrorModel, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - resp, err := SendHTTPRequest(http.MethodGet, p.path+"/tags", nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodGet, p.path+"/tags", http.NoBody, true, clientTimeout) if err != nil { return nil, nil, err } diff --git a/internal/wrappers/results-pdf-http.go b/internal/wrappers/results-pdf-http.go index f348e10a6..42c9133f4 100644 --- a/internal/wrappers/results-pdf-http.go +++ b/internal/wrappers/results-pdf-http.go @@ -108,8 +108,8 @@ func (r *PdfHTTPWrapper) CheckPdfReportStatus(reportID string) (*PdfPollingRespo func (r *PdfHTTPWrapper) DownloadPdfReport(reportID, targetFile string) error { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - url := fmt.Sprintf("%s/%s/download", r.path, reportID) - resp, err := SendHTTPRequest(http.MethodGet, url, nil, true, clientTimeout) + customURL := fmt.Sprintf("%s/%s/download", r.path, reportID) + resp, err := SendHTTPRequest(http.MethodGet, customURL, http.NoBody, true, clientTimeout) if err != nil { return err } diff --git a/internal/wrappers/results-sbom-http.go b/internal/wrappers/results-sbom-http.go index 9e428167c..3de1752da 100644 --- a/internal/wrappers/results-sbom-http.go +++ b/internal/wrappers/results-sbom-http.go @@ -137,8 +137,8 @@ func (r *SbomHTTPWrapper) GetSbomReportStatus(reportID string) (*SbomPollingResp func (r *SbomHTTPWrapper) DownloadSbomReport(reportID, targetFile string) error { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - url := fmt.Sprintf("%s/%s/%s/%s", r.path, "requests", reportID, "download") - resp, err := SendHTTPRequest(http.MethodGet, url, nil, true, clientTimeout) + customURL := fmt.Sprintf("%s/%s/%s/%s", r.path, "requests", reportID, "download") + resp, err := SendHTTPRequest(http.MethodGet, customURL, http.NoBody, true, clientTimeout) if err != nil { return err } diff --git a/internal/wrappers/risks-overview-http.go b/internal/wrappers/risks-overview-http.go index 8c7c9fa38..f2e2dfb19 100644 --- a/internal/wrappers/risks-overview-http.go +++ b/internal/wrappers/risks-overview-http.go @@ -27,7 +27,7 @@ func (r *RisksOverviewHTTPWrapper) GetAllAPISecRisksByScanID(scanID string) ( ) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) path := fmt.Sprintf(r.path, scanID) - resp, err := SendHTTPRequest(http.MethodGet, path, nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodGet, path, http.NoBody, true, clientTimeout) if err != nil { return nil, nil, err } diff --git a/internal/wrappers/scans-http.go b/internal/wrappers/scans-http.go index b3f75913f..ebd477043 100644 --- a/internal/wrappers/scans-http.go +++ b/internal/wrappers/scans-http.go @@ -76,7 +76,7 @@ func (s *ScansHTTPWrapper) Get(params map[string]string) (*ScansCollectionRespon func (s *ScansHTTPWrapper) GetByID(scanID string) (*ScanResponseModel, *ErrorModel, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - resp, err := SendHTTPRequest(http.MethodGet, s.path+"/"+scanID, nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodGet, s.path+"/"+scanID, http.NoBody, true, clientTimeout) if err != nil { return nil, nil, err } @@ -86,7 +86,7 @@ func (s *ScansHTTPWrapper) GetByID(scanID string) (*ScanResponseModel, *ErrorMod func (s *ScansHTTPWrapper) GetWorkflowByID(scanID string) ([]*ScanTaskResponseModel, *ErrorModel, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) path := fmt.Sprintf("%s/%s/workflow", s.path, scanID) - resp, err := SendHTTPRequest(http.MethodGet, path, nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodGet, path, http.NoBody, true, clientTimeout) if err != nil { return nil, nil, err } @@ -124,7 +124,7 @@ func handleWorkflowResponseWithBody(resp *http.Response, err error) ([]*ScanTask func (s *ScansHTTPWrapper) Delete(scanID string) (*ErrorModel, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - resp, err := SendHTTPRequest(http.MethodDelete, s.path+"/"+scanID, nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodDelete, s.path+"/"+scanID, http.NoBody, true, clientTimeout) if err != nil { return nil, err } @@ -150,7 +150,7 @@ func (s *ScansHTTPWrapper) Cancel(scanID string) (*ErrorModel, error) { func (s *ScansHTTPWrapper) Tags() (map[string][]string, *ErrorModel, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) - resp, err := SendHTTPRequest(http.MethodGet, s.path+"/tags", nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodGet, s.path+"/tags", http.NoBody, true, clientTimeout) if err != nil { return nil, nil, err } diff --git a/internal/wrappers/tenant-http.go b/internal/wrappers/tenant-http.go index 4dfd9156b..65ecd7282 100644 --- a/internal/wrappers/tenant-http.go +++ b/internal/wrappers/tenant-http.go @@ -26,7 +26,7 @@ func (r *TenantConfigurationHTTPWrapper) GetTenantConfiguration() ( ) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) // add the path parameter to the path - resp, err := SendHTTPRequest(http.MethodGet, r.path, nil, true, clientTimeout) + resp, err := SendHTTPRequest(http.MethodGet, r.path, http.NoBody, true, clientTimeout) if err != nil { return nil, nil, err } From a098fe99f116b478a371501cc514ac1fddffcb31 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 16:32:59 +0100 Subject: [PATCH 05/23] more refactoring --- internal/wrappers/bitbucket-http.go | 2 +- internal/wrappers/tenant-http.go | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index c351d2a30..e68d9fe23 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -156,7 +156,7 @@ func (g *BitBucketHTTPWrapper) getFromBitBucket( } if len(token) > 0 { - req.Header.Add(AuthorizationHeader, fmt.Sprintf(basicFormat, token)) + enrichWithOath2Credentials(req, token, basicFormat) } q := req.URL.Query() diff --git a/internal/wrappers/tenant-http.go b/internal/wrappers/tenant-http.go index 65ecd7282..e605961d0 100644 --- a/internal/wrappers/tenant-http.go +++ b/internal/wrappers/tenant-http.go @@ -30,12 +30,10 @@ func (r *TenantConfigurationHTTPWrapper) GetTenantConfiguration() ( if err != nil { return nil, nil, err } - - decoder := json.NewDecoder(resp.Body) - defer func() { _ = resp.Body.Close() }() + decoder := json.NewDecoder(resp.Body) switch resp.StatusCode { case http.StatusBadRequest, http.StatusInternalServerError: From 9ad369321c0673e2f7d7f0b84b767de54ffc2e86 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 17:54:26 +0100 Subject: [PATCH 06/23] refactoring --- internal/wrappers/projects-http.go | 4 ++-- internal/wrappers/response.go | 6 ------ internal/wrappers/scans-http.go | 11 ++++++----- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/internal/wrappers/projects-http.go b/internal/wrappers/projects-http.go index e1d8eec54..8a2c89fbf 100644 --- a/internal/wrappers/projects-http.go +++ b/internal/wrappers/projects-http.go @@ -166,6 +166,7 @@ func (p *ProjectsHTTPWrapper) Delete(projectID string) (*ErrorModel, error) { if err != nil { return nil, err } + defer resp.Body.Close() return handleProjectResponseWithNoBody(resp, err, http.StatusNoContent) } @@ -178,11 +179,10 @@ func (p *ProjectsHTTPWrapper) Tags() ( if err != nil { return nil, nil, err } + defer resp.Body.Close() decoder := json.NewDecoder(resp.Body) - defer resp.Body.Close() - switch resp.StatusCode { case http.StatusBadRequest, http.StatusInternalServerError: errorModel := ErrorModel{} diff --git a/internal/wrappers/response.go b/internal/wrappers/response.go index 1b2a78022..30b347a6c 100644 --- a/internal/wrappers/response.go +++ b/internal/wrappers/response.go @@ -18,8 +18,6 @@ func handleScanResponseWithNoBody(resp *http.Response, err error, } decoder := json.NewDecoder(resp.Body) - defer resp.Body.Close() - switch resp.StatusCode { case http.StatusBadRequest, http.StatusInternalServerError, http.StatusNotFound: errorModel := ErrorModel{} @@ -43,8 +41,6 @@ func handleScanResponseWithBody(resp *http.Response, err error, } decoder := json.NewDecoder(resp.Body) - defer resp.Body.Close() - switch resp.StatusCode { case http.StatusBadRequest, http.StatusInternalServerError: errorModel := ErrorModel{} @@ -75,8 +71,6 @@ func handleProjectResponseWithNoBody(resp *http.Response, err error, } decoder := json.NewDecoder(resp.Body) - defer resp.Body.Close() - switch resp.StatusCode { case http.StatusBadRequest, http.StatusInternalServerError: errorModel := ErrorModel{} diff --git a/internal/wrappers/scans-http.go b/internal/wrappers/scans-http.go index ebd477043..5fe00946a 100644 --- a/internal/wrappers/scans-http.go +++ b/internal/wrappers/scans-http.go @@ -39,6 +39,7 @@ func (s *ScansHTTPWrapper) Create(model *Scan) (*ScanResponseModel, *ErrorModel, if err != nil { return nil, nil, err } + defer resp.Body.Close() return handleScanResponseWithBody(resp, err, http.StatusCreated) } @@ -80,6 +81,7 @@ func (s *ScansHTTPWrapper) GetByID(scanID string) (*ScanResponseModel, *ErrorMod if err != nil { return nil, nil, err } + defer resp.Body.Close() return handleScanResponseWithBody(resp, err, http.StatusOK) } @@ -90,6 +92,7 @@ func (s *ScansHTTPWrapper) GetWorkflowByID(scanID string) ([]*ScanTaskResponseMo if err != nil { return nil, nil, err } + defer resp.Body.Close() return handleWorkflowResponseWithBody(resp, err) } @@ -99,8 +102,6 @@ func handleWorkflowResponseWithBody(resp *http.Response, err error) ([]*ScanTask } decoder := json.NewDecoder(resp.Body) - defer resp.Body.Close() - switch resp.StatusCode { case http.StatusBadRequest, http.StatusInternalServerError: errorModel := ErrorModel{} @@ -128,6 +129,7 @@ func (s *ScansHTTPWrapper) Delete(scanID string) (*ErrorModel, error) { if err != nil { return nil, err } + defer resp.Body.Close() return handleScanResponseWithNoBody(resp, err, http.StatusNoContent) } @@ -144,7 +146,7 @@ func (s *ScansHTTPWrapper) Cancel(scanID string) (*ErrorModel, error) { if err != nil { return nil, err } - + defer resp.Body.Close() return handleScanResponseWithNoBody(resp, err, http.StatusNoContent) } @@ -154,9 +156,8 @@ func (s *ScansHTTPWrapper) Tags() (map[string][]string, *ErrorModel, error) { if err != nil { return nil, nil, err } - decoder := json.NewDecoder(resp.Body) - defer resp.Body.Close() + decoder := json.NewDecoder(resp.Body) switch resp.StatusCode { case http.StatusBadRequest, http.StatusInternalServerError: From 7547408643a6c371cd1a537579db3c66ce0e2b71 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 18:02:05 +0100 Subject: [PATCH 07/23] refactoring --- internal/wrappers/projects-http.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/wrappers/projects-http.go b/internal/wrappers/projects-http.go index 8a2c89fbf..d369bf55c 100644 --- a/internal/wrappers/projects-http.go +++ b/internal/wrappers/projects-http.go @@ -33,6 +33,7 @@ func (p *ProjectsHTTPWrapper) Create(model *Project) (*ProjectResponseModel, *Er if err != nil { return nil, nil, err } + defer resp.Body.Close() return handleProjectResponseWithBody(resp, err, http.StatusCreated) } @@ -47,7 +48,7 @@ func (p *ProjectsHTTPWrapper) Update(projectID string, model *Project) error { if err != nil { return err } - resp.Body.Close() + defer resp.Body.Close() switch resp.StatusCode { case http.StatusNoContent: return nil @@ -71,7 +72,7 @@ func (p *ProjectsHTTPWrapper) UpdateConfiguration(projectID string, configuratio if err != nil { return nil, err } - + defer resp.Body.Close() return handleProjectResponseWithNoBody(resp, err, http.StatusNoContent) } @@ -121,6 +122,7 @@ func (p *ProjectsHTTPWrapper) GetByID(projectID string) ( if err != nil { return nil, nil, err } + defer resp.Body.Close() return handleProjectResponseWithBody(resp, err, http.StatusOK) } @@ -131,7 +133,6 @@ func (p *ProjectsHTTPWrapper) GetBranchesByID(projectID string, params map[strin params["limit"] = limitValue resp, err := SendHTTPRequestWithQueryParams(http.MethodGet, p.path+request, params, nil, clientTimeout) - if err != nil { return nil, nil, err } From 08e7a8a273d17ace6efd8dfdd5cff68affeed62c Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 21:18:46 +0100 Subject: [PATCH 08/23] linter test --- internal/wrappers/response.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/wrappers/response.go b/internal/wrappers/response.go index 30b347a6c..3beee23d9 100644 --- a/internal/wrappers/response.go +++ b/internal/wrappers/response.go @@ -11,6 +11,8 @@ const ( failedToParseErr = "Failed to parse error response" ) +// nolint:bodyclose + func handleScanResponseWithNoBody(resp *http.Response, err error, successStatusCode int) (*ErrorModel, error) { if err != nil { From ad930308c9b55de9d75532495e7fbc42cb5ae29e Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 21:26:53 +0100 Subject: [PATCH 09/23] linter test --- internal/wrappers/response.go | 3 +-- internal/wrappers/results-pdf-http.go | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/wrappers/response.go b/internal/wrappers/response.go index 3beee23d9..3a2bb95fa 100644 --- a/internal/wrappers/response.go +++ b/internal/wrappers/response.go @@ -1,3 +1,4 @@ +// nolint:bodyclose package wrappers import ( @@ -11,8 +12,6 @@ const ( failedToParseErr = "Failed to parse error response" ) -// nolint:bodyclose - func handleScanResponseWithNoBody(resp *http.Response, err error, successStatusCode int) (*ErrorModel, error) { if err != nil { diff --git a/internal/wrappers/results-pdf-http.go b/internal/wrappers/results-pdf-http.go index 42c9133f4..6f0d3320b 100644 --- a/internal/wrappers/results-pdf-http.go +++ b/internal/wrappers/results-pdf-http.go @@ -1,3 +1,4 @@ +// nolint:bodyclose package wrappers import ( From 5cc1d7a402b1be67f9a29ed59cbf01f8f0bc56f7 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 21:32:11 +0100 Subject: [PATCH 10/23] linter test --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e160d4053..3fb37101f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,7 @@ jobs: env: GOPRIVATE: "github.com/checkmarxDev/*" steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v3 - name: Set up Go 1.17.x uses: actions/setup-go@v4 with: @@ -117,7 +117,7 @@ jobs: - name: Setup git run: git config --global url."https://${{ secrets.PERSONAL_ACCESS_TOKEN }}:@github.com/".insteadOf "https://github.com" - name: golangci-lint - uses: golangci/golangci-lint-action@v3.4.0 + uses: golangci/golangci-lint-action@v3 with: version: v1.52.2 args: -c .golangci.yml From 18c798e4d28285d2984dc337d875b70345fe22cc Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 21:39:08 +0100 Subject: [PATCH 11/23] linter test --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fb37101f..bf52b1e43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,8 +117,8 @@ jobs: - name: Setup git run: git config --global url."https://${{ secrets.PERSONAL_ACCESS_TOKEN }}:@github.com/".insteadOf "https://github.com" - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v3.7.0 with: - version: v1.52.2 + version: v1.54.2 args: -c .golangci.yml only-new-issues: true From c21d0aeb52a94252d3504447d2bdcabd16a61306 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 21:41:47 +0100 Subject: [PATCH 12/23] nolint test to figure out where is the bodyclose error e.e --- internal/wrappers/projects-http.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/wrappers/projects-http.go b/internal/wrappers/projects-http.go index d369bf55c..8ee1e6a14 100644 --- a/internal/wrappers/projects-http.go +++ b/internal/wrappers/projects-http.go @@ -1,3 +1,4 @@ +// nolint:bodyclose package wrappers import ( From e41d2993700677b6764cb3a34fbe775f36d8241c Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 21:44:25 +0100 Subject: [PATCH 13/23] nolint test to figure out where is the bodyclose error e.e --- internal/wrappers/logs-http.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/wrappers/logs-http.go b/internal/wrappers/logs-http.go index b94d2b474..af59ae61a 100644 --- a/internal/wrappers/logs-http.go +++ b/internal/wrappers/logs-http.go @@ -1,3 +1,4 @@ +// nolint:bodyclose package wrappers import ( From 6c9b3c5dc076a05d7bfd1be47c10c4def7f715bf Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 22:06:53 +0100 Subject: [PATCH 14/23] linter fix e.e --- .github/workflows/ci.yml | 6 +++--- internal/wrappers/bfl-http.go | 5 +---- internal/wrappers/learn-more-http.go | 5 +---- internal/wrappers/logs-http.go | 1 - internal/wrappers/pr-http.go | 1 + internal/wrappers/projects-http.go | 1 - internal/wrappers/response.go | 1 - internal/wrappers/results-pdf-http.go | 1 - 8 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf52b1e43..e160d4053 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,7 +109,7 @@ jobs: env: GOPRIVATE: "github.com/checkmarxDev/*" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2.3.4 - name: Set up Go 1.17.x uses: actions/setup-go@v4 with: @@ -117,8 +117,8 @@ jobs: - name: Setup git run: git config --global url."https://${{ secrets.PERSONAL_ACCESS_TOKEN }}:@github.com/".insteadOf "https://github.com" - name: golangci-lint - uses: golangci/golangci-lint-action@v3.7.0 + uses: golangci/golangci-lint-action@v3.4.0 with: - version: v1.54.2 + version: v1.52.2 args: -c .golangci.yml only-new-issues: true diff --git a/internal/wrappers/bfl-http.go b/internal/wrappers/bfl-http.go index 1aa578f68..4baf33d63 100644 --- a/internal/wrappers/bfl-http.go +++ b/internal/wrappers/bfl-http.go @@ -33,6 +33,7 @@ func (r *BflHTTPWrapper) GetBflByScanIDAndQueryID(params map[string]string) ( clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) log.Println(fmt.Sprintf("Fetching the best fix location for QueryID: %s", params[commonParams.QueryIDQueryParam])) resp, err := SendHTTPRequestWithQueryParams(http.MethodGet, r.path, params, nil, clientTimeout) + defer resp.Body.Close() return handleBflResponseWithBody(resp, err) } @@ -43,10 +44,6 @@ func handleBflResponseWithBody(resp *http.Response, err error) (*BFLResponseMode decoder := json.NewDecoder(resp.Body) - defer func() { - _ = resp.Body.Close() - }() - switch resp.StatusCode { case http.StatusBadRequest, http.StatusInternalServerError: errorModel := WebError{} diff --git a/internal/wrappers/learn-more-http.go b/internal/wrappers/learn-more-http.go index 918f6b130..5957a1376 100644 --- a/internal/wrappers/learn-more-http.go +++ b/internal/wrappers/learn-more-http.go @@ -32,6 +32,7 @@ func (r *LearnMoreHTTPWrapper) GetLearnMoreDetails(params map[string]string) ( clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) // add the path parameter to the path resp, err := SendHTTPRequestWithQueryParams(http.MethodGet, r.path, params, nil, clientTimeout) + defer resp.Body.Close() return handleResponse(resp, err, params[commonParams.QueryIDQueryParam]) } @@ -42,10 +43,6 @@ func handleResponse(resp *http.Response, err error, queryID string) (*[]*LearnMo decoder := json.NewDecoder(resp.Body) - defer func() { - _ = resp.Body.Close() - }() - switch resp.StatusCode { case http.StatusBadRequest, http.StatusInternalServerError: errorModel := WebError{} diff --git a/internal/wrappers/logs-http.go b/internal/wrappers/logs-http.go index af59ae61a..b94d2b474 100644 --- a/internal/wrappers/logs-http.go +++ b/internal/wrappers/logs-http.go @@ -1,4 +1,3 @@ -// nolint:bodyclose package wrappers import ( diff --git a/internal/wrappers/pr-http.go b/internal/wrappers/pr-http.go index 77085112a..7ebfa305c 100644 --- a/internal/wrappers/pr-http.go +++ b/internal/wrappers/pr-http.go @@ -36,6 +36,7 @@ func (r *PRHTTPWrapper) PostPRDecoration(model *PRModel) ( return "", nil, err } resp, err := SendHTTPRequestWithJSONContentType(http.MethodPost, r.githubPath, bytes.NewBuffer(jsonBytes), true, clientTimeout) + defer resp.Body.Close() return handlePRResponseWithBody(resp, err) } diff --git a/internal/wrappers/projects-http.go b/internal/wrappers/projects-http.go index 8ee1e6a14..d369bf55c 100644 --- a/internal/wrappers/projects-http.go +++ b/internal/wrappers/projects-http.go @@ -1,4 +1,3 @@ -// nolint:bodyclose package wrappers import ( diff --git a/internal/wrappers/response.go b/internal/wrappers/response.go index 3a2bb95fa..30b347a6c 100644 --- a/internal/wrappers/response.go +++ b/internal/wrappers/response.go @@ -1,4 +1,3 @@ -// nolint:bodyclose package wrappers import ( diff --git a/internal/wrappers/results-pdf-http.go b/internal/wrappers/results-pdf-http.go index 6f0d3320b..42c9133f4 100644 --- a/internal/wrappers/results-pdf-http.go +++ b/internal/wrappers/results-pdf-http.go @@ -1,4 +1,3 @@ -// nolint:bodyclose package wrappers import ( From 803735702447ce24647c606c753ed491a5dbd5a0 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 22:13:11 +0100 Subject: [PATCH 15/23] linter update --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e160d4053..241807be0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,8 +117,8 @@ jobs: - name: Setup git run: git config --global url."https://${{ secrets.PERSONAL_ACCESS_TOKEN }}:@github.com/".insteadOf "https://github.com" - name: golangci-lint - uses: golangci/golangci-lint-action@v3.4.0 + uses: golangci/golangci-lint-action@v3 with: - version: v1.52.2 + version: v1.54.2 args: -c .golangci.yml only-new-issues: true From 78d489c51a32516c3a84b3f57b7b6d47a10fcc86 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Thu, 7 Sep 2023 22:24:34 +0100 Subject: [PATCH 16/23] linter --- internal/wrappers/azure-http.go | 2 +- internal/wrappers/bfl-http.go | 3 +++ internal/wrappers/bitbucket-http.go | 2 +- .../wrappers/bitbucketserver/bitbucket-server-http.go | 2 +- internal/wrappers/learn-more-http.go | 3 +++ internal/wrappers/pr-http.go | 3 +++ internal/wrappers/predicates-http.go | 8 ++++++-- 7 files changed, 18 insertions(+), 5 deletions(-) diff --git a/internal/wrappers/azure-http.go b/internal/wrappers/azure-http.go index 89f55277c..adaf959e7 100644 --- a/internal/wrappers/azure-http.go +++ b/internal/wrappers/azure-http.go @@ -111,7 +111,7 @@ func (g *AzureHTTPWrapper) get( queryParams map[string]string, authFormat string, ) (bool, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) + req, err := http.NewRequest(http.MethodGet, url, http.NoBody) if err != nil { return false, err } diff --git a/internal/wrappers/bfl-http.go b/internal/wrappers/bfl-http.go index 4baf33d63..b1c223b91 100644 --- a/internal/wrappers/bfl-http.go +++ b/internal/wrappers/bfl-http.go @@ -33,6 +33,9 @@ func (r *BflHTTPWrapper) GetBflByScanIDAndQueryID(params map[string]string) ( clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) log.Println(fmt.Sprintf("Fetching the best fix location for QueryID: %s", params[commonParams.QueryIDQueryParam])) resp, err := SendHTTPRequestWithQueryParams(http.MethodGet, r.path, params, nil, clientTimeout) + if err != nil { + return nil, nil, err + } defer resp.Body.Close() return handleBflResponseWithBody(resp, err) } diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index e68d9fe23..1acf2eca1 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -150,7 +150,7 @@ func (g *BitBucketHTTPWrapper) getFromBitBucket( logger.PrintIfVerbose(fmt.Sprintf("Request to %s", url)) - req, err := http.NewRequest(http.MethodGet, url, nil) + req, err := http.NewRequest(http.MethodGet, url, http.NoBody) if err != nil { return err } diff --git a/internal/wrappers/bitbucketserver/bitbucket-server-http.go b/internal/wrappers/bitbucketserver/bitbucket-server-http.go index d40ef77ff..b37b5f8d1 100644 --- a/internal/wrappers/bitbucketserver/bitbucket-server-http.go +++ b/internal/wrappers/bitbucketserver/bitbucket-server-http.go @@ -148,7 +148,7 @@ func getBitBucketServer( ) error { var err error - req, err := http.NewRequest(http.MethodGet, url, nil) + req, err := http.NewRequest(http.MethodGet, url, http.NoBody) if err != nil { return err } diff --git a/internal/wrappers/learn-more-http.go b/internal/wrappers/learn-more-http.go index 5957a1376..3d2a16060 100644 --- a/internal/wrappers/learn-more-http.go +++ b/internal/wrappers/learn-more-http.go @@ -32,6 +32,9 @@ func (r *LearnMoreHTTPWrapper) GetLearnMoreDetails(params map[string]string) ( clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) // add the path parameter to the path resp, err := SendHTTPRequestWithQueryParams(http.MethodGet, r.path, params, nil, clientTimeout) + if err != nil { + return nil, nil, err + } defer resp.Body.Close() return handleResponse(resp, err, params[commonParams.QueryIDQueryParam]) } diff --git a/internal/wrappers/pr-http.go b/internal/wrappers/pr-http.go index 7ebfa305c..2dbd0d512 100644 --- a/internal/wrappers/pr-http.go +++ b/internal/wrappers/pr-http.go @@ -36,6 +36,9 @@ func (r *PRHTTPWrapper) PostPRDecoration(model *PRModel) ( return "", nil, err } resp, err := SendHTTPRequestWithJSONContentType(http.MethodPost, r.githubPath, bytes.NewBuffer(jsonBytes), true, clientTimeout) + if err != nil { + return "", nil, err + } defer resp.Body.Close() return handlePRResponseWithBody(resp, err) } diff --git a/internal/wrappers/predicates-http.go b/internal/wrappers/predicates-http.go index d853c1188..882fe6f7b 100644 --- a/internal/wrappers/predicates-http.go +++ b/internal/wrappers/predicates-http.go @@ -47,8 +47,12 @@ func (r *ResultsPredicatesHTTPWrapper) GetAllPredicatesForSimilarityID(similarit var request = "/" + similarityID + "?project-ids=" + projectID logger.PrintIfVerbose(fmt.Sprintf("Sending GET request to %s", r.path+request)) - - return handleResponseWithBody(SendHTTPRequest(http.MethodGet, r.path+request, http.NoBody, true, clientTimeout)) + resp, err := SendHTTPRequest(http.MethodGet, r.path+request, http.NoBody, true, clientTimeout) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + return handleResponseWithBody(resp, err) } func (r *ResultsPredicatesHTTPWrapper) SetPath(newPath string) { From 96550d3b56ab6b46c7859a3154c9558313939bb3 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Fri, 8 Sep 2023 11:57:10 +0100 Subject: [PATCH 17/23] testing pre-commit --- internal/wrappers/bitbucket-http.go | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index 1acf2eca1..07e868545 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -150,28 +150,11 @@ func (g *BitBucketHTTPWrapper) getFromBitBucket( logger.PrintIfVerbose(fmt.Sprintf("Request to %s", url)) - req, err := http.NewRequest(http.MethodGet, url, http.NoBody) + resp, err := GetWithQueryParams(g.client, url, token, basicFormat, queryParams) if err != nil { return err } - - if len(token) > 0 { - enrichWithOath2Credentials(req, token, basicFormat) - } - - q := req.URL.Query() - for k, v := range queryParams { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - resp, err := g.client.Do(req) - if err != nil { - return err - } - - defer func() { - _ = resp.Body.Close() - }() + //defer resp.Body.Close() switch resp.StatusCode { case http.StatusOK: err = json.NewDecoder(resp.Body).Decode(target) From f1356679de383c0ab87f7daa9566e1a3f9f3c2dc Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Fri, 8 Sep 2023 11:58:02 +0100 Subject: [PATCH 18/23] testing pre-commit --- internal/wrappers/bitbucket-http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index 07e868545..f840f76c2 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -154,7 +154,7 @@ func (g *BitBucketHTTPWrapper) getFromBitBucket( if err != nil { return err } - //defer resp.Body.Close() + defer resp.Body.Close() switch resp.StatusCode { case http.StatusOK: err = json.NewDecoder(resp.Body).Decode(target) From 161605871e28c292ea84849d5e463f05ffe1765d Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Fri, 8 Sep 2023 14:13:05 +0100 Subject: [PATCH 19/23] testing pre-commit --- internal/wrappers/bitbucket-http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index f840f76c2..5c9cd44e3 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -147,7 +147,7 @@ func (g *BitBucketHTTPWrapper) getFromBitBucket( url, token string, target interface{}, queryParams map[string]string, ) error { var err error - + // test logger.PrintIfVerbose(fmt.Sprintf("Request to %s", url)) resp, err := GetWithQueryParams(g.client, url, token, basicFormat, queryParams) From c140aaa2198bf71b3f3ab718c6857ce0ff4e3120 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Fri, 8 Sep 2023 17:33:04 +0100 Subject: [PATCH 20/23] - Adding support to custom requets in GetWithQueryParams - Refactoring github-http functions --- internal/wrappers/bitbucket-http.go | 2 +- internal/wrappers/client.go | 16 +++-- internal/wrappers/github-http.go | 100 +++++++++------------------- 3 files changed, 43 insertions(+), 75 deletions(-) diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index 5c9cd44e3..f840f76c2 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -147,7 +147,7 @@ func (g *BitBucketHTTPWrapper) getFromBitBucket( url, token string, target interface{}, queryParams map[string]string, ) error { var err error - // test + logger.PrintIfVerbose(fmt.Sprintf("Request to %s", url)) resp, err := GetWithQueryParams(g.client, url, token, basicFormat, queryParams) diff --git a/internal/wrappers/client.go b/internal/wrappers/client.go index e6fc4002f..634a464a5 100644 --- a/internal/wrappers/client.go +++ b/internal/wrappers/client.go @@ -355,17 +355,21 @@ func GetWithQueryParams(client *http.Client, urlAddress, token, authFormat strin if err != nil { return nil, err } + return GetWithQueryParamsAndCustomRequest(client, req, urlAddress, token, authFormat, queryParams) +} + +// GetWithQueryParamsAndCustomRequest used when we need to add custom headers to the request +func GetWithQueryParamsAndCustomRequest(client *http.Client, customReq *http.Request, urlAddress, token, authFormat string, queryParams map[string]string) (*http.Response, error) { if len(token) > 0 { - enrichWithOath2Credentials(req, token, authFormat) + enrichWithOath2Credentials(customReq, token, authFormat) } - - q := req.URL.Query() + q := customReq.URL.Query() for k, v := range queryParams { q.Add(k, v) } - req.URL.RawQuery = q.Encode() - req = addReqMonitor(req) - return request(client, req, true) + customReq.URL.RawQuery = q.Encode() + customReq = addReqMonitor(customReq) + return request(client, customReq, true) } func GetAccessToken() (string, error) { authURI, err := getAuthURI() diff --git a/internal/wrappers/github-http.go b/internal/wrappers/github-http.go index c09aadf80..c8f153065 100644 --- a/internal/wrappers/github-http.go +++ b/internal/wrappers/github-http.go @@ -161,10 +161,7 @@ func (g *GitHubHTTPWrapper) getTemplates() error { } func (g *GitHubHTTPWrapper) get(url string, target interface{}) error { - resp, err := get(g.client, url, target, map[string]string{}) - - closeBody(resp) - + _, err := get(g.client, url, target, map[string]string{}) return err } @@ -205,7 +202,7 @@ func collectPage( return "", err } - defer closeBody(resp) + defer resp.Body.Close() *pageCollection = append(*pageCollection, holder...) next := getNextPageLink(resp) @@ -229,71 +226,38 @@ func getNextPageLink(resp *http.Response) string { } func get(client *http.Client, url string, target interface{}, queryParams map[string]string) (*http.Response, error) { - var err error - var count uint8 - - for count < retryLimit { - var currentError error - - req, currentError := http.NewRequest(http.MethodGet, url, http.NoBody) - if currentError != nil { - return nil, currentError - } - - req.Header.Add(acceptHeader, apiVersion) - - token := viper.GetString(params.SCMTokenFlag) - if len(token) > 0 { - req.Header.Add(AuthorizationHeader, fmt.Sprintf(tokenFormat, token)) - } + req, err := http.NewRequest(http.MethodGet, url, http.NoBody) + if err != nil { + return nil, err + } + req.Header.Add(acceptHeader, apiVersion) + token := viper.GetString(params.SCMTokenFlag) + logger.PrintRequest(req) + resp, err := GetWithQueryParamsAndCustomRequest(client, req, url, token, tokenFormat, queryParams) + if err != nil { + return nil, err + } + defer resp.Body.Close() + logger.PrintResponse(resp, true) - q := req.URL.Query() - for k, v := range queryParams { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - - logger.PrintRequest(req) - resp, currentError := client.Do(req) - if currentError != nil { - count++ - logger.PrintIfVerbose(fmt.Sprintf("Request to %s dropped, retrying", req.URL)) - err = currentError - continue + switch resp.StatusCode { + case http.StatusOK: + logger.PrintIfVerbose(fmt.Sprintf("Request to URL %s OK", req.URL)) + err = json.NewDecoder(resp.Body).Decode(target) + if err != nil { + return nil, err } - - logger.PrintResponse(resp, true) - - switch resp.StatusCode { - case http.StatusOK: - logger.PrintIfVerbose(fmt.Sprintf("Request to URL %s OK", req.URL)) - currentError = json.NewDecoder(resp.Body).Decode(target) - closeBody(resp) - if currentError != nil { - return nil, currentError - } - case http.StatusConflict: - logger.PrintIfVerbose(fmt.Sprintf("Found empty repository in %s", req.URL)) - closeBody(resp) - return nil, nil - default: - body, currentError := io.ReadAll(resp.Body) - closeBody(resp) - if currentError != nil { - logger.PrintIfVerbose(currentError.Error()) - return nil, currentError - } - message := fmt.Sprintf("Code %d %s", resp.StatusCode, string(body)) - return nil, errors.New(message) + case http.StatusConflict: + logger.PrintIfVerbose(fmt.Sprintf("Found empty repository in %s", req.URL)) + return nil, nil + default: + body, err := io.ReadAll(resp.Body) + if err != nil { + logger.PrintIfVerbose(err.Error()) + return nil, err } - return resp, nil - } - - return nil, err -} - -func closeBody(resp *http.Response) { - if resp != nil && resp.Body != nil { - _ = resp.Body.Close() + message := fmt.Sprintf("Code %d %s", resp.StatusCode, string(body)) + return nil, errors.New(message) } + return resp, nil } From 76dc79cdb9ae93e269c53c14765aabdfccea1a89 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Fri, 8 Sep 2023 17:48:10 +0100 Subject: [PATCH 21/23] - Refactoring github-http functions --- internal/wrappers/github-http.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/wrappers/github-http.go b/internal/wrappers/github-http.go index c8f153065..575b6c74f 100644 --- a/internal/wrappers/github-http.go +++ b/internal/wrappers/github-http.go @@ -161,7 +161,10 @@ func (g *GitHubHTTPWrapper) getTemplates() error { } func (g *GitHubHTTPWrapper) get(url string, target interface{}) error { - _, err := get(g.client, url, target, map[string]string{}) + resp, err := get(g.client, url, target, map[string]string{}) + if err != nil { + defer resp.Body.Close() + } return err } From 8db065745278d570d8b77966b2ced83e4700995f Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Fri, 8 Sep 2023 18:58:21 +0100 Subject: [PATCH 22/23] - Refactoring gitlab-http and azure-http functions --- internal/wrappers/azure-http.go | 23 +--------- internal/wrappers/client.go | 1 + internal/wrappers/gitlab-http.go | 78 ++++++++++++-------------------- 3 files changed, 31 insertions(+), 71 deletions(-) diff --git a/internal/wrappers/azure-http.go b/internal/wrappers/azure-http.go index adaf959e7..48d489693 100644 --- a/internal/wrappers/azure-http.go +++ b/internal/wrappers/azure-http.go @@ -111,31 +111,12 @@ func (g *AzureHTTPWrapper) get( queryParams map[string]string, authFormat string, ) (bool, error) { - req, err := http.NewRequest(http.MethodGet, url, http.NoBody) - if err != nil { - return false, err - } - - if len(token) > 0 { - req.Header.Add(AuthorizationHeader, fmt.Sprintf(authFormat, token)) - } - - q := req.URL.Query() - for k, v := range queryParams { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() - resp, err := g.client.Do(req) + resp, err := GetWithQueryParams(g.client, url, token, authFormat, queryParams) if err != nil { return false, err } - - logger.PrintRequest(req) - - defer func() { - _ = resp.Body.Close() - }() + defer resp.Body.Close() logger.PrintResponse(resp, true) diff --git a/internal/wrappers/client.go b/internal/wrappers/client.go index 634a464a5..4242c4acb 100644 --- a/internal/wrappers/client.go +++ b/internal/wrappers/client.go @@ -355,6 +355,7 @@ func GetWithQueryParams(client *http.Client, urlAddress, token, authFormat strin if err != nil { return nil, err } + logger.PrintRequest(req) return GetWithQueryParamsAndCustomRequest(client, req, urlAddress, token, authFormat, queryParams) } diff --git a/internal/wrappers/gitlab-http.go b/internal/wrappers/gitlab-http.go index d3ae2ad3c..82ca98001 100644 --- a/internal/wrappers/gitlab-http.go +++ b/internal/wrappers/gitlab-http.go @@ -128,57 +128,41 @@ func (g *GitLabHTTPWrapper) GetGitLabProjects(gitLabGroupName string, queryParam func getFromGitLab( client *http.Client, requestURL string, target interface{}, queryParams map[string]string, ) (*http.Response, error) { - var err error - var count uint8 - for count < retryLimitGitLab { - var currentError error - req, currentError := http.NewRequest(http.MethodGet, requestURL, http.NoBody) - if currentError != nil { - return nil, currentError - } + req, err := http.NewRequest(http.MethodGet, requestURL, http.NoBody) + if err != nil { + return nil, err + } - token := viper.GetString(params.SCMTokenFlag) - if len(token) > 0 { - req.Header.Add(gitLabAuthorizationHeader, fmt.Sprintf(gitLabTokenFormat, token)) - } - q := req.URL.Query() - for k, v := range queryParams { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() + token := viper.GetString(params.SCMTokenFlag) - logger.PrintRequest(req) + logger.PrintRequest(req) - resp, currentError := client.Do(req) - if currentError != nil { - count++ - logger.PrintIfVerbose(fmt.Sprintf("Request to %s dropped, retrying", req.URL)) - err = currentError - continue - } + resp, err := GetWithQueryParamsAndCustomRequest(client, req, requestURL, token, bearearFormat, queryParams) + if err != nil { + return nil, err + } + defer resp.Body.Close() - logger.PrintResponse(resp, true) + logger.PrintResponse(resp, true) - switch resp.StatusCode { - case http.StatusOK: - currentError = json.NewDecoder(resp.Body).Decode(target) - closeResponseBody(resp) - if currentError != nil { - return nil, currentError - } - default: - body, currentError := io.ReadAll(resp.Body) - closeResponseBody(resp) - if currentError != nil { - logger.PrintIfVerbose(currentError.Error()) - return nil, currentError - } - message := fmt.Sprintf("Code %d %s", resp.StatusCode, string(body)) - return nil, errors.New(message) + switch resp.StatusCode { + case http.StatusOK: + err = json.NewDecoder(resp.Body).Decode(target) + if err != nil { + return nil, err + } + default: + body, currentError := io.ReadAll(resp.Body) + if currentError != nil { + logger.PrintIfVerbose(currentError.Error()) + return nil, currentError } - return resp, nil + message := fmt.Sprintf("Code %d %s", resp.StatusCode, string(body)) + return nil, errors.New(message) } + return resp, nil + return nil, err } @@ -217,7 +201,7 @@ func collectPageForGitLab( if err != nil { return "", err } - defer closeResponseBody(resp) + defer resp.Body.Close() *pageCollection = append(*pageCollection, holder...) nextPageURL := getNextPage(resp) @@ -238,9 +222,3 @@ func getNextPage(resp *http.Response) string { } return "" } - -func closeResponseBody(resp *http.Response) { - if resp != nil && resp.Body != nil { - _ = resp.Body.Close() - } -} From 55f6cb533186a2cb66357566b406a732cf0c3815 Mon Sep 17 00:00:00 2001 From: igorlombacx Date: Mon, 11 Sep 2023 13:42:00 +0100 Subject: [PATCH 23/23] mistype fix --- internal/wrappers/client.go | 10 +++++----- internal/wrappers/gitlab-http.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/wrappers/client.go b/internal/wrappers/client.go index 4242c4acb..27841f2b8 100644 --- a/internal/wrappers/client.go +++ b/internal/wrappers/client.go @@ -37,7 +37,7 @@ const ( MissingTenant = "Failed to authenticate - please provide tenant" jwtError = "Error retrieving %s from jwt token" basicFormat = "Basic %s" - bearearFormat = "Bearer %s" + bearerFormat = "Bearer %s" contentTypeHeader = "Content-Type" formURLContentType = "application/x-www-form-urlencoded" jsonContentType = "application/json" @@ -202,7 +202,7 @@ func SendHTTPRequestByFullURLContentLength( client := GetClient(timeout) setAgentName(req) if auth { - enrichWithOath2Credentials(req, accessToken, bearearFormat) + enrichWithOath2Credentials(req, accessToken, bearerFormat) } req = addReqMonitor(req) @@ -298,7 +298,7 @@ func HTTPRequestWithQueryParams( q.Add(k, v) } req.URL.RawQuery = q.Encode() - enrichWithOath2Credentials(req, accessToken, bearearFormat) + enrichWithOath2Credentials(req, accessToken, bearerFormat) var resp *http.Response resp, err = request(client, req, printBody) if err != nil { @@ -343,7 +343,7 @@ func SendHTTPRequestWithJSONContentType(method, path string, body io.Reader, aut return nil, err } if auth { - enrichWithOath2Credentials(req, accessToken, bearearFormat) + enrichWithOath2Credentials(req, accessToken, bearerFormat) } req = addReqMonitor(req) @@ -415,7 +415,7 @@ func enrichWithPasswordCredentials( "failed to authenticate", ) } - enrichWithOath2Credentials(request, accessToken, bearearFormat) + enrichWithOath2Credentials(request, accessToken, bearerFormat) return nil } diff --git a/internal/wrappers/gitlab-http.go b/internal/wrappers/gitlab-http.go index 82ca98001..70e5d3080 100644 --- a/internal/wrappers/gitlab-http.go +++ b/internal/wrappers/gitlab-http.go @@ -138,7 +138,7 @@ func getFromGitLab( logger.PrintRequest(req) - resp, err := GetWithQueryParamsAndCustomRequest(client, req, requestURL, token, bearearFormat, queryParams) + resp, err := GetWithQueryParamsAndCustomRequest(client, req, requestURL, token, bearerFormat, queryParams) if err != nil { return nil, err }