Timeout and retry flags support for bitbucket, azure, github and gitlab wrappers

.github/workflows/ci.yml

@@ -117,8 +117,8 @@ jobs:
       - name: Setup git
         run: git config --global url."https://${{ secrets.PERSONAL_ACCESS_TOKEN }}:@github.com/".insteadOf "https://github.com"
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3.4.0
+        uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.52.2
+          version: v1.54.2
           args: -c .golangci.yml
           only-new-issues: true

internal/commands/scarealtime/sca-realtime-utils.go

@@ -162,7 +162,7 @@ func downloadSCAResolverHashFile(scaResolverHashURL, scaResolverZipFileNameHash
 func downloadFile(downloadURLPath, fileName string) error {
 	logger.PrintIfVerbose("Downloading " + fileName + " from: " + downloadURLPath)
 
-	response, err := wrappers.SendHTTPRequestByFullURL(http.MethodGet, downloadURLPath, nil, false, 0, "", true)
+	response, err := wrappers.SendHTTPRequestByFullURL(http.MethodGet, downloadURLPath, http.NoBody, false, 0, "", true)
 	if err != nil {
 		return errors.Errorf("Invoking HTTP request to upload file failed - %s", err.Error())
 	}

internal/wrappers/azure-http.go

@@ -30,7 +30,6 @@ const (
 	azureTop             = "$top"
 	azurePage            = "$skip"
 	azureLayoutTime      = "2006-01-02"
-	basicFormat          = "Basic %s"
 	failedAuth           = "failed Azure Authentication"
 	unauthorized         = "unauthorized: verify if the organization you provided is correct"
 	azurePageLenValue    = 100
@@ -112,33 +111,12 @@ func (g *AzureHTTPWrapper) get(
 	queryParams map[string]string,
 	authFormat string,
 ) (bool, error) {
-	var err error
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
+	resp, err := GetWithQueryParams(g.client, url, token, authFormat, queryParams)
 	if err != nil {
 		return false, err
 	}
-
-	if len(token) > 0 {
-		req.Header.Add(AuthorizationHeader, fmt.Sprintf(authFormat, token))
-	}
-
-	q := req.URL.Query()
-	for k, v := range queryParams {
-		q.Add(k, v)
-	}
-	req.URL.RawQuery = q.Encode()
-	resp, err := g.client.Do(req)
-
-	if err != nil {
-		return false, err
-	}
-
-	logger.PrintRequest(req)
-
-	defer func() {
-		_ = resp.Body.Close()
-	}()
+	defer resp.Body.Close()
 
 	logger.PrintResponse(resp, true)
 

internal/wrappers/bfl-http.go

@@ -33,6 +33,10 @@ func (r *BflHTTPWrapper) GetBflByScanIDAndQueryID(params map[string]string) (
 	clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey)
 	log.Println(fmt.Sprintf("Fetching the best fix location for QueryID: %s", params[commonParams.QueryIDQueryParam]))
 	resp, err := SendHTTPRequestWithQueryParams(http.MethodGet, r.path, params, nil, clientTimeout)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer resp.Body.Close()
 	return handleBflResponseWithBody(resp, err)
 }
 
@@ -43,10 +47,6 @@ func handleBflResponseWithBody(resp *http.Response, err error) (*BFLResponseMode
 
 	decoder := json.NewDecoder(resp.Body)
 
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-
 	switch resp.StatusCode {
 	case http.StatusBadRequest, http.StatusInternalServerError:
 		errorModel := WebError{}

internal/wrappers/bitbucket-http.go

@@ -150,28 +150,11 @@ func (g *BitBucketHTTPWrapper) getFromBitBucket(
 
 	logger.PrintIfVerbose(fmt.Sprintf("Request to %s", url))
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
+	resp, err := GetWithQueryParams(g.client, url, token, basicFormat, queryParams)
 	if err != nil {
 		return err
 	}
-
-	if len(token) > 0 {
-		req.Header.Add(AuthorizationHeader, fmt.Sprintf(basicFormat, token))
-	}
-
-	q := req.URL.Query()
-	for k, v := range queryParams {
-		q.Add(k, v)
-	}
-	req.URL.RawQuery = q.Encode()
-	resp, err := g.client.Do(req)
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		_ = resp.Body.Close()
-	}()
+	defer resp.Body.Close()
 	switch resp.StatusCode {
 	case http.StatusOK:
 		err = json.NewDecoder(resp.Body).Decode(target)
@@ -277,52 +260,30 @@ func collectPageBitBucket(
 }
 
 func getBitBucket(client *http.Client, token, url string, target interface{}, queryParams map[string]string) error {
-	var err error
-
-	req, err := http.NewRequest(http.MethodGet, url, nil)
-	if err != nil {
-		return err
-	}
-	if len(token) > 0 {
-		req.Header.Add(AuthorizationHeader, fmt.Sprintf(basicFormat, token))
-	}
-
-	q := req.URL.Query()
-	for k, v := range queryParams {
-		q.Add(k, v)
-	}
-
-	req.URL.RawQuery = q.Encode()
-	resp, err := client.Do(req)
+	resp, err := GetWithQueryParams(client, url, token, basicFormat, queryParams)
 	if err != nil {
 		return err
 	}
-	logger.PrintRequest(req)
-
-	defer func() {
-		_ = resp.Body.Close()
-	}()
-
-	logger.PrintResponse(resp, true)
+	defer resp.Body.Close()
 
 	switch resp.StatusCode {
 	case http.StatusOK:
 		err = json.NewDecoder(resp.Body).Decode(target)
 		if err != nil {
 			return err
 		}
-		// State sent when expired token
+	// State sent when expired token
 	case http.StatusUnauthorized:
 		err = errors.New(failedBitbucketAuth)
 		return err
-		// State sent when no token is provided
+	// State sent when no token is provided
 	case http.StatusForbidden:
 		err = errors.New(failedBitbucketAuth)
 		return err
 	case http.StatusNotFound:
 		err = errors.New(failedBitbucketNotFound)
 		return err
-		// Case the commit/project does not exist in the organization
+	// Case the commit/project does not exist in the organization
 	default:
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {

internal/wrappers/bitbucketserver/bitbucket-server-http.go

@@ -148,7 +148,7 @@ func getBitBucketServer(
 ) error {
 	var err error
 
-	req, err := http.NewRequest(http.MethodGet, url, nil)
+	req, err := http.NewRequest(http.MethodGet, url, http.NoBody)
 	if err != nil {
 		return err
 	}

internal/wrappers/client.go

@@ -29,12 +29,18 @@ const (
 	NoTimeout               = 0
 	ntlmProxyToken          = "ntlm"
 	checkmarxURLError       = "Could not reach provided Checkmarx server"
+	invalidCredentialsError = "Provided credentials are invalid"
 	APIKeyDecodeErrorFormat = "Token decoding error: %s"
 	tryPrintOffset          = 2
 	retryLimitPrintOffset   = 1
 	MissingURI              = "When using client-id and client-secret please provide base-uri or base-auth-uri"
 	MissingTenant           = "Failed to authenticate - please provide tenant"
 	jwtError                = "Error retrieving %s from jwt token"
+	basicFormat             = "Basic %s"
+	bearearFormat           = "Bearer %s"
+	contentTypeHeader       = "Content-Type"
+	formURLContentType      = "application/x-www-form-urlencoded"
+	jsonContentType         = "application/json"
 )
 
 type ClientCredentialsInfo struct {
@@ -196,16 +202,12 @@ func SendHTTPRequestByFullURLContentLength(
 	client := GetClient(timeout)
 	setAgentName(req)
 	if auth {
-		enrichWithOath2Credentials(req, accessToken)
+		enrichWithOath2Credentials(req, accessToken, bearearFormat)
 	}
 
 	req = addReqMonitor(req)
-	var resp *http.Response
-	resp, err = request(client, req, bodyPrint)
-	if err != nil {
-		return nil, err
-	}
-	return resp, nil
+
+	return request(client, req, bodyPrint)
 }
 
 func addReqMonitor(req *http.Request) *http.Request {
@@ -254,19 +256,13 @@ func SendHTTPRequestPasswordAuth(method string, body io.Reader, timeout uint, us
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Add("content-type", "application/json")
+	req.Header.Add(contentTypeHeader, jsonContentType)
 	err = enrichWithPasswordCredentials(req, username, password, adminClientID, adminClientSecret)
 	if err != nil {
 		return nil, err
 	}
-	var resp *http.Response
-
 	req = addReqMonitor(req)
-	resp, err = doRequest(client, req)
-	if err != nil {
-		return nil, err
-	}
-	return resp, nil
+	return doRequest(client, req)
 }
 
 func SendPrivateHTTPRequestWithQueryParams(
@@ -302,7 +298,7 @@ func HTTPRequestWithQueryParams(
 		q.Add(k, v)
 	}
 	req.URL.RawQuery = q.Encode()
-	enrichWithOath2Credentials(req, accessToken)
+	enrichWithOath2Credentials(req, accessToken, bearearFormat)
 	var resp *http.Response
 	resp, err = request(client, req, printBody)
 	if err != nil {
@@ -327,8 +323,8 @@ func addTenantAuthURI(baseAuthURI string) (string, error) {
 	return fmt.Sprintf("%s/%s", strings.Trim(baseAuthURI, "/"), authPath), nil
 }
 
-func enrichWithOath2Credentials(request *http.Request, accessToken string) {
-	request.Header.Add("Authorization", "Bearer "+accessToken)
+func enrichWithOath2Credentials(request *http.Request, accessToken, authFormat string) {
+	request.Header.Add(AuthorizationHeader, fmt.Sprintf(authFormat, accessToken))
 }
 
 func SendHTTPRequestWithJSONContentType(method, path string, body io.Reader, auth bool, timeout uint) (
@@ -342,23 +338,40 @@ func SendHTTPRequestWithJSONContentType(method, path string, body io.Reader, aut
 	req, err := http.NewRequest(method, fullURL, body)
 	client := GetClient(timeout)
 	setAgentName(req)
-	req.Header.Add("Content-Type", "application/json")
+	req.Header.Add("Content-Type", jsonContentType)
 	if err != nil {
 		return nil, err
 	}
 	if auth {
-		enrichWithOath2Credentials(req, accessToken)
+		enrichWithOath2Credentials(req, accessToken, bearearFormat)
 	}
 
 	req = addReqMonitor(req)
-	var resp *http.Response
-	resp, err = doRequest(client, req)
+	return doRequest(client, req)
+}
+
+func GetWithQueryParams(client *http.Client, urlAddress, token, authFormat string, queryParams map[string]string) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodGet, urlAddress, http.NoBody)
 	if err != nil {
 		return nil, err
 	}
-	return resp, nil
+	logger.PrintRequest(req)
+	return GetWithQueryParamsAndCustomRequest(client, req, urlAddress, token, authFormat, queryParams)
 }
 
+// GetWithQueryParamsAndCustomRequest used when we need to add custom headers to the request
+func GetWithQueryParamsAndCustomRequest(client *http.Client, customReq *http.Request, urlAddress, token, authFormat string, queryParams map[string]string) (*http.Response, error) {
+	if len(token) > 0 {
+		enrichWithOath2Credentials(customReq, token, authFormat)
+	}
+	q := customReq.URL.Query()
+	for k, v := range queryParams {
+		q.Add(k, v)
+	}
+	customReq.URL.RawQuery = q.Encode()
+	customReq = addReqMonitor(customReq)
+	return request(client, customReq, true)
+}
 func GetAccessToken() (string, error) {
 	authURI, err := getAuthURI()
 	if err != nil {
@@ -402,8 +415,7 @@ func enrichWithPasswordCredentials(
 			"failed to authenticate",
 		)
 	}
-
-	request.Header.Add("Authorization", "Bearer "+accessToken)
+	enrichWithOath2Credentials(request, accessToken, bearearFormat)
 	return nil
 }
 
@@ -458,7 +470,7 @@ func getNewToken(credentialsPayload, authServerURI string) (string, error) {
 		return "", err
 	}
 	req = addReqMonitor(req)
-	req.Header.Add("content-type", "application/x-www-form-urlencoded")
+	req.Header.Add(contentTypeHeader, formURLContentType)
 	clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey)
 	client := GetClient(clientTimeout)
 
@@ -468,13 +480,13 @@ func getNewToken(credentialsPayload, authServerURI string) (string, error) {
 		return "", errors.Errorf("%s %s", checkmarxURLError, authURL)
 	}
 	if res.StatusCode == http.StatusBadRequest {
-		return "", errors.Errorf("%v %s \n", res.StatusCode, "Provided credentials are invalid")
+		return "", errors.Errorf("%d %s \n", res.StatusCode, invalidCredentialsError)
 	}
 	if res.StatusCode == http.StatusNotFound {
-		return "", errors.Errorf("%v %s \n", res.StatusCode, "Provided Tenant Name is invalid")
+		return "", errors.Errorf("%d %s \n", res.StatusCode, "Provided Tenant Name is invalid")
 	}
 	if res.StatusCode == http.StatusUnauthorized {
-		return "", errors.Errorf("%v %s \n", res.StatusCode, "Provided credentials are invalid")
+		return "", errors.Errorf("%d %s \n", res.StatusCode, invalidCredentialsError)
 	}
 
 	body, _ := ioutil.ReadAll(res.Body)
@@ -486,7 +498,7 @@ func getNewToken(credentialsPayload, authServerURI string) (string, error) {
 			return "", err
 		}
 
-		return "", errors.Errorf("%v %s %s", res.StatusCode, credentialsErr.Error, credentialsErr.Description)
+		return "", errors.Errorf("%d %s %s", res.StatusCode, credentialsErr.Error, credentialsErr.Description)
 	}
 
 	defer func() {