Skip to content

Commit

Permalink
Timeout and retry flags support for bitbucket, azure, github and gitl…
Browse files Browse the repository at this point in the history
…ab wrappers (#576)

* - GetWithQueryParams function to create requests with retry flag

* - refactoring client.go

* - refactoring client.go - linter

* - Adding support to custom requets in GetWithQueryParams

* - Refactoring github-http functions

* - Refactoring gitlab-http and azure-http functions

* - mistype fix
  • Loading branch information
igorlombacx authored Sep 12, 2023
1 parent 6f52362 commit ce18723
Show file tree
Hide file tree
Showing 20 changed files with 162 additions and 264 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,8 @@ jobs:
- name: Setup git
run: git config --global url."https://${{ secrets.PERSONAL_ACCESS_TOKEN }}:@github.com/".insteadOf "https://github.com"
- name: golangci-lint
uses: golangci/golangci-lint-action@v3.4.0
uses: golangci/golangci-lint-action@v3
with:
version: v1.52.2
version: v1.54.2
args: -c .golangci.yml
only-new-issues: true
2 changes: 1 addition & 1 deletion internal/commands/scarealtime/sca-realtime-utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ func downloadSCAResolverHashFile(scaResolverHashURL, scaResolverZipFileNameHash
func downloadFile(downloadURLPath, fileName string) error {
logger.PrintIfVerbose("Downloading " + fileName + " from: " + downloadURLPath)

response, err := wrappers.SendHTTPRequestByFullURL(http.MethodGet, downloadURLPath, nil, false, 0, "", true)
response, err := wrappers.SendHTTPRequestByFullURL(http.MethodGet, downloadURLPath, http.NoBody, false, 0, "", true)
if err != nil {
return errors.Errorf("Invoking HTTP request to upload file failed - %s", err.Error())
}
Expand Down
26 changes: 2 additions & 24 deletions internal/wrappers/azure-http.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ const (
azureTop = "$top"
azurePage = "$skip"
azureLayoutTime = "2006-01-02"
basicFormat = "Basic %s"
failedAuth = "failed Azure Authentication"
unauthorized = "unauthorized: verify if the organization you provided is correct"
azurePageLenValue = 100
Expand Down Expand Up @@ -112,33 +111,12 @@ func (g *AzureHTTPWrapper) get(
queryParams map[string]string,
authFormat string,
) (bool, error) {
var err error

req, err := http.NewRequest(http.MethodGet, url, nil)
resp, err := GetWithQueryParams(g.client, url, token, authFormat, queryParams)
if err != nil {
return false, err
}

if len(token) > 0 {
req.Header.Add(AuthorizationHeader, fmt.Sprintf(authFormat, token))
}

q := req.URL.Query()
for k, v := range queryParams {
q.Add(k, v)
}
req.URL.RawQuery = q.Encode()
resp, err := g.client.Do(req)

if err != nil {
return false, err
}

logger.PrintRequest(req)

defer func() {
_ = resp.Body.Close()
}()
defer resp.Body.Close()

logger.PrintResponse(resp, true)

Expand Down
8 changes: 4 additions & 4 deletions internal/wrappers/bfl-http.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ func (r *BflHTTPWrapper) GetBflByScanIDAndQueryID(params map[string]string) (
clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey)
log.Println(fmt.Sprintf("Fetching the best fix location for QueryID: %s", params[commonParams.QueryIDQueryParam]))
resp, err := SendHTTPRequestWithQueryParams(http.MethodGet, r.path, params, nil, clientTimeout)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()
return handleBflResponseWithBody(resp, err)
}

Expand All @@ -43,10 +47,6 @@ func handleBflResponseWithBody(resp *http.Response, err error) (*BFLResponseMode

decoder := json.NewDecoder(resp.Body)

defer func() {
_ = resp.Body.Close()
}()

switch resp.StatusCode {
case http.StatusBadRequest, http.StatusInternalServerError:
errorModel := WebError{}
Expand Down
53 changes: 7 additions & 46 deletions internal/wrappers/bitbucket-http.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,28 +150,11 @@ func (g *BitBucketHTTPWrapper) getFromBitBucket(

logger.PrintIfVerbose(fmt.Sprintf("Request to %s", url))

req, err := http.NewRequest(http.MethodGet, url, nil)
resp, err := GetWithQueryParams(g.client, url, token, basicFormat, queryParams)
if err != nil {
return err
}

if len(token) > 0 {
req.Header.Add(AuthorizationHeader, fmt.Sprintf(basicFormat, token))
}

q := req.URL.Query()
for k, v := range queryParams {
q.Add(k, v)
}
req.URL.RawQuery = q.Encode()
resp, err := g.client.Do(req)
if err != nil {
return err
}

defer func() {
_ = resp.Body.Close()
}()
defer resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
err = json.NewDecoder(resp.Body).Decode(target)
Expand Down Expand Up @@ -277,52 +260,30 @@ func collectPageBitBucket(
}

func getBitBucket(client *http.Client, token, url string, target interface{}, queryParams map[string]string) error {
var err error

req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return err
}
if len(token) > 0 {
req.Header.Add(AuthorizationHeader, fmt.Sprintf(basicFormat, token))
}

q := req.URL.Query()
for k, v := range queryParams {
q.Add(k, v)
}

req.URL.RawQuery = q.Encode()
resp, err := client.Do(req)
resp, err := GetWithQueryParams(client, url, token, basicFormat, queryParams)
if err != nil {
return err
}
logger.PrintRequest(req)

defer func() {
_ = resp.Body.Close()
}()

logger.PrintResponse(resp, true)
defer resp.Body.Close()

switch resp.StatusCode {
case http.StatusOK:
err = json.NewDecoder(resp.Body).Decode(target)
if err != nil {
return err
}
// State sent when expired token
// State sent when expired token
case http.StatusUnauthorized:
err = errors.New(failedBitbucketAuth)
return err
// State sent when no token is provided
// State sent when no token is provided
case http.StatusForbidden:
err = errors.New(failedBitbucketAuth)
return err
case http.StatusNotFound:
err = errors.New(failedBitbucketNotFound)
return err
// Case the commit/project does not exist in the organization
// Case the commit/project does not exist in the organization
default:
body, err := io.ReadAll(resp.Body)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion internal/wrappers/bitbucketserver/bitbucket-server-http.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func getBitBucketServer(
) error {
var err error

req, err := http.NewRequest(http.MethodGet, url, nil)
req, err := http.NewRequest(http.MethodGet, url, http.NoBody)
if err != nil {
return err
}
Expand Down
72 changes: 42 additions & 30 deletions internal/wrappers/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,18 @@ const (
NoTimeout = 0
ntlmProxyToken = "ntlm"
checkmarxURLError = "Could not reach provided Checkmarx server"
invalidCredentialsError = "Provided credentials are invalid"
APIKeyDecodeErrorFormat = "Token decoding error: %s"
tryPrintOffset = 2
retryLimitPrintOffset = 1
MissingURI = "When using client-id and client-secret please provide base-uri or base-auth-uri"
MissingTenant = "Failed to authenticate - please provide tenant"
jwtError = "Error retrieving %s from jwt token"
basicFormat = "Basic %s"
bearerFormat = "Bearer %s"
contentTypeHeader = "Content-Type"
formURLContentType = "application/x-www-form-urlencoded"
jsonContentType = "application/json"
)

type ClientCredentialsInfo struct {
Expand Down Expand Up @@ -196,16 +202,12 @@ func SendHTTPRequestByFullURLContentLength(
client := GetClient(timeout)
setAgentName(req)
if auth {
enrichWithOath2Credentials(req, accessToken)
enrichWithOath2Credentials(req, accessToken, bearerFormat)
}

req = addReqMonitor(req)
var resp *http.Response
resp, err = request(client, req, bodyPrint)
if err != nil {
return nil, err
}
return resp, nil

return request(client, req, bodyPrint)
}

func addReqMonitor(req *http.Request) *http.Request {
Expand Down Expand Up @@ -254,19 +256,13 @@ func SendHTTPRequestPasswordAuth(method string, body io.Reader, timeout uint, us
if err != nil {
return nil, err
}
req.Header.Add("content-type", "application/json")
req.Header.Add(contentTypeHeader, jsonContentType)
err = enrichWithPasswordCredentials(req, username, password, adminClientID, adminClientSecret)
if err != nil {
return nil, err
}
var resp *http.Response

req = addReqMonitor(req)
resp, err = doRequest(client, req)
if err != nil {
return nil, err
}
return resp, nil
return doRequest(client, req)
}

func SendPrivateHTTPRequestWithQueryParams(
Expand Down Expand Up @@ -302,7 +298,7 @@ func HTTPRequestWithQueryParams(
q.Add(k, v)
}
req.URL.RawQuery = q.Encode()
enrichWithOath2Credentials(req, accessToken)
enrichWithOath2Credentials(req, accessToken, bearerFormat)
var resp *http.Response
resp, err = request(client, req, printBody)
if err != nil {
Expand All @@ -327,8 +323,8 @@ func addTenantAuthURI(baseAuthURI string) (string, error) {
return fmt.Sprintf("%s/%s", strings.Trim(baseAuthURI, "/"), authPath), nil
}

func enrichWithOath2Credentials(request *http.Request, accessToken string) {
request.Header.Add("Authorization", "Bearer "+accessToken)
func enrichWithOath2Credentials(request *http.Request, accessToken, authFormat string) {
request.Header.Add(AuthorizationHeader, fmt.Sprintf(authFormat, accessToken))
}

func SendHTTPRequestWithJSONContentType(method, path string, body io.Reader, auth bool, timeout uint) (
Expand All @@ -342,23 +338,40 @@ func SendHTTPRequestWithJSONContentType(method, path string, body io.Reader, aut
req, err := http.NewRequest(method, fullURL, body)
client := GetClient(timeout)
setAgentName(req)
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Content-Type", jsonContentType)
if err != nil {
return nil, err
}
if auth {
enrichWithOath2Credentials(req, accessToken)
enrichWithOath2Credentials(req, accessToken, bearerFormat)
}

req = addReqMonitor(req)
var resp *http.Response
resp, err = doRequest(client, req)
return doRequest(client, req)
}

func GetWithQueryParams(client *http.Client, urlAddress, token, authFormat string, queryParams map[string]string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodGet, urlAddress, http.NoBody)
if err != nil {
return nil, err
}
return resp, nil
logger.PrintRequest(req)
return GetWithQueryParamsAndCustomRequest(client, req, urlAddress, token, authFormat, queryParams)
}

// GetWithQueryParamsAndCustomRequest used when we need to add custom headers to the request
func GetWithQueryParamsAndCustomRequest(client *http.Client, customReq *http.Request, urlAddress, token, authFormat string, queryParams map[string]string) (*http.Response, error) {
if len(token) > 0 {
enrichWithOath2Credentials(customReq, token, authFormat)
}
q := customReq.URL.Query()
for k, v := range queryParams {
q.Add(k, v)
}
customReq.URL.RawQuery = q.Encode()
customReq = addReqMonitor(customReq)
return request(client, customReq, true)
}
func GetAccessToken() (string, error) {
authURI, err := getAuthURI()
if err != nil {
Expand Down Expand Up @@ -402,8 +415,7 @@ func enrichWithPasswordCredentials(
"failed to authenticate",
)
}

request.Header.Add("Authorization", "Bearer "+accessToken)
enrichWithOath2Credentials(request, accessToken, bearerFormat)
return nil
}

Expand Down Expand Up @@ -458,7 +470,7 @@ func getNewToken(credentialsPayload, authServerURI string) (string, error) {
return "", err
}
req = addReqMonitor(req)
req.Header.Add("content-type", "application/x-www-form-urlencoded")
req.Header.Add(contentTypeHeader, formURLContentType)
clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey)
client := GetClient(clientTimeout)

Expand All @@ -468,13 +480,13 @@ func getNewToken(credentialsPayload, authServerURI string) (string, error) {
return "", errors.Errorf("%s %s", checkmarxURLError, authURL)
}
if res.StatusCode == http.StatusBadRequest {
return "", errors.Errorf("%v %s \n", res.StatusCode, "Provided credentials are invalid")
return "", errors.Errorf("%d %s \n", res.StatusCode, invalidCredentialsError)
}
if res.StatusCode == http.StatusNotFound {
return "", errors.Errorf("%v %s \n", res.StatusCode, "Provided Tenant Name is invalid")
return "", errors.Errorf("%d %s \n", res.StatusCode, "Provided Tenant Name is invalid")
}
if res.StatusCode == http.StatusUnauthorized {
return "", errors.Errorf("%v %s \n", res.StatusCode, "Provided credentials are invalid")
return "", errors.Errorf("%d %s \n", res.StatusCode, invalidCredentialsError)
}

body, _ := ioutil.ReadAll(res.Body)
Expand All @@ -486,7 +498,7 @@ func getNewToken(credentialsPayload, authServerURI string) (string, error) {
return "", err
}

return "", errors.Errorf("%v %s %s", res.StatusCode, credentialsErr.Error, credentialsErr.Description)
return "", errors.Errorf("%d %s %s", res.StatusCode, credentialsErr.Error, credentialsErr.Description)
}

defer func() {
Expand Down
Loading

0 comments on commit ce18723

Please sign in to comment.