Convert SSH clone urls to HTTPS (#358)

commands/utils/git.go

@@ -7,6 +7,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing/protocol/packp/capability"
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/plumbing/transport/client"
+	"github.com/jfrog/froggit-go/vcsutils"
 	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"net/http"
 	"os"
@@ -26,7 +27,13 @@ const (
 	refFormat = "refs/heads/%s:refs/heads/%[1]s"
 
 	// Timout is seconds for the git operations performed by the go-git client.
-	goGitTimeoutSeconds = 60
+	goGitTimeoutSeconds = 120
+
+	// Https clone url formats for each service provider
+	githubHttpsFormat          = "%s/%s/%s.git"
+	gitLabHttpsFormat          = "%s/%s/%s.git"
+	bitbucketServerHttpsFormat = "%s/scm/%s/%s.git"
+	azureDevopsHttpsFormat     = "https://%s@%s%s/_git/%s"
 )
 
 type GitManager struct {
@@ -42,6 +49,8 @@ type GitManager struct {
 	dryRunRepoPath string
 	// Custom naming formats
 	customTemplates CustomTemplates
+	// Git details
+	git *Git
 }
 
 type CustomTemplates struct {
@@ -64,7 +73,7 @@ func NewGitManager(dryRun bool, clonedRepoPath, projectPath, remoteName, token,
 	if err != nil {
 		return nil, err
 	}
-	return &GitManager{repository: repository, dryRunRepoPath: clonedRepoPath, remoteName: remoteName, auth: basicAuth, dryRun: dryRun, customTemplates: templates}, nil
+	return &GitManager{repository: repository, dryRunRepoPath: clonedRepoPath, remoteName: remoteName, auth: basicAuth, dryRun: dryRun, customTemplates: templates, git: g}, nil
 }
 
 func (gm *GitManager) Checkout(branchName string) error {
@@ -80,39 +89,50 @@ func (gm *GitManager) Clone(destinationPath, branchName string) error {
 		// "Clone" the repository from the testdata folder
 		return gm.dryRunClone(destinationPath)
 	}
-	// Gets the remote repo url from the current .git dir
-	gitRemote, err := gm.repository.Remote(gm.remoteName)
+	gitRemoteUrl, err := gm.getRemoteUrl()
 	if err != nil {
-		return fmt.Errorf("'git remote %s' failed with error: %s", gm.remoteName, err.Error())
-	}
-	if len(gitRemote.Config().URLs) < 1 {
-		return errors.New("failed to find git remote URL")
+		return err
 	}
-	repoURL := gitRemote.Config().URLs[0]
-
 	transport.UnsupportedCapabilities = []capability.Capability{
 		capability.ThinPack,
 	}
 	if branchName == "" {
 		log.Debug("Since no branch name was set, assuming 'master' as the default branch")
 		branchName = "master"
 	}
-	log.Debug(fmt.Sprintf("Cloning repository with these details:\nClone url: %s remote name: %s, branch: %s", repoURL, gm.remoteName, getFullBranchName(branchName)))
+	log.Debug(fmt.Sprintf("Cloning repository with these details:\nClone url: %s remote name: %s, branch: %s", gitRemoteUrl, gm.remoteName, getFullBranchName(branchName)))
 	cloneOptions := &git.CloneOptions{
-		URL:           repoURL,
+		URL:           gitRemoteUrl,
 		Auth:          gm.auth,
 		RemoteName:    gm.remoteName,
 		ReferenceName: getFullBranchName(branchName),
 	}
 	repo, err := git.PlainClone(destinationPath, false, cloneOptions)
 	if err != nil {
-		return fmt.Errorf("'git clone %s from %s' failed with error: %s", branchName, repoURL, err.Error())
+		return fmt.Errorf("'git clone %s from %s' failed with error: %s", branchName, gitRemoteUrl, err.Error())
 	}
 	gm.repository = repo
-	log.Debug(fmt.Sprintf("Project cloned from %s to %s", repoURL, destinationPath))
+	log.Debug(fmt.Sprintf("Project cloned from %s to %s", gitRemoteUrl, destinationPath))
 	return nil
 }
 
+func (gm *GitManager) getRemoteUrl() (string, error) {
+	// Gets the remote repo url from the current .git dir
+	gitRemote, err := gm.repository.Remote(gm.remoteName)
+	if err != nil {
+		return "", fmt.Errorf("'git remote %s' failed with error: %s", gm.remoteName, err.Error())
+	}
+	if len(gitRemote.Config().URLs) < 1 {
+		return "", errors.New("failed to find git remote URL")
+	}
+	remoteUrl := gitRemote.Config().URLs[0]
+	if strings.HasPrefix(remoteUrl, "https://") {
+		return remoteUrl, nil
+	}
+	// Handle SSH clone urls
+	return gm.generateHTTPSCloneUrl()
+}
+
 func (gm *GitManager) CreateBranchAndCheckout(branchName string) error {
 	err := gm.createBranchAndCheckout(branchName, true)
 	if err != nil {
@@ -334,6 +354,25 @@ func (gm *GitManager) dryRunClone(destination string) error {
 	return nil
 }
 
+// Construct HTTPS clone url from the provided git info.
+// Frogbot already has an access token with sufficient permissions to clone with HTTPS,
+// in case we encounter SSH clone url, we generate HTTPS url instead.
+func (gm *GitManager) generateHTTPSCloneUrl() (url string, err error) {
+	switch gm.git.GitProvider {
+	case vcsutils.GitHub:
+		return fmt.Sprintf(githubHttpsFormat, gm.git.ApiEndpoint, gm.git.RepoOwner, gm.git.RepoName), nil
+	case vcsutils.GitLab:
+		return fmt.Sprintf(gitLabHttpsFormat, gm.git.ApiEndpoint, gm.git.GitProject, gm.git.RepoName), nil
+	case vcsutils.BitbucketServer:
+		return fmt.Sprintf(bitbucketServerHttpsFormat, gm.git.ApiEndpoint, gm.git.RepoOwner, gm.git.RepoName), nil
+	case vcsutils.AzureRepos:
+		azureEndpointWithoutHttps := strings.Join(strings.Split(gm.git.ApiEndpoint, "https://")[1:], "")
+		return fmt.Sprintf(azureDevopsHttpsFormat, gm.git.RepoOwner, azureEndpointWithoutHttps, gm.git.GitProject, gm.git.RepoName), nil
+	default:
+		return "", fmt.Errorf("unsupported version control provider: %s", gm.git.GitProvider.String())
+	}
+}
+
 func toBasicAuth(token, username string) *githttp.BasicAuth {
 	// The username can be anything except for an empty string
 	if username == "" {

commands/utils/git_test.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"github.com/jfrog/froggit-go/vcsutils"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"github.com/stretchr/testify/assert"
 	"testing"
@@ -187,3 +188,86 @@ func TestGitManager_GenerateAggregatedCommitMessage(t *testing.T) {
 		})
 	}
 }
+
+func TestConvertSSHtoHTTPS(t *testing.T) {
+	testsCases := []struct {
+		repoName    string
+		repoOwner   string
+		projectName string
+		expected    string
+		apiEndpoint string
+		vcsProvider vcsutils.VcsProvider
+	}{
+		{
+			repoName:    "npmExample",
+			repoOwner:   "repoOwner",
+			expected:    "https://github.com/repoOwner/npmExample.git",
+			apiEndpoint: "https://github.com",
+			vcsProvider: vcsutils.GitHub,
+		}, {
+			repoName:    "npmExample",
+			repoOwner:   "repoOwner",
+			expected:    "https://api.github.com/repoOwner/npmExample.git",
+			apiEndpoint: "https://api.github.com",
+			vcsProvider: vcsutils.GitHub,
+		},
+		{
+			repoName:    "npmProject",
+			projectName: "myTest5551218",
+			apiEndpoint: "https://gitlab.com",
+			expected:    "https://gitlab.com/myTest5551218/npmProject.git",
+			vcsProvider: vcsutils.GitLab,
+		}, {
+			repoName:    "onPremProject",
+			projectName: "myTest5551218",
+			apiEndpoint: "https://gitlab.example.com",
+			expected:    "https://gitlab.example.com/myTest5551218/onPremProject.git",
+			vcsProvider: vcsutils.GitLab,
+		},
+		{
+			repoName:    "npmExample",
+			projectName: "firstProject",
+			repoOwner:   "azureReposOwner",
+			apiEndpoint: "https://dev.azure.com/azureReposOwner/",
+			expected:    "https://[email protected]/azureReposOwner/firstProject/_git/npmExample",
+			vcsProvider: vcsutils.AzureRepos,
+		}, {
+			repoName:    "npmExample",
+			projectName: "onPremProject",
+			repoOwner:   "organization",
+			apiEndpoint: "https://your-server-name:port/organization/",
+			expected:    "https://organization@your-server-name:port/organization/onPremProject/_git/npmExample",
+			vcsProvider: vcsutils.AzureRepos,
+		},
+		{
+			repoName:    "npmExample",
+			repoOwner:   "~bitbucketServerOwner", // Bitbucket server private projects owners start with ~ prefix.
+			apiEndpoint: "https://git.company.info",
+			expected:    "https://git.company.info/scm/~bitbucketServerOwner/npmExample.git",
+			vcsProvider: vcsutils.BitbucketServer,
+		}, {
+			repoName:    "npmExample",
+			repoOwner:   "bitbucketServerOwner", // Public on prem repo
+			apiEndpoint: "https://git.company.info",
+			expected:    "https://git.company.info/scm/bitbucketServerOwner/npmExample.git",
+			vcsProvider: vcsutils.BitbucketServer,
+		}, {
+			repoName:    "notSupported",
+			repoOwner:   "cloudOwner",
+			expected:    "",
+			vcsProvider: vcsutils.BitbucketCloud,
+		},
+	}
+	for _, test := range testsCases {
+		t.Run(test.vcsProvider.String(), func(t *testing.T) {
+			gm := GitManager{git: &Git{GitProvider: test.vcsProvider, RepoName: test.repoName, RepoOwner: test.repoOwner, GitProject: test.projectName, ApiEndpoint: test.apiEndpoint}}
+			remoteUrl, err := gm.generateHTTPSCloneUrl()
+			if remoteUrl == "" {
+				assert.Equal(t, err.Error(), "unsupported version control provider: Bitbucket Cloud")
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, test.expected, remoteUrl)
+			}
+		})
+	}
+}

commands/utils/params.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	xrutils "github.com/jfrog/jfrog-cli-core/v2/xray/utils"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -280,6 +281,9 @@ func extractGitParamsFromEnv() (*Git, error) {
 	if err = readParamFromEnv(GitApiEndpointEnv, &gitParams.ApiEndpoint); err != nil && !e.IsMissingEnvErr(err) {
 		return nil, err
 	}
+	if err = verifyValidApiEndpoint(gitParams.ApiEndpoint); err != nil {
+		return nil, err
+	}
 	if gitParams.GitProvider, err = extractVcsProviderFromEnv(); err != nil {
 		return nil, err
 	}
@@ -321,6 +325,21 @@ func extractGitParamsFromEnv() (*Git, error) {
 	return &gitParams, err
 }
 
+func verifyValidApiEndpoint(apiEndpoint string) error {
+	// Empty string will resolve to default values.
+	if apiEndpoint == "" {
+		return nil
+	}
+	parsedUrl, err := url.Parse(apiEndpoint)
+	if err != nil {
+		return err
+	}
+	if parsedUrl.Scheme == "" {
+		return errors.New("the given API endpoint is invalid. Please note that the API endpoint format should be provided with the 'HTTPS' protocol as a prefix")
+	}
+	return nil
+}
+
 func readParamFromEnv(envKey string, paramValue *string) error {
 	*paramValue = getTrimmedEnv(envKey)
 	if *paramValue == "" {

commands/utils/params_test.go

@@ -410,3 +410,26 @@ func TestFrogbotConfigAggregator_UnmarshalYaml(t *testing.T) {
 	assert.ElementsMatch(t, []string{"watch-1", "watch-2"}, thirdRepo.Watches)
 	assert.Equal(t, "proj", thirdRepo.JFrogProjectKey)
 }
+
+func TestVerifyValidApiEndpoint(t *testing.T) {
+	testsCases := []struct {
+		endpointUrl   string
+		expectedError bool
+	}{
+		{endpointUrl: "https://git.company.info"},
+		{endpointUrl: "http://git.company.info"},
+		{endpointUrl: "justAString", expectedError: true},
+		{endpointUrl: ""},
+		{endpointUrl: "git.company.info", expectedError: true},
+	}
+	for _, test := range testsCases {
+		t.Run(test.endpointUrl, func(t *testing.T) {
+			err := verifyValidApiEndpoint(test.endpointUrl)
+			if test.expectedError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

docs/install-bitbucket-server.md

@@ -67,11 +67,12 @@
                JF_GIT_PROVIDER= "bitbucketServer"
                
                // [Mandatory]
-               // Username of the Bitbucket account
+               // Username of the account associated with the token
                JF_GIT_USERNAME= ""
                
                // [Mandatory]
                // Bitbucket project namespace
+               // Private projects should start with the prefix: "~"
                JF_GIT_OWNER= ""
                
                // [Mandatory]