Merge pull request #452 from malinthaprasan/gitint

VCS command improvements

import-export-cli/cmd/root.go

@@ -133,7 +133,8 @@ func createConfigFiles() {
 	if !utils.IsFileExist(utils.MainConfigFilePath) {
 		var mainConfig = new(utils.MainConfig)
 		mainConfig.Config = utils.Config{utils.DefaultHttpRequestTimeout,
-			utils.DefaultExportDirPath, k8sUtils.DefaultKubernetesMode, utils.DefaultTokenType}
+			utils.DefaultExportDirPath, k8sUtils.DefaultKubernetesMode, utils.DefaultTokenType,
+			false, ""}
 		utils.WriteConfigFile(mainConfig, utils.MainConfigFilePath)
 	}
 

import-export-cli/cmd/set.go

@@ -30,20 +30,29 @@ var flagHttpRequestTimeout int
 var flagExportDirectory string
 var flagKubernetesMode string
 
+var flagVCSDeletionEnabled bool
+var flagVCSConfigPath string
+
+const flagVCSConfigPathName = "vcs-config-path"
+
 // Set command related Info
 const setCmdLiteral = "set"
 const setCmdShortDesc = "Set configuration parameters"
 
 const setCmdLongDesc = `Set configuration parameters. Use at least one of the following flags
 * --http-request-timeout <time-in-milli-seconds>
 * --export-directory <path-to-directory-where-apis-should-be-saved>
-* --mode <mode-of-apictl>`
+* --mode <mode-of-apictl>
+* --vcs-deletion-enabled <enable-or-disable-project-deletion-via-vcs>
+* --vcs-config-path <path-to-custom-vcs-config-file>`
 
 const setCmdExamples = utils.ProjectName + ` ` + setCmdLiteral + ` --http-request-timeout 3600 --export-directory /home/user/exported-apis
 ` + utils.ProjectName + ` ` + setCmdLiteral + ` --http-request-timeout 5000 --export-directory C:\Documents\exported
 ` + utils.ProjectName + ` ` + setCmdLiteral + ` --http-request-timeout 5000
 ` + utils.ProjectName + ` ` + setCmdLiteral + ` --mode kubernetes
-` + utils.ProjectName + ` ` + setCmdLiteral + ` --mode default`
+` + utils.ProjectName + ` ` + setCmdLiteral + ` --mode default
+` + utils.ProjectName + ` ` + setCmdLiteral + ` --vcs-deletion-enabled=true
+` + utils.ProjectName + ` ` + setCmdLiteral + ` --vcs-config-path /home/user/custom/vcs-config.yaml`
 
 // SetCmd represents the 'set' command
 var SetCmd = &cobra.Command{
@@ -53,11 +62,11 @@ var SetCmd = &cobra.Command{
 	Example: setCmdExamples,
 	Run: func(cmd *cobra.Command, args []string) {
 		utils.Logln(utils.LogPrefixInfo + setCmdLiteral + " called")
-		executeSetCmd(utils.MainConfigFilePath, utils.ExportDirectory)
+		executeSetCmd(utils.MainConfigFilePath, cmd)
 	},
 }
 
-func executeSetCmd(mainConfigFilePath, exportDirectory string) {
+func executeSetCmd(mainConfigFilePath string, cmd *cobra.Command) {
 	// read the existing config vars
 	configVars := utils.GetMainConfigFromFile(mainConfigFilePath)
 	//Change Http Request timeout
@@ -101,6 +110,20 @@ func executeSetCmd(mainConfigFilePath, exportDirectory string) {
 		}
 	}
 
+	//VCS configs
+	if configVars.Config.VCSDeletionEnabled != flagVCSDeletionEnabled {
+		if flagVCSDeletionEnabled {
+			fmt.Println("Project deletion is enabled in VCS")
+		} else {
+			fmt.Println("Project deletion is disabled in VCS")
+		}
+		configVars.Config.VCSDeletionEnabled = flagVCSDeletionEnabled
+	}
+	if cmd.Flags().Changed(flagVCSConfigPathName) {
+		configVars.Config.VCSConfigFilePath = flagVCSConfigPath
+		fmt.Println("VCS config file path is set to : " + flagVCSConfigPath)
+	}
+
 	utils.WriteConfigFile(configVars, mainConfigFilePath)
 }
 
@@ -129,4 +152,8 @@ func init() {
 	SetCmd.Flags().StringVarP(&flagKubernetesMode, "mode", "m", utils.DefaultEnvironmentName, "If mode is set to \"k8s\", apictl "+
 		"is capable of executing Kubectl commands. For example \"apictl get pods\" -> \"kubectl get pods\". To go back "+
 		"to the default mode, set the mode to \"default\"")
+	SetCmd.Flags().BoolVar(&flagVCSDeletionEnabled, "vcs-deletion-enabled", false,
+		"Specifies whether project deletion is allowed during deployment.")
+	SetCmd.Flags().StringVar(&flagVCSConfigPath, flagVCSConfigPathName, "",
+		"Path to the VCS Configuration yaml file which keeps the VCS meta data")
 }

import-export-cli/cmd/vcsDeploy.go

@@ -24,6 +24,7 @@ import (
 	"github.com/wso2/product-apim-tooling/import-export-cli/credentials"
 	"github.com/wso2/product-apim-tooling/import-export-cli/git"
 	"github.com/wso2/product-apim-tooling/import-export-cli/utils"
+	"os"
 )
 
 var flagVCSDeployEnvName string    // name of the environment the project changes need to be deployed
@@ -48,6 +49,11 @@ var DeployCmd = &cobra.Command{
 	Example: deployCmdExamples,
 	Run: func(cmd *cobra.Command, args []string) {
 		utils.Logln(utils.LogPrefixInfo + deployCmdLiteral + " called")
+		if !utils.EnvExistsInMainConfigFile(flagVCSDeployEnvName, utils.MainConfigFilePath) {
+			fmt.Println(flagVCSDeployEnvName, "does not exists. Add it using add-env")
+			os.Exit(1)
+		}
+
 		credential, err := getCredentials(flagVCSDeployEnvName)
 		if err != nil {
 			utils.HandleErrorAndExit("Error getting credentials", err)

import-export-cli/cmd/vcsStatus.go

@@ -24,6 +24,7 @@ import (
 	"github.com/wso2/product-apim-tooling/import-export-cli/git"
 	"github.com/wso2/product-apim-tooling/import-export-cli/specs/params"
 	"github.com/wso2/product-apim-tooling/import-export-cli/utils"
+	"os"
 	"strconv"
 )
 
@@ -45,6 +46,11 @@ var VCSStatusCmd = &cobra.Command{
 	Example: vcsStatusCmdCmdExamples,
 	Run: func(cmd *cobra.Command, args []string) {
 		utils.Logln(utils.LogPrefixInfo + vcsStatusCmdLiteral + " called")
+		if !utils.EnvExistsInMainConfigFile(flagVCSStatusEnvName, utils.MainConfigFilePath) {
+			fmt.Println(flagVCSStatusEnvName, "does not exists. Add it using add-env")
+			os.Exit(1)
+		}
+
 		_, totalProjectsToUpdate, updatedProjectsPerType := git.GetStatus(flagVCSStatusEnvName, git.FromRevTypeLastAttempted)
 		if totalProjectsToUpdate == 0 {
 			fmt.Println("Everything is up-to-date")

import-export-cli/docs/apictl_set.md

@@ -8,6 +8,8 @@ Set configuration parameters. Use at least one of the following flags
 * --http-request-timeout <time-in-milli-seconds>
 * --export-directory <path-to-directory-where-apis-should-be-saved>
 * --mode <mode-of-apictl>
+* --vcs-deletion-enabled <enable-or-disable-project-deletion-via-vcs>
+* --vcs-config-path <path-to-custom-vcs-config-file>
 
 ```
 apictl set [flags]
@@ -21,6 +23,8 @@ apictl set --http-request-timeout 5000 --export-directory C:\Documents\exported
 apictl set --http-request-timeout 5000
 apictl set --mode kubernetes
 apictl set --mode default
+apictl set --vcs-deletion-enabled=true
+apictl set --vcs-config-path /home/user/custom/vcs-config.yaml
 ```
 
 ### Options
@@ -30,6 +34,8 @@ apictl set --mode default
   -h, --help                       help for set
       --http-request-timeout int   Timeout for HTTP Client (default 10000)
   -m, --mode string                If mode is set to "k8s", apictl is capable of executing Kubectl commands. For example "apictl get pods" -> "kubectl get pods". To go back to the default mode, set the mode to "default" (default "default")
+      --vcs-config-path string     Path to the VCS Configuration yaml file which keeps the VCS meta data
+      --vcs-deletion-enabled       Specifies whether project deletion is allowed during deployment.
 ```
 
 ### Options inherited from parent commands

import-export-cli/git/consts.go

@@ -31,4 +31,6 @@ const VCSRepoInfoFileName = "vcs.yaml"
 const FromRevTypeLastAttempted = "last_attempted"
 const FromRevTypeLastSuccessful = "last_successful"
 
+const lastSuccessfulCommitsToKeep = 15
+
 var VCSConfigFilePath = filepath.Join(utils.ConfigDirPath, VCSConfigFileName)

import-export-cli/git/gitUtils.go

@@ -57,6 +57,10 @@ func getVCSConfigFromFileSilently(filePath string) *VCSConfig {
 // Returns Environment, the environment specific VCS configuration
 // Returns bool, whether the environment is available in the VCS configuration or not
 func getVCSEnvironmentDetails(repoId, environment string) (VCSConfig, Environment, bool)  {
+    mainConfig := utils.GetMainConfigFromFile(utils.MainConfigFilePath)
+    if mainConfig.Config.VCSConfigFilePath != "" {
+        VCSConfigFilePath = mainConfig.Config.VCSConfigFilePath
+    }
     vcsConfig := getVCSConfigFromFileSilently(VCSConfigFilePath)
     if vcsConfig.Repos == nil {
         vcsConfig.Repos = make(map[string]Repo)
@@ -71,22 +75,27 @@ func getVCSEnvironmentDetails(repoId, environment string) (VCSConfig, Environmen
 // Returns string, id of the git repository (located in vcs.yaml)
 // Returns int, the total number of projects to deploy
 // Returns map[string][]*params.ProjectParams, the details of the projects that needs to deploy
-func GetStatus(environment, fromRevType string) (string, int, map[string][]*params.ProjectParams){
+func GetStatus(environment, fromRevType string) (string, int, map[string][]*params.ProjectParams) {
     var envRevision string
+    mainConfig := utils.GetMainConfigFromFile(utils.MainConfigFilePath)
     repoId, err := getRepoId()
     if err != nil {
         utils.HandleErrorAndExit("Error while retrieving repository id", err)
     }
     if repoId == "" {
-        utils.HandleErrorAndExit("The repository info: vcs.yaml is not found in the repository root. " +
+        utils.HandleErrorAndExit("The repository info: vcs.yaml is not found in the repository root. "+
             "If this is the first time you are using this repo, please initialize it with 'vcs init'.", nil)
     }
     _, envVCSConfig, hasEnv := getVCSEnvironmentDetails(repoId, environment)
     if hasEnv {
         if fromRevType == FromRevTypeLastAttempted {
             envRevision = envVCSConfig.LastAttemptedRev
-        } else if fromRevType == FromRevTypeLastSuccessful{
-            envRevision = envVCSConfig.LastSuccessfulRev
+        } else if fromRevType == FromRevTypeLastSuccessful {
+            if len(envVCSConfig.LastSuccessfulRev) > 0 {
+                envRevision = envVCSConfig.LastSuccessfulRev[0]
+            } else {
+                envRevision = ""
+            }
         }
     }
 
@@ -98,9 +107,12 @@ func GetStatus(environment, fromRevType string) (string, int, map[string][]*para
     var changedFiles string
     if envRevision == "" {
         changedFiles, _ = executeGitCommand("ls-tree", "-r", "HEAD", "--name-only", "--full-tree")
-    } else {
+    } else if mainConfig.Config.VCSDeletionEnabled {
         changedFiles, _ = executeGitCommand("diff", "--name-only", envRevision)
+    } else {
+        changedFiles, _ = executeGitCommand("diff", "--diff-filter=d", "--name-only", envRevision)
     }
+
     changedFileList := strings.Split(changedFiles,"\n")
     // remove the last empty element
     if len(changedFileList) > 0 {
@@ -170,13 +182,14 @@ func Rollback(accessToken, environment string) error {
         return errors.New("Nothing to rollback")
     }
 
-    if envVCSConfig.LastSuccessfulRev == "" {
+    if len(envVCSConfig.LastSuccessfulRev) == 0 {
         return errors.New("Failed to rollback as there are no previous successful revisions")
     }
+    lastSuccessfulRevision := envVCSConfig.LastSuccessfulRev[0]
 
     currentBranch := getCurrentBranch()
-    tmpBranchName := "tmp-" + envVCSConfig.LastSuccessfulRev[0:8]
-    checkoutNewBranchFromRevision(tmpBranchName, envVCSConfig.LastSuccessfulRev)
+    tmpBranchName := "tmp-" + lastSuccessfulRevision[0:8]
+    checkoutNewBranchFromRevision(tmpBranchName, lastSuccessfulRevision)
     deployUpdatedProjects(accessToken, repoId, environment, totalProjectsToUpdate, updatedProjectsPerType)
     checkoutBranch(currentBranch)
     deleteTmpBranch(tmpBranchName)
@@ -422,7 +435,11 @@ func updateVCSConfig(repoId, environment string, failedProjects map[string][]*pa
     envVCSConfig.FailedProjects = failedProjects
 
     if len(failedProjects) == 0 {
-        envVCSConfig.LastSuccessfulRev = envVCSConfig.LastAttemptedRev
+        if len(envVCSConfig.LastSuccessfulRev) == 0 || len(envVCSConfig.LastSuccessfulRev) > 0 &&
+            envVCSConfig.LastSuccessfulRev[0] != envVCSConfig.LastAttemptedRev {
+            persistedLast := envVCSConfig.LastSuccessfulRev[0:utils.Min(lastSuccessfulCommitsToKeep-1, len(envVCSConfig.LastSuccessfulRev))]
+            envVCSConfig.LastSuccessfulRev = append([]string{envVCSConfig.LastAttemptedRev}, persistedLast...)
+        }
     }
     _, hasRepo := vcsConfig.Repos[repoId]
     if !hasRepo {
@@ -456,18 +473,26 @@ func DeployChangedFiles(accessToken, environment string) map[string][]*params.Pr
         deployUpdatedProjects(accessToken, repoId, environment, totalProjectsToUpdate, updatedProjectsPerType)
 
     if hasDeletedProjects {
+        //check whether project deletion is disabled
+        mainConfig := utils.GetMainConfigFromFile(utils.MainConfigFilePath)
+        if !mainConfig.Config.VCSDeletionEnabled {
+            utils.HandleErrorAndExit("Error: there are projects to delete while project " +
+                "deletion is disabled via VCS", nil)
+        }
+
         // work on deleted files
         _, envVCSConfig, hasEnv := getVCSEnvironmentDetails(repoId, environment)
-        if !hasEnv || envVCSConfig.LastSuccessfulRev == "" {
+        if !hasEnv || len(envVCSConfig.LastSuccessfulRev) == 0 {
             utils.HandleErrorAndExit("Error: there are projects to delete but no last successful "+
                 "revision available in vcs config (vcs_config.yaml)", nil)
             return nil
         }
         currentBranch := getCurrentBranch()
-        tmpBranchName := "tmp-" + envVCSConfig.LastSuccessfulRev[0:8]
+        lastSuccessfulRev := envVCSConfig.LastSuccessfulRev[0]
+        tmpBranchName := "tmp-" + lastSuccessfulRev[0:8]
 
         fmt.Println("\nDeleting projects ..")
-        checkoutNewBranchFromRevision(tmpBranchName, envVCSConfig.LastSuccessfulRev)
+        checkoutNewBranchFromRevision(tmpBranchName, lastSuccessfulRev)
         failedProjects = deployProjectDeletions(accessToken, environment, deletedProjectsPerType, failedProjects)
         checkoutBranch(currentBranch)
         deleteTmpBranch(tmpBranchName)
@@ -676,9 +701,22 @@ func getSubPaths(parent string, path string) (paths []string) {
 // Executes the give git command as args list and returns the output
 func executeGitCommand(args ...string) (string, error) {
     cmd := exec.Command(Git, args...)
+
+    if utils.VerboseModeEnabled() {
+        utils.Logln("Executing command: " + Git + " " + strings.Join(args, " "))
+    }
+
     var errBuf bytes.Buffer
     cmd.Stderr = io.MultiWriter(os.Stderr, &errBuf)
 
     output, err := cmd.Output()
+
+    if utils.VerboseModeEnabled() {
+        if err != nil {
+            utils.HandleErrorAndContinue("Error occurred while executing command: ", err)
+        } else {
+            utils.Logln("Output : " + string(output))
+        }
+    }
     return string(output), err
 }

import-export-cli/git/vcsParams.go

@@ -8,7 +8,7 @@ type VCSConfig struct {
 
 type Environment struct {
     LastAttemptedRev  string                             `yaml:"lastAttemptedRev"`
-    LastSuccessfulRev string                             `yaml:"lastSuccessfulRev"`
+    LastSuccessfulRev []string                           `yaml:"lastSuccessfulRev"`
     FailedProjects    map[string][]*params.ProjectParams `yaml:"failedProjects"`
 }
 

import-export-cli/shell-completions/apictl_bash_completions.sh

@@ -1626,6 +1626,11 @@ _apictl_set()
     two_word_flags+=("--mode")
     two_word_flags+=("-m")
     local_nonpersistent_flags+=("--mode=")
+    flags+=("--vcs-config-path=")
+    two_word_flags+=("--vcs-config-path")
+    local_nonpersistent_flags+=("--vcs-config-path=")
+    flags+=("--vcs-deletion-enabled")
+    local_nonpersistent_flags+=("--vcs-deletion-enabled")
     flags+=("--insecure")
     flags+=("-k")
     flags+=("--verbose")

import-export-cli/shell-completions/apictl_zsh_completions.sh

@@ -778,6 +778,8 @@ function _apictl_set {
     '(-h --help)'{-h,--help}'[help for set]' \
     '--http-request-timeout[Timeout for HTTP Client]:' \
     '(-m --mode)'{-m,--mode}'[If mode is set to "k8s", apictl is capable of executing Kubectl commands. For example "apictl get pods" -> "kubectl get pods". To go back to the default mode, set the mode to "default"]:' \
+    '--vcs-config-path[Path to the VCS Configuration yaml file which keeps the VCS meta data]:' \
+    '--vcs-deletion-enabled[Specifies whether project deletion is allowed during deployment.]' \
     '(-k --insecure)'{-k,--insecure}'[Allow connections to SSL endpoints without certs]' \
     '--verbose[Enable verbose mode]'
 }

import-export-cli/utils/structs.go

@@ -38,6 +38,8 @@ type Config struct {
 	ExportDirectory    string `yaml:"export_directory"`
 	KubernetesMode     bool   `yaml:"kubernetes_mode"`
 	TokenType          string `yaml:"token_type"`
+	VCSDeletionEnabled bool   `yaml:"vcs_deletion_enabled"`
+	VCSConfigFilePath  string `yaml:"vcs_config_file_path"`
 }
 
 type EnvKeys struct {

import-export-cli/utils/utils.go

@@ -222,3 +222,11 @@ func SetToK8sMode() {
 	configVars.Config.KubernetesMode = true
 	WriteConfigFile(configVars, MainConfigFilePath)
 }
+
+// returns min of two ints
+func Min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}