diff --git a/import-export-cli/cmd/root.go b/import-export-cli/cmd/root.go index 141f825cb..0beb49367 100644 --- a/import-export-cli/cmd/root.go +++ b/import-export-cli/cmd/root.go @@ -133,7 +133,8 @@ func createConfigFiles() { if !utils.IsFileExist(utils.MainConfigFilePath) { var mainConfig = new(utils.MainConfig) mainConfig.Config = utils.Config{utils.DefaultHttpRequestTimeout, - utils.DefaultExportDirPath, k8sUtils.DefaultKubernetesMode, utils.DefaultTokenType} + utils.DefaultExportDirPath, k8sUtils.DefaultKubernetesMode, utils.DefaultTokenType, + false, ""} utils.WriteConfigFile(mainConfig, utils.MainConfigFilePath) } diff --git a/import-export-cli/cmd/set.go b/import-export-cli/cmd/set.go index caa5cfd3a..61043a4d4 100644 --- a/import-export-cli/cmd/set.go +++ b/import-export-cli/cmd/set.go @@ -30,6 +30,11 @@ var flagHttpRequestTimeout int var flagExportDirectory string var flagKubernetesMode string +var flagVCSDeletionEnabled bool +var flagVCSConfigPath string + +const flagVCSConfigPathName = "vcs-config-path" + // Set command related Info const setCmdLiteral = "set" const setCmdShortDesc = "Set configuration parameters" @@ -37,13 +42,17 @@ const setCmdShortDesc = "Set configuration parameters" const setCmdLongDesc = `Set configuration parameters. Use at least one of the following flags * --http-request-timeout * --export-directory -* --mode ` +* --mode +* --vcs-deletion-enabled +* --vcs-config-path ` const setCmdExamples = utils.ProjectName + ` ` + setCmdLiteral + ` --http-request-timeout 3600 --export-directory /home/user/exported-apis ` + utils.ProjectName + ` ` + setCmdLiteral + ` --http-request-timeout 5000 --export-directory C:\Documents\exported ` + utils.ProjectName + ` ` + setCmdLiteral + ` --http-request-timeout 5000 ` + utils.ProjectName + ` ` + setCmdLiteral + ` --mode kubernetes -` + utils.ProjectName + ` ` + setCmdLiteral + ` --mode default` +` + utils.ProjectName + ` ` + setCmdLiteral + ` --mode default +` + utils.ProjectName + ` ` + setCmdLiteral + ` --vcs-deletion-enabled=true +` + utils.ProjectName + ` ` + setCmdLiteral + ` --vcs-config-path /home/user/custom/vcs-config.yaml` // SetCmd represents the 'set' command var SetCmd = &cobra.Command{ @@ -53,11 +62,11 @@ var SetCmd = &cobra.Command{ Example: setCmdExamples, Run: func(cmd *cobra.Command, args []string) { utils.Logln(utils.LogPrefixInfo + setCmdLiteral + " called") - executeSetCmd(utils.MainConfigFilePath, utils.ExportDirectory) + executeSetCmd(utils.MainConfigFilePath, cmd) }, } -func executeSetCmd(mainConfigFilePath, exportDirectory string) { +func executeSetCmd(mainConfigFilePath string, cmd *cobra.Command) { // read the existing config vars configVars := utils.GetMainConfigFromFile(mainConfigFilePath) //Change Http Request timeout @@ -101,6 +110,20 @@ func executeSetCmd(mainConfigFilePath, exportDirectory string) { } } + //VCS configs + if configVars.Config.VCSDeletionEnabled != flagVCSDeletionEnabled { + if flagVCSDeletionEnabled { + fmt.Println("Project deletion is enabled in VCS") + } else { + fmt.Println("Project deletion is disabled in VCS") + } + configVars.Config.VCSDeletionEnabled = flagVCSDeletionEnabled + } + if cmd.Flags().Changed(flagVCSConfigPathName) { + configVars.Config.VCSConfigFilePath = flagVCSConfigPath + fmt.Println("VCS config file path is set to : " + flagVCSConfigPath) + } + utils.WriteConfigFile(configVars, mainConfigFilePath) } @@ -129,4 +152,8 @@ func init() { SetCmd.Flags().StringVarP(&flagKubernetesMode, "mode", "m", utils.DefaultEnvironmentName, "If mode is set to \"k8s\", apictl "+ "is capable of executing Kubectl commands. For example \"apictl get pods\" -> \"kubectl get pods\". To go back "+ "to the default mode, set the mode to \"default\"") + SetCmd.Flags().BoolVar(&flagVCSDeletionEnabled, "vcs-deletion-enabled", false, + "Specifies whether project deletion is allowed during deployment.") + SetCmd.Flags().StringVar(&flagVCSConfigPath, flagVCSConfigPathName, "", + "Path to the VCS Configuration yaml file which keeps the VCS meta data") } diff --git a/import-export-cli/cmd/vcsDeploy.go b/import-export-cli/cmd/vcsDeploy.go index ff6b8f7d9..f9a1ddf04 100644 --- a/import-export-cli/cmd/vcsDeploy.go +++ b/import-export-cli/cmd/vcsDeploy.go @@ -24,6 +24,7 @@ import ( "github.com/wso2/product-apim-tooling/import-export-cli/credentials" "github.com/wso2/product-apim-tooling/import-export-cli/git" "github.com/wso2/product-apim-tooling/import-export-cli/utils" + "os" ) var flagVCSDeployEnvName string // name of the environment the project changes need to be deployed @@ -48,6 +49,11 @@ var DeployCmd = &cobra.Command{ Example: deployCmdExamples, Run: func(cmd *cobra.Command, args []string) { utils.Logln(utils.LogPrefixInfo + deployCmdLiteral + " called") + if !utils.EnvExistsInMainConfigFile(flagVCSDeployEnvName, utils.MainConfigFilePath) { + fmt.Println(flagVCSDeployEnvName, "does not exists. Add it using add-env") + os.Exit(1) + } + credential, err := getCredentials(flagVCSDeployEnvName) if err != nil { utils.HandleErrorAndExit("Error getting credentials", err) diff --git a/import-export-cli/cmd/vcsStatus.go b/import-export-cli/cmd/vcsStatus.go index e748d5028..64a19156e 100644 --- a/import-export-cli/cmd/vcsStatus.go +++ b/import-export-cli/cmd/vcsStatus.go @@ -24,6 +24,7 @@ import ( "github.com/wso2/product-apim-tooling/import-export-cli/git" "github.com/wso2/product-apim-tooling/import-export-cli/specs/params" "github.com/wso2/product-apim-tooling/import-export-cli/utils" + "os" "strconv" ) @@ -45,6 +46,11 @@ var VCSStatusCmd = &cobra.Command{ Example: vcsStatusCmdCmdExamples, Run: func(cmd *cobra.Command, args []string) { utils.Logln(utils.LogPrefixInfo + vcsStatusCmdLiteral + " called") + if !utils.EnvExistsInMainConfigFile(flagVCSStatusEnvName, utils.MainConfigFilePath) { + fmt.Println(flagVCSStatusEnvName, "does not exists. Add it using add-env") + os.Exit(1) + } + _, totalProjectsToUpdate, updatedProjectsPerType := git.GetStatus(flagVCSStatusEnvName, git.FromRevTypeLastAttempted) if totalProjectsToUpdate == 0 { fmt.Println("Everything is up-to-date") diff --git a/import-export-cli/docs/apictl_set.md b/import-export-cli/docs/apictl_set.md index 974e97681..ee575bb51 100644 --- a/import-export-cli/docs/apictl_set.md +++ b/import-export-cli/docs/apictl_set.md @@ -8,6 +8,8 @@ Set configuration parameters. Use at least one of the following flags * --http-request-timeout * --export-directory * --mode +* --vcs-deletion-enabled +* --vcs-config-path ``` apictl set [flags] @@ -21,6 +23,8 @@ apictl set --http-request-timeout 5000 --export-directory C:\Documents\exported apictl set --http-request-timeout 5000 apictl set --mode kubernetes apictl set --mode default +apictl set --vcs-deletion-enabled=true +apictl set --vcs-config-path /home/user/custom/vcs-config.yaml ``` ### Options @@ -30,6 +34,8 @@ apictl set --mode default -h, --help help for set --http-request-timeout int Timeout for HTTP Client (default 10000) -m, --mode string If mode is set to "k8s", apictl is capable of executing Kubectl commands. For example "apictl get pods" -> "kubectl get pods". To go back to the default mode, set the mode to "default" (default "default") + --vcs-config-path string Path to the VCS Configuration yaml file which keeps the VCS meta data + --vcs-deletion-enabled Specifies whether project deletion is allowed during deployment. ``` ### Options inherited from parent commands diff --git a/import-export-cli/git/consts.go b/import-export-cli/git/consts.go index 27f93168b..2590d580d 100644 --- a/import-export-cli/git/consts.go +++ b/import-export-cli/git/consts.go @@ -31,4 +31,6 @@ const VCSRepoInfoFileName = "vcs.yaml" const FromRevTypeLastAttempted = "last_attempted" const FromRevTypeLastSuccessful = "last_successful" +const lastSuccessfulCommitsToKeep = 15 + var VCSConfigFilePath = filepath.Join(utils.ConfigDirPath, VCSConfigFileName) \ No newline at end of file diff --git a/import-export-cli/git/gitUtils.go b/import-export-cli/git/gitUtils.go index 5b9bcfca2..dda7d5de6 100644 --- a/import-export-cli/git/gitUtils.go +++ b/import-export-cli/git/gitUtils.go @@ -57,6 +57,10 @@ func getVCSConfigFromFileSilently(filePath string) *VCSConfig { // Returns Environment, the environment specific VCS configuration // Returns bool, whether the environment is available in the VCS configuration or not func getVCSEnvironmentDetails(repoId, environment string) (VCSConfig, Environment, bool) { + mainConfig := utils.GetMainConfigFromFile(utils.MainConfigFilePath) + if mainConfig.Config.VCSConfigFilePath != "" { + VCSConfigFilePath = mainConfig.Config.VCSConfigFilePath + } vcsConfig := getVCSConfigFromFileSilently(VCSConfigFilePath) if vcsConfig.Repos == nil { vcsConfig.Repos = make(map[string]Repo) @@ -71,22 +75,27 @@ func getVCSEnvironmentDetails(repoId, environment string) (VCSConfig, Environmen // Returns string, id of the git repository (located in vcs.yaml) // Returns int, the total number of projects to deploy // Returns map[string][]*params.ProjectParams, the details of the projects that needs to deploy -func GetStatus(environment, fromRevType string) (string, int, map[string][]*params.ProjectParams){ +func GetStatus(environment, fromRevType string) (string, int, map[string][]*params.ProjectParams) { var envRevision string + mainConfig := utils.GetMainConfigFromFile(utils.MainConfigFilePath) repoId, err := getRepoId() if err != nil { utils.HandleErrorAndExit("Error while retrieving repository id", err) } if repoId == "" { - utils.HandleErrorAndExit("The repository info: vcs.yaml is not found in the repository root. " + + utils.HandleErrorAndExit("The repository info: vcs.yaml is not found in the repository root. "+ "If this is the first time you are using this repo, please initialize it with 'vcs init'.", nil) } _, envVCSConfig, hasEnv := getVCSEnvironmentDetails(repoId, environment) if hasEnv { if fromRevType == FromRevTypeLastAttempted { envRevision = envVCSConfig.LastAttemptedRev - } else if fromRevType == FromRevTypeLastSuccessful{ - envRevision = envVCSConfig.LastSuccessfulRev + } else if fromRevType == FromRevTypeLastSuccessful { + if len(envVCSConfig.LastSuccessfulRev) > 0 { + envRevision = envVCSConfig.LastSuccessfulRev[0] + } else { + envRevision = "" + } } } @@ -98,9 +107,12 @@ func GetStatus(environment, fromRevType string) (string, int, map[string][]*para var changedFiles string if envRevision == "" { changedFiles, _ = executeGitCommand("ls-tree", "-r", "HEAD", "--name-only", "--full-tree") - } else { + } else if mainConfig.Config.VCSDeletionEnabled { changedFiles, _ = executeGitCommand("diff", "--name-only", envRevision) + } else { + changedFiles, _ = executeGitCommand("diff", "--diff-filter=d", "--name-only", envRevision) } + changedFileList := strings.Split(changedFiles,"\n") // remove the last empty element if len(changedFileList) > 0 { @@ -170,13 +182,14 @@ func Rollback(accessToken, environment string) error { return errors.New("Nothing to rollback") } - if envVCSConfig.LastSuccessfulRev == "" { + if len(envVCSConfig.LastSuccessfulRev) == 0 { return errors.New("Failed to rollback as there are no previous successful revisions") } + lastSuccessfulRevision := envVCSConfig.LastSuccessfulRev[0] currentBranch := getCurrentBranch() - tmpBranchName := "tmp-" + envVCSConfig.LastSuccessfulRev[0:8] - checkoutNewBranchFromRevision(tmpBranchName, envVCSConfig.LastSuccessfulRev) + tmpBranchName := "tmp-" + lastSuccessfulRevision[0:8] + checkoutNewBranchFromRevision(tmpBranchName, lastSuccessfulRevision) deployUpdatedProjects(accessToken, repoId, environment, totalProjectsToUpdate, updatedProjectsPerType) checkoutBranch(currentBranch) deleteTmpBranch(tmpBranchName) @@ -422,7 +435,11 @@ func updateVCSConfig(repoId, environment string, failedProjects map[string][]*pa envVCSConfig.FailedProjects = failedProjects if len(failedProjects) == 0 { - envVCSConfig.LastSuccessfulRev = envVCSConfig.LastAttemptedRev + if len(envVCSConfig.LastSuccessfulRev) == 0 || len(envVCSConfig.LastSuccessfulRev) > 0 && + envVCSConfig.LastSuccessfulRev[0] != envVCSConfig.LastAttemptedRev { + persistedLast := envVCSConfig.LastSuccessfulRev[0:utils.Min(lastSuccessfulCommitsToKeep-1, len(envVCSConfig.LastSuccessfulRev))] + envVCSConfig.LastSuccessfulRev = append([]string{envVCSConfig.LastAttemptedRev}, persistedLast...) + } } _, hasRepo := vcsConfig.Repos[repoId] if !hasRepo { @@ -456,18 +473,26 @@ func DeployChangedFiles(accessToken, environment string) map[string][]*params.Pr deployUpdatedProjects(accessToken, repoId, environment, totalProjectsToUpdate, updatedProjectsPerType) if hasDeletedProjects { + //check whether project deletion is disabled + mainConfig := utils.GetMainConfigFromFile(utils.MainConfigFilePath) + if !mainConfig.Config.VCSDeletionEnabled { + utils.HandleErrorAndExit("Error: there are projects to delete while project " + + "deletion is disabled via VCS", nil) + } + // work on deleted files _, envVCSConfig, hasEnv := getVCSEnvironmentDetails(repoId, environment) - if !hasEnv || envVCSConfig.LastSuccessfulRev == "" { + if !hasEnv || len(envVCSConfig.LastSuccessfulRev) == 0 { utils.HandleErrorAndExit("Error: there are projects to delete but no last successful "+ "revision available in vcs config (vcs_config.yaml)", nil) return nil } currentBranch := getCurrentBranch() - tmpBranchName := "tmp-" + envVCSConfig.LastSuccessfulRev[0:8] + lastSuccessfulRev := envVCSConfig.LastSuccessfulRev[0] + tmpBranchName := "tmp-" + lastSuccessfulRev[0:8] fmt.Println("\nDeleting projects ..") - checkoutNewBranchFromRevision(tmpBranchName, envVCSConfig.LastSuccessfulRev) + checkoutNewBranchFromRevision(tmpBranchName, lastSuccessfulRev) failedProjects = deployProjectDeletions(accessToken, environment, deletedProjectsPerType, failedProjects) checkoutBranch(currentBranch) deleteTmpBranch(tmpBranchName) @@ -676,9 +701,22 @@ func getSubPaths(parent string, path string) (paths []string) { // Executes the give git command as args list and returns the output func executeGitCommand(args ...string) (string, error) { cmd := exec.Command(Git, args...) + + if utils.VerboseModeEnabled() { + utils.Logln("Executing command: " + Git + " " + strings.Join(args, " ")) + } + var errBuf bytes.Buffer cmd.Stderr = io.MultiWriter(os.Stderr, &errBuf) output, err := cmd.Output() + + if utils.VerboseModeEnabled() { + if err != nil { + utils.HandleErrorAndContinue("Error occurred while executing command: ", err) + } else { + utils.Logln("Output : " + string(output)) + } + } return string(output), err } diff --git a/import-export-cli/git/vcsParams.go b/import-export-cli/git/vcsParams.go index 43b4e0944..137a31c2d 100644 --- a/import-export-cli/git/vcsParams.go +++ b/import-export-cli/git/vcsParams.go @@ -8,7 +8,7 @@ type VCSConfig struct { type Environment struct { LastAttemptedRev string `yaml:"lastAttemptedRev"` - LastSuccessfulRev string `yaml:"lastSuccessfulRev"` + LastSuccessfulRev []string `yaml:"lastSuccessfulRev"` FailedProjects map[string][]*params.ProjectParams `yaml:"failedProjects"` } diff --git a/import-export-cli/shell-completions/apictl_bash_completions.sh b/import-export-cli/shell-completions/apictl_bash_completions.sh index 832432389..e37c673f6 100644 --- a/import-export-cli/shell-completions/apictl_bash_completions.sh +++ b/import-export-cli/shell-completions/apictl_bash_completions.sh @@ -1626,6 +1626,11 @@ _apictl_set() two_word_flags+=("--mode") two_word_flags+=("-m") local_nonpersistent_flags+=("--mode=") + flags+=("--vcs-config-path=") + two_word_flags+=("--vcs-config-path") + local_nonpersistent_flags+=("--vcs-config-path=") + flags+=("--vcs-deletion-enabled") + local_nonpersistent_flags+=("--vcs-deletion-enabled") flags+=("--insecure") flags+=("-k") flags+=("--verbose") diff --git a/import-export-cli/shell-completions/apictl_zsh_completions.sh b/import-export-cli/shell-completions/apictl_zsh_completions.sh index ad80235ba..67a8493fe 100644 --- a/import-export-cli/shell-completions/apictl_zsh_completions.sh +++ b/import-export-cli/shell-completions/apictl_zsh_completions.sh @@ -778,6 +778,8 @@ function _apictl_set { '(-h --help)'{-h,--help}'[help for set]' \ '--http-request-timeout[Timeout for HTTP Client]:' \ '(-m --mode)'{-m,--mode}'[If mode is set to "k8s", apictl is capable of executing Kubectl commands. For example "apictl get pods" -> "kubectl get pods". To go back to the default mode, set the mode to "default"]:' \ + '--vcs-config-path[Path to the VCS Configuration yaml file which keeps the VCS meta data]:' \ + '--vcs-deletion-enabled[Specifies whether project deletion is allowed during deployment.]' \ '(-k --insecure)'{-k,--insecure}'[Allow connections to SSL endpoints without certs]' \ '--verbose[Enable verbose mode]' } diff --git a/import-export-cli/utils/structs.go b/import-export-cli/utils/structs.go index ce06e7309..ae56d11df 100644 --- a/import-export-cli/utils/structs.go +++ b/import-export-cli/utils/structs.go @@ -38,6 +38,8 @@ type Config struct { ExportDirectory string `yaml:"export_directory"` KubernetesMode bool `yaml:"kubernetes_mode"` TokenType string `yaml:"token_type"` + VCSDeletionEnabled bool `yaml:"vcs_deletion_enabled"` + VCSConfigFilePath string `yaml:"vcs_config_file_path"` } type EnvKeys struct { diff --git a/import-export-cli/utils/utils.go b/import-export-cli/utils/utils.go index 2b6e775a7..904da6859 100644 --- a/import-export-cli/utils/utils.go +++ b/import-export-cli/utils/utils.go @@ -222,3 +222,11 @@ func SetToK8sMode() { configVars.Config.KubernetesMode = true WriteConfigFile(configVars, MainConfigFilePath) } + +// returns min of two ints +func Min(a, b int) int { + if a < b { + return a + } + return b +}