From c488ef2c205113960eaa16c15fee9f9f585d36a8 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Fri, 7 Jul 2023 10:01:07 +1000 Subject: [PATCH 1/4] refactor: combined task output to single stream to retain order --- internal/lagoon/tasks.go | 43 +++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/internal/lagoon/tasks.go b/internal/lagoon/tasks.go index b7ad721d..3317a155 100644 --- a/internal/lagoon/tasks.go +++ b/internal/lagoon/tasks.go @@ -121,17 +121,14 @@ func ExecuteTaskInEnvironment(task Task) error { command = append(command, "-c") command = append(command, task.Command) - stdout, stderr, err := ExecTaskInPod(task, command, false) //(task.Service, task.Namespace, command, false, task.Container, task.ScaleWaitTime, task.ScaleMaxIterations) + output, err := ExecTaskInPod(task, command, false) //(task.Service, task.Namespace, command, false, task.Container, task.ScaleWaitTime, task.ScaleMaxIterations) if err != nil { fmt.Printf("Failed to execute task `%v` due to reason `%v`\n", task.Name, err.Error()) } - if len(stdout) > 0 { - fmt.Printf("*** Task STDOUT ***\n %v \n *** STDOUT Ends ***\n", stdout) - } - if len(stderr) > 0 { - fmt.Printf("*** Task STDERR ***\n %v \n *** STDERR Ends ***\n", stderr) + if len(output) > 0 { + fmt.Printf("*** Task output ***\n %v \n *** output ends ***\n", output) } return err @@ -142,16 +139,16 @@ func ExecTaskInPod( task Task, command []string, tty bool, -) (string, string, error) { +) (string, error) { restCfg, err := getConfig() if err != nil { - return "", "", err + return "", err } clientset, err := GetK8sClient(restCfg) if err != nil { - return "", "", fmt.Errorf("unable to create client: %v", err) + return "", fmt.Errorf("unable to create client: %v", err) } depClient := clientset.AppsV1().Deployments(task.Namespace) @@ -162,11 +159,11 @@ func ExecTaskInPod( LabelSelector: lagoonServiceLabel, }) if err != nil { - return "", "", err + return "", err } if len(deployments.Items) == 0 { - return "", "", &DeploymentMissingError{ErrorText: "No deployments found matching label: " + lagoonServiceLabel} + return "", &DeploymentMissingError{ErrorText: "No deployments found matching label: " + lagoonServiceLabel} } deployment := &deployments.Items[0] @@ -176,14 +173,14 @@ func ExecTaskInPod( numIterations := 1 for ; !podReady; numIterations++ { if numIterations >= task.ScaleMaxIterations { //break if there's some reason we can't scale the pod - return "", "", errors.New("Failed to scale pods for " + deployment.Name) + return "", errors.New("Failed to scale pods for " + deployment.Name) } if deployment.Status.ReadyReplicas == 0 { fmt.Println(fmt.Sprintf("No ready replicas found, scaling up. Attempt %d/%d", numIterations, task.ScaleMaxIterations)) scale, err := clientset.AppsV1().Deployments(task.Namespace).GetScale(context.TODO(), deployment.Name, v1.GetOptions{}) if err != nil { - return "", "", err + return "", err } if scale.Spec.Replicas == 0 { @@ -193,7 +190,7 @@ func ExecTaskInPod( time.Sleep(time.Second * time.Duration(task.ScaleWaitTime)) deployment, err = depClient.Get(context.TODO(), deployment.Name, v1.GetOptions{}) if err != nil { - return "", "", err + return "", err } } else { podReady = true @@ -208,7 +205,7 @@ func ExecTaskInPod( }) if err != nil { - return "", "", err + return "", err } var pod corev1.Pod @@ -229,7 +226,7 @@ func ExecTaskInPod( } } if !foundRunningPod { - return "", "", &PodScalingError{ + return "", &PodScalingError{ ErrorText: "Unable to find running Pod for namespace: " + task.Namespace, } } @@ -250,7 +247,7 @@ func ExecTaskInPod( scheme := runtime.NewScheme() if err := corev1.AddToScheme(scheme); err != nil { - return "", "", fmt.Errorf("error adding to scheme: %v", err) + return "", fmt.Errorf("error adding to scheme: %v", err) } if len(command) == 0 { command = []string{"sh"} @@ -267,20 +264,20 @@ func ExecTaskInPod( exec, err := remotecommand.NewSPDYExecutor(restCfg, "POST", req.URL()) if err != nil { - return "", "", fmt.Errorf("error while creating Executor: %v", err) + return "", fmt.Errorf("error while creating Executor: %v", err) } - var stdout, stderr bytes.Buffer + var output bytes.Buffer err = exec.Stream(remotecommand.StreamOptions{ - Stdout: &stdout, - Stderr: &stderr, + Stdout: &output, + Stderr: &output, Tty: tty, }) if err != nil { - return stdout.String(), stderr.String(), fmt.Errorf("Error returned: %v", err) + return output.String(), fmt.Errorf("Error returned: %v", err) } - return stdout.String(), stderr.String(), nil + return output.String(), nil } From 09a5d031d609003d9c0dc4d0d37479589d0f4965 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Fri, 29 Sep 2023 17:35:35 +1000 Subject: [PATCH 2/4] chore: refactor output buffer --- internal/lagoon/tasks.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/lagoon/tasks.go b/internal/lagoon/tasks.go index 3317a155..6f282ac4 100644 --- a/internal/lagoon/tasks.go +++ b/internal/lagoon/tasks.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "io/ioutil" "strconv" "time" @@ -267,12 +268,16 @@ func ExecTaskInPod( return "", fmt.Errorf("error while creating Executor: %v", err) } - var output bytes.Buffer + var stdOut, stdErr bytes.Buffer err = exec.Stream(remotecommand.StreamOptions{ - Stdout: &output, - Stderr: &output, + Stdout: &stdOut, + Stderr: &stdErr, Tty: tty, }) + buffers := []io.Reader{&stdOut, &stdErr} + var output bytes.Buffer + reader := io.MultiReader(buffers...) + output.ReadFrom(reader) if err != nil { return output.String(), fmt.Errorf("Error returned: %v", err) } From 781e2e309e9ba08251560d05dbabbd7d4a73f893 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Fri, 29 Sep 2023 19:25:26 +1000 Subject: [PATCH 3/4] refactor: stream to os out, but utilise accordions for tasks --- cmd/tasks_run.go | 25 +++++++------ cmd/tasks_run_test.go | 15 +++++--- internal/lagoon/tasks.go | 54 +++++++++++++-------------- legacy/build-deploy-docker-compose.sh | 10 ++++- 4 files changed, 57 insertions(+), 47 deletions(-) diff --git a/cmd/tasks_run.go b/cmd/tasks_run.go index 6c01ab0e..da88672d 100644 --- a/cmd/tasks_run.go +++ b/cmd/tasks_run.go @@ -4,13 +4,14 @@ import ( "context" "errors" "fmt" + "io/ioutil" + "os" + "strings" + "github.com/spf13/cobra" "github.com/uselagoon/build-deploy-tool/internal/generator" "github.com/uselagoon/build-deploy-tool/internal/lagoon" "github.com/uselagoon/build-deploy-tool/internal/tasklib" - "io/ioutil" - "os" - "strings" ) var runPreRollout, runPostRollout, outOfClusterConfig bool @@ -31,7 +32,7 @@ var taskCmd = &cobra.Command{ // unidleThenRun is a wrapper around 'runCleanTaskInEnvironment' used for pre-rollout tasks // We actually want to unidle the namespace before running pre-rollout tasks, // so we wrap the usual task runner before calling it. -func unidleThenRun(namespace string, incoming lagoon.Task) error { +func unidleThenRun(namespace string, prePost string, incoming lagoon.Task) error { fmt.Printf("Unidling namespace with RequiresEnvironment: %v, ScaleMaxIterations:%v and ScaleWaitTime:%v\n", incoming.RequiresEnvironment, incoming.ScaleMaxIterations, incoming.ScaleWaitTime) err := lagoon.UnidleNamespace(context.TODO(), namespace, incoming.ScaleMaxIterations, incoming.ScaleWaitTime) if err != nil { @@ -47,7 +48,7 @@ func unidleThenRun(namespace string, incoming lagoon.Task) error { return fmt.Errorf("There was a problem when unidling the environment for pre-rollout tasks: %v", err.Error()) } } - return runCleanTaskInEnvironment(namespace, incoming) + return runCleanTaskInEnvironment(namespace, prePost, incoming) } var tasksPreRun = &cobra.Command{ @@ -65,7 +66,7 @@ var tasksPreRun = &cobra.Command{ } fmt.Println("Executing Pre-rollout Tasks") - taskIterator, err := iterateTaskGenerator(true, unidleThenRun, buildValues, true) + taskIterator, err := iterateTaskGenerator(true, unidleThenRun, buildValues, "Pre-Rollout", true) if err != nil { fmt.Println("Pre-rollout Tasks Failed with the following error: ", err.Error()) os.Exit(1) @@ -97,7 +98,7 @@ var tasksPostRun = &cobra.Command{ fmt.Println("Executing Post-rollout Tasks") - taskIterator, err := iterateTaskGenerator(false, runCleanTaskInEnvironment, buildValues, true) + taskIterator, err := iterateTaskGenerator(false, runCleanTaskInEnvironment, buildValues, "Post-Rollout", true) if err != nil { fmt.Println("Pre-rollout Tasks Failed with the following error: ", err.Error()) os.Exit(1) @@ -159,7 +160,7 @@ type iterateTaskFuncType func(tasklib.TaskEnvironment, []lagoon.Task) (bool, err // that lets the resulting function reference values as part of the closure, thereby cleaning up the definition a bit. // so, the variables passed into the factor (eg. allowDeployMissingErrors, etc.) determine the way the function behaves, // without needing to pass those into the call to the returned function itself. -func iterateTaskGenerator(allowDeployMissingErrors bool, taskRunner runTaskInEnvironmentFuncType, buildValues generator.BuildValues, debug bool) (iterateTaskFuncType, error) { +func iterateTaskGenerator(allowDeployMissingErrors bool, taskRunner runTaskInEnvironmentFuncType, buildValues generator.BuildValues, prePost string, debug bool) (iterateTaskFuncType, error) { var retErr error namespace := buildValues.Namespace if namespace == "" { @@ -188,7 +189,7 @@ func iterateTaskGenerator(allowDeployMissingErrors bool, taskRunner runTaskInEnv return true, err } if runTask { - err := taskRunner(namespace, task) + err := taskRunner(namespace, prePost, task) if err != nil { switch e := err.(type) { case *lagoon.DeploymentMissingError: @@ -242,12 +243,12 @@ func evaluateWhenConditionsForTaskInEnvironment(environment tasklib.TaskEnvironm return retBool, nil } -type runTaskInEnvironmentFuncType func(namespace string, incoming lagoon.Task) error +type runTaskInEnvironmentFuncType func(namespace string, prePost string, incoming lagoon.Task) error // runCleanTaskInEnvironment implements runTaskInEnvironmentFuncType and will // 1. make sure the task we pass to the execution environment is free of any data we don't want (hence the new task) // 2. will actually execute the task in the environment. -func runCleanTaskInEnvironment(namespace string, incoming lagoon.Task) error { +func runCleanTaskInEnvironment(namespace string, prePost string, incoming lagoon.Task) error { task := lagoon.NewTask() task.Command = incoming.Command task.Namespace = namespace @@ -257,7 +258,7 @@ func runCleanTaskInEnvironment(namespace string, incoming lagoon.Task) error { task.Name = incoming.Name task.ScaleMaxIterations = incoming.ScaleMaxIterations task.ScaleWaitTime = incoming.ScaleWaitTime - err := lagoon.ExecuteTaskInEnvironment(task) + err := lagoon.ExecuteTaskInEnvironment(task, prePost) return err } diff --git a/cmd/tasks_run_test.go b/cmd/tasks_run_test.go index fe9a58b3..06fbfe12 100644 --- a/cmd/tasks_run_test.go +++ b/cmd/tasks_run_test.go @@ -166,13 +166,14 @@ func Test_iterateTaskGenerator(t *testing.T) { tests := []struct { name string debug bool + prePost string args args wantError bool }{ {name: "Runs with no errors", args: args{ allowDeployMissingErrors: true, - taskRunner: func(namespace string, incoming lagoon.Task) error { + taskRunner: func(namespace string, prePost string, incoming lagoon.Task) error { return nil }, tasks: []lagoon.Task{ @@ -180,12 +181,13 @@ func Test_iterateTaskGenerator(t *testing.T) { }, buildValues: generator.BuildValues{Namespace: "empty"}, }, + prePost: "PreRollout", wantError: false, }, {name: "Allows deploy missing errors and keeps rolling (pre rollout case)", args: args{ allowDeployMissingErrors: true, - taskRunner: func(namespace string, incoming lagoon.Task) error { + taskRunner: func(namespace string, prePost string, incoming lagoon.Task) error { return &lagoon.DeploymentMissingError{} }, tasks: []lagoon.Task{ @@ -193,12 +195,13 @@ func Test_iterateTaskGenerator(t *testing.T) { }, buildValues: generator.BuildValues{Namespace: "empty"}, }, + prePost: "PreRollout", wantError: false, }, {name: "Does not allow deploy missing errors and stops with error (post rollout)", args: args{ allowDeployMissingErrors: false, - taskRunner: func(namespace string, incoming lagoon.Task) error { + taskRunner: func(namespace string, prePost string, incoming lagoon.Task) error { return &lagoon.DeploymentMissingError{} }, tasks: []lagoon.Task{ @@ -206,12 +209,13 @@ func Test_iterateTaskGenerator(t *testing.T) { }, buildValues: generator.BuildValues{Namespace: "empty"}, }, + prePost: "PostRollout", wantError: true, }, {name: "Allows deploy missing errors but stops with any other error (pre rollout)", args: args{ allowDeployMissingErrors: true, - taskRunner: func(namespace string, incoming lagoon.Task) error { + taskRunner: func(namespace string, prePost string, incoming lagoon.Task) error { return &lagoon.PodScalingError{} }, tasks: []lagoon.Task{ @@ -219,12 +223,13 @@ func Test_iterateTaskGenerator(t *testing.T) { }, buildValues: generator.BuildValues{Namespace: "empty"}, }, + prePost: "PostRollout", wantError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, _ := iterateTaskGenerator(tt.args.allowDeployMissingErrors, tt.args.taskRunner, tt.args.buildValues, tt.debug) + got, _ := iterateTaskGenerator(tt.args.allowDeployMissingErrors, tt.args.taskRunner, tt.args.buildValues, tt.prePost, tt.debug) _, err := got(tasklib.TaskEnvironment{}, tt.args.tasks) if tt.wantError && err == nil { diff --git a/internal/lagoon/tasks.go b/internal/lagoon/tasks.go index 6f282ac4..1562e84f 100644 --- a/internal/lagoon/tasks.go +++ b/internal/lagoon/tasks.go @@ -1,12 +1,11 @@ package lagoon import ( - "bytes" "context" "errors" "fmt" - "io" "io/ioutil" + "os" "strconv" "time" @@ -111,7 +110,7 @@ func getConfig() (*rest.Config, error) { } // ExecuteTaskInEnvironment . -func ExecuteTaskInEnvironment(task Task) error { +func ExecuteTaskInEnvironment(task Task, prePost string) error { command := make([]string, 0, 5) if task.Shell != "" { command = append(command, task.Shell) @@ -122,15 +121,19 @@ func ExecuteTaskInEnvironment(task Task) error { command = append(command, "-c") command = append(command, task.Command) - output, err := ExecTaskInPod(task, command, false) //(task.Service, task.Namespace, command, false, task.Container, task.ScaleWaitTime, task.ScaleMaxIterations) + fmt.Printf("##############################################\nBEGIN %s %s\n##############################################\n", prePost, task.Name) + st := time.Now() + + err := ExecTaskInPod(task, command, false) //(task.Service, task.Namespace, command, false, task.Container, task.ScaleWaitTime, task.ScaleMaxIterations) if err != nil { fmt.Printf("Failed to execute task `%v` due to reason `%v`\n", task.Name, err.Error()) } - if len(output) > 0 { - fmt.Printf("*** Task output ***\n %v \n *** output ends ***\n", output) - } + et := time.Now() + diff := time.Time{}.Add(et.Sub(st)) + tz, _ := et.Zone() + fmt.Printf("##############################################\nSTEP %s %s: Completed at %s (%s) Duration %s Elapsed %s\n##############################################\n", prePost, task.Name, et.Format("2006-01-02 15:04:05"), tz, diff.Format("15:04:05"), diff.Format("15:04:05")) return err } @@ -140,16 +143,16 @@ func ExecTaskInPod( task Task, command []string, tty bool, -) (string, error) { +) error { restCfg, err := getConfig() if err != nil { - return "", err + return err } clientset, err := GetK8sClient(restCfg) if err != nil { - return "", fmt.Errorf("unable to create client: %v", err) + return fmt.Errorf("unable to create client: %v", err) } depClient := clientset.AppsV1().Deployments(task.Namespace) @@ -160,11 +163,11 @@ func ExecTaskInPod( LabelSelector: lagoonServiceLabel, }) if err != nil { - return "", err + return err } if len(deployments.Items) == 0 { - return "", &DeploymentMissingError{ErrorText: "No deployments found matching label: " + lagoonServiceLabel} + return &DeploymentMissingError{ErrorText: "No deployments found matching label: " + lagoonServiceLabel} } deployment := &deployments.Items[0] @@ -174,14 +177,14 @@ func ExecTaskInPod( numIterations := 1 for ; !podReady; numIterations++ { if numIterations >= task.ScaleMaxIterations { //break if there's some reason we can't scale the pod - return "", errors.New("Failed to scale pods for " + deployment.Name) + return errors.New("Failed to scale pods for " + deployment.Name) } if deployment.Status.ReadyReplicas == 0 { fmt.Println(fmt.Sprintf("No ready replicas found, scaling up. Attempt %d/%d", numIterations, task.ScaleMaxIterations)) scale, err := clientset.AppsV1().Deployments(task.Namespace).GetScale(context.TODO(), deployment.Name, v1.GetOptions{}) if err != nil { - return "", err + return err } if scale.Spec.Replicas == 0 { @@ -191,7 +194,7 @@ func ExecTaskInPod( time.Sleep(time.Second * time.Duration(task.ScaleWaitTime)) deployment, err = depClient.Get(context.TODO(), deployment.Name, v1.GetOptions{}) if err != nil { - return "", err + return err } } else { podReady = true @@ -206,7 +209,7 @@ func ExecTaskInPod( }) if err != nil { - return "", err + return err } var pod corev1.Pod @@ -227,7 +230,7 @@ func ExecTaskInPod( } } if !foundRunningPod { - return "", &PodScalingError{ + return &PodScalingError{ ErrorText: "Unable to find running Pod for namespace: " + task.Namespace, } } @@ -248,7 +251,7 @@ func ExecTaskInPod( scheme := runtime.NewScheme() if err := corev1.AddToScheme(scheme); err != nil { - return "", fmt.Errorf("error adding to scheme: %v", err) + return fmt.Errorf("error adding to scheme: %v", err) } if len(command) == 0 { command = []string{"sh"} @@ -265,24 +268,19 @@ func ExecTaskInPod( exec, err := remotecommand.NewSPDYExecutor(restCfg, "POST", req.URL()) if err != nil { - return "", fmt.Errorf("error while creating Executor: %v", err) + return fmt.Errorf("error while creating Executor: %v", err) } - var stdOut, stdErr bytes.Buffer err = exec.Stream(remotecommand.StreamOptions{ - Stdout: &stdOut, - Stderr: &stdErr, + Stdout: os.Stdout, + Stderr: os.Stderr, Tty: tty, }) - buffers := []io.Reader{&stdOut, &stdErr} - var output bytes.Buffer - reader := io.MultiReader(buffers...) - output.ReadFrom(reader) if err != nil { - return output.String(), fmt.Errorf("Error returned: %v", err) + return fmt.Errorf("Error returned: %v", err) } - return output.String(), nil + return nil } diff --git a/legacy/build-deploy-docker-compose.sh b/legacy/build-deploy-docker-compose.sh index 8dd9e3d4..81fde247 100755 --- a/legacy/build-deploy-docker-compose.sh +++ b/legacy/build-deploy-docker-compose.sh @@ -806,11 +806,14 @@ if [ "${LAGOON_PREROLLOUT_DISABLED}" != "true" ]; then build-deploy-tool tasks pre-rollout else echo "pre-rollout tasks are currently disabled LAGOON_PREROLLOUT_DISABLED is set to true" + set +x + currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" + patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "preRolloutsCompleted" "Pre-Rollout Tasks" + set -x fi set +x currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" -patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "preRolloutsCompleted" "Pre-Rollout Tasks" previousStepEnd=${currentStepEnd} beginBuildStep "Service Configuration Phase 1" "serviceConfigurationPhase1" set -x @@ -1638,11 +1641,14 @@ if [ "${LAGOON_POSTROLLOUT_DISABLED}" != "true" ]; then build-deploy-tool tasks post-rollout else echo "post-rollout tasks are currently disabled LAGOON_POSTROLLOUT_DISABLED is set to true" + set +x + currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" + patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "postRolloutsCompleted" "Post-Rollout Tasks" + set -x fi set +x currentStepEnd="$(date +"%Y-%m-%d %H:%M:%S")" -patchBuildStep "${buildStartTime}" "${previousStepEnd}" "${currentStepEnd}" "${NAMESPACE}" "postRolloutsCompleted" "Post-Rollout Tasks" previousStepEnd=${currentStepEnd} beginBuildStep "Build and Deploy" "finalizingBuild" set -x From 03057e2844ce6ff4489aabebd19ea592e934aa91 Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 3 Oct 2023 18:35:31 +1100 Subject: [PATCH 4/4] fix: check service account token first --- cmd/tasks_run.go | 3 +-- internal/lagoon/tasks.go | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/tasks_run.go b/cmd/tasks_run.go index da88672d..9f9132cf 100644 --- a/cmd/tasks_run.go +++ b/cmd/tasks_run.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io/ioutil" "os" "strings" @@ -169,7 +168,7 @@ func iterateTaskGenerator(allowDeployMissingErrors bool, taskRunner runTaskInEnv if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { retErr = fmt.Errorf("A target namespace is required to run pre/post-rollout tasks") } - nsb, err := ioutil.ReadFile(filename) + nsb, err := os.ReadFile(filename) if err != nil { retErr = err } diff --git a/internal/lagoon/tasks.go b/internal/lagoon/tasks.go index 1562e84f..b2ec5201 100644 --- a/internal/lagoon/tasks.go +++ b/internal/lagoon/tasks.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io/ioutil" "os" "strconv" "time" @@ -85,9 +84,9 @@ func getConfig() (*rest.Config, error) { if *kubeconfig == "" { //Fall back on out of cluster // read the deployer token. - token, err := ioutil.ReadFile("/var/run/secrets/lagoon/deployer/token") + token, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") if err != nil { - token, err = ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") + token, err = os.ReadFile("/var/run/secrets/lagoon/deployer/token") if err != nil { return nil, err }