feat: use entrypoint sh for container executor (#4162)

* feat: use entrypoint sh for container executor

* feat: add container shell field

* feat: create dynamically entrypoint

* fix: script paths

* feat: add docker inspection

* feat: add secret creds to skopeo

* Revert "Auxiliary commit to revert individual files from b2b1282"

This reverts commit e0c9f0ff94756e81de76139429a1bef665995ebe.

* fix: generate mock

* fix: pass registry parameter

* fix: add const for script names

* feat: add const for entrypoint

* feat: add unit tests for skopeo secrets

api/v1/testkube.yaml

@@ -3845,15 +3845,19 @@ components:
  description: configuration parameters for storing test artifacts
  preRunScript:
  type: string
- description: script to run before test execution (not supported for container executors)
+ description: script to run before test execution
  example: "echo -n '$SECRET_ENV' > ./secret_file"
  postRunScript:
  type: string
- description: script to run after test execution (not supported for container executors)
+ description: script to run after test execution
  example: "sleep 30" 
  runningContext:
  $ref: "#/components/schemas/RunningContext"
  description: running context for the test execution
+ containerShell:
+ type: string
+ description: shell used in container executor
+ example: "/bin/sh" 
 
  Artifact:
  type: object
@@ -4348,11 +4352,11 @@ components:
  description: adjusting parameters for test content
  preRunScript:
  type: string
- description: script to run before test execution (not supported for container executors)
+ description: script to run before test execution
  example: "echo -n '$SECRET_ENV' > ./secret_file"
  postRunScript:
  type: string
- description: script to run after test execution (not supported for container executors)
+ description: script to run after test execution
  example: "sleep 30"
  scraperTemplate:
  type: string

build/api-server/Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1
 ARG ALPINE_IMAGE
 FROM ${ALPINE_IMAGE}
-RUN apk --no-cache add ca-certificates libssl1.1 git
+RUN apk --no-cache add ca-certificates libssl1.1 git skopeo
 WORKDIR /root/
 COPY testkube-api-server /bin/app
 USER 1001

contrib/executor/init/pkg/runner/runner.go

@@ -4,19 +4,27 @@ import (
  "context"
  "os"
  "path/filepath"
+ "strings"
 
  "github.com/pkg/errors"
 
  "github.com/kubeshop/testkube/pkg/api/v1/testkube"
  "github.com/kubeshop/testkube/pkg/envs"
  "github.com/kubeshop/testkube/pkg/executor"
+ "github.com/kubeshop/testkube/pkg/executor/containerexecutor"
  "github.com/kubeshop/testkube/pkg/executor/content"
  "github.com/kubeshop/testkube/pkg/executor/output"
  "github.com/kubeshop/testkube/pkg/executor/runner"
  "github.com/kubeshop/testkube/pkg/storage/minio"
  "github.com/kubeshop/testkube/pkg/ui"
 )
 
+const (
+ defaultShell = "/bin/sh"
+ preRunScriptName = "prerun.sh"
+ postRunScriptName = "postrun.sh"
+)
+
 // NewRunner creates init runner
 func NewRunner(params envs.Params) *InitRunner {
  dir := os.Getenv("RUNNER_DATADIR")
@@ -66,6 +74,51 @@ func (r *InitRunner) Run(ctx context.Context, execution testkube.Execution) (res
  return result, errors.Errorf("could not fetch test content: %v", err)
  }
 
+ if execution.PreRunScript != "" || execution.PostRunScript != "" {
+ command := "#!" + defaultShell
+ if execution.ContainerShell != "" {
+ command = "#!" + execution.ContainerShell
+ }
+ command += "\n"
+
+ if execution.PreRunScript != "" {
+ command += filepath.Join(r.dir, preRunScriptName) + "\n"
+ }
+
+ if len(execution.Command) != 0 {
+ command += strings.Join(execution.Command, " ")
+ command += " \"$@\"\n"
+ }
+
+ if execution.PostRunScript != "" {
+ command += filepath.Join(r.dir, postRunScriptName) + "\n"
+ }
+
+ var scripts = []struct {
+ file string
+ data string
+ comment string
+ }{
+ {preRunScriptName, execution.PreRunScript, "prerun"},
+ {containerexecutor.EntrypointScriptName, command, "entrypoint"},
+ {postRunScriptName, execution.PostRunScript, "postrun"},
+ }
+
+ for _, script := range scripts {
+ if script.data == "" {
+ continue
+ }
+
+ file := filepath.Join(r.dir, script.file)
+ output.PrintLogf("%s Creating %s script...", ui.IconWorld, script.comment)
+ if err = os.WriteFile(file, []byte(script.data), 0755); err != nil {
+ output.PrintLogf("%s Could not create %s script %s: %s", ui.IconCross, script.comment, file, err.Error())
+ return result, errors.Errorf("could not create %s script %s: %v", script.comment, file, err)
+ }
+ output.PrintLogf("%s %s script created", ui.IconCheckMark, script.comment)
+ }
+ }
+
  // TODO: write a proper cloud implementation
  // add copy files in case object storage is set
  if r.Params.Endpoint != "" && !r.Params.CloudMode {

pkg/api/v1/testkube/model_execution.go

@@ -64,9 +64,11 @@ type Execution struct {
  // minio bucket name to get uploads from
  BucketName string `json:"bucketName,omitempty"`
  ArtifactRequest *ArtifactRequest `json:"artifactRequest,omitempty"`
- // script to run before test execution (not supported for container executors)
+ // script to run before test execution
  PreRunScript string `json:"preRunScript,omitempty"`
- // script to run after test execution (not supported for container executors)
+ // script to run after test execution
  PostRunScript string `json:"postRunScript,omitempty"`
  RunningContext *RunningContext `json:"runningContext,omitempty"`
+ // shell used in container executor
+ ContainerShell string `json:"containerShell,omitempty"`
 }

pkg/api/v1/testkube/model_execution_request.go

@@ -70,9 +70,9 @@ type ExecutionRequest struct {
  // cron job template extensions
  CronJobTemplate string `json:"cronJobTemplate,omitempty"`
  ContentRequest *TestContentRequest `json:"contentRequest,omitempty"`
- // script to run before test execution (not supported for container executors)
+ // script to run before test execution
  PreRunScript string `json:"preRunScript,omitempty"`
- // script to run after test execution (not supported for container executors)
+ // script to run after test execution
  PostRunScript string `json:"postRunScript,omitempty"`
  // scraper template extensions
  ScraperTemplate string `json:"scraperTemplate,omitempty"`

pkg/api/v1/testkube/model_execution_update_request.go

@@ -70,9 +70,9 @@ type ExecutionUpdateRequest struct {
  // cron job template extensions
  CronJobTemplate *string `json:"cronJobTemplate,omitempty"`
  ContentRequest **TestContentUpdateRequest `json:"contentRequest,omitempty"`
- // script to run before test execution (not supported for container executors)
+ // script to run before test execution
  PreRunScript *string `json:"preRunScript,omitempty"`
- // script to run after test execution (not supported for container executors)
+ // script to run after test execution
  PostRunScript *string `json:"postRunScript,omitempty"`
  // scraper template extensions
  ScraperTemplate *string `json:"scraperTemplate,omitempty"`

pkg/api/v1/testkube/model_test_trigger_condition_spec.go

@@ -14,6 +14,6 @@ type TestTriggerConditionSpec struct {
  Conditions []TestTriggerCondition `json:"conditions,omitempty"`
  // duration in seconds the test trigger waits for conditions, until its stopped
  Timeout int32 `json:"timeout,omitempty"`
- // duration in seconds the test trigger waits between condition check
+ // duration in seconds the test trigger waits between condition checks
  Delay int32 `json:"delay,omitempty"`
 }

pkg/executor/containerexecutor/containerexecutor.go

@@ -279,7 +279,7 @@ func (c *ContainerExecutor) ExecuteSync(ctx context.Context, execution *testkube
 func (c *ContainerExecutor) createJob(ctx context.Context, execution testkube.Execution, options client.ExecuteOptions) (*JobOptions, error) {
  jobsClient := c.clientSet.BatchV1().Jobs(c.namespace)
 
- jobOptions, err := NewJobOptions(c.images, c.templates, c.serviceAccountName, c.registry, c.clusterID, execution, options)
+ jobOptions, err := NewJobOptions(c.log, c.images, c.templates, c.serviceAccountName, c.registry, c.clusterID, execution, options)
  if err != nil {
  return nil, err
  }

pkg/executor/containerexecutor/containerexecutor_test.go

@@ -155,6 +155,7 @@ func TestNewExecutorJobSpecWithWorkingDirRelative(t *testing.T) {
  t.Parallel()
 
  jobOptions, _ := NewJobOptions(
+ logger(),
  executor.Images{},
  executor.Templates{},
  "",
@@ -189,6 +190,7 @@ func TestNewExecutorJobSpecWithWorkingDirAbsolute(t *testing.T) {
  t.Parallel()
 
  jobOptions, _ := NewJobOptions(
+ logger(),
  executor.Images{},
  executor.Templates{},
  "",
@@ -223,6 +225,7 @@ func TestNewExecutorJobSpecWithoutWorkingDir(t *testing.T) {
  t.Parallel()
 
  jobOptions, _ := NewJobOptions(
+ logger(),
  executor.Images{},
  executor.Templates{},
  "",

pkg/executor/containerexecutor/tmpl.go

@@ -4,6 +4,7 @@ import (
  "bytes"
  "encoding/json"
  "fmt"
+ "path/filepath"
  "strings"
  "text/template"
 
@@ -20,6 +21,13 @@ import (
  "github.com/kubeshop/testkube/pkg/executor"
  "github.com/kubeshop/testkube/pkg/executor/client"
  "github.com/kubeshop/testkube/pkg/executor/env"
+ "github.com/kubeshop/testkube/pkg/secret"
+ "github.com/kubeshop/testkube/pkg/skopeo"
+)
+
+const (
+ // EntrypointScriptName is entrypoint script name
+ EntrypointScriptName = "entrypoint.sh"
 )
 
 //go:embed templates/job.tmpl
@@ -196,15 +204,64 @@ func NewPersistentVolumeClaimSpec(log *zap.SugaredLogger, options *JobOptions) (
  return &pvc, nil
 }
 
+// InspectDockerImage inspects docker image
+func InspectDockerImage(namespace, registry, image string, imageSecrets []string) ([]string, string, error) {
+ inspector := skopeo.NewClient()
+ if len(imageSecrets) != 0 {
+ secretClient, err := secret.NewClient(namespace)
+ if err != nil {
+ return nil, "", err
+ }
+
+ var secrets []corev1.Secret
+ for _, imageSecret := range imageSecrets {
+ object, err := secretClient.GetObject(imageSecret)
+ if err != nil {
+ return nil, "", err
+ }
+
+ secrets = append(secrets, *object)
+ }
+
+ inspector, err = skopeo.NewClientFromSecrets(secrets, registry)
+ if err != nil {
+ return nil, "", err
+ }
+ }
+
+ dockerImage, err := inspector.Inspect(image)
+ if err != nil {
+ return nil, "", err
+ }
+
+ return append(dockerImage.Config.Entrypoint, dockerImage.Config.Cmd...), dockerImage.Shell, nil
+}
+
 // NewJobOptions provides job options for templates
-func NewJobOptions(images executor.Images, templates executor.Templates, serviceAccountName, registry, clusterID string,
- execution testkube.Execution, options client.ExecuteOptions) (*JobOptions, error) {
+func NewJobOptions(log *zap.SugaredLogger, images executor.Images, templates executor.Templates,
+ serviceAccountName, registry, clusterID string, execution testkube.Execution, options client.ExecuteOptions) (*JobOptions, error) {
+ jobOptions := NewJobOptionsFromExecutionOptions(options)
+ if execution.PreRunScript != "" || execution.PostRunScript != "" {
+ jobOptions.Command = []string{filepath.Join(executor.VolumeDir, EntrypointScriptName)}
+ if jobOptions.Image != "" {
+ cmd, shell, err := InspectDockerImage(jobOptions.Namespace, registry, jobOptions.Image, jobOptions.ImagePullSecrets)
+ if err == nil {
+ if len(execution.Command) == 0 {
+ execution.Command = cmd
+ }
+
+ execution.ContainerShell = shell
+ } else {
+ log.Errorw("Docker image inspection error", "error", err)
+ }
+ }
+ }
+
  jsn, err := json.Marshal(execution)
  if err != nil {
  return nil, err
  }
 
- jobOptions := NewJobOptionsFromExecutionOptions(options)
  jobOptions.Name = execution.Id
  jobOptions.Namespace = execution.TestNamespace
  jobOptions.TestName = execution.TestName

pkg/scheduler/test_scheduler_test.go

@@ -20,8 +20,10 @@ import (
 )
 
 func TestParamsNilAssign(t *testing.T) {
+ t.Parallel()
 
  t.Run("merge two maps", func(t *testing.T) {
+ t.Parallel()
 
  p1 := map[string]testkube.Variable{"p1": testkube.NewBasicVariable("p1", "1")}
  p2 := map[string]testkube.Variable{"p2": testkube.NewBasicVariable("p2", "2")}
@@ -33,6 +35,7 @@ func TestParamsNilAssign(t *testing.T) {
  })
 
  t.Run("merge two maps with override", func(t *testing.T) {
+ t.Parallel()
 
  p1 := map[string]testkube.Variable{"p1": testkube.NewBasicVariable("p1", "1")}
  p2 := map[string]testkube.Variable{"p1": testkube.NewBasicVariable("p1", "2")}
@@ -44,6 +47,7 @@ func TestParamsNilAssign(t *testing.T) {
  })
 
  t.Run("merge with nil map", func(t *testing.T) {
+ t.Parallel()
 
  p2 := map[string]testkube.Variable{"p2": testkube.NewBasicVariable("p2", "2")}
 
@@ -56,6 +60,8 @@ func TestParamsNilAssign(t *testing.T) {
 }
 
 func TestGetExecuteOptions(t *testing.T) {
+ t.Parallel()
+
  mockCtrl := gomock.NewController(t)
  defer mockCtrl.Finish()
 

pkg/secret/client.go

@@ -19,6 +19,7 @@ const testkubeTestSecretLabel = "tests-secrets"
 //go:generate mockgen -destination=./mock_client.go -package=secret "github.com/kubeshop/testkube/pkg/secret" Interface
 type Interface interface {
  Get(id string) (map[string]string, error)
+ GetObject(id string) (*v1.Secret, error)
  List() (map[string]map[string]string, error)
  Create(id string, labels, stringData map[string]string) error
  Apply(id string, labels, stringData map[string]string) error
@@ -66,6 +67,19 @@ func (c *Client) Get(id string) (map[string]string, error) {
  return stringData, nil
 }
 
+// GetObject is a method to retrieve an existing secret object
+func (c *Client) GetObject(id string) (*v1.Secret, error) {
+ secretsClient := c.ClientSet.CoreV1().Secrets(c.Namespace)
+ ctx := context.Background()
+
+ secretSpec, err := secretsClient.Get(ctx, id, metav1.GetOptions{})
+ if err != nil {
+ return nil, err
+ }
+
+ return secretSpec, nil
+}
+
 // List is a method to retrieve all existing secrets
 func (c *Client) List() (map[string]map[string]string, error) {
  secretsClient := c.ClientSet.CoreV1().Secrets(c.Namespace)