chore!: make minio client optional and improve initialization

e2e/bittwister/suite_setup_test.go

@@ -7,7 +7,8 @@
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 
+	"github.com/celestiaorg/knuu/e2e"
 	"github.com/celestiaorg/knuu/pkg/knuu"
 )
 
 type Suite struct {
@@ -16,11 +17,13 @@
 }
 
 func (s *Suite) SetupSuite() {
-	var (
-		err error
-		ctx = context.Background()
-	)
-	s.Knuu, err = knuu.New(ctx, knuu.Options{ProxyEnabled: true})
+	ctx := context.Background()
+
+	var err error
+	s.Knuu, err = knuu.New(ctx, knuu.Options{
+		ProxyEnabled: true,
+		TestScope:    e2e.DefaultTestScope(),
+	})
 	s.Require().NoError(err)
 	s.T().Logf("Scope: %s", s.Knuu.Scope())
 	s.Knuu.HandleStopSignal(ctx)

e2e/system/build_from_git_test.go

@@ -2,13 +2,18 @@ package system
 
 import (
 	"context"
+	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/celestiaorg/knuu/e2e"
 	"github.com/celestiaorg/knuu/pkg/builder"
 	"github.com/celestiaorg/knuu/pkg/instance"
+	"github.com/celestiaorg/knuu/pkg/k8s"
 	"github.com/celestiaorg/knuu/pkg/knuu"
+	"github.com/celestiaorg/knuu/pkg/minio"
 )
 
 func TestBuildFromGit(t *testing.T) {
@@ -18,7 +23,66 @@ func TestBuildFromGit(t *testing.T) {
 	ctx := context.Background()
 
 	// The default image builder is kaniko here
-	kn, err := knuu.New(ctx, knuu.Options{})
+	kn, err := knuu.New(ctx, knuu.Options{TestScope: e2e.DefaultTestScope()})
+	require.NoError(t, err, "Error creating knuu")
+
+	target, err := kn.NewInstance("git-builder")
+	require.NoError(t, err, "Error creating instance")
+
+	t.Log("Building the image")
+
+	// This is a blocking call which builds the image from git repo
+	err = target.SetGitRepo(ctx, builder.GitContext{
+		Repo:     "https://github.com/celestiaorg/knuu.git",
+		Branch:   "test/build-from-git", // This branch has a Dockerfile and is protected as to not be deleted
+		Username: "",
+		Password: "",
+	})
+	require.NoError(t, err, "Error setting git repo")
+
+	t.Log("Image built")
+
+	t.Cleanup(func() {
+		if err := target.Destroy(ctx); err != nil {
+			t.Logf("Error destroying instance: %v", err)
+		}
+	})
+
+	require.NoError(t, target.Commit())
+
+	t.Logf("Starting instance")
+
+	assert.NoError(t, target.Start(ctx))
+
+	t.Logf("Instance started")
+
+	// The file is created by the dockerfile in the repo,
+	// so to make sure it is built correctly, we check the file
+	data, err := target.GetFileBytes(ctx, "/test.txt")
+	require.NoError(t, err, "Error getting file bytes")
+
+	data = []byte(strings.TrimSpace(string(data)))
+	assert.Equal(t, []byte("Hello, World!"), data, "File bytes do not match")
+}
+func TestBuildFromGitWithModifications(t *testing.T) {
+	t.Parallel()
+
+	// Setup
+	ctx := context.Background()
+
+	k8sClient, err := k8s.NewClient(ctx, e2e.DefaultTestScope())
+	require.NoError(t, err, "Error creating k8s client")
+
+	// Since we are modifying the git repo,
+	// we need to setup minio to allow the builder to push the changes
+	minioClient, err := minio.New(ctx, k8sClient)
+	require.NoError(t, err, "Error creating minio client")
+
+	// The default image builder is kaniko here
+	kn, err := knuu.New(ctx, knuu.Options{
+		K8sClient:   k8sClient,
+		MinioClient: minioClient,
+	})
 	require.NoError(t, err, "Error creating knuu")
 
 	sampleInstance, err := kn.NewInstance("git-builder")

e2e/tshark/tshark_test.go

@@ -11,8 +11,11 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/celestiaorg/knuu/e2e"
 	"github.com/celestiaorg/knuu/pkg/instance"
+	"github.com/celestiaorg/knuu/pkg/k8s"
 	"github.com/celestiaorg/knuu/pkg/knuu"
+	"github.com/celestiaorg/knuu/pkg/minio"
 )
 
 const (
@@ -26,7 +29,13 @@ func TestTshark(t *testing.T) {
 
 	ctx := context.Background()
 
-	kn, err := knuu.New(ctx, knuu.Options{})
+	k8sClient, err := k8s.NewClient(ctx, e2e.DefaultTestScope())
+	require.NoError(t, err, "error creating k8s client")
+
+	minioClient, err := minio.New(ctx, k8sClient)
+	require.NoError(t, err, "error creating minio client")
+
+	kn, err := knuu.New(ctx, knuu.Options{MinioClient: minioClient, K8sClient: k8sClient})
 	require.NoError(t, err, "error creating knuu")
 
 	defer func() {
@@ -47,10 +56,6 @@ func TestTshark(t *testing.T) {
 	err = target.SetCommand("sleep", "infinity")
 	require.NoError(t, err, "error setting command")
 
-	t.Log("deploying minio as s3 backend")
-	err = kn.MinioClient.DeployMinio(ctx)
-	require.NoError(t, err, "error deploying minio")
-
 	t.Log("getting minio configs")
 	minioConf, err := kn.MinioClient.GetConfigs(ctx)
 	require.NoError(t, err, "error getting S3 (minio) configs")
@@ -93,7 +98,7 @@ func TestTshark(t *testing.T) {
 	_, err = target.ExecuteCommand(ctx, "ping", "-c", "4", "google.com")
 	require.NoError(t, err, "error executing command")
 
-	url, err := kn.MinioClient.GetMinioURL(ctx, fileKey, s3BucketName)
+	url, err := kn.MinioClient.GetURL(ctx, fileKey, s3BucketName)
 	require.NoError(t, err, "error getting minio url")
 
 	resp, err := http.Get(url)

e2e/utils.go

@@ -0,0 +1,11 @@
+package e2e
+
+import (
+	"fmt"
+	"time"
+)
+
+func DefaultTestScope() string {
+	t := time.Now()
+	return fmt.Sprintf("%s-%03d", t.Format("20060102-150405"), t.Nanosecond()/1e6)
+}

pkg/builder/kaniko/errors.go

@@ -30,4 +30,5 @@ var (
 	ErrMinioDeploymentFailed            = errors.New("MinioDeploymentFailed", "Minio deployment failed")
 	ErrDeletingMinioContent             = errors.New("DeletingMinioContent", "error deleting Minio content")
 	ErrParsingQuantity                  = errors.New("ParsingQuantity", "error parsing quantity")
+	ErrMinioFailedToGetDeployment       = errors.New("MinioFailedToGetDeployment", "Minio failed to get deployment")
 )

pkg/builder/kaniko/kaniko.go

@@ -13,9 +13,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/celestiaorg/knuu/pkg/builder"
-	"github.com/celestiaorg/knuu/pkg/k8s"
-	"github.com/celestiaorg/knuu/pkg/minio"
 	"github.com/celestiaorg/knuu/pkg/names"
+	"github.com/celestiaorg/knuu/pkg/system"
 )
 
 const (
@@ -31,9 +30,8 @@ const (
 )
 
 type Kaniko struct {
-	K8s         k8s.KubeManager
-	Minio       *minio.Minio // Minio service to store the build context if it's a directory
-	ContentName string       // Name of the content pushed to Minio
+	system.SystemDependencies
+	ContentName string // Name of the content pushed to Minio
 }
 
 var _ builder.Builder = &Kaniko{}
@@ -44,7 +42,7 @@ func (k *Kaniko) Build(ctx context.Context, b *builder.BuilderOptions) (logs str
 		return "", ErrPreparingJob.Wrap(err)
 	}
 
-	cJob, err := k.K8s.Clientset().BatchV1().Jobs(k.K8s.Namespace()).Create(ctx, job, metav1.CreateOptions{})
+	cJob, err := k.K8sClient.Clientset().BatchV1().Jobs(k.K8sClient.Namespace()).Create(ctx, job, metav1.CreateOptions{})
 	if err != nil {
 		return "", ErrCreatingJob.Wrap(err)
 	}
@@ -76,7 +74,7 @@ func (k *Kaniko) Build(ctx context.Context, b *builder.BuilderOptions) (logs str
 }
 
 func (k *Kaniko) waitForJobCompletion(ctx context.Context, job *batchv1.Job) (*batchv1.Job, error) {
-	watcher, err := k.K8s.Clientset().BatchV1().Jobs(k.K8s.Namespace()).Watch(ctx, metav1.ListOptions{
+	watcher, err := k.K8sClient.Clientset().BatchV1().Jobs(k.K8sClient.Namespace()).Watch(ctx, metav1.ListOptions{
 		FieldSelector: fmt.Sprintf("metadata.name=%s", job.Name),
 	})
 	if err != nil {
@@ -107,7 +105,7 @@ func (k *Kaniko) waitForJobCompletion(ctx context.Context, job *batchv1.Job) (*b
 }
 
 func (k *Kaniko) firstPodFromJob(ctx context.Context, job *batchv1.Job) (*v1.Pod, error) {
-	podList, err := k.K8s.Clientset().CoreV1().Pods(k.K8s.Namespace()).List(ctx, metav1.ListOptions{
+	podList, err := k.K8sClient.Clientset().CoreV1().Pods(k.K8sClient.Namespace()).List(ctx, metav1.ListOptions{
 		LabelSelector: fmt.Sprintf("job-name=%s", job.Name),
 	})
 	if err != nil {
@@ -130,7 +128,7 @@ func (k *Kaniko) containerLogs(ctx context.Context, pod *v1.Pod) (string, error)
 		Container: pod.Spec.Containers[0].Name,
 	}
 
-	req := k.K8s.Clientset().CoreV1().Pods(k.K8s.Namespace()).GetLogs(pod.Name, &logOptions)
+	req := k.K8sClient.Clientset().CoreV1().Pods(k.K8sClient.Namespace()).GetLogs(pod.Name, &logOptions)
 	logs, err := req.DoRaw(ctx)
 	if err != nil {
 		return "", err
@@ -140,7 +138,7 @@ func (k *Kaniko) containerLogs(ctx context.Context, pod *v1.Pod) (string, error)
 }
 
 func (k *Kaniko) cleanup(ctx context.Context, job *batchv1.Job) error {
-	err := k.K8s.Clientset().BatchV1().Jobs(k.K8s.Namespace()).
+	err := k.K8sClient.Clientset().BatchV1().Jobs(k.K8sClient.Namespace()).
 		Delete(ctx, job.Name, metav1.DeleteOptions{
 			PropagationPolicy: &[]metav1.DeletionPropagation{metav1.DeletePropagationBackground}[0],
 		})
@@ -149,7 +147,7 @@ func (k *Kaniko) cleanup(ctx context.Context, job *batchv1.Job) error {
 	}
 
 	// Delete the associated Pods
-	err = k.K8s.Clientset().CoreV1().Pods(k.K8s.Namespace()).
+	err = k.K8sClient.Clientset().CoreV1().Pods(k.K8sClient.Namespace()).
 		DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
 			LabelSelector: fmt.Sprintf("job-name=%s", job.Name),
 		})
@@ -159,7 +157,7 @@ func (k *Kaniko) cleanup(ctx context.Context, job *batchv1.Job) error {
 
 	// Delete the content pushed to Minio
 	if k.ContentName != "" {
-		if err := k.Minio.DeleteFromMinio(ctx, k.ContentName, MinioBucketName); err != nil {
+		if err := k.MinioClient.Delete(ctx, k.ContentName, MinioBucketName); err != nil {
 			return ErrDeletingMinioContent.Wrap(err)
 		}
 	}
@@ -248,10 +246,6 @@ func (k *Kaniko) prepareJob(ctx context.Context, b *builder.BuilderOptions) (*ba
 // As kaniko also supports directly tar.gz archives, no need to extract it,
 // we just need to set the context to tar://<path-to-archive>
 func (k *Kaniko) mountDir(ctx context.Context, bCtx string, job *batchv1.Job) (*batchv1.Job, error) {
-	if k.Minio == nil {
-		return nil, ErrMinioNotConfigured
-	}
-
 	// Create the tar.gz archive
 	archiveData, err := createTarGz(builder.GetDirFromBuildContext(bCtx))
 	if err != nil {
@@ -263,15 +257,11 @@ func (k *Kaniko) mountDir(ctx context.Context, bCtx string, job *batchv1.Job) (*
 	hash.Write(archiveData)
 	k.ContentName = hex.EncodeToString(hash.Sum(nil))
 
-	if err := k.Minio.DeployMinio(ctx); err != nil {
-		return nil, ErrMinioDeploymentFailed.Wrap(err)
-	}
-
-	if err := k.Minio.PushToMinio(ctx, bytes.NewReader(archiveData), k.ContentName, MinioBucketName); err != nil {
+	if err := k.MinioClient.Push(ctx, bytes.NewReader(archiveData), k.ContentName, MinioBucketName); err != nil {
 		return nil, err
 	}
 
-	s3URL, err := k.Minio.GetMinioURL(ctx, k.ContentName, MinioBucketName)
+	s3URL, err := k.MinioClient.GetURL(ctx, k.ContentName, MinioBucketName)
 	if err != nil {
 		return nil, err
 	}

pkg/builder/kaniko/kaniko_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/celestiaorg/knuu/pkg/builder"
 	"github.com/celestiaorg/knuu/pkg/k8s"
+	"github.com/celestiaorg/knuu/pkg/system"
 )
 
 const (
@@ -28,7 +29,9 @@ func TestKanikoBuilder(t *testing.T) {
 	k8sClient, err := k8s.NewClientCustom(context.Background(), k8sCS, k8sCS.Discovery(), nil, k8sNamespace)
 	require.NoError(t, err)
 	kb := &Kaniko{
-		K8s: k8sClient,
+		SystemDependencies: system.SystemDependencies{
+			K8sClient: k8sClient,
+		},
 	}
 	ctx := context.Background()
 

pkg/knuu/errors.go

@@ -205,4 +205,6 @@ var (
 	ErrCannotGetTraefikEndpoint                  = errors.New("CannotGetTraefikEndpoint", "cannot get traefik endpoint")
 	ErrGettingProxyURL                           = errors.New("GettingProxyURL", "error getting proxy URL for service '%s'")
 	ErrTraefikAPINotAvailable                    = errors.New("TraefikAPINotAvailable", "traefik API is not available")
+	ErrTestScopeNotSet                           = errors.New("TestScopeNotSet", "test scope is not set")
+	ErrK8sClientNotSet                           = errors.New("K8sClientNotSet", "k8s client is not set")
 )