feat(private-containers): private registries

refs akash-network/support#54

Signed-off-by: Adam Bozanich <[email protected]>

cluster/kube/apply.go

@@ -86,6 +86,29 @@ func applyNetPolicies(ctx context.Context, kc kubernetes.Interface, b builder.Ne
 // 	return err
 // }
 
+func applyServiceCredentials(ctx context.Context, kc kubernetes.Interface, b builder.ServiceCredentials) error {
+	obj, err := kc.CoreV1().Secrets(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{})
+	metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "secrets-get", err, errors.IsNotFound)
+
+	switch {
+	case err == nil:
+		obj, err = b.Update(obj)
+		if err == nil {
+			_, err = kc.CoreV1().Secrets(b.NS()).Update(ctx, obj, metav1.UpdateOptions{})
+			metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-get", err)
+
+		}
+	case errors.IsNotFound(err):
+		obj, err = b.Create()
+		if err == nil {
+			_, err = kc.CoreV1().Secrets(b.NS()).Create(ctx, obj, metav1.CreateOptions{})
+			metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-create", err)
+		}
+	}
+	return err
+
+}
+
 func applyDeployment(ctx context.Context, kc kubernetes.Interface, b builder.Deployment) error {
 	obj, err := kc.AppsV1().Deployments(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{})
 	metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "deployments-get", err, errors.IsNotFound)

cluster/kube/builder/service_credentials.go

@@ -0,0 +1,110 @@
+package builder
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	mani "github.com/akash-network/akash-api/go/manifest/v2beta2"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type ServiceCredentials interface {
+	NS() string
+	Name() string
+	Create() (*corev1.Secret, error)
+	Update(obj *corev1.Secret) (*corev1.Secret, error)
+}
+
+type serviceCredentials struct {
+	ns          string
+	serviceName string
+	credentials *mani.ServiceImageCredentials
+	// TODO: labels for deleting
+	// labels      map[string]string
+}
+
+func NewServiceCredentials(ns string, serviceName string, credentials *mani.ServiceImageCredentials) ServiceCredentials {
+	return serviceCredentials{
+		ns:          ns,
+		serviceName: serviceName,
+		credentials: credentials,
+	}
+}
+
+func (b serviceCredentials) NS() string {
+	return b.ns
+}
+
+func (b serviceCredentials) Name() string {
+	return fmt.Sprintf("docker-creds-%v", b.serviceName)
+}
+
+func (b serviceCredentials) Create() (*corev1.Secret, error) {
+	// see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298
+
+	data, err := b.encodeSecret()
+	if err != nil {
+		return nil, err
+	}
+
+	obj := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: b.ns,
+			Name:      b.Name(),
+		},
+		Data: map[string][]byte{
+			corev1.DockerConfigJsonKey: data,
+		},
+		Type: corev1.SecretTypeDockerConfigJson,
+	}
+
+	return obj, nil
+}
+
+func (b serviceCredentials) Update(obj *corev1.Secret) (*corev1.Secret, error) {
+	// see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298
+
+	data, err := b.encodeSecret()
+	if err != nil {
+		return nil, err
+	}
+
+	obj.Data = map[string][]byte{
+		corev1.DockerConfigJsonKey: data,
+	}
+	obj.Type = corev1.SecretTypeDockerConfigJson
+	return obj, nil
+}
+
+type dockerCredentialsEntry struct {
+	Username string `json:"username,omitempty"`
+	Password string `json:"password,omitempty"`
+	Email    string `json:"email,omitempty"`
+	Auth     string `json:"auth,omitempty"`
+}
+
+type dockerCredentials struct {
+	Auths map[string]dockerCredentialsEntry `json:"auths"`
+}
+
+func (b serviceCredentials) encodeSecret() ([]byte, error) {
+	entry := dockerCredentialsEntry{
+		Username: b.credentials.Username,
+		Password: b.credentials.Password,
+		Email:    b.credentials.Email,
+		Auth:     encodeAuth(b.credentials.Username, b.credentials.Password),
+	}
+	creds := dockerCredentials{
+		Auths: map[string]dockerCredentialsEntry{
+			b.credentials.Host: entry,
+		},
+	}
+	return json.Marshal(creds)
+}
+
+func encodeAuth(username, password string) string {
+	value := username + ":" + password
+	return base64.StdEncoding.EncodeToString([]byte(value))
+}

cluster/kube/builder/workload.go

@@ -26,6 +26,7 @@ const (
 type workloadBase interface {
 	builderBase
 	Name() string
+	NS() string
 }
 
 type Workload struct {
@@ -54,6 +55,10 @@ func (b *Workload) Name() string {
 	return b.deployment.ManifestGroup().Services[b.serviceIdx].Name
 }
 
+func (b *Workload) NS() string {
+	return LidNS(b.deployment.LeaseID())
+}
+
 func (b *Workload) container() corev1.Container {
 	falseValue := false
 
@@ -317,11 +322,20 @@ func (b *Workload) labels() map[string]string {
 }
 
 func (b *Workload) imagePullSecrets() []corev1.LocalObjectReference {
-	if b.settings.DockerImagePullSecretsName == "" {
+
+	sname := b.settings.DockerImagePullSecretsName
+
+	// TODO: fix akash-api proto-gen
+	service := &b.deployment.ManifestGroup().Services[b.serviceIdx]
+	if service.Credentials != nil {
+		sname = NewServiceCredentials(b.NS(), b.Name(), service.Credentials).Name()
+	}
+
+	if sname == "" {
 		return nil
 	}
 
-	return []corev1.LocalObjectReference{{Name: b.settings.DockerImagePullSecretsName}}
+	return []corev1.LocalObjectReference{{Name: sname}}
 }
 
 func (b *Workload) addEnvVarsForDeployment(envVarsAlreadyAdded map[string]int, env []corev1.EnvVar) []corev1.EnvVar {

cluster/kube/client.go

@@ -190,6 +190,7 @@ type deploymentService struct {
 	statefulSet   builder.StatefulSet
 	localService  builder.Service
 	globalService builder.Service
+	credentials   builder.ServiceCredentials
 }
 
 type deploymentApplies struct {
@@ -268,6 +269,10 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err
 
 		svc := &deploymentService{}
 
+		if service.Credentials != nil {
+			svc.credentials = builder.NewServiceCredentials(workload.NS(), workload.Name(), service.Credentials)
+		}
+
 		persistent := false
 		for i := range service.Resources.Storage {
 			attrVal := service.Resources.Storage[i].Attributes.Find(sdl.StorageAttributePersistent)
@@ -291,6 +296,7 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err
 
 		svc.localService = builder.BuildService(workload, false)
 		svc.globalService = builder.BuildService(workload, true)
+
 	}
 
 	if err := applyNS(ctx, c.kc, applies.ns); err != nil {
@@ -320,6 +326,13 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err
 		applyObjs := applies.services[svcIdx]
 		service := &group.Services[svcIdx]
 
+		if applyObjs.credentials != nil {
+			if err = applyServiceCredentials(ctx, c.kc, applyObjs.credentials); err != nil {
+				c.log.Error("applying credentials", "err", err, "lease", lid, "service", service.Name)
+				return err
+			}
+		}
+
 		if applyObjs.statefulSet != nil {
 			if err = applyStatefulSet(ctx, c.kc, applyObjs.statefulSet); err != nil {
 				c.log.Error("applying statefulSet", "err", err, "lease", lid, "service", service.Name)