Skip to content

Commit

Permalink
feat(private-containers): private registries
Browse files Browse the repository at this point in the history
refs akash-network/support#54

Signed-off-by: Adam Bozanich <[email protected]>
  • Loading branch information
boz committed Apr 13, 2024
1 parent a41a724 commit 8b98979
Show file tree
Hide file tree
Showing 6 changed files with 166 additions and 5 deletions.
23 changes: 23 additions & 0 deletions cluster/kube/apply.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,29 @@ func applyNetPolicies(ctx context.Context, kc kubernetes.Interface, b builder.Ne
// return err
// }

func applyServiceCredentials(ctx context.Context, kc kubernetes.Interface, b builder.ServiceCredentials) error {
obj, err := kc.CoreV1().Secrets(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{})
metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "secrets-get", err, errors.IsNotFound)

switch {
case err == nil:
obj, err = b.Update(obj)
if err == nil {
_, err = kc.CoreV1().Secrets(b.NS()).Update(ctx, obj, metav1.UpdateOptions{})
metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-get", err)

}
case errors.IsNotFound(err):
obj, err = b.Create()
if err == nil {
_, err = kc.CoreV1().Secrets(b.NS()).Create(ctx, obj, metav1.CreateOptions{})
metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-create", err)
}
}
return err

}

func applyDeployment(ctx context.Context, kc kubernetes.Interface, b builder.Deployment) error {
obj, err := kc.AppsV1().Deployments(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{})
metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "deployments-get", err, errors.IsNotFound)
Expand Down
111 changes: 111 additions & 0 deletions cluster/kube/builder/service_credentials.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package builder

import (
"encoding/base64"
"encoding/json"
"fmt"
"strings"

mani "github.com/akash-network/akash-api/go/manifest/v2beta2"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type ServiceCredentials interface {
NS() string
Name() string
Create() (*corev1.Secret, error)
Update(obj *corev1.Secret) (*corev1.Secret, error)
}

type serviceCredentials struct {
ns string
serviceName string
credentials *mani.ServiceImageCredentials
// TODO: labels for deleting
// labels map[string]string
}

func NewServiceCredentials(ns string, serviceName string, credentials *mani.ServiceImageCredentials) ServiceCredentials {
return serviceCredentials{
ns: ns,
serviceName: serviceName,
credentials: credentials,
}
}

func (b serviceCredentials) NS() string {
return b.ns
}

func (b serviceCredentials) Name() string {
return fmt.Sprintf("docker-creds-%v", b.serviceName)
}

func (b serviceCredentials) Create() (*corev1.Secret, error) {
// see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298

data, err := b.encodeSecret()
if err != nil {
return nil, err
}

obj := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Namespace: b.ns,
Name: b.Name(),
},
Data: map[string][]byte{
corev1.DockerConfigJsonKey: data,
},
Type: corev1.SecretTypeDockerConfigJson,
}

return obj, nil
}

func (b serviceCredentials) Update(obj *corev1.Secret) (*corev1.Secret, error) {
// see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298

data, err := b.encodeSecret()
if err != nil {
return nil, err
}

obj.Data = map[string][]byte{
corev1.DockerConfigJsonKey: data,
}
obj.Type = corev1.SecretTypeDockerConfigJson
return obj, nil
}

type dockerCredentialsEntry struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
Email string `json:"email,omitempty"`
Auth string `json:"auth,omitempty"`
}

type dockerCredentials struct {
Auths map[string]dockerCredentialsEntry `json:"auths"`
}

func (b serviceCredentials) encodeSecret() ([]byte, error) {
entry := dockerCredentialsEntry{
Username: strings.TrimSpace(b.credentials.Username),
Password: strings.TrimSpace(b.credentials.Password),
Email: strings.TrimSpace(b.credentials.Email),
Auth: encodeAuth(strings.TrimSpace(b.credentials.Username), strings.TrimSpace(b.credentials.Password)),
}
creds := dockerCredentials{
Auths: map[string]dockerCredentialsEntry{
b.credentials.Host: entry,
},
}
return json.Marshal(creds)
}

func encodeAuth(username, password string) string {
value := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(value))
}
18 changes: 16 additions & 2 deletions cluster/kube/builder/workload.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const (
type workloadBase interface {
builderBase
Name() string
NS() string
}

type Workload struct {
Expand Down Expand Up @@ -54,6 +55,10 @@ func (b *Workload) Name() string {
return b.deployment.ManifestGroup().Services[b.serviceIdx].Name
}

func (b *Workload) NS() string {
return LidNS(b.deployment.LeaseID())
}

func (b *Workload) container() corev1.Container {
falseValue := false

Expand Down Expand Up @@ -362,11 +367,20 @@ func (b *Workload) labels() map[string]string {
}

func (b *Workload) imagePullSecrets() []corev1.LocalObjectReference {
if b.settings.DockerImagePullSecretsName == "" {

sname := b.settings.DockerImagePullSecretsName

// TODO: fix akash-api proto-gen
service := &b.deployment.ManifestGroup().Services[b.serviceIdx]
if service.Credentials != nil {
sname = NewServiceCredentials(b.NS(), b.Name(), service.Credentials).Name()
}

if sname == "" {
return nil
}

return []corev1.LocalObjectReference{{Name: b.settings.DockerImagePullSecretsName}}
return []corev1.LocalObjectReference{{Name: sname}}
}

func (b *Workload) addEnvVarsForDeployment(envVarsAlreadyAdded map[string]int, env []corev1.EnvVar) []corev1.EnvVar {
Expand Down
13 changes: 13 additions & 0 deletions cluster/kube/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ type deploymentService struct {
statefulSet builder.StatefulSet
localService builder.Service
globalService builder.Service
credentials builder.ServiceCredentials
}

type deploymentApplies struct {
Expand Down Expand Up @@ -268,6 +269,10 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err

svc := &deploymentService{}

if service.Credentials != nil {
svc.credentials = builder.NewServiceCredentials(workload.NS(), workload.Name(), service.Credentials)
}

persistent := false
for i := range service.Resources.Storage {
attrVal := service.Resources.Storage[i].Attributes.Find(sdl.StorageAttributePersistent)
Expand All @@ -291,6 +296,7 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err

svc.localService = builder.BuildService(workload, false)
svc.globalService = builder.BuildService(workload, true)

}

if err := applyNS(ctx, c.kc, applies.ns); err != nil {
Expand Down Expand Up @@ -320,6 +326,13 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err
applyObjs := applies.services[svcIdx]
service := &group.Services[svcIdx]

if applyObjs.credentials != nil {
if err = applyServiceCredentials(ctx, c.kc, applyObjs.credentials); err != nil {
c.log.Error("applying credentials", "err", err, "lease", lid, "service", service.Name)
return err
}
}

if applyObjs.statefulSet != nil {
if err = applyStatefulSet(ctx, c.kc, applyObjs.statefulSet); err != nil {
c.log.Error("applying statefulSet", "err", err, "lease", lid, "service", service.Name)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/akash-network/provider
go 1.21

require (
github.com/akash-network/akash-api v0.0.61
github.com/akash-network/akash-api v0.0.62
github.com/akash-network/node v0.32.3
github.com/avast/retry-go/v4 v4.5.0
github.com/blang/semver/v4 v4.0.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -197,8 +197,8 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
github.com/akash-network/akash-api v0.0.61 h1:Hj/IBr9cFMsFs4VjymLZCoX/5dNfZSFd4iFwUSqhBtQ=
github.com/akash-network/akash-api v0.0.61/go.mod h1:pNr61L4+0sheol7ZK0HjgK3rxpIAbYBGq1w1oH4B0+M=
github.com/akash-network/akash-api v0.0.62 h1:o/V/iwlrel++5xo1UTlGO8aASKWTXm3to5+Z3IyYZUw=
github.com/akash-network/akash-api v0.0.62/go.mod h1:pNr61L4+0sheol7ZK0HjgK3rxpIAbYBGq1w1oH4B0+M=
github.com/akash-network/cometbft v0.34.27-akash h1:V1dApDOr8Ee7BJzYyQ7Z9VBtrAul4+baMeA6C49dje0=
github.com/akash-network/cometbft v0.34.27-akash/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw=
github.com/akash-network/ledger-go v0.14.3 h1:LCEFkTfgGA2xFMN2CtiKvXKE7dh0QSM77PJHCpSkaAo=
Expand Down

0 comments on commit 8b98979

Please sign in to comment.