From ebf3c5023f3b766d32b2073810f05b78f06a0e64 Mon Sep 17 00:00:00 2001 From: Dean Hunter Date: Wed, 24 Apr 2024 13:59:05 +0200 Subject: [PATCH] feat: enhance build process with docker authentication Adds support for automatic docker authentication during the build process. If no credentials are found, the build will attempt to authenticate using the provided username, password, and destination registry. This ensures images can be pushed to private registries without manual setup. Additionally, refactors the Login command to use the new authentication helper method and adds placeholders for other commands. Updates the Dockerfile to include a shell command to keep the container running. --- .gitignore | 1 + Builder/dockerfile | 4 ++- Client/auth/auth.go | 3 +- Client/go.mod | 11 +++---- Client/go.sum | 12 ++++---- Client/kaniko/cmd.go | 61 +++++++++++++++++++++++++++++------- Client/kaniko/kaniko.go | 12 ++++++++ Client/shared/shared.go | 16 ++++++++++ How-To/azure_devops.md | 68 +++++++++++++++++++++++++++++++++++++++++ readme.md | 5 ++- 10 files changed, 165 insertions(+), 28 deletions(-) create mode 100644 How-To/azure_devops.md diff --git a/.gitignore b/.gitignore index 010eec7..9345158 100644 --- a/.gitignore +++ b/.gitignore @@ -17,5 +17,6 @@ # Dependency directories (remove the comment below to include it) # vendor/ dockerbuilder.sh +deploy_tmp.yaml # Go workspace file go.work \ No newline at end of file diff --git a/Builder/dockerfile b/Builder/dockerfile index e34bf82..85e5f3f 100644 --- a/Builder/dockerfile +++ b/Builder/dockerfile @@ -13,4 +13,6 @@ COPY --from=build /go/bin/docker /kaniko/ COPY --from=kaniko-executor /kaniko/* /kaniko/ COPY --from=kaniko-executor /etc/nsswitch.conf /etc/nsswitch.conf COPY Example Example -ENV PATH="/kaniko:${PATH}" \ No newline at end of file +ENV PATH="/kaniko:${PATH}" + +CMD ["sh", "-c", "tail -f /dev/null"] \ No newline at end of file diff --git a/Client/auth/auth.go b/Client/auth/auth.go index f08e212..61b1e37 100644 --- a/Client/auth/auth.go +++ b/Client/auth/auth.go @@ -9,6 +9,7 @@ import ( "os" "net/http" environment "github.com/CoreViewInc/CoreNiko/environment" + shared "github.com/CoreViewInc/CoreNiko/shared" ) type DockerAuth struct { @@ -137,7 +138,7 @@ func New(envProvider *environment.EnvProvider) *DockerAuth { func ReadEncodedCredentialsFromFile() (string, string, error) { configPath := "/kaniko/.docker/config.json" if _, err := os.Stat(configPath); os.IsNotExist(err) { - return "", "", fmt.Errorf("config file does not exist at %s", configPath) + return "", "", shared.FileNotFoundError{Path: configPath} // Return the custom error here } data, err := ioutil.ReadFile(configPath) diff --git a/Client/go.mod b/Client/go.mod index 7cac220..29ca77f 100644 --- a/Client/go.mod +++ b/Client/go.mod @@ -2,14 +2,15 @@ module github.com/CoreViewInc/CoreNiko go 1.21 - require ( + github.com/google/uuid v1.3.0 github.com/spf13/afero v1.6.0 github.com/spf13/cobra v1.3.0 go.etcd.io/bbolt v1.3.8 k8s.io/api v0.29.3 k8s.io/apimachinery v0.29.3 k8s.io/client-go v0.29.3 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b ) require ( @@ -23,7 +24,6 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -33,10 +33,10 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -46,7 +46,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/Client/go.sum b/Client/go.sum index 350432a..d6fdcc4 100644 --- a/Client/go.sum +++ b/Client/go.sum @@ -484,8 +484,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -581,11 +581,11 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/Client/kaniko/cmd.go b/Client/kaniko/cmd.go index 55d9223..e6474fe 100644 --- a/Client/kaniko/cmd.go +++ b/Client/kaniko/cmd.go @@ -7,6 +7,7 @@ import ( "errors" "path/filepath" "os" + "log" auth "github.com/CoreViewInc/CoreNiko/auth" environment "github.com/CoreViewInc/CoreNiko/environment" io "github.com/CoreViewInc/CoreNiko/io" @@ -111,7 +112,6 @@ func (kd *KanikoDocker) BuildImage(options shared.BuildOptions, contextPath, doc dockerfilePath = filepath.Join(cwd, "Dockerfile") } - fmt.Println(contextPath+" : "+dockerfilePath) if err := kd.CopyContextAndDockerfile(contextPath, dockerfilePath, newdir); err != nil { return err } @@ -130,8 +130,33 @@ func (kd *KanikoDocker) BuildImage(options shared.BuildOptions, contextPath, doc return err } for _,stage := range stages{ + + //check docker session has logged in previously, or use credentials plus destination as target + _, _, err := auth.ReadEncodedCredentialsFromFile() + if err != nil { + if shared.IsFileNotFoundError(err) { + fmt.Println("No docker login performed, using environment variables.") + parsed_tag,err := kd.ParseDockerImageTag(stage) + if err!=nil{ + fmt.Println("error getting url from tag") + return err + } + registry := parsed_tag.Registry + if len(registry) >0{ + err = kd.manuallogin("username","password",registry,false) + if err !=nil{ + fmt.Println("return heeerexxxx") + return err + } + } + } else { + return fmt.Errorf("reading encoded credentials from file: %w", err) + } + } + + // now everything is configured we can actually execute the build kanikoExecutor.Destination[0] = stage - _, _, err := kanikoExecutor.Execute() + _, _, err = kanikoExecutor.Execute() if err != nil { return err } @@ -144,45 +169,59 @@ func (kd *KanikoDocker) BuildImage(options shared.BuildOptions, contextPath, doc } func (kd *KanikoDocker) TagImage(args []string) { - fmt.Println("Placeholder - tag") + fmt.Println("Tag") } func (kd *KanikoDocker) PushImage(args []string) { - fmt.Println("Placeholder - push") + fmt.Println("Push") } -//called from cli to do a docker login, generates required config file for kaniko -func (kd *KanikoDocker) Login(args []string,username string,password,url string) { +func (kd *KanikoDocker) manuallogin(username string,password,url string,validate bool) error { dockerauth := auth.New(kd.Env) if len(username)>0 && len(password)>0{ dockerauth = auth.NewUserPassAuth(username, password,url) - _, err := dockerauth.VerifyCredentials() - if err!=nil{ - panic(err) + if validate{ + _, err := dockerauth.VerifyCredentials() + if err!=nil{ + return err + } } }else{ fmt.Println("No passowrd or username received!") } err := dockerauth.CreateDockerConfigJSON() if err !=nil{ - panic(err) + return err + } + return nil +} + +func (kd *KanikoDocker) Login(args []string, username string, password string, url string) { + fmt.Println("Login") + err := kd.manuallogin(username, password, url,true) + if err != nil { + log.Println(err) + os.Exit(1) } } func (kd *KanikoDocker) InspectImage(args []string) (string, error){ + fmt.Println("Inspect") return "",nil } func (kd *KanikoDocker) PullImage(imageName string) error { + fmt.Println("Pull") return nil } func (kd *KanikoDocker) ListImages(args []string) (string, error) { + fmt.Println("Ls") return "",nil } func (kd *KanikoDocker) ImageHistory(args []string) (string, error) { - return "placeholder",nil //temporary to provide a debuggable value + return "",nil //temporary to provide a debuggable value } diff --git a/Client/kaniko/kaniko.go b/Client/kaniko/kaniko.go index 838e874..427f045 100644 --- a/Client/kaniko/kaniko.go +++ b/Client/kaniko/kaniko.go @@ -30,6 +30,18 @@ func (ke *KanikoExecutor) SetRootDir(value string) { ke.RootDir = value } +func (ke *KanikoExecutor) GetArg(argName string) (string, bool) { + args := ke.buildArgs() + + fmt.Println(args) + for i := 0; i < len(args); i++ { + if args[i] == argName && i+1 < len(args) { + return args[i+1], true + } + } + return "", false +} + func (ke *KanikoExecutor) UpdateArg(argName, newValue string) { args := ke.buildArgs() updated := false diff --git a/Client/shared/shared.go b/Client/shared/shared.go index b56951a..b959b26 100644 --- a/Client/shared/shared.go +++ b/Client/shared/shared.go @@ -9,6 +9,21 @@ import ( "math/rand" ) +type FileNotFoundError struct { + Path string +} + +// Error implements the error interface for FileNotFoundError. +func (e FileNotFoundError) Error() string { + return fmt.Sprintf("config file does not exist at %s", e.Path) +} + +// IsFileNotFoundError checks if an error is of the FileNotFoundError type. +func IsFileNotFoundError(err error) bool { + _, ok := err.(FileNotFoundError) + return ok +} + // Defintion a docker cli must implement type DockerBuilder interface { BuildImage(options BuildOptions, contextPath string, dockerfilePath string) error @@ -29,6 +44,7 @@ type ExecutorInterface interface { UpdateArg(argName, argValue string) GetRootDir() string SetRootDir(string) + GetArg(string) (string, bool) } // BuildOptions defines options for the docker build command. diff --git a/How-To/azure_devops.md b/How-To/azure_devops.md new file mode 100644 index 0000000..1941ab9 --- /dev/null +++ b/How-To/azure_devops.md @@ -0,0 +1,68 @@ +# Setting Up NFS-Compatible Storage for Azure DevOps Build Agents in AKS + +When running Docker builds with your own build agents in Azure DevOps within a Kubernetes cluster, like Azure Kubernetes Service (AKS), you'll need to prepare for some specific technical considerations. A common challenge is the AKS default storage drivers' lack of support for the Network File System (NFS), which can cause build process errors. + +## Understanding the Issue + +The default storage drivers in AKS often result in errors during Docker builds, such as: +``` +error: chmod on /azp/_work/1/s/.git/config.lock failed: Operation not permitted fatal: could not set 'core.filemode' to 'false'. +``` +These errors occur because the default storage options don't support certain filesystem operations, like changing file permissions or ownership, which are critical for tools like Git. + +## Solution: Enabling NFS-Compatible Storage + +To ensure Docker builds go smoothly within Kubernetes, it's recommended to use an NFS-compatible storage driver. NFS supports the shared access to files and directories that is essential for the interaction between the build agent and executing pods. + +### Step 1: Create an NFS Server + +Establish an NFS server, either self-managed or through a managed NFS service such as Azure Files. + +### Step 2: Configure the NFS Storage Class + +Introduce a new storage class in your Kubernetes cluster using the NFS provisioner to dynamically provision NFS-based persistent volumes. + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: nfs-storage +provisioner: kubernetes.io/nfs +parameters: + server: + path: /exported/path +Step 3: Update the Build Agent Deployment +Modify your build agent's deployment configuration to use the NFS storage class for persistent storage, ensuring the working directory is on an NFS-compatible volume. + +yaml +Copy code + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: build-agent +spec: + template: + spec: + containers: + - name: build-agent + image: your-build-agent-image + volumeMounts: + - name: work-dir + mountPath: /azp/_work + volumes: + - name: work-dir + persistentVolumeClaim: + claimName: build-agent-pvc +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: build-agent-pvc +spec: + accessModes: + - ReadWriteOnce + storageClassName: nfs-storage + resources: + requests: + storage: 10Gi \ No newline at end of file diff --git a/readme.md b/readme.md index cb4b4c2..d254dbc 100644 --- a/readme.md +++ b/readme.md @@ -1,9 +1,8 @@ -[![Default](https://github.com/CoreViewInc/CoreNiko/actions/workflows/go.yml/badge.svg)](https://github.com/CoreViewInc/CoreNiko/actions/workflows/go.yml) -![Develop](https://github.com/CoreViewInc/CoreNiko/actions/workflows/go.yml/badge.svg?branch=develop) +[![Develop](https://github.com/CoreViewInc/CoreNiko/actions/workflows/go.yml/badge.svg)](https://github.com/CoreViewInc/CoreNiko/actions/workflows/go.yml) # CoreNiko -**CoreNiko** The CoreNiko project aims to wrap Kaniko in an easy to deploy solution to simplify scalable builds in Kubernetes. The key goal of the project is to allow developers to setup scalable build agents quickly using kubernetes with a docker like CLI tool with as close to 1-1 compatibility as possible. The project is currently in Alpha state. +**CoreNiko** aims to wrap Kaniko in an easy to deploy solution to simplify scalable builds in Kubernetes. The key goal of the project is to allow developers to setup scalable build agents quickly using kubernetes with a docker like CLI tool with as close to 1-1 compatibility as possible. The project is currently in Alpha state. ### Key Features: