feat: optimize image build kaniko-->buildkit (#1709)

builder/build/build.go

@@ -81,8 +81,8 @@ type Response struct {
 
 //Request build input
 type Request struct {
- KanikoImage  string
- KanikoArgs  []string
+ BuildKitImage string
+ BuildKitArgs []string
  RbdNamespace string
  GRDataPVCName string
  CachePVCName string

builder/build/code_build.go

@@ -142,7 +142,7 @@ func (s *slugBuild) buildRunnerImage(slugPackage string) (string, error) {
  return "", fmt.Errorf("pull image %s: %v", builder.RUNNERIMAGENAME, err)
  }
  logrus.Infof("pull image %s successfully.", builder.RUNNERIMAGENAME)
- err := sources.ImageBuild(s.re.Arch, cacheDir, "", "", s.re.RbdNamespace, s.re.ServiceID, s.re.DeployVersion, s.re.Logger, "run-build", "", s.re.KanikoImage, s.re.KanikoArgs)
+ err := sources.ImageBuild(s.re.Arch, cacheDir, "", "", s.re.RbdNamespace, s.re.ServiceID, s.re.DeployVersion, s.re.Logger, "run-build", "", s.re.BuildKitImage, s.re.BuildKitArgs, s.re.KubeClient)
  if err != nil {
  s.re.Logger.Error(fmt.Sprintf("build image %s of new version failure", imageName), map[string]string{"step": "builder-exector", "status": "failure"})
  logrus.Errorf("build image error: %s", err.Error())
@@ -555,7 +555,7 @@ func (s *slugBuild) HandleNodeJsDir(re *Request) error {
  return err
  }
  defer filePtr.Close()
- if dir, ok := re.BuildEnvs["DIST_DIR"]; !ok || dir == ""{
+ if dir, ok := re.BuildEnvs["DIST_DIR"]; !ok || dir == "" {
  re.BuildEnvs["DIST_DIR"] = "dist"
  }
 

builder/build/dockerfile_build.go

@@ -24,6 +24,7 @@ import (
  "encoding/json"
  "fmt"
  "github.com/eapache/channels"
+ "github.com/goodrain/rainbond/builder"
  jobc "github.com/goodrain/rainbond/builder/job"
  "github.com/goodrain/rainbond/builder/sources"
  "github.com/goodrain/rainbond/db"
@@ -83,7 +84,7 @@ func (d *dockerfileBuild) stopPreBuildJob(re *Request) error {
  return nil
 }
 
-// Use kaniko to create a job to build an image
+// Use buildkit to create a job to build an image
 func (d *dockerfileBuild) runBuildJob(re *Request, buildImageName string) error {
  name := fmt.Sprintf("%s-%s", re.ServiceID, re.DeployVersion)
  namespace := re.RbdNamespace
@@ -133,26 +134,44 @@ func (d *dockerfileBuild) runBuildJob(re *Request, buildImageName string) error
  if err != nil {
  return err
  }
- volumes, mounts := d.createVolumeAndMount(re, secret.Name)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ err = sources.PrepareBuildKitTomlCM(ctx, re.KubeClient, re.RbdNamespace)
+ if err != nil {
+ return err
+ }
+ volumes, mounts := d.createVolumeAndMount(re, secret.Name, re.ServiceID)
  podSpec.Volumes = volumes
+ privileged := true
  container := corev1.Container{
- Name: name,
- //2022.11.4: latest==1.9.1
- Image: re.KanikoImage,
+ Name: name,
+ Image: re.BuildKitImage,
  Stdin: true,
  StdinOnce: true,
- Args: []string{fmt.Sprintf("--context=%v", re.SourceDir), fmt.Sprintf("--destination=%s", buildImageName), "--skip-tls-verify", "--reproducible"},
+ Command: []string{"buildctl-daemonless.sh"},
+ Args: []string{
+ "build",
+ "--frontend",
+ "dockerfile.v0",
+ "--local",
+ fmt.Sprintf("context=%v", re.SourceDir),
+ "--local",
+ fmt.Sprintf("dockerfile=%v", re.SourceDir),
+ "--output",
+ fmt.Sprintf("type=image,name=%s,push=true", buildImageName),
+ },
+ SecurityContext: &corev1.SecurityContext{
+ Privileged: &privileged,
+ },
  }
- if len(re.KanikoArgs) > 0 {
- container.Args = append(container.Args, re.KanikoArgs...)
+ if len(re.BuildKitArgs) > 0 {
+ container.Args = append(container.Args, re.BuildKitArgs...)
  }
  container.VolumeMounts = mounts
  podSpec.Containers = append(podSpec.Containers, container)
  job.Spec = podSpec
  writer := re.Logger.GetWriter("builder", "info")
  reChan := channels.NewRingChannel(10)
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
 
  logrus.Debugf("create job[name: %s; namespace: %s]", job.Name, job.Namespace)
  err = jobc.GetJobController().ExecJob(ctx, &job, writer, reChan)
@@ -168,7 +187,8 @@ func (d *dockerfileBuild) runBuildJob(re *Request, buildImageName string) error
  return nil
 }
 
-func (d *dockerfileBuild) createVolumeAndMount(re *Request, secretName string) (volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) {
+func (d *dockerfileBuild) createVolumeAndMount(re *Request, secretName, ServiceID string) (volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) {
+ hostPathType := corev1.HostPathDirectoryOrCreate
  hostsFilePathType := corev1.HostPathFile
  dockerfileBuildVolume := corev1.Volume{
  Name: "dockerfile-build",
@@ -191,7 +211,30 @@ func (d *dockerfileBuild) createVolumeAndMount(re *Request, secretName string) (
  volumes = []corev1.Volume{
  dockerfileBuildVolume,
  {
- Name: "kaniko-secret",
+ Name: "buildkittoml",
+ VolumeSource: corev1.VolumeSource{
+ ConfigMap: &corev1.ConfigMapVolumeSource{
+ LocalObjectReference: corev1.LocalObjectReference{Name: builder.REGISTRYDOMAIN},
+ Items: []corev1.KeyToPath{
+ {
+ Key: "buildkittoml",
+ Path: "buildkitd.toml",
+ },
+ },
+ },
+ },
+ },
+ {
+ Name: "buildkit-db",
+ VolumeSource: corev1.VolumeSource{
+ HostPath: &corev1.HostPathVolumeSource{
+ Path: fmt.Sprintf("/cache/buildkit-cache/%v", ServiceID),
+ Type: &hostPathType,
+ },
+ },
+ },
+ {
+ Name: "buildkit-secret",
  VolumeSource: corev1.VolumeSource{
  Secret: &corev1.SecretVolumeSource{
  SecretName: secretName,
@@ -220,13 +263,21 @@ func (d *dockerfileBuild) createVolumeAndMount(re *Request, secretName string) (
  MountPath: "/cache",
  },
  {
- Name: "kaniko-secret",
- MountPath: "/kaniko/.docker",
+ Name: "buildkit-secret",
+ MountPath: "/root/.docker",
  },
  {
  Name: "hosts",
  MountPath: "/etc/hosts",
  },
+ {
+ Name: "buildkittoml",
+ MountPath: "/etc/buildkit",
+ },
+ {
+ Name: "buildkit-db",
+ MountPath: "/var/lib/buildkit",
+ },
  }
  return volumes, volumeMounts
 }

builder/build/netcore_build.go

@@ -71,7 +71,7 @@ func (d *netcoreBuild) Build(re *Request) (*Response, error) {
  return nil, fmt.Errorf("write default dockerfile error:%s", err.Error())
  }
  // build image
- err := sources.ImageBuild(re.Arch, d.sourceDir, re.CachePVCName, re.CacheMode, re.RbdNamespace, re.ServiceID, re.DeployVersion, re.Logger, "nc-build", "", re.KanikoImage, re.KanikoArgs)
+ err := sources.ImageBuild(re.Arch, d.sourceDir, re.CachePVCName, re.CacheMode, re.RbdNamespace, re.ServiceID, re.DeployVersion, re.Logger, "nc-build", "", re.BuildKitImage, re.BuildKitArgs, re.KubeClient)
  if err != nil {
  re.Logger.Error(fmt.Sprintf("build image %s failure, find log in rbd-chaos", d.buildImageName), map[string]string{"step": "builder-exector", "status": "failure"})
  logrus.Errorf("build image error: %s", err.Error())

builder/exector/build_from_sourcecode_run.go

@@ -61,8 +61,8 @@ type SourceCodeBuildItem struct {
  CacheDir string `json:"cache_dir"`
  TGZDir string `json:"tgz_dir"`
  ImageClient sources.ImageClient
- KanikoImage  string
- KanikoArgs  []string
+ BuildKitImage string
+ BuildKitArgs []string
  KubeClient kubernetes.Interface
  RbdNamespace string
  RbdRepoName string
@@ -326,8 +326,8 @@ func (i *SourceCodeBuildItem) codeBuild() (*build.Response, error) {
  return nil, err
  }
  buildReq := &build.Request{
- KanikoArgs: i.KanikoArgs,
- KanikoImage:  i.KanikoImage,
+ BuildKitImage: i.BuildKitImage,
+ BuildKitArgs: i.BuildKitArgs,
  RbdNamespace: i.RbdNamespace,
  SourceDir: i.RepoInfo.GetCodeBuildAbsPath(),
  CacheDir: i.CacheDir,

builder/exector/exector.go

@@ -24,6 +24,7 @@ import (
  "github.com/goodrain/rainbond/builder/sources"
  "runtime"
  "runtime/debug"
+ "strings"
  "sync"
  "time"
 
@@ -117,9 +118,10 @@ func NewManager(conf option.Config, mqc mqclient.MQClient) (Manager, error) {
  return nil, err
  }
  logrus.Infof("The maximum number of concurrent build tasks supported by the current node is %d", maxConcurrentTask)
+
  return &exectorManager{
- KanikoImage:  conf.KanikoImage,
- KanikoArgs:  conf.KanikoArgs,
+ BuildKitImage: conf.BuildKitImage,
+ BuildKitArgs: strings.Split(conf.BuildKitArgs, "&"),
  KubeClient: kubeClient,
  EtcdCli: etcdCli,
  mqClient: mqc,
@@ -133,8 +135,8 @@ func NewManager(conf option.Config, mqc mqclient.MQClient) (Manager, error) {
 }
 
 type exectorManager struct {
- KanikoImage  string
- KanikoArgs  []string
+ BuildKitImage string
+ BuildKitArgs []string
  KubeClient kubernetes.Interface
  EtcdCli *clientv3.Client
  tasks chan *pb.TaskMessage
@@ -344,8 +346,8 @@ func (e *exectorManager) buildFromImage(task *pb.TaskMessage) {
 func (e *exectorManager) buildFromSourceCode(task *pb.TaskMessage) {
  i := NewSouceCodeBuildItem(task.TaskBody)
  i.ImageClient = e.imageClient
- i.KanikoImage = e.KanikoImage
- i.KanikoArgs = e.KanikoArgs
+ i.BuildKitImage = e.BuildKitImage
+ i.BuildKitArgs = e.BuildKitArgs
  i.KubeClient = e.KubeClient
  i.RbdNamespace = e.cfg.RbdNamespace
  i.RbdRepoName = e.cfg.RbdRepoName

builder/exector/plugin_dockerfile.go

@@ -105,7 +105,7 @@ func (e *exectorManager) runD(t *model.BuildPluginTaskBody, logger event.Logger)
  n1 := strings.Split(mm[len(mm)-1], ".")[0]
  buildImageName := fmt.Sprintf(builder.REGISTRYDOMAIN+"/plugin_%s_%s:%s", n1, t.PluginID, t.DeployVersion)
  logger.Info("start build image", map[string]string{"step": "builder-exector"})
- err := sources.ImageBuild("", sourceDir, "", "", "rbd-system", t.PluginID, t.DeployVersion, logger, "plug-build", buildImageName, e.KanikoImage, e.KanikoArgs)
+ err := sources.ImageBuild("", sourceDir, "", "", "rbd-system", t.PluginID, t.DeployVersion, logger, "plug-build", buildImageName, e.BuildKitImage, e.BuildKitArgs, e.KubeClient)
  if err != nil {
  logger.Error(fmt.Sprintf("build image %s failure,find log in rbd-chaos", buildImageName), map[string]string{"step": "builder-exector", "status": "failure"})
  logrus.Errorf("[plugin]build image error: %s", err.Error())