From bc12aad8733065b2e7eaebd3be83f89cbba5dd37 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Tue, 18 Jun 2024 22:37:29 +0200 Subject: [PATCH] Trying to add a registry to push the images. --- .gitattributes | 1 + .github/workflows/test.yml | 6 +- .gitignore | 4 + Dockerfile | 4 +- README.md | 10 +- docker-compose.yml | 29 +++-- docker/daemon.json | 3 + main_backup.go | 255 ------------------------------------- pkg/staging/helper.go | 40 +++--- registry/config.yml | 20 +++ registry/data/.gitkeep | 0 test/test.sh | 24 ++-- 12 files changed, 97 insertions(+), 299 deletions(-) create mode 100644 .gitattributes create mode 100644 docker/daemon.json delete mode 100644 main_backup.go create mode 100644 registry/config.yml create mode 100644 registry/data/.gitkeep diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d207b18 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.go text eol=lf diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 412710c..216552d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,8 +18,12 @@ jobs: with: fetch-depth: 0 + # TODO: + # - name: Lint + # run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.59.1 golangci-lint run -v --enable-all + - name: Build - run: docker compose build + run: docker compose build - name: Test Binary run: ./test/test.sh binary diff --git a/.gitignore b/.gitignore index eca411f..7c961ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,10 @@ # Main executable main +# Registry shared folder +registry/data/** +!registry/data/.gitkeep + # Binaries for programs and plugins *.exe *.exe~ diff --git a/Dockerfile b/Dockerfile index a37091e..21fc68a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ COPY --from=builder_binary --chown=user:user /builder /home/user/builder RUN printf '#!/bin/sh\n\ export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\ -/home/user/builder $@ | buildctl-daemonless.sh build --output type=image,name=docker.io/metacall/builder_output,push=false\n'\ +/home/user/builder $@ | buildctl-daemonless.sh build --output type=image,name=registry:5000/metacall/builder_output,push=true\n'\ >> /home/user/builder.sh \ && chmod 700 /home/user/builder.sh \ && chmod 700 /home/user/builder @@ -47,7 +47,7 @@ COPY --from=builder_binary --chown=root:root /builder /home/builder RUN apk add --no-cache docker RUN printf '#!/bin/sh\n\ -/home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build --output type=docker,name=docker.io/metacall/builder_output | docker load\n'\ +/home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build --output type=image,name=registry:5000/metacall/builder_output,push=true\n'\ >> /home/builder.sh \ && chmod 700 /home/builder.sh \ && chmod 700 /home/builder diff --git a/README.md b/README.md index c1639b6..2aeda3e 100644 --- a/README.md +++ b/README.md @@ -45,13 +45,19 @@ BUILDER_ARGS="runtime py" docker compose up --exit-code-from client client Rootless: ```sh -BUILDER_ARGS="runtime node" docker compose up rootless +BUILDER_ARGS="runtime node" docker compose up --exit-code-from rootless rootless ``` You can also run the builder binary only: ```sh -BUILDER_ARGS="runtime rb" docker compose up binary +BUILDER_ARGS="runtime rb" docker compose up --exit-code-from binary binary +``` + +## Linter + +```sh +docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.59.1 golangci-lint run -v --enable-all ``` ## Useful Tools diff --git a/docker-compose.yml b/docker-compose.yml index cba04cc..6ccf9b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3.7" -volumes: - buildkit: +# volumes: +# buildkit: services: binary: @@ -24,7 +24,9 @@ services: security_opt: - "seccomp=unconfined" - "apparmor=unconfined" - network_mode: "host" + # network_mode: "host" + depends_on: + - registry client: image: metacall/builder_client @@ -36,21 +38,32 @@ services: entrypoint: /home/builder.sh ${BUILDER_ARGS:-} volumes: - /var/run/docker.sock:/var/run/docker.sock - - buildkit:/run/user/1000/buildkit/buildkitd.sock + # - buildkit:/run/user/1000/buildkit/buildkitd.sock depends_on: - buildkit + - registry buildkit: image: moby/buildkit container_name: metacall_builder_buildkit privileged: true - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - /run/containerd/containerd.sock:/run/containerd/containerd.sock - - buildkit:/run/buildkit/buildkitd.sock + # volumes: + # - /var/run/docker.sock:/var/run/docker.sock + # - /run/containerd/containerd.sock:/run/containerd/containerd.sock + # - buildkit:/run/buildkit/buildkitd.sock healthcheck: test: buildctl debug workers interval: 5s timeout: 30s retries: 3 start_period: 1s + + registry: + image: registry:2.8 + container_name: metacall_builder_registry + restart: always + ports: + - 5000:5000 + volumes: + - ./registry/config.yml:/etc/docker/registry/config.yml:ro + - ./registry/data:/var/lib/registry:rw diff --git a/docker/daemon.json b/docker/daemon.json new file mode 100644 index 0000000..e69c41c --- /dev/null +++ b/docker/daemon.json @@ -0,0 +1,3 @@ +{ + "registry-mirrors": ["http://metacall_builder_registry:5000/"] +} diff --git a/main_backup.go b/main_backup.go deleted file mode 100644 index a876989..0000000 --- a/main_backup.go +++ /dev/null @@ -1,255 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "log" - "os" - "runtime" - - "github.com/moby/buildkit/client/llb" - _ "github.com/moby/buildkit/util/progress" -) - -type LanguageType int - -const ( - Python LanguageType = iota - Node - TypeScript - Ruby - C - CSharp - Java - WebAssembly - Cobol - File - Mock - RPC -) - -var LanguageMap = map[string]LanguageType{ - "py": Python, - "node": Node, - "ts": TypeScript, - "rb": Ruby, - "c": C, - "cs": CSharp, - "java": Java, - "wasm": WebAssembly, - "cob": Cobol, - "file": File, - "mock": Mock, - "rpc": RPC, -} - -var envDepsFuncMap = map[LanguageType]func(llb.State) llb.State{ - Python: buildPyEnv, - Node: buildNodeEnv, - // TypeScript: buildTS, - Ruby: buildRubyEnv, - CSharp: buildCSharpEnv, - Java: buildJavaEnv, - WebAssembly: buildWasmEnv, - C: buildCEnv, - // Cobol: buildCobol, - // File: buildFile, - RPC: buildRPCEnv, -} - -var runtimeDepsFuncMap = map[LanguageType]func(llb.State) llb.State{ - Python: buildPyRuntime, - Node: buildNodeRuntime, - // TypeScript: buildTSRuntime, - Ruby: buildRubyRuntime, - // CSharp: buildCSharpRuntime, - // Java: buildJavaRuntime, - // WebAssembly: buildWasmRuntime, - // Cobol: buildCobolRuntime, - // File: buildFile, - RPC: buildRpcRuntime, -} - -var LanguageKeys = (func() []string { - keys := make([]string, len(LanguageMap)) - for k, v := range LanguageMap { - keys[v] = k - } - return keys -})() - -func (lang LanguageType) String() string { - return LanguageKeys[lang] -} - -type BuildOptions struct { - version string - languages []LanguageType -} - -func ParseLanguages(args []string) ([]LanguageType, error) { - languages := make([]LanguageType, len(args)) - - for i, lang := range args { - langId, found := LanguageMap[lang] - - if !found { - return nil, fmt.Errorf("invalid language, the key '%s' does not exist", lang) - } - - languages[i] = langId - } - - return languages, nil -} - -func buildNodeEnv(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get -y --no-install-recommends install python3 g++ make nodejs npm")).Root() -} - -func buildPyEnv(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get -y --no-install-recommends install python3 python3-dev python3-pip")).Root() -} - -func buildRubyEnv(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get -y --no-install-recommends install ruby2.7 ruby2.7-dev")).Root() -} - -func buildRPCEnv(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get -y --no-install-recommends install libcurl4-openssl-dev")).Root() -} - -func buildJavaEnv(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get -y --no-install-recommends install default-jdk")).Root() -} - -func buildCSharpEnv(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get install wget")). - Run(llb.Shlex("wget https://packages.microsoft.com/config/debian/11/packages-microsoft-prod.deb -O packages-microsoft-prod.deb")). - Run(llb.Shlex("dpkg -i packages-microsoft-prod.deb")). - Run(llb.Shlex("rm packages-microsoft-prod.deb")). - Run(llb.Shlex("apt-get install -y apt-transport-https")). - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get install -y dotnet-sdk-5.0")). - Run(llb.Shlex("apt-get -y remove wget")). - Run(llb.Shlex("apt-get -y autoremove --purge")).Root() - -} - -func buildWasmEnv(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get -y --no-install-recommends install xz-utils wget")). - Run(llb.Shlex("wget https://wasmtime.dev/install.sh")). - Run(llb.Shlex("bash install.sh")). - Run(llb.Shlex("rm install.sh")). - Run(llb.Shlex("apt-get -y remove wget xz-utils")). - Run(llb.Shlex("apt-get -y autoremove --purge")).Root() -} - -func buildCEnv(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get -y --no-install-recommends install cmake build-essential")).Root() -} - -func buildDevDepsBase(baseImg llb.State, version string) llb.State { - return baseImg. - Run(llb.Shlex("apt-get update")). - Run(llb.Shlex("apt-get -y --no-install-recommends install git wget gpg apt-transport-https ca-certificates cmake build-essential")). - Run(llb.Shlexf("git clone --depth 1 --single-branch --branch=%v https://github.com/metacall/core.git", version)). - Run(llb.Shlex("mkdir core/build")). - Dir("core/build"). - Run(llb.Shlex("cmake -DOPTION_BUILD_SCRIPTS=OFF -DOPTION_BUILD_EXAMPLES=OFF -DOPTION_BUILD_TESTS=OFF -DOPTION_BUILD_DOCS=OFF -DOPTION_FORK_SAFE=OFF ..")). - Run(llb.Shlexf("cmake --build . -j %v --target install", runtime.NumCPU())).Root() -} - -func buildPyRuntime(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get -y install --no-install-recommends libpython3.9")). - Run(llb.Shlex("apt-mark hold libpython3.9")).Root() -} - -func buildNodeRuntime(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get -y --no-install-recommends install libnode72")). - Run(llb.Shlex("apt-mark hold libnode72")).Root() -} - -func buildRubyRuntime(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get -y --no-install-recommends install libruby2.7")). - Run(llb.Shlex("apt-mark hold libruby2.7")).Root() -} - -func buildRpcRuntime(baseImg llb.State) llb.State { - return baseImg. - Run(llb.Shlex("apt-get -y --no-install-recommends install libcurl4")).Root() -} - -func buildlangdevimage(devDepsLang llb.State, devDepsBase llb.State) llb.State { - merged := llb.Merge([]llb.State{devDepsLang, devDepsBase}) - finalImg := merged.Dir("core/build").Run(llb.Shlex("cmake -DOPTION_BUILD_LOADERS_PY=On -DOPTION_BUILD_LOADERS_NODE=On ..")). - Run(llb.Shlexf("make -j%v", runtime.NumCPU())).Root() - return finalImg - -} - -func buildImg(langs []LanguageType, version string) { - - base := llb.Image("docker.io/library/debian:trixie-slim") - - devDepsBase := buildDevDepsBase(base, version) - - devDepsLang := base - - for _, v := range langs { - devDepsLang = envDepsFuncMap[v](devDepsLang) - } - - devLang := buildlangdevimage(devDepsLang, devDepsBase) - devLangBuild := llb.Diff(devDepsBase, devLang) - - runtimeDeps := base.Run(llb.Shlex("apt-get update")).Root() - for _, v := range langs { - runtimeDeps = runtimeDepsFuncMap[v](runtimeDeps) - } - - finalImg := llb.Merge([]llb.State{devLangBuild, devDepsBase, runtimeDeps}) - - langdepsllb, err := finalImg.Marshal(context.TODO(), llb.LinuxAmd64) - - if err != nil { - log.Fatal(err) - } - llb.WriteTo(langdepsllb, os.Stdout) - -} - -func main() { - var opt BuildOptions - var err error - flag.StringVar(&opt.version, "version", "develop", "MetaCall version to build with") - flag.Parse() - opt.languages, err = ParseLanguages(flag.Args()) - - if err != nil { - log.Fatal(err) - } - - buildImg(opt.languages, opt.version) - -} diff --git a/pkg/staging/helper.go b/pkg/staging/helper.go index e4fa994..a461b72 100644 --- a/pkg/staging/helper.go +++ b/pkg/staging/helper.go @@ -7,12 +7,6 @@ import ( "github.com/moby/buildkit/client/llb" ) -func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State { - return dst.With( - copyMultiple(src, srcpaths, dstpath), - ) -} - func validateArgs(args []string) (string, error) { cmdArgs := []string{} for _, arg := range args { @@ -25,20 +19,6 @@ func validateArgs(args []string) (string, error) { return strings.Join(cmdArgs, " "), nil } -func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption { - var stateOptions []llb.StateOption - for _, srcPath := range srcPaths { - stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath)) - } - - return func(s llb.State) llb.State { - for _, stateOption := range stateOptions { - s = stateOption(s) - } - return s - } -} - func copyFrom(src llb.State, srcPath, destPath string) llb.StateOption { return func(s llb.State) llb.State { return copy(src, srcPath, s, destPath) @@ -52,3 +32,23 @@ func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.St CreateDestPath: true, })) } + +// func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State { +// return dst.With( +// copyMultiple(src, srcpaths, dstpath), +// ) +// } + +// func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption { +// var stateOptions []llb.StateOption +// for _, srcPath := range srcPaths { +// stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath)) +// } + +// return func(s llb.State) llb.State { +// for _, stateOption := range stateOptions { +// s = stateOption(s) +// } +// return s +// } +// } diff --git a/registry/config.yml b/registry/config.yml new file mode 100644 index 0000000..762d686 --- /dev/null +++ b/registry/config.yml @@ -0,0 +1,20 @@ +version: 0.1 +log: + fields: + service: registry +storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +proxy: + remoteurl: https://registry-1.docker.io diff --git a/registry/data/.gitkeep b/registry/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/test.sh b/test/test.sh index ef7d9cf..e1cc97e 100755 --- a/test/test.sh +++ b/test/test.sh @@ -14,19 +14,21 @@ fi DOCKER_SERVICE=${1:-rootless} -# Build them separately -for mode in deps dev runtime; do - for lang in py node rb; do - echo "Building ${mode} mode with ${lang} language." - export BUILDER_ARGS="${mode} ${lang}" - ${DOCKER_CMD} up --exit-code-from ${DOCKER_SERVICE} ${DOCKER_SERVICE} - ${DOCKER_CMD} down - done -done +# TODO: +# # Build them separately +# for mode in deps dev runtime; do +# for lang in py node rb; do +# echo "Building ${mode} mode with ${lang} language." +# export BUILDER_ARGS="${mode} ${lang}" +# ${DOCKER_CMD} up --exit-code-from ${DOCKER_SERVICE} ${DOCKER_SERVICE} +# ${DOCKER_CMD} down +# done +# done -# Build the all together +# Build the languages all together echo "Building runtime mode with all languages." -export BUILDER_ARGS="runtime py node rb" +# export BUILDER_ARGS="runtime py node rb" +export BUILDER_ARGS="runtime node" ${DOCKER_CMD} up --exit-code-from ${DOCKER_SERVICE} ${DOCKER_SERVICE} ${DOCKER_CMD} down