From ba486a5f6aead6d24bd659e5628a0a45da7d65df Mon Sep 17 00:00:00 2001 From: Ashish <93992470+ashpect@users.noreply.github.com> Date: Wed, 28 Aug 2024 03:44:47 +0530 Subject: [PATCH 1/2] builder_cache (#29) * mounting a volume to buildkit * squash last 12 commits * remove toml * remove volume * retrigger tests * making import export dynamic * set the startup flag * Add tests support for startup & refactoring * testing * testing * testing * testing * testing * testing * testing * testing * testing * testing * was sh all along * fix startup string to bool * removing ts for testing as building it alone fails * runtime error while building c * now go failed * rust failed * just py,rb,node * just py,rb,node * trigger all the tests * trigger all the tests * trigger all the tests * trigger all the tests * trigger all the tests * trigger all the tests startup with cli, i am soooo dumb * i think this might work now * final testing * retrigger * what happens if i trigger tests with only runtime build * retrigger with backtrace added * add diff registry for run time/dev to isolate and retrigger tests with both open * disable backtrace like before, this test should fail * triggering only the startup tests with backtrace enabled * enable backtrace,both issues resolved, very demure, very mindful * cleanup and enable all tests * only add startup-rootless and startup-client as tests * Seperated tests --- .github/workflows/{test.yml => langTest.yml} | 2 +- .github/workflows/startuptest.yml | 38 ++++++++++ Dockerfile | 12 +++- cmd/builder/flags.go | 5 ++ cmd/builder/runtime.go | 10 ++- docker-compose.yml | 14 ++-- pkg/env/env.go | 4 +- pkg/staging/stages.go | 27 ++++--- {hack => scripts}/buildctl.sh | 0 scripts/builder.sh | 20 ++++++ test/test.sh | 74 ++++++++++++++++---- 11 files changed, 167 insertions(+), 39 deletions(-) rename .github/workflows/{test.yml => langTest.yml} (95%) create mode 100644 .github/workflows/startuptest.yml rename {hack => scripts}/buildctl.sh (100%) create mode 100644 scripts/builder.sh diff --git a/.github/workflows/test.yml b/.github/workflows/langTest.yml similarity index 95% rename from .github/workflows/test.yml rename to .github/workflows/langTest.yml index 7f2f6c1..7bc95a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/langTest.yml @@ -34,4 +34,4 @@ jobs: run: ./test/test.sh rootless - name: Test Daemon - run: ./test/test.sh client + run: ./test/test.sh client \ No newline at end of file diff --git a/.github/workflows/startuptest.yml b/.github/workflows/startuptest.yml new file mode 100644 index 0000000..8102638 --- /dev/null +++ b/.github/workflows/startuptest.yml @@ -0,0 +1,38 @@ +name: Build and Test MetaCall Builder for startup flow +# Seperate because the space is limited in the github actions + +on: + push: + branches: + - master + tags: + - "v*.*.*" + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # TODO: + # - name: Lint + # run: docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.59.1 golangci-lint run -v --enable-all + + - name: Build + run: docker compose build + + - name: Test Binary + env: + BUILDER_ARGS: "runtime rb" + run: docker compose up --exit-code-from binary binary + + - name: Test Startup-Rootless + run: ./test/test.sh rootless startup + + - name: Test Startup-Daemon + run: ./test/test.sh client startup diff --git a/Dockerfile b/Dockerfile index 341cae9..4a7f5aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,9 +32,13 @@ FROM moby/buildkit:master-rootless AS builder_rootless COPY --from=builder_binary --chown=user:user /builder /home/user/builder +# TO-DO : Replace with copying from scripts locally RUN printf '#!/bin/sh\n\ export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\ -/home/user/builder $@ | buildctl-daemonless.sh build --output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\ +/home/user/builder $@ | buildctl-daemonless.sh build \ +--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \ +--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \ +--output type=image,name=registry:5000/metacall/builder_output_$1,push=true,registry.insecure=true\n'\ >> /home/user/builder.sh \ && chmod 700 /home/user/builder.sh \ && chmod 700 /home/user/builder @@ -46,8 +50,12 @@ COPY --from=builder_binary --chown=root:root /builder /home/builder RUN apk add --no-cache docker +# TO-DO : Replace with copying from scripts locally RUN printf '#!/bin/sh\n\ -/home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build --output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\ +/home/builder $@ | buildctl --addr="docker-container://metacall_builder_buildkit" build \ +--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \ +--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \ +--output type=image,name=registry:5000/metacall/builder_output_$1,push=true,registry.insecure=true\n'\ >> /home/builder.sh \ && chmod 700 /home/builder.sh \ && chmod 700 /home/builder diff --git a/cmd/builder/flags.go b/cmd/builder/flags.go index dde37b6..97cbb52 100644 --- a/cmd/builder/flags.go +++ b/cmd/builder/flags.go @@ -6,8 +6,13 @@ import ( type RuntimeImageFlags struct { MetacallCli bool + Startup bool } func (i *RuntimeImageFlags) Set(cmd *cobra.Command) { cmd.Flags().BoolVar(&i.MetacallCli, "cli", false, "set to also get metacall cli in the runtime image") } + +func (i *RuntimeImageFlags) SetStartup(cmd *cobra.Command) { + cmd.Flags().BoolVar(&i.Startup, "startup", false, "startup flag to be used for building image with all languages") +} \ No newline at end of file diff --git a/cmd/builder/runtime.go b/cmd/builder/runtime.go index b3b2986..8728d00 100644 --- a/cmd/builder/runtime.go +++ b/cmd/builder/runtime.go @@ -21,15 +21,21 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { Use: "runtime", Short: "Build runtime image for MetaCall", RunE: func(cmd *cobra.Command, args []string) error { + if o.RuntimeImageFlags.MetacallCli { args = append(args, "node") } + + if o.RuntimeImageFlags.Startup { + args = staging.GetAllLanguages() + } + base := cmd.Context().Value(baseKey{}).(llb.State) devBaseLang := staging.DevBase(base, branch, args) devImage := staging.MergeStates(devBaseLang) - runtimeLang:= staging.RuntimeBase(base, branch, args) + runtimeLang := staging.RuntimeBase(base, branch, args) runtimeImage := staging.MergeStates(runtimeLang) if o.RuntimeImageFlags.MetacallCli { @@ -40,7 +46,6 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { if err != nil { return err } - cmd.SetContext(context.WithValue(cmd.Context(), finalKey{}, runtime)) return nil @@ -48,6 +53,7 @@ func NewRuntimeCmd(o *RuntimeOptions) *cobra.Command { Example: `"builder runtime -b develop --cli nodejs typescript go rust wasm java c cobol"`, } o.RuntimeImageFlags.Set(cmd) + o.RuntimeImageFlags.SetStartup(cmd) return cmd } diff --git a/docker-compose.yml b/docker-compose.yml index 7450344..373d619 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,5 @@ version: "3.7" -# volumes: -# buildkit: - services: binary: image: metacall/builder_binary @@ -21,6 +18,10 @@ services: dockerfile: Dockerfile target: builder_rootless entrypoint: /home/user/builder.sh ${BUILDER_ARGS:-} + # entrypoint: /bin/sh -c "sleep infinity" + environment: + IMPORT_REGISTRY: ${IMPORT_REGISTRY:-registry:5000/metacall/builder_startup} # Default won't be needed once we copy the script + EXPORT_REGISTRY: ${EXPORT_REGISTRY:-registry:5000/metacall/builder_dump} security_opt: - "seccomp=unconfined" - "apparmor=unconfined" @@ -37,6 +38,9 @@ services: dockerfile: Dockerfile target: builder_client entrypoint: /home/builder.sh ${BUILDER_ARGS:-} + environment: + IMPORT_REGISTRY: ${IMPORT_REGISTRY:-testing} + EXPORT_REGISTRY: ${EXPORT_REGISTRY:-testing} volumes: - /var/run/docker.sock:/var/run/docker.sock # - buildkit:/run/user/1000/buildkit/buildkitd.sock @@ -51,10 +55,6 @@ services: container_name: metacall_builder_buildkit restart: always privileged: true - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock - # - /run/containerd/containerd.sock:/run/containerd/containerd.sock - # - buildkit:/run/buildkit/buildkitd.sock healthcheck: test: buildctl debug workers interval: 10s diff --git a/pkg/env/env.go b/pkg/env/env.go index eb50444..f194953 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -137,8 +137,8 @@ func (e Env) MetaCallBuild(arg string) Env { } func (e Env) MetacallRuntime(arg string) Env { - e.state = e.state.Run(llb.Shlexf("bash /usr/local/metacall/tools/metacall-runtime.sh base ports clean %v", arg)). - Root() // TODO: Add backtrace + e.state = e.state.Run(llb.Shlexf("bash /usr/local/metacall/tools/metacall-runtime.sh base backtrace ports clean %v", arg)). + Root() return e } diff --git a/pkg/staging/stages.go b/pkg/staging/stages.go index 710090c..2801777 100644 --- a/pkg/staging/stages.go +++ b/pkg/staging/stages.go @@ -6,9 +6,8 @@ import ( ) var languageMap = map[string]string{ - "base": "base", - "py": "python", - "rb": "ruby", + "py": "python", + "rb": "ruby", // "netcore": "netcore", // "netcore2": "netcore2", // "netcore5": "netcore5", @@ -22,15 +21,15 @@ var languageMap = map[string]string{ // "v8rep52": "v8rep52", // "v8rep51": "v8rep51", "node": "nodejs", - "ts": "typescript", + // "ts": "typescript", // "file": "file", // "rpc": "rpc", - "wasm": "wasm", - "java": "java", - "c": "c", - "cob": "cobol", - "go": "go", - "rs": "rust", + // "wasm": "wasm", + // "java": "java", + // // "c": "c", + // "cob": "cobol", + // "go": "go", + // "rs": "rust", // "backtrace" : "backtrace", // "sandbox" : "sandbox", } @@ -135,3 +134,11 @@ func MergeStates(individualLangStates map[string]llb.State) llb.State { } return llb.Merge(states) } + +func GetAllLanguages() []string { + langs := []string{} + for lang := range languageMap { + langs = append(langs, lang) + } + return langs +} diff --git a/hack/buildctl.sh b/scripts/buildctl.sh similarity index 100% rename from hack/buildctl.sh rename to scripts/buildctl.sh diff --git a/scripts/builder.sh b/scripts/builder.sh new file mode 100644 index 0000000..56b224f --- /dev/null +++ b/scripts/builder.sh @@ -0,0 +1,20 @@ +#!/bin/sh\n\ +export BUILDKITD_FLAGS=--oci-worker-no-process-sandbox\n\ + +if [ -z "$EXPORT_REGISTRY" ]; then + EXPORT_REGISTRY="registry:5000/metacall/builder_cache" +fi + +if [ -z "$IMPORT_REGISTRY" ]; then + IMPORT_REGISTRY="registry:5000/metacall/builder_startup" +fi + +if [ -z "$BUILDER_BINARY" ]; then + BUILDER_BINARY="/home/user/builder" +fi + + +${BUILDER_BINARY} $@ | buildctl-daemonless.sh build \ +--export-cache type=registry,ref=${EXPORT_REGISTRY},registry.insecure=true \ +--import-cache type=registry,ref=${IMPORT_REGISTRY},registry.insecure=true \ +--output type=image,name=registry:5000/metacall/builder_output,push=true,registry.insecure=true\n'\ \ No newline at end of file diff --git a/test/test.sh b/test/test.sh index d6e7164..a11b581 100755 --- a/test/test.sh +++ b/test/test.sh @@ -16,34 +16,46 @@ else fi DOCKER_SERVICE=${1:-rootless} +TEST_TYPE=${2:-default} build() { ${DOCKER_CMD} up --exit-code-from ${DOCKER_SERVICE} ${DOCKER_SERVICE} } -test() { - build - +setupRegistry(){ + ${DOCKER_CMD} up -d registry while [ ! "$(docker inspect --format '{{json .State.Health.Status }}' metacall_builder_registry)" = "\"healthy\"" ]; do sleep 5 done +} - DOCKER_OUTPUT=`docker run --rm -v ./test/suites:/test -t localhost:5000/metacall/builder_output sh -c "metacallcli test/$1"` +cleanup(){ + ${DOCKER_CMD} down +} + +test() { + + build + setupRegistry + + first_arg=$(echo $BUILDER_ARGS | cut -d ' ' -f 1) + + DOCKER_OUTPUT=`docker run --rm -v ./test/suites:/test -t localhost:5000/metacall/builder_output_$first_arg sh -c "metacallcli test/$1"` DOCKER_OUTPUT=`echo ${DOCKER_OUTPUT} | tr -d '\r\n'` EXPECTED_OUTPUT=`echo $2 | tr -d '\r\n'` + TEST_NAME=`echo $1` if [ "${DOCKER_OUTPUT}" = "${EXPECTED_OUTPUT}" ]; then - echo "Test passed: $1" + echo "Test passed: ${TEST_NAME}" else - echo "Failed to run test: $1" + echo "Failed to run test: ${TEST_NAME}" echo "Expected output was: '${EXPECTED_OUTPUT}'" echo "Test output was: '${DOCKER_OUTPUT}'" exit 1 fi - ${DOCKER_CMD} down } # TODO: @@ -57,12 +69,44 @@ test() { # done # done -# Build the dev image with NodeJS language -echo "Building dev mode with NodeJS language." -export BUILDER_ARGS="dev node" -test node/test.js "0123456789" +defaultTests(){ + # Build the dev image with NodeJS language + echo "Building dev mode with NodeJS language." + export BUILDER_ARGS="dev node" + export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" + export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" + test node/test.js "0123456789" + cleanup + + # Build the cli image with languages all together + echo "Building cli mode with all languages." + export BUILDER_ARGS="runtime --cli py node rb" + export IMPORT_REGISTRY="registry:5000/metacall/builder_cache" + export EXPORT_REGISTRY="registry:5000/metacall/builder_cache" + test node/test.js "0123456789" + cleanup +} + +startupTests(){ + # Build the startup image with all languages + echo "Building all languages in startup mode." + export BUILDER_ARGS="runtime --cli --startup" + export EXPORT_REGISTRY="registry:5000/metacall/builder_startup" + export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" + test node/test.js "0123456789" + + + # Testing the cache registry + echo "Building cli mode with node and py languages." + export BUILDER_ARGS="runtime --cli py node" + export IMPORT_REGISTRY="registry:5000/metacall/builder_startup" + export EXPORT_REGISTRY="registry:5000/metacall/builder_dump" # To not able to rewrite the cache + test node/test.js "0123456789" # Should be quicker since all caches are already built + cleanup +} -# Build the cli image with languages all together -echo "Building cli mode with all languages." -export BUILDER_ARGS="runtime --cli py node rb" -test node/test.js "0123456789" +if [ "${TEST_TYPE}" = "default" ]; then + defaultTests +elif [ "${TEST_TYPE}" = "startup" ]; then + startupTests +fi From 697eba9f0d79f631ed49c24189fe3d8f2c58a83c Mon Sep 17 00:00:00 2001 From: Ashish <93992470+ashpect@users.noreply.github.com> Date: Wed, 28 Aug 2024 03:47:40 +0530 Subject: [PATCH 2/2] Linting_tests (#34) * mounting a volume to buildkit * squash last 12 commits * remove toml * remove volume * retrigger tests * making import export dynamic * set the startup flag * Add tests support for startup & refactoring * testing * testing * testing * testing * testing * testing * testing * testing * testing * testing * was sh all along * fix startup string to bool * removing ts for testing as building it alone fails * runtime error while building c * now go failed * rust failed * just py,rb,node * just py,rb,node * trigger all the tests * trigger all the tests * trigger all the tests * trigger all the tests * trigger all the tests * trigger all the tests startup with cli, i am soooo dumb * i think this might work now * final testing * retrigger * what happens if i trigger tests with only runtime build * retrigger with backtrace added * add diff registry for run time/dev to isolate and retrigger tests with both open * disable backtrace like before, this test should fail * triggering only the startup tests with backtrace enabled * enable backtrace,both issues resolved, very demure, very mindful * cleanup and enable all tests * only add startup-rootless and startup-client as tests * Seperated tests * linting tests * fix lint --- .github/workflows/golangci-lint.yml | 30 +++++++++++++++++++++++ pkg/staging/helper.go | 38 ++++++++++++++--------------- 2 files changed, 49 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/golangci-lint.yml diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml new file mode 100644 index 0000000..f208e48 --- /dev/null +++ b/.github/workflows/golangci-lint.yml @@ -0,0 +1,30 @@ +name: golangci-lint +on: + push: + branches: + - main + - master + pull_request: + +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + pull-requests: read + +jobs: + golangci: + strategy: + matrix: + go: [stable] + os: [ubuntu-latest, macos-latest, windows-latest] + name: lint + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + - name: golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: v1.60 \ No newline at end of file diff --git a/pkg/staging/helper.go b/pkg/staging/helper.go index 6331357..dcad8dd 100644 --- a/pkg/staging/helper.go +++ b/pkg/staging/helper.go @@ -41,22 +41,22 @@ func copy(src llb.State, srcPath string, dest llb.State, destPath string) llb.St })) } -func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State { - return dst.With( - copyMultiple(src, srcpaths, dstpath), - ) -} - -func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption { - var stateOptions []llb.StateOption - for _, srcPath := range srcPaths { - stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath)) - } - - return func(s llb.State) llb.State { - for _, stateOption := range stateOptions { - s = stateOption(s) - } - return s - } -} +// Functions which might be useful later, left as comments here : +// func copyForStates(src llb.State, dst llb.State, srcpaths []string, dstpath string) llb.State { +// return dst.With( +// copyMultiple(src, srcpaths, dstpath), +// ) +// } + +// func copyMultiple(src llb.State, srcPaths []string, destPath string) llb.StateOption { +// var stateOptions []llb.StateOption +// for _, srcPath := range srcPaths { +// stateOptions = append(stateOptions, copyFrom(src, srcPath, destPath)) +// } +// return func(s llb.State) llb.State { +// for _, stateOption := range stateOptions { +// s = stateOption(s) +// } +// return s +// } +// }