Merge pull request #33 from nervosnetwork/v1.0.0-rc1

Build v1.0.0-rc1

.github/workflows/docker-publish.yml

@@ -1,42 +1,48 @@
 name: Docker
 
 on:
-  # TODO: build nightly image
+  # Build v1.0.x image every 12 hours
   # schedule:
-  #   - cron: '22 3 * * *'
+  # - cron: '22 */12 * * *'
   push:
-    branches: [ main, 'docker-publish*' ]
+    branches: [ 'dev', 'v1*' ]
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
-  pull_request:
-    branches: [ main ]
 
 env:
   # Use docker.io for Docker Hub if empty
-  REGISTRY: ${{ github.repository_owner != 'nervosnetwork' && 'ghcr.io/' || '' }}
+  REGISTRY: ${{ github.ref_type != 'tag' && 'ghcr.io/' || '' }}
   # github.repository as <account>/<repo>
   IMAGE_NAME: godwoken-prebuilds
 
 
 jobs:
   docker-build-push:
     runs-on: ubuntu-latest
+    # Map the meta step outputs to this job outputs
+    outputs:
+      image_name: ${{ steps.result.outputs.image_name }}
+      image_tag: ${{ steps.result.outputs.image_tag }}
+    # If you specify the access for any of these scopes, all of those that are not specified are set to none.
     permissions:
       contents: read
       packages: write
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            ~/.cargo/bin
-            target
-          key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-cargo-
 
       - name: Install Rust components
         run: rustup component add rustfmt && rustup component add clippy
@@ -56,33 +62,116 @@ jobs:
 
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
-      # The GHP_CRT secret is password or personal access token with `write:packages` access scope
+      # GitHub automatically creates a unique GITHUB_TOKEN secret to use in this workflow.
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v1
         with:
           registry: ${{ env.REGISTRY }}
-          username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }}
-          password: ${{ secrets.DOCKERHUB_TOKEN || secrets.GHP_CRT }}
+          username: ${{ github.ref_type != 'tag' && github.repository_owner || secrets.DOCKERHUB_USERNAME }}
+          password: ${{ github.ref_type != 'tag' && secrets.GITHUB_TOKEN || secrets.DOCKERHUB_TOKEN }}
+
+      - name: Prepare components
+        id: prepare
+        run: |
+          make prepare-repos
+          echo "Record the component's reference to the outputs of this step"
+          cat versions
+
+      - name: Print the references of components
+        run: |
+          echo ref.component.godwoken=${{ steps.prepare.outputs.GODWOKEN_REF }}
+          echo ref.component.godwoken-scripts=${{ steps.prepare.outputs.GODWOKEN_SCRIPTS_REF }}
+          echo ref.component.godwoken-polyjuice=${{ steps.prepare.outputs.POLYJUICE_REF }}
+          echo ref.component.ckb-production-scripts=${{ steps.prepare.outputs.OMNI_LOCK_REF }}
+
+      - name: Cache of component.ckb-production-scripts
+        id: ckb-production-scripts-cache
+        uses: actions/cache@v3
+        with:
+          path: build/ckb-production-scripts/build/omni_lock
+          key: component.ckb-production-scripts-${{ steps.prepare.outputs.ckb-production-scripts-sha1 }}
+      - name: Build omni_lock
+        if: steps.ckb-production-scripts-cache.outputs.cache-hit != 'true'
+        working-directory: build/ckb-production-scripts
+        run: make all-via-docker
+
+      - name: Cache of component.godwoken-scripts
+        id: godwoken-scripts-cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            build/godwoken-scripts/build/release/*
+            build/godwoken-scripts/c/build/*-generator
+            build/godwoken-scripts/c/build/*-validator
+            build/godwoken-scripts/c/build/account_locks/*
+          key: component.godwoken-scripts-${{ steps.prepare.outputs.godwoken-scripts-sha1 }}
+      - name: Build godwoken-scripts
+        if: steps.godwoken-scripts-cache.outputs.cache-hit != 'true'
+        working-directory: build/godwoken-scripts
+        run: cd c && make && cd .. && capsule build --release --debug-output
+
+      - name: Cache of component.godwoken-polyjuice
+        id: godwoken-polyjuice-cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            build/godwoken-polyjuice/build/*generator*
+            build/godwoken-polyjuice/build/*validator*
+          key: component.godwoken-polyjuice-${{ steps.prepare.outputs.godwoken-polyjuice-sha1 }}
+      - name: Build godwoken-polyjuice
+        if: steps.godwoken-polyjuice-cache.outputs.cache-hit != 'true'
+        working-directory: build/godwoken-polyjuice
+        run: make all-via-docker
+
+      - name: Cache of component.godwoken
+        id: godwoken-cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            build/godwoken/target/release/godwoken
+            build/godwoken/target/release/gw-tools
+          key: component.godwoken-${{ steps.prepare.outputs.godwoken-sha1 }}
+      - name: Build godwoken
+        if: steps.godwoken-cache.outputs.cache-hit != 'true'
+        working-directory: build/godwoken
+        run: rustup component add rustfmt && cargo build --release
 
       # Extract metadata (tags, labels) for Docker
       # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
         uses: docker/metadata-action@v3
         with:
-          images: ${{ env.REGISTRY }}${{ github.repository_owner == 'nervosnetwork' && 'nervos' || github.repository_owner }}/${{ env.IMAGE_NAME }}
-
-      - name: build
-        run: make build-components
+          images: ${{ env.REGISTRY }}${{ startsWith(github.ref, 'refs/tags') && github.repository_owner == 'nervosnetwork' && 'nervos' || github.repository_owner }}/${{ env.IMAGE_NAME }}
+          # dynamically set date as a suffix
+          tags: |
+            type=ref,event=branch,suffix=-{{date 'YYYYMMDDHHmm'}}
+            type=ref,event=branch
+          labels: |
+            maintainer=Nervos Core Dev <[email protected]>
+            org.opencontainers.image.authors=Nervos Core Dev <[email protected]>
+            source.component.godwoken=https://github.com/nervosnetwork/godwoken
+            source.component.godwoken-scripts-Scripts=https://github.com/nervosnetwork/godwoken-scripts
+            source.component.godwoken-polyjuice=https://github.com/nervosnetwork/godwoken-polyjuice
+            source.component.ckb-production-scripts=https://github.com/nervosnetwork/ckb-production-scripts
+            ref.component.godwoken=${{ steps.prepare.outputs.GODWOKEN_REF }}
+            ref.component.godwoken-sha1=${{ steps.prepare.outputs.godwoken-sha1 }}
+            ref.component.godwoken-scripts=${{ steps.prepare.outputs.GODWOKEN_SCRIPTS_REF }}
+            ref.component.godwoken-scripts-sha1=${{ steps.prepare.outputs.godwoken-scripts-sha1 }}
+            ref.component.godwoken-polyjuice=${{ steps.prepare.outputs.POLYJUICE_REF }}
+            ref.component.godwoken-polyjuice-sha1=${{ steps.prepare.outputs.godwoken-polyjuice-sha1 }}
+            ref.component.ckb-production-scripts=${{ steps.prepare.outputs.OMNI_LOCK_REF }}
+            ref.component.ckb-production-scripts-sha1=${{ steps.prepare.outputs.ckb-production-scripts-sha1 }}
 
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image to ${{ env.REGISTRY }}${{ github.repository_owner }}/${{ env.IMAGE_NAME }}
-        if: ${{ github.repository_owner != 'nervosnetwork'}}
+        if: ${{ github.ref_type != 'tag' }}
         uses: docker/build-push-action@v2
         with:
           context: .
+          file: Dockerfile.fast
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -94,17 +183,33 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
+          file: Dockerfile.fast
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
-      # - name: Check versions of the binaries used by Godwoken
-      #   run: |
-      # FIXME: /lib/x86_64-linux-gnu/libm.so.6: version `GLIBC_2.29' not found (required by godwoken)
-      #     docker run --rm godwoken-prebuilds godwoken --version
-      #     docker run --rm godwoken-prebuilds gw-tools --version
-      ###
-      #     docker run --rm godwoken-prebuilds ckb --version
-      #     docker run --rm godwoken-prebuilds ckb-cli --version
-      #     docker run --rm godwoken-prebuilds ckb-indexer --version
-      #     docker run --rm godwoken-prebuilds find /scripts
+      - name: Check versions of the binaries in ${{ fromJSON(steps.meta.outputs.json).tags[0] }}
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          IMAGE: ${{ fromJSON(steps.meta.outputs.json).tags[0] }}
+        run: |
+          docker run --rm ${{ env.IMAGE }} godwoken --version
+          docker run --rm ${{ env.IMAGE }} gw-tools --version
+          docker run --rm ${{ env.IMAGE }} ckb --version
+          docker run --rm ${{ env.IMAGE }} ckb-cli --version
+          docker run --rm ${{ env.IMAGE }} ckb-indexer --version
+          docker run --rm ${{ env.IMAGE }} find /scripts -type f -exec sha1sum {} \;
+
+      - name: Record image info to the outputs of this jobs
+        id: result
+        run: |
+          echo "::set-output name=image_name::`echo ${{ fromJSON(steps.meta.outputs.json).tags[0] }} | awk -F ':' '{print $1}'`"
+          echo "::set-output name=image_tag::`echo ${{ fromJSON(steps.meta.outputs.json).tags[0] }} | awk -F ':' '{print $NF}'`"
+
+  integration-test:
+    needs: docker-build-push
+    uses: nervosnetwork/godwoken-tests/.github/workflows/reusable-integration-test-v1.yml@v1
+    with:
+      gw_prebuild_image_name: ${{ needs.docker-build-push.outputs.image_name }}
+      gw_prebuild_image_tag: ${{ needs.docker-build-push.outputs.image_tag }}
+      # gw_prebuild_image_digest: ${{ needs.docker-build-push.outputs.image_digest }}

Dockerfile

@@ -1,5 +1,4 @@
 FROM rust:1 as builder
-MAINTAINER Xuejie Xiao <[email protected]>
 
 RUN apt-get update
 RUN apt-get -y install --no-install-recommends llvm-dev clang libclang-dev libssl-dev
@@ -18,11 +17,9 @@ RUN cd /ckb-indexer && curl -LO https://github.com/nervosnetwork/ckb-indexer/rel
 RUN cd /ckb-indexer && unzip ckb-indexer-0.3.2-linux.zip && tar xzf ckb-indexer-linux-x86_64.tar.gz
 
 FROM ubuntu:21.04
-MAINTAINER Xuejie Xiao <[email protected]>
 
 RUN mkdir -p /scripts/godwoken-scripts \
- && mkdir -p /scripts/godwoken-polyjuice \
- && mkdir -p /scripts/clerkb
+ && mkdir -p /scripts/godwoken-polyjuice
 
 RUN apt-get update \
  && apt-get dist-upgrade -y \
@@ -45,12 +42,11 @@ COPY build/godwoken-scripts/c/build/*-generator /scripts/godwoken-scripts/
 COPY build/godwoken-scripts/c/build/*-validator /scripts/godwoken-scripts/
 COPY build/godwoken-scripts/c/build/account_locks/* /scripts/godwoken-scripts/
 
-# /scripts/godwoken-polyjuice
-COPY build/godwoken-polyjuice/build/generator* /scripts/godwoken-polyjuice/
-COPY build/godwoken-polyjuice/build/validator* /scripts/godwoken-polyjuice/
+# /scripts/omni-lock
+COPY build/ckb-production-scripts/build/omni_lock /scripts/godwoken-scripts/
 
-# /scripts/clerkb
-COPY build/clerkb/build/debug/poa /scripts/clerkb/
-COPY build/clerkb/build/debug/state /scripts/clerkb/
+# /scripts/godwoken-polyjuice
+COPY build/godwoken-polyjuice/build/*generator* /scripts/godwoken-polyjuice/
+COPY build/godwoken-polyjuice/build/*validator* /scripts/godwoken-polyjuice/
 
 CMD [ "godwoken", "--version" ]

Dockerfile.fast

@@ -0,0 +1,46 @@
+FROM ubuntu:21.04
+
+RUN mkdir -p /scripts/godwoken-scripts \
+ && mkdir -p /scripts/godwoken-polyjuice \
+ && mkdir /ckb \
+ && mkdir /ckb-indexer
+
+RUN apt-get update \
+ && apt-get dist-upgrade -y \
+ && apt-get install -y curl unzip \
+ && apt-get clean \
+ && echo 'Finished installing OS updates'
+
+# ckb
+RUN cd /ckb \
+ && curl -LO https://github.com/nervosnetwork/ckb/releases/download/v0.101.7/ckb_v0.101.7_x86_64-unknown-linux-gnu.tar.gz \
+ && tar xzf ckb_v0.101.7_x86_64-unknown-linux-gnu.tar.gz \
+ && cp ckb_v0.101.7_x86_64-unknown-linux-gnu/ckb /bin/ckb \
+ && cp ckb_v0.101.7_x86_64-unknown-linux-gnu/ckb-cli /bin/ckb-cli \
+ && rm -rf /ckb
+
+# ckb-indexer
+RUN cd /ckb-indexer \
+ && curl -LO https://github.com/nervosnetwork/ckb-indexer/releases/download/v0.3.2/ckb-indexer-0.3.2-linux.zip \
+ && unzip ckb-indexer-0.3.2-linux.zip && tar xzf ckb-indexer-linux-x86_64.tar.gz \
+ && cp /ckb-indexer/ckb-indexer /bin/ckb-indexer \
+ && rm -rf /ckb-indexer
+
+# /scripts/omni-lock
+COPY build/ckb-production-scripts/build/omni_lock /scripts/godwoken-scripts/
+
+# /scripts/godwoken-scripts
+COPY build/godwoken-scripts/build/release/* /scripts/godwoken-scripts/
+COPY build/godwoken-scripts/c/build/*-generator /scripts/godwoken-scripts/
+COPY build/godwoken-scripts/c/build/*-validator /scripts/godwoken-scripts/
+COPY build/godwoken-scripts/c/build/account_locks/* /scripts/godwoken-scripts/
+
+# /scripts/godwoken-polyjuice
+COPY build/godwoken-polyjuice/build/*generator* /scripts/godwoken-polyjuice/
+COPY build/godwoken-polyjuice/build/*validator* /scripts/godwoken-polyjuice/
+
+# godwoken
+COPY build/godwoken/target/release/godwoken /bin/godwoken
+COPY build/godwoken/target/release/gw-tools /bin/gw-tools
+
+CMD [ "godwoken", "--version" ]

Makefile

@@ -4,37 +4,40 @@ SHELL := /bin/bash
 GODWOKEN_REPO := https://github.com/nervosnetwork/godwoken.git
 GODWOKEN_SCRIPTS_REPO := https://github.com/nervosnetwork/godwoken-scripts.git
 POLYJUICE_REPO := https://github.com/nervosnetwork/godwoken-polyjuice.git
-CLERKB_REPO := https://github.com/nervosnetwork/clerkb.git
+OMNI_LOCK_REPO := https://github.com/nervosnetwork/ckb-production-scripts.git
 
 # components tags
-GODWOKEN_REF := v0.10.3
-GODWOKEN_SCRIPTS_REF := v0.8.4
-POLYJUICE_REF := v0.8.10-aot
-CLERKB_REF := v0.4.0
+GODWOKEN_REF := v1.0.0-rc1
+GODWOKEN_SCRIPTS_REF := v1.0.0-rc1
+POLYJUICE_REF := v1.0.0-rc1
+OMNI_LOCK_REF := rc_lock
 
 define prepare_repo
-	if [ -d "build/$(3)" ]; then\
-		cd build/$(3);\
-		git reset --hard;\
-		git fetch --all;\
-		git checkout $(2);\
-		git submodule update --init --recursive;\
-	else\
-		git clone --recursive $(1) -b $(2) build/$(3);\
+	if [ ! -d "build/$(3)" ]; then\
+		git clone --depth=1 $(1) build/$(3);\
 	fi
+	cd build/$(3);\
+	git fetch origin $(2);\
+	git checkout FETCH_HEAD;\
+	git submodule update --init --recursive --depth=1;\
+	echo "::set-output name=$(3)-sha1::$$(git rev-parse HEAD)" >> ../../versions
 endef
 
 prepare-repos:
 	mkdir -p build
 	$(call prepare_repo,$(GODWOKEN_REPO),$(GODWOKEN_REF),godwoken)
+	echo "::set-output name=GODWOKEN_REF::$(GODWOKEN_REF) $$(cd build/godwoken && git rev-parse --short HEAD)" >> versions
 	$(call prepare_repo,$(GODWOKEN_SCRIPTS_REPO),$(GODWOKEN_SCRIPTS_REF),godwoken-scripts)
+	echo "::set-output name=GODWOKEN_SCRIPTS_REF::$(GODWOKEN_SCRIPTS_REF) $$(cd build/godwoken-scripts && git rev-parse --short HEAD)" >> versions
 	$(call prepare_repo,$(POLYJUICE_REPO),$(POLYJUICE_REF),godwoken-polyjuice)
-	$(call prepare_repo,$(CLERKB_REPO),$(CLERKB_REF),clerkb)
+	echo "::set-output name=POLYJUICE_REF::$(POLYJUICE_REF) $$(cd build/godwoken-polyjuice && git rev-parse --short HEAD)" >> versions
+	$(call prepare_repo,$(OMNI_LOCK_REPO),$(OMNI_LOCK_REF),ckb-production-scripts)
+	echo "::set-output name=OMNI_LOCK_REF::$(OMNI_LOCK_REF) $$(cd build/ckb-production-scripts && git rev-parse --short HEAD)" >> versions
 
 build-components: prepare-repos
 	cd build/godwoken-polyjuice && make dist && cd -
 	cd build/godwoken-scripts && cd c && make && cd .. && capsule build --release --debug-output && cd ../..
-	cd build/clerkb && yarn && make all-via-docker && cd ../..
+	cd build/ckb-production-scripts && make all-via-docker
 
 build-push:
 	make build-components
@@ -57,19 +60,14 @@ test:
 test-files:
 	echo "start checking build result..."
 # compare scripts files
-	make test-clerkb-files
 	make test-scripts-files
 	make test-polyjuice-files
 # compare bin files
 	cd `pwd`/test-result/bin && ./godwoken --version && ./gw-tools --version
 	[ -e "test-result" ] && rm -rf test-result
 
-test-clerkb-files:
-	source tool.sh && check_clerkb_files_exists
-
 test-scripts-files:
 	source tool.sh && check_scripts_files_exists
 
 test-polyjuice-files:
 	source tool.sh && check_polyjuice_files_exists 
-