Merge branch 'refs/heads/temp' into feat/wasm-extra-fields

.devcontainer/Dockerfile

@@ -44,16 +44,30 @@ ENV PATH=$CARGO_HOME/bin:$PATH
 COPY rust-toolchain.toml .
 RUN TOOLCHAIN_VERSION="$(grep channel rust-toolchain.toml | awk '{print $3}' | tr -d '"')" && \
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
-    --profile minimal \
     -y \
     --default-toolchain "${TOOLCHAIN_VERSION}" \
     --target wasm32-unknown-unknown
 
-# Install wasm-bindgen-cli in the same profile as other components, to sacrifice some performance & disk space to gain
-# better build caching
-RUN if [[ -z "${SCCACHE_MEMCACHED}" ]] ; then unset SCCACHE_MEMCACHED ; fi ; \
-    RUSTFLAGS="-C target-feature=-crt-static" \
-    # Meanwhile if you want to update wasm-bindgen you also need to update version in:
-    #  - packages/wasm-dpp/Cargo.toml
-    #  - packages/wasm-dpp/scripts/build-wasm.sh
-    cargo install [email protected] --locked
+# Download and install cargo-binstall
+ENV BINSTALL_VERSION=1.10.11
+RUN set -ex; \
+    if [ "$TARGETARCH" = "amd64" ]; then \
+        CARGO_BINSTALL_ARCH="x86_64-unknown-linux-musl"; \
+    elif [ "$TARGETARCH" = "arm64" ] || [ "$TARGETARCH" = "aarch64" ]; then \
+        CARGO_BINSTALL_ARCH="aarch64-unknown-linux-musl"; \
+    else \
+        echo "Unsupported architecture: $TARGETARCH"; exit 1; \
+    fi; \
+    DOWNLOAD_URL="https://github.com/cargo-bins/cargo-binstall/releases/download/v${BINSTALL_VERSION}/cargo-binstall-${CARGO_BINSTALL_ARCH}.tgz"; \
+    curl -L --fail --show-error "$DOWNLOAD_URL" -o /tmp/cargo-binstall.tgz; \
+    tar -xzf /tmp/cargo-binstall.tgz -C /tmp cargo-binstall; \
+    chmod +x /tmp/cargo-binstall; \
+    /tmp/cargo-binstall -y --force cargo-binstall; \
+    rm /tmp/cargo-binstall; \
+    cargo binstall -V
+
+RUN cargo binstall [email protected] --locked \
+                                           --no-discover-github-token \
+                                           --disable-telemetry \
+                                           --no-track \
+                                           --no-confirm

.dockerignore

@@ -2,6 +2,7 @@
 .idea
 .vscode
 .nyc_output
+.github
 *.log
 Dockerfile
 **/*.spec.ts

.github/actions/aws_credentials/action.yaml

@@ -0,0 +1,49 @@
+---
+name: "aws_credentials"
+description: |
+  Configure .aws/credentials file with provided access key ID and secret access key.
+
+  This action creates a credentials file in ${HOME}/.aws/credentials with the provided access key ID and secret access key.
+  It also sets AWS_PROFILE and AWS_SHARED_CREDENTIALS_FILE environment variables to use this profile.
+
+  It can conflict with other actions that define AWS credentials or set AWS_PROFILE env variable.
+  Explicitly set AWS_PROFILE=sccache and unset AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in case
+  of conflicting settings.
+inputs:
+  access_key_id:
+    description: Access key ID
+    required: true
+  secret_access_key:
+    description: Secret access key
+    required: true
+  profile:
+    description: AWS profile to use; set AWS_PROFILE env variable to use this profile
+    default: "default"
+
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials
+      shell: bash
+      run: |
+        mkdir -p "${HOME}/.aws"
+        cat >> ${HOME}/.aws/credentials << EOF
+        [${{ inputs.profile }}]
+        aws_access_key_id=${{ inputs.access_key_id }}
+        aws_secret_access_key=${{ inputs.secret_access_key }}
+        EOF
+        chmod -R go-rwx ${HOME}/.aws
+
+    - name: Set env variables
+      shell: bash
+      run: |
+        # Exit on any error  
+        set -euo pipefail  
+        # Validate AWS_PROFILE is not empty  
+        if [ -z "${{ inputs.profile }}" ]; then  
+          echo "Error: AWS_PROFILE cannot be empty"
+          exit 1
+        fi
+        # Export variables        
+        echo "AWS_PROFILE=${{ inputs.profile }}" >> $GITHUB_ENV
+        echo "AWS_SHARED_CREDENTIALS_FILE=${HOME}/.aws/credentials" >> $GITHUB_ENV

.github/actions/aws_ecr_login/action.yaml

@@ -0,0 +1,42 @@
+---
+# Login to AWS ECR
+name: "aws_ecr_login"
+description: "Login to AWS ECR to store Docker containers"
+inputs:
+  aws_account_id:
+    description: AWS account ID (AWS_ACCOUNT_ID)
+    required: true
+  aws_access_key_id:
+    description: Access key ID (AWS_ACCESS_KEY_ID)
+    required: true
+  aws_secret_access_key:
+    description: Secret access key (AWS_SECRET_ACCESS_KEY)
+    required: true
+  aws_region:
+    description: AWS region to use (AWS_REGION)
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Configure AWS credentials and bucket region
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ inputs.aws_access_key_id }}
+        aws-secret-access-key: ${{ inputs.aws_secret_access_key }}
+        aws-region: ${{ inputs.aws_region }}
+
+    - name: Login to ECR
+      run: |
+        aws ecr get-login-password \
+          --region ${{ inputs.aws_region }} | docker login --username AWS --password-stdin ${{ inputs.aws_account_id }}.dkr.ecr.${{ inputs.aws_region  }}.amazonaws.com
+      shell: bash
+
+    # Unset AWS credentials to avoid conflicts, as we prefer credentials from ~/.aws/credentials to authenticate
+    - name: Unset AWS credentials to avoid conflicts
+      shell: bash
+      run: |
+        echo AWS_DEFAULT_REGION='' >> $GITHUB_ENV
+        echo AWS_REGION='' >> $GITHUB_ENV
+        echo AWS_ACCESS_KEY_ID='' >> $GITHUB_ENV
+        echo AWS_SECRET_ACCESS_KEY='' >> $GITHUB_ENV

.github/actions/docker/action.yaml

@@ -1,6 +1,13 @@
 ---
 name: "Build and push docker image"
-description: "Build and push docker image by digest with Rust caching"
+description: |
+  Build and push docker image by digest with extensive caching.
+
+  This action builds and pushes a Docker image to Docker Hub.
+  It uses caching for Rust dependencies and Docker layers.
+  It also provides sccache settings to the docker builder for caching Rust compilation.
+
+  Layers cache and sccache will use the same credentials and S3 bucket, but different prefixes.
 inputs:
   image_name:
     description: Name of image in Docker Hub, like `drive`
@@ -22,27 +29,27 @@ inputs:
     description: User name to use when pushing images to Docker Hub
   dockerhub_token:
     description: Docker Hub token to use
-  cache_mounts:
-    description: Load cache mounts cache
-    default: |
-      cargo_registry_index
-      cargo_registry_cache
-      cargo_git
   cargo_profile:
     description: Cargo build profile, i.e release or dev
     default: dev
-  bucket:
-    description: S3 bucket to use for caching, must match runner define in `runs-on`
-    default: multi-runner-cache-x1xibo9c
-  region:
+  cache_bucket:
+    description: S3 bucket to use for caching (both sccache and layer cache)
+    required: true
+  cache_region:
     description: S3 bucket region
     required: true
-  aws_access_key_id:
-    description: AWS access key ID
+  cache_endpoint:
+    description: S3 endpoint to use for caching
     required: true
-  aws_secret_access_key:
-    description: AWS secret access key
+  cache_access_key_id:
+    description: Access key ID for s3 cache
     required: true
+  cache_secret_access_key:
+    description: Secret access key for s3 cache
+    required: true
+  cache_to_name:
+    description: "Save cache to name manifest (should be used only on default branch)"
+    default: "false"
 outputs:
   digest:
     value: ${{ steps.docker_build.outputs.digest }}
@@ -65,17 +72,10 @@ runs:
         install: true
         driver-opts: env.BUILDKIT_STEP_LOG_MAX_SIZE=10485760
         cleanup: false
-        config-inline: |
+        buildkitd-config-inline: |
           [worker.oci]
             gc = false
 
-    - name: Load Docker mount cache
-      uses: dcginfra/buildkit-cache-dance/inject@s5cmd
-      if: ${{ inputs.cache_mounts != '' }}
-      with:
-        bucket: ${{ inputs.bucket }}
-        mounts: ${{ inputs.cache_mounts }}
-
     - name: Set Docker tags and labels from image tag
       id: docker_meta
       uses: docker/metadata-action@v5
@@ -90,33 +90,109 @@ runs:
       id: layer_cache_settings
       with:
         name: ${{ inputs.image_name }}
-        bucket: ${{ inputs.bucket }}
+        region: ${{ inputs.cache_region }}
+        bucket: ${{ inputs.cache_bucket }}
+        endpoint: ${{ inputs.cache_endpoint }}
+        prefix: "cache-layers/${{ inputs.platform }}/"
+        cache_to_name: ${{ inputs.cache_to_name }}
+        s3_access_key_id: ${{ inputs.cache_access_key_id }}
+        s3_secret_access_key: ${{ inputs.cache_secret_access_key }}
+
+    - name: Set HOME variable to github context
+      shell: bash
+      run: echo "HOME=$HOME" >> $GITHUB_ENV
+
+    - name: Cargo cache for Docker
+      uses: actions/cache@v4
+      id: cargo-cache
+      with:
+        path: |
+          ${{ env.HOME }}/cargo-cache-registry-index
+          ${{ env.HOME }}/cargo-cache-registry-cache
+          ${{ env.HOME }}/cargo-cache-git-db
+        key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-cargo-
+
+    - name: Inject cargo cache into docker
+      uses: reproducible-containers/[email protected]
+      with:
+        cache-map: |
+          {
+            "${{ env.HOME }}/cargo-cache-registry-index": {
+              "target": "/root/.cargo/registry/index",
+              "id": "cargo_registry_index"
+            },
+            "${{ env.HOME }}/cargo-cache-registry-cache": {
+              "target": "/root/.cargo/registry/cache",
+              "id": "cargo_registry_cache"
+            },
+            "${{ env.HOME }}/cargo-cache-git-db": {
+              "target": "/root/.cargo/git/db",
+              "id": "cargo_git"
+            }
+          }
+        skip-extraction: ${{ steps.cargo-cache.outputs.cache-hit }}
+
+    - name: Yarn unplugged cache for Docker
+      uses: actions/cache@v4
+      id: yarn-cache
+      with:
+        path: ${{ env.HOME }}/yarn-unplugged-cache
+        key: ${{ inputs.platform }}-yarn-unplugged-${{ hashFiles('yarn.lock') }}
+        restore-keys: |
+          ${{ inputs.platform }}-yarn-unplugged-
+
+    - name: Set arch
+      id: arch
+      uses: actions/github-script@v6
+      with:
+        result-encoding: "string"
+        script: return '${{ inputs.platform }}'.replace('linux/', '');
+
+    - name: Inject cargo cache into docker
+      uses: reproducible-containers/[email protected]
+      with:
+        cache-map: |
+          {
+            "${{ env.HOME }}/yarn-unplugged-cache": {
+              "target": "/tmp/unplugged",
+              "id": "unplugged_${{ steps.arch.outputs.result }}"
+            }
+          }
+        skip-extraction: ${{ steps.yarn-cache.outputs.cache-hit }}
+
+    - name: Configure sccache settings
+      uses: ./.github/actions/sccache
+      id: sccache
+      with:
+        bucket: ${{ inputs.cache_bucket }}
+        region: ${{ inputs.cache_region }}
+        endpoint: ${{ inputs.cache_endpoint }}
+        access_key_id: ${{ inputs.cache_access_key_id }}
+        secret_access_key: ${{ inputs.cache_secret_access_key }}
+        platform: ${{ inputs.platform }}
+        install: false
 
     - name: Build and push Docker image ${{ inputs.image }}
       id: docker_build
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
+      env:
+        # AWS profile to be used by layer cache; sccache settings are passed via build-args
+        AWS_PROFILE: ${{ steps.layer_cache_settings.outputs.aws_profile }}
       with:
         context: .
         builder: ${{ steps.buildx.outputs.name }}
         target: ${{ inputs.target }}
         labels: ${{ steps.docker_meta.outputs.labels }}
+        push: ${{ inputs.push_tags }}
         tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }}
         platforms: ${{ inputs.platform }}
+        secret-files: |
+          AWS=${{ env.HOME }}/.aws/credentials
         build-args: |
           CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }}
-          RUSTC_WRAPPER=sccache
-          SCCACHE_BUCKET=${{ inputs.bucket }}
-          SCCACHE_REGION=${{ inputs.region }}
-          SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache
-          AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }}
-          AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }}
+          ${{ steps.sccache.outputs.env_vars }}
         cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }}
         cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}
         outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true
-
-    - name: Save Docker mount cache
-      uses: dcginfra/buildkit-cache-dance/extract@s5cmd
-      if: ${{ inputs.cache_mounts != '' }}
-      with:
-        bucket: ${{ inputs.bucket }}
-        mounts: ${{ inputs.cache_mounts }}

.github/actions/librocksdb/action.yaml

@@ -0,0 +1,49 @@
+---
+# This action builds and caches librocksdb. If we find that this solution consumes too much time, we can consider
+# prebuilding librocksdb outside of the pipeline (eg. in the grovedb release process), publish as an artifact, and
+# download it in the pipeline.
+name: "librocksdb"
+description: "Build and install librocksdb"
+inputs:
+  version:
+    description: RocksDB version, eg. "8.10.2"
+    required: false
+    default: "8.10.2"
+  force:
+    description: Force rebuild
+    required: false
+    default: "false"
+
+runs:
+  using: composite
+  steps:
+    - name: Cache librocksdb
+      uses: actions/cache@v4
+      id: librocksdb-cache
+      with:
+        key: librocksdb/${{ inputs.version }}/${{ runner.os }}/${{ runner.arch }}
+        path: /opt/rocksdb
+
+    - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' || inputs.force == 'true' }}
+      shell: bash
+      name: Build librocksdb
+      run: |
+        set -ex
+        WORKDIR=/tmp/rocksdb-build
+        mkdir -p ${WORKDIR}/rocksdb
+        mkdir -p /opt/rocksdb/usr/local/lib/
+        pushd ${WORKDIR}/rocksdb
+
+        # building rocksdb
+        git clone https://github.com/facebook/rocksdb.git -b v${{ inputs.version }} --depth 1 .
+        make -j$(nproc) static_lib
+        make DESTDIR=/opt/rocksdb install-static
+        set +x
+
+        echo Done.
+        echo Configuration:
+        echo
+        echo "ROCKSDB_STATIC='/opt/rocksdb/usr/local/lib/librocksdb.a'"
+        echo "ROCKSDB_LIB_DIR='/opt/rocksdb/usr/local/lib'"
+
+        popd