Merge pull request #2 from ice-blockchain/ion-sync-upstream

Ion sync upstream

.github/workflows/build-ion-linux-android-tonlib.yml

@@ -17,7 +17,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y build-essential git cmake ninja-build automake libtool texinfo autoconf libgflags-dev \
         zlib1g-dev libssl-dev libreadline-dev libmicrohttpd-dev pkg-config libgsl-dev python3 python3-dev \
-        libtool autoconf libsodium-dev libsecp256k1-dev
+        libtool autoconf libsodium-dev libsecp256k1-dev liblz4-dev
 
     - name: Build TON
       run: |
@@ -29,4 +29,4 @@ jobs:
       uses: actions/upload-artifact@master
       with:
         name: tonlib-android
-        path: artifacts
+        path: artifacts

.github/workflows/build-ion-linux-x86-64-shared.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Install system libraries
       run: |
         sudo apt-get update
-        sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev
+        sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev
 
     - name: Install clang-16
       run: |

.github/workflows/build-ton-macos-arm64-shared.yml

@@ -0,0 +1,25 @@
+name: MacOS TON build (shared, arm64)
+
+on: [push,workflow_dispatch,workflow_call]
+
+jobs:
+  build:
+    runs-on: macos-14
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Build TON
+        run: |
+          cp assembly/native/build-macos-shared.sh .
+          chmod +x build-macos-shared.sh
+          ./build-macos-shared.sh -t -a
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@master
+        with:
+          name: ion-binaries-macos-14
+          path: artifacts

.github/workflows/create-release.yml

@@ -192,6 +192,14 @@ jobs:
           asset_name: tonlibjson.dll
           tag: ${{ steps.tag.outputs.TAG }}
 
+      - name: Upload Windows 2019 single artifact - libemulator
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-win-binaries/emulator.dll
+          asset_name: libemulator.dll
+          tag: ${{ steps.tag.outputs.TAG }}
+
       - name: Upload Windows 2019 single artifact - tonlib-cli
         uses: svenstaro/upload-release-action@v2
         with:
@@ -274,6 +282,14 @@ jobs:
           asset_name: tonlibjson-mac-x86-64.dylib
           tag: ${{ steps.tag.outputs.TAG }}
 
+      - name: Upload Mac x86-64 single artifact - libemulator
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-x86_64-macos-binaries/libemulator.dylib
+          asset_name: libemulator-mac-x86-64.dylib
+          tag: ${{ steps.tag.outputs.TAG }}
+
       - name: Upload Mac x86-64 single artifact - tonlib-cli
         uses: svenstaro/upload-release-action@v2
         with:
@@ -282,6 +298,97 @@ jobs:
           asset_name: tonlib-cli-mac-x86-64
           tag: ${{ steps.tag.outputs.TAG }}
 
+
+      # mac arm64
+
+      - name: Upload Mac arm64 artifacts
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries.zip
+          asset_name: ion-mac-arm64.zip
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - fift
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/fift
+          asset_name: fift-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - func
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/func
+          asset_name: func-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - lite-client
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/lite-client
+          asset_name: lite-client-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - rldp-http-proxy
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/rldp-http-proxy
+          asset_name: rldp-http-proxy-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - http-proxy
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/http-proxy
+          asset_name: http-proxy-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - storage-daemon-cli
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/storage-daemon-cli
+          asset_name: storage-daemon-cli-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - storage-daemon
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/storage-daemon
+          asset_name: storage-daemon-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - tonlibjson
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/libtonlibjson.dylib
+          asset_name: tonlibjson-mac-arm64.dylib
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - libemulator
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/libemulator.dylib
+          asset_name: libemulator-mac-arm64.dylib
+          tag: ${{ steps.tag.outputs.TAG }}
+
+      - name: Upload Mac arm64 single artifact - tonlib-cli
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-arm64-macos-binaries/tonlib-cli
+          asset_name: tonlib-cli-mac-arm64
+          tag: ${{ steps.tag.outputs.TAG }}
+
       # linux x86-64
 
       - name: Upload Linux x86-64 artifacts
@@ -356,6 +463,14 @@ jobs:
           asset_name: tonlibjson-linux-x86_64.so
           tag: ${{ steps.tag.outputs.TAG }}
 
+      - name: Upload Linux x86-64 single artifact - libemulator
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: artifacts/ion-x86_64-linux-binaries/libemulator.so
+          asset_name: libemulator-linux-x86_64.so
+          tag: ${{ steps.tag.outputs.TAG }}
+
       - name: Upload Linux x86-64 single artifact - tonlib-cli
         uses: svenstaro/upload-release-action@v2
         with:

.github/workflows/ton-arm64-macos.yml

@@ -0,0 +1,37 @@
+name: MacOS TON build (portable, arm64)
+
+on: [push,workflow_dispatch,workflow_call]
+
+jobs:
+  build:
+    runs-on: macos-14
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: 'recursive'
+
+    - uses: cachix/install-nix-action@v23
+      with:
+        extra_nix_config: |
+          access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build TON
+      run: |
+        cp assembly/nix/build-macos-nix.sh .
+        chmod +x build-macos-nix.sh
+        ./build-macos-nix.sh -t
+
+    - name: Simple binaries test
+      run: |
+        sudo mv /nix/store /nix/store2
+        artifacts/validator-engine -V
+        artifacts/lite-client -V
+        artifacts/fift -V
+        artifacts/func -V
+
+    - name: Upload artifacts
+      uses: actions/upload-artifact@master
+      with:
+        name: ton-arm64-macos-binaries
+        path: artifacts

CMakeLists.txt

@@ -234,7 +234,7 @@ if (THREADS_HAVE_PTHREAD_ARG)
 endif()
 
 if (TON_USE_JEMALLOC)
-  find_package(JeMalloc REQUIRED)
+  find_package(jemalloc REQUIRED)
 endif()
 
 set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \

Changelog.md

@@ -1,3 +1,30 @@
+## 2024.06 Update
+
+1. Make Jemalloc default allocator
+2. Add candidate broadcasting and caching
+3. Limit per address speed for external messages broadcast by reasonably large number 
+4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks
+5. Extended statistics and logs for celldb usage, session stats, persistent state serialization
+6. Tonlib and explorer fixes
+7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all`
+8. Add valiator-console command to stop persistent state serialization
+9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only.
+
+
+## 2024.04 Update
+
+1. Emulator: Single call optimized runGetMethod added
+2. Tonlib: a series of proof improvements, also breaking Change in `liteServer.getAllShardsInfo` method (see below)
+3. DB: usage statistics now collected, outdated persistent states are not serialized
+4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests
+5. Network: lz4 compression of block candidates (disabled by default).
+6. Overlays: add custom overlays
+7. Transaction Executor: fixed issue with due_payment collection
+
+* `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block.
+
+Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue).
+
 ## 2024.03 Update
 
 1. Preparatory (not enabled yet) code for pre-compiled smart-contract.

Dockerfile

@@ -1,6 +1,6 @@
 FROM ubuntu:22.04 as builder
 RUN apt-get update && \
-	DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev pkg-config autoconf automake libtool && \
+	DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev && \
 	rm -rf /var/lib/apt/lists/*
 ENV CC clang
 ENV CXX clang++
@@ -14,12 +14,12 @@ COPY ./ ./
 
 RUN mkdir build && \
 	cd build && \
-	cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. && \
+	cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \
 	ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
 
 FROM ubuntu:22.04
 RUN apt-get update && \
-    apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev && \
+    apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev && \
     rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /var/ion-work/db && \