Merge branch 'testnet' into accelerator

.github/workflows/build-ton-wasm-emscripten.yml

@@ -19,7 +19,7 @@ jobs:
 
  - name: Build TON WASM artifacts
  run: |
- cd assembly/wasm
+ cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
  chmod +x fift-func-wasm-build-ubuntu.sh
  ./fift-func-wasm-build-ubuntu.sh -a
 

assembly/cicd/jenkins/test-builds.groovy

@@ -219,7 +219,7 @@ pipeline {
  steps {
  timeout(time: 180, unit: 'MINUTES') {
  sh '''
- cd assembly/wasm
+ cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
  chmod +x fift-func-wasm-build-ubuntu.sh
  ./fift-func-wasm-build-ubuntu.sh -a
  '''

assembly/wasm/fift-func-wasm-build-ubuntu.sh

@@ -1,5 +1,3 @@
-# The script builds funcfift compiler to WASM
-
 # Execute these prerequisites first
 # sudo apt update
 # sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \
@@ -11,10 +9,12 @@
 # sudo ./llvm.sh 16 all
 
 with_artifacts=false
+scratch_new=false
 
-while getopts 'a' flag; do
+while getopts 'af' flag; do
  case "${flag}" in
  a) with_artifacts=true ;;
+ f) scratch_new=true ;;
  *) break
  ;;
  esac
@@ -24,131 +24,155 @@ export CC=$(which clang-16)
 export CXX=$(which clang++-16)
 export CCACHE_DISABLE=1
 
-cd ../..
-rm -rf openssl zlib emsdk secp256k1 libsodium build
 echo `pwd`
+if [ "$scratch_new" = true ]; then
+ echo Compiling openssl zlib lz4 emsdk secp256k1 libsodium emsdk ton
+ rm -rf openssl zlib lz4 emsdk secp256k1 libsodium build
+fi
 
-git clone https://github.com/openssl/openssl.git
-cd openssl
-git checkout checkout openssl-3.1.4
-./config
-make -j16
-OPENSSL_DIR=`pwd`
-cd ..
-
-git clone https://github.com/madler/zlib.git
-cd zlib
-git checkout v1.3.1
-ZLIB_DIR=`pwd`
-cd ..
-
-git clone https://github.com/lz4/lz4.git
-cd lz4
-git checkout v1.9.4
-LZ4_DIR=`pwd`
-cd ..
-
-git clone https://github.com/bitcoin-core/secp256k1.git
-cd secp256k1
-git checkout v0.3.2
-./autogen.sh
-SECP256K1_DIR=`pwd`
-cd ..
-
-git clone https://github.com/jedisct1/libsodium
-cd libsodium
-git checkout 1.0.18-RELEASE
-SODIUM_DIR=`pwd`
-cd ..
-
-mkdir build
-cd build
-cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
--DCMAKE_CXX_STANDARD=17 \
--DOPENSSL_FOUND=1 \
--DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
--DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
--DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
--DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.so \
--DTON_USE_ABSEIL=OFF ..
-
-test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
-
-ninja fift smc-envelope
 
-test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
+if [ ! -d "openssl" ]; then
+ git clone https://github.com/openssl/openssl.git
+ cd openssl
+ git checkout openssl-3.1.4
+ ./config
+ make -j16
+ OPENSSL_DIR=`pwd`
+ cd ..
+else
+ OPENSSL_DIR=`pwd`/openssl
+ echo Using compiled openssl at $OPENSSL_DIR
+fi
 
-rm -rf *
+if [ ! -d "build" ]; then
+ mkdir build
+ cd build
+ cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_CXX_STANDARD=17 \
+ -DOPENSSL_FOUND=1 \
+ -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
+ -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
+ -DTON_USE_ABSEIL=OFF ..
+
+ test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
+ ninja fift smc-envelope
+ test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
+ rm -rf *
+ cd ..
+else
+ echo cleaning build...
+ rm -rf build/*
+fi
 
-cd ..
+if [ ! -d "emsdk" ]; then
+ git clone https://github.com/emscripten-core/emsdk.git
+echo
+ echo Using cloned emsdk
+fi
 
-git clone https://github.com/emscripten-core/emsdk.git
 cd emsdk
 ./emsdk install 3.1.19
 ./emsdk activate 3.1.19
 EMSDK_DIR=`pwd`
-ls $EMSDK_DIR
 
 . $EMSDK_DIR/emsdk_env.sh
 export CC=$(which emcc)
 export CXX=$(which em++)
 export CCACHE_DISABLE=1
 
-cd ../openssl
-
-make clean
-emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
-sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
-sed -i 's/-ldl//g' Makefile
-sed -i 's/-O3/-Os/g' Makefile
-emmake make depend
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
-
-cd ../zlib
-
-emconfigure ./configure --static
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
+cd ..
 
-cd ../lz4
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
+if [ ! -f "openssl/openssl_em" ]; then
+ cd openssl
+ make clean
+ emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
+ sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
+ sed -i 's/-ldl//g' Makefile
+ sed -i 's/-O3/-Os/g' Makefile
+ emmake make depend
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
+ touch openssl_em
+ cd ..
+else
+ echo Using compiled openssl with emscripten
+fi
 
-cd ../secp256k1
+if [ ! -d "zlib" ]; then
+ git clone https://github.com/madler/zlib.git
+ cd zlib
+ git checkout v1.3.1
+ ZLIB_DIR=`pwd`
+ emconfigure ./configure --static
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
+ cd ..
+else
+ ZLIB_DIR=`pwd`/zlib
+ echo Using compiled zlib with emscripten at $ZLIB_DIR
+fi
 
-emconfigure ./configure --enable-module-recovery
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
+if [ ! -d "lz4" ]; then
+ git clone https://github.com/lz4/lz4.git
+ cd lz4
+ git checkout v1.9.4
+ LZ4_DIR=`pwd`
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
+ cd ..
+else
+ LZ4_DIR=`pwd`/lz4
+ echo Using compiled lz4 with emscripten at $LZ4_DIR
+fi
 
-cd ../libsodium
+if [ ! -d "secp256k1" ]; then
+ git clone https://github.com/bitcoin-core/secp256k1.git
+ cd secp256k1
+ git checkout v0.3.2
+ ./autogen.sh
+ SECP256K1_DIR=`pwd`
+ emconfigure ./configure --enable-module-recovery
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
+ cd ..
+else
+ SECP256K1_DIR=`pwd`/secp256k1
+ echo Using compiled secp256k1 with emscripten at $SECP256K1_DIR
+fi
 
-emconfigure ./configure --disable-ssp
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
+if [ ! -d "libsodium" ]; then
+ git clone https://github.com/jedisct1/libsodium
+ cd libsodium
+ git checkout 1.0.18-RELEASE
+ SODIUM_DIR=`pwd`
+ emconfigure ./configure --disable-ssp
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
+ cd ..
+else
+ SODIUM_DIR=`pwd`/libsodium
+ echo Using compiled libsodium with emscripten at $SODIUM_DIR
+fi
 
-cd ../build
+cd build
 
-emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release \
+emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
 -DZLIB_FOUND=1 \
 -DZLIB_LIBRARIES=$ZLIB_DIR/libz.a \
 -DZLIB_INCLUDE_DIR=$ZLIB_DIR \
 -DLZ4_FOUND=1 \
 -DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \
 -DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \
 -DOPENSSL_FOUND=1 \
--DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
 -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
 -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \
--DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.a \
 -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \
 -DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \
--DSECP256K1_FOUND=1 \
 -DSECP256K1_INCLUDE_DIR=$SECP256K1_DIR/include \
 -DSECP256K1_LIBRARY=$SECP256K1_DIR/.libs/libsecp256k1.a \
 -DSODIUM_INCLUDE_DIR=$SODIUM_DIR/src/libsodium/include \
 -DSODIUM_LIBRARY_RELEASE=$SODIUM_DIR/src/libsodium/.libs/libsodium.a \
--DSODIUM_USE_STATIC_LIBS=ON ..
+..
 
 test $? -eq 0 || { echo "Can't configure TON with emmake "; exit 1; }
 cp -R ../crypto/smartcont ../crypto/fift/lib crypto

crypto/CMakeLists.txt

@@ -358,7 +358,8 @@ target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto)
 
 add_library(fift-lib STATIC ${FIFT_SOURCE})
 target_include_directories(fift-lib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
-target_link_libraries(fift-lib PUBLIC ton_crypto ton_db tdutils ton_block)
+target_link_libraries(fift-lib PUBLIC ton_crypto tdutils ton_block)
+
 if (USE_EMSCRIPTEN)
  target_link_options(fift-lib PRIVATE -fexceptions)
  target_compile_options(fift-lib PRIVATE -fexceptions)

crypto/block/transaction.cpp

@@ -1661,7 +1661,12 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
  std::unique_ptr<StringLoggerTail> logger;
  auto vm_log = vm::VmLog();
  if (cfg.with_vm_log) {
- size_t log_max_size = cfg.vm_log_verbosity > 0 ? 1024 * 1024 : 256;
+ size_t log_max_size = 256;
+ if (cfg.vm_log_verbosity > 4) {
+ log_max_size = 32 << 20;
+ } else if (cfg.vm_log_verbosity > 0) {
+ log_max_size = 1 << 20;
+ }
  logger = std::make_unique<StringLoggerTail>(log_max_size);
  vm_log.log_interface = logger.get();
  vm_log.log_options = td::LogOptions(VERBOSITY_NAME(DEBUG), true, false);
@@ -1673,6 +1678,7 @@ bool Transaction::prepare_compute_phase(const ComputePhaseConfig& cfg) {
  vm_log.log_mask |= vm::VmLog::DumpStack;
  if (cfg.vm_log_verbosity > 4) {
  vm_log.log_mask |= vm::VmLog::DumpStackVerbose;
+ vm_log.log_mask |= vm::VmLog::DumpC5;
  }
  }
  }

crypto/common/bitstring.cpp

@@ -640,11 +640,11 @@ long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, con
  return bits;
 }
 
-long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end) {
+long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end) {
  const char* ptr = str;
- while (ptr < str_end && buff_size && (*ptr == '0' || *ptr == '1')) {
+ while (ptr < str_end && buff_size_bits && (*ptr == '0' || *ptr == '1')) {
  *buff++ = (bool)(*ptr++ & 1);
- --buff_size;
+ --buff_size_bits;
  }
  return td::narrow_cast<long>(ptr == str_end ? ptr - str : str - ptr - 1);
 }

crypto/common/bitstring.h

@@ -58,7 +58,7 @@ unsigned long long bits_load_long_top(ConstBitPtr from, unsigned top_bits);
 long long bits_load_long(ConstBitPtr from, unsigned bits);
 unsigned long long bits_load_ulong(ConstBitPtr from, unsigned bits);
 long parse_bitstring_hex_literal(unsigned char* buff, std::size_t buff_size, const char* str, const char* str_end);
-long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size, const char* str, const char* str_end);
+long parse_bitstring_binary_literal(BitPtr buff, std::size_t buff_size_bits, const char* str, const char* str_end);
 
 void bits_sha256(BitPtr to, ConstBitPtr from, std::size_t size);
 

crypto/fift/Fift.cpp

@@ -49,7 +49,6 @@ td::Result<int> Fift::interpret_istream(std::istream& stream, std::string curren
 }
 
 td::Result<int> Fift::do_interpret(IntCtx& ctx, bool is_interactive) {
- ctx.ton_db = &config_.ton_db;
  ctx.source_lookup = &config_.source_lookup;
  ctx.dictionary = ctx.main_dictionary = ctx.context = config_.dictionary;
  ctx.output_stream = config_.output_stream;

crypto/fift/Fift.h

@@ -19,7 +19,6 @@
 #pragma once
 
 #include "SourceLookup.h"
-#include "vm/db/TonDb.h"
 #include "Dictionary.h"
 
 #include "td/utils/Status.h"
@@ -31,13 +30,11 @@ struct Fift {
  public:
  struct Config {
  fift::SourceLookup source_lookup;
- vm::TonDb ton_db;
  fift::Dictionary dictionary;
  std::ostream* output_stream{&std::cout};
  std::ostream* error_stream{&std::cerr};
  bool show_backtrace{true};
  };
- // Fift must own ton_db and dictionary, no concurrent access is allowed
  explicit Fift(Config config);
 
  td::Result<int> interpret_file(std::string fname, std::string current_dir, bool interactive = false);