Remove rocksdb usage from fift (#1064)

* add github action for macOS 14 (arm64, M1)

* add github action (portable) for macOS 14 (arm64, M1)

* rename macOS arm64 output artifact

* Update libsodium on windows

* Compile libsodium

* Update build-windows.bat

* use upgraded libsodium 1.0.20; use compiled static libsodium for Windows instead of precompiled;

* revert libsodium 1.0.20; use compiled static libsodium for Windows instead of precompiled;

* use upgraded libsodium 1.0.20; use compiled static libsodium for Windows instead of precompiled;

* fix libsodium version 1.0.19; use compiled static libsodium for Windows instead of precompiled;

* try 1.0.20 libsodium precompiled on github

* try 1.0.18 libsodium precompiled on github

* try windows build on win server 2019

* and use PlatformToolset=v142

* use cmake -G "Visual Studio 16 2019"

* fix path to msvc 2019 on github

* separate github windows build on win server 2019 and build on win server 2022

* Update assembly/native/build-windows-2019.bat

add retry mechanism

Co-authored-by: Dr. Awesome Doge <[email protected]>

* add test-emulator; disable test groovy pipeline

* trigger all gh actions

* fix win build

* call test-emulator

* remove usage of rocksdb in fift-lib

* put back some code for test-db

* fix test-emulator

* remove usage of db-path parameter in fift

* some func adjustments

* fix checkout of openssl in fift-func-wasm-build-ubuntu.sh

* typo

* improve wasm build script for quicker turn around

* remove sENVIRONMENT=web,worker for funcfiftlib. will be added later.

* remove sENVIRONMENT=web,worker for funcfiftlib. will be added later.

* remove sENVIRONMENT=web,worker for funcfiftlib. will be added later.

* minor adjustments

* remove -d option in fift; optimize fift-lib usage

* reduce tondb usage

---------

Co-authored-by: neodiX <[email protected]>
Co-authored-by: Dr. Awesome Doge <[email protected]>
Co-authored-by: ms <[email protected]>

.github/workflows/build-ton-wasm-emscripten.yml

@@ -19,7 +19,7 @@ jobs:
 
  - name: Build TON WASM artifacts
  run: |
- cd assembly/wasm
+ cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
  chmod +x fift-func-wasm-build-ubuntu.sh
  ./fift-func-wasm-build-ubuntu.sh -a
 

assembly/cicd/jenkins/test-builds.groovy

@@ -219,7 +219,7 @@ pipeline {
  steps {
  timeout(time: 180, unit: 'MINUTES') {
  sh '''
- cd assembly/wasm
+ cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
  chmod +x fift-func-wasm-build-ubuntu.sh
  ./fift-func-wasm-build-ubuntu.sh -a
  '''

assembly/wasm/fift-func-wasm-build-ubuntu.sh

@@ -1,5 +1,3 @@
-# The script builds funcfift compiler to WASM
-
 # Execute these prerequisites first
 # sudo apt update
 # sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \
@@ -11,10 +9,12 @@
 # sudo ./llvm.sh 16 all
 
 with_artifacts=false
+scratch_new=false
 
-while getopts 'a' flag; do
+while getopts 'af' flag; do
  case "${flag}" in
  a) with_artifacts=true ;;
+ f) scratch_new=true ;;
  *) break
  ;;
  esac
@@ -24,131 +24,155 @@ export CC=$(which clang-16)
 export CXX=$(which clang++-16)
 export CCACHE_DISABLE=1
 
-cd ../..
-rm -rf openssl zlib emsdk secp256k1 libsodium build
 echo `pwd`
+if [ "$scratch_new" = true ]; then
+ echo Compiling openssl zlib lz4 emsdk secp256k1 libsodium emsdk ton
+ rm -rf openssl zlib lz4 emsdk secp256k1 libsodium build
+fi
 
-git clone https://github.com/openssl/openssl.git
-cd openssl
-git checkout checkout openssl-3.1.4
-./config
-make -j16
-OPENSSL_DIR=`pwd`
-cd ..
-
-git clone https://github.com/madler/zlib.git
-cd zlib
-git checkout v1.3.1
-ZLIB_DIR=`pwd`
-cd ..
-
-git clone https://github.com/lz4/lz4.git
-cd lz4
-git checkout v1.9.4
-LZ4_DIR=`pwd`
-cd ..
-
-git clone https://github.com/bitcoin-core/secp256k1.git
-cd secp256k1
-git checkout v0.3.2
-./autogen.sh
-SECP256K1_DIR=`pwd`
-cd ..
-
-git clone https://github.com/jedisct1/libsodium
-cd libsodium
-git checkout 1.0.18-RELEASE
-SODIUM_DIR=`pwd`
-cd ..
-
-mkdir build
-cd build
-cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
--DCMAKE_CXX_STANDARD=17 \
--DOPENSSL_FOUND=1 \
--DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
--DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
--DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
--DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.so \
--DTON_USE_ABSEIL=OFF ..
-
-test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
-
-ninja fift smc-envelope
 
-test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
+if [ ! -d "openssl" ]; then
+ git clone https://github.com/openssl/openssl.git
+ cd openssl
+ git checkout openssl-3.1.4
+ ./config
+ make -j16
+ OPENSSL_DIR=`pwd`
+ cd ..
+else
+ OPENSSL_DIR=`pwd`/openssl
+ echo Using compiled openssl at $OPENSSL_DIR
+fi
 
-rm -rf *
+if [ ! -d "build" ]; then
+ mkdir build
+ cd build
+ cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_CXX_STANDARD=17 \
+ -DOPENSSL_FOUND=1 \
+ -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
+ -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
+ -DTON_USE_ABSEIL=OFF ..
+
+ test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
+ ninja fift smc-envelope
+ test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
+ rm -rf *
+ cd ..
+else
+ echo cleaning build...
+ rm -rf build/*
+fi
 
-cd ..
+if [ ! -d "emsdk" ]; then
+ git clone https://github.com/emscripten-core/emsdk.git
+echo
+ echo Using cloned emsdk
+fi
 
-git clone https://github.com/emscripten-core/emsdk.git
 cd emsdk
 ./emsdk install 3.1.19
 ./emsdk activate 3.1.19
 EMSDK_DIR=`pwd`
-ls $EMSDK_DIR
 
 . $EMSDK_DIR/emsdk_env.sh
 export CC=$(which emcc)
 export CXX=$(which em++)
 export CCACHE_DISABLE=1
 
-cd ../openssl
-
-make clean
-emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
-sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
-sed -i 's/-ldl//g' Makefile
-sed -i 's/-O3/-Os/g' Makefile
-emmake make depend
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
-
-cd ../zlib
-
-emconfigure ./configure --static
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
+cd ..
 
-cd ../lz4
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
+if [ ! -f "openssl/openssl_em" ]; then
+ cd openssl
+ make clean
+ emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
+ sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
+ sed -i 's/-ldl//g' Makefile
+ sed -i 's/-O3/-Os/g' Makefile
+ emmake make depend
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
+ touch openssl_em
+ cd ..
+else
+ echo Using compiled openssl with emscripten
+fi
 
-cd ../secp256k1
+if [ ! -d "zlib" ]; then
+ git clone https://github.com/madler/zlib.git
+ cd zlib
+ git checkout v1.3.1
+ ZLIB_DIR=`pwd`
+ emconfigure ./configure --static
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
+ cd ..
+else
+ ZLIB_DIR=`pwd`/zlib
+ echo Using compiled zlib with emscripten at $ZLIB_DIR
+fi
 
-emconfigure ./configure --enable-module-recovery
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
+if [ ! -d "lz4" ]; then
+ git clone https://github.com/lz4/lz4.git
+ cd lz4
+ git checkout v1.9.4
+ LZ4_DIR=`pwd`
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
+ cd ..
+else
+ LZ4_DIR=`pwd`/lz4
+ echo Using compiled lz4 with emscripten at $LZ4_DIR
+fi
 
-cd ../libsodium
+if [ ! -d "secp256k1" ]; then
+ git clone https://github.com/bitcoin-core/secp256k1.git
+ cd secp256k1
+ git checkout v0.3.2
+ ./autogen.sh
+ SECP256K1_DIR=`pwd`
+ emconfigure ./configure --enable-module-recovery
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
+ cd ..
+else
+ SECP256K1_DIR=`pwd`/secp256k1
+ echo Using compiled secp256k1 with emscripten at $SECP256K1_DIR
+fi
 
-emconfigure ./configure --disable-ssp
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
+if [ ! -d "libsodium" ]; then
+ git clone https://github.com/jedisct1/libsodium
+ cd libsodium
+ git checkout 1.0.18-RELEASE
+ SODIUM_DIR=`pwd`
+ emconfigure ./configure --disable-ssp
+ emmake make -j16
+ test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
+ cd ..
+else
+ SODIUM_DIR=`pwd`/libsodium
+ echo Using compiled libsodium with emscripten at $SODIUM_DIR
+fi
 
-cd ../build
+cd build
 
-emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release \
+emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
 -DZLIB_FOUND=1 \
 -DZLIB_LIBRARIES=$ZLIB_DIR/libz.a \
 -DZLIB_INCLUDE_DIR=$ZLIB_DIR \
 -DLZ4_FOUND=1 \
 -DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \
 -DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \
 -DOPENSSL_FOUND=1 \
--DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
 -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
 -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \
--DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.a \
 -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \
 -DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \
--DSECP256K1_FOUND=1 \
 -DSECP256K1_INCLUDE_DIR=$SECP256K1_DIR/include \
 -DSECP256K1_LIBRARY=$SECP256K1_DIR/.libs/libsecp256k1.a \
 -DSODIUM_INCLUDE_DIR=$SODIUM_DIR/src/libsodium/include \
 -DSODIUM_LIBRARY_RELEASE=$SODIUM_DIR/src/libsodium/.libs/libsodium.a \
--DSODIUM_USE_STATIC_LIBS=ON ..
+..
 
 test $? -eq 0 || { echo "Can't configure TON with emmake "; exit 1; }
 cp -R ../crypto/smartcont ../crypto/fift/lib crypto

crypto/CMakeLists.txt

@@ -358,7 +358,8 @@ target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto)
 
 add_library(fift-lib STATIC ${FIFT_SOURCE})
 target_include_directories(fift-lib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
-target_link_libraries(fift-lib PUBLIC ton_crypto ton_db tdutils ton_block)
+target_link_libraries(fift-lib PUBLIC ton_crypto tdutils ton_block)
+
 if (USE_EMSCRIPTEN)
  target_link_options(fift-lib PRIVATE -fexceptions)
  target_compile_options(fift-lib PRIVATE -fexceptions)

crypto/fift/Fift.cpp

@@ -49,7 +49,6 @@ td::Result<int> Fift::interpret_istream(std::istream& stream, std::string curren
 }
 
 td::Result<int> Fift::do_interpret(IntCtx& ctx, bool is_interactive) {
- ctx.ton_db = &config_.ton_db;
  ctx.source_lookup = &config_.source_lookup;
  ctx.dictionary = ctx.main_dictionary = ctx.context = config_.dictionary;
  ctx.output_stream = config_.output_stream;

crypto/fift/Fift.h

@@ -19,7 +19,6 @@
 #pragma once
 
 #include "SourceLookup.h"
-#include "vm/db/TonDb.h"
 #include "Dictionary.h"
 
 #include "td/utils/Status.h"
@@ -31,13 +30,11 @@ struct Fift {
  public:
  struct Config {
  fift::SourceLookup source_lookup;
- vm::TonDb ton_db;
  fift::Dictionary dictionary;
  std::ostream* output_stream{&std::cout};
  std::ostream* error_stream{&std::cerr};
  bool show_backtrace{true};
  };
- // Fift must own ton_db and dictionary, no concurrent access is allowed
  explicit Fift(Config config);
 
  td::Result<int> interpret_file(std::string fname, std::string current_dir, bool interactive = false);

crypto/fift/fift-main.cpp

@@ -46,8 +46,6 @@
 #include "SourceLookup.h"
 #include "words.h"
 
-#include "vm/db/TonDb.h"
-
 #include "td/utils/logging.h"
 #include "td/utils/misc.h"
 #include "td/utils/Parser.h"
@@ -65,7 +63,6 @@ void usage(const char* progname) {
  "\t-I<source-search-path>\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, "
  "$FIFTPATH is used instead.\n"
  "\t-L<library-fif-file>\tPre-loads a library source file\n"
- "\t-d<ton-db-path>\tUse a ton database\n"
  "\t-s\tScript mode: use first argument as a fift source file and import remaining arguments as $n)\n"
  "\t-v<verbosity-level>\tSet verbosity level\n"
  "\t-V<version>\tShow fift build information\n";
@@ -94,13 +91,12 @@ int main(int argc, char* const argv[]) {
  bool script_mode = false;
  std::vector<std::string> library_source_files, source_list;
  std::vector<std::string> source_include_path;
- std::string ton_db_path;
 
  fift::Fift::Config config;
 
  int i;
  int new_verbosity_level = VERBOSITY_NAME(INFO);
- while (!script_mode && (i = getopt(argc, argv, "hinI:L:d:sv:V")) != -1) {
+ while (!script_mode && (i = getopt(argc, argv, "hinI:L:sv:V")) != -1) {
  switch (i) {
  case 'i':
  interactive = true;
@@ -115,9 +111,6 @@ int main(int argc, char* const argv[]) {
  case 'L':
  library_source_files.emplace_back(optarg);
  break;
- case 'd':
- ton_db_path = optarg;
- break;
  case 's':
  script_mode = true;
  break;
@@ -158,16 +151,6 @@ int main(int argc, char* const argv[]) {
  config.source_lookup.add_include_path(path);
  }
 
- if (!ton_db_path.empty()) {
- auto r_ton_db = vm::TonDbImpl::open(ton_db_path);
- if (r_ton_db.is_error()) {
- LOG(ERROR) << "Error opening ton database: " << r_ton_db.error().to_string();
- std::exit(2);
- }
- config.ton_db = r_ton_db.move_as_ok();
- // FIXME //std::atexit([&] { config.ton_db.reset(); });
- }
-
  fift::init_words_common(config.dictionary);
  fift::init_words_vm(config.dictionary, true); // enable vm debug
  fift::init_words_ton(config.dictionary);

crypto/fift/utils.cpp

@@ -22,6 +22,8 @@
 #include "td/utils/filesystem.h"
 #include "td/utils/misc.h"
 #include "td/utils/port/path.h"
+#include "vm/boc.h"
+#include <map>
 
 namespace fift {
 namespace {