diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 9aeb67f532c..7d3ec45658d 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -189,7 +189,7 @@ jobs: run: | mkdir -p ${{ needs.preamble.outputs.folder_build }} echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key - plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM" "CLIENTTOOLS_ONLY") + plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED" "PLATFORM" "CLIENTTOOLS_ONLY") for plugin in "${plugins[@]}"; do sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index e7bfab4a02f..24560aa6331 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -128,7 +128,7 @@ jobs: mkdir -p ${{ github.workspace }}/build mkdir -p ${{ github.workspace }}/.ccache declare -a plugins - plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }}) + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED"' }}) for plugin in "${plugins[@]}"; do sudo rm -f ${{ github.workspace }}/build/CMakeCache.txt sudo rm -rf ${{ github.workspace }}/build/CMakeFiles diff --git a/.github/workflows/build-gh_runner.yml b/.github/workflows/build-gh_runner.yml index 48823c004c5..2b352e2cb99 100644 --- a/.github/workflows/build-gh_runner.yml +++ b/.github/workflows/build-gh_runner.yml @@ -205,7 +205,7 @@ jobs: mkdir -p ${{ github.workspace }}/LN mkdir -p ${{ github.workspace }}/build declare -a plugins - plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }}) + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED"' }}) for plugin in "${plugins[@]}"; do rm -f ./build/CMakeCache.txt rm -rf ./build/CMakeFiles diff --git a/CMakeLists.txt b/CMakeLists.txt index b26f33ec7e0..621f77deaaa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -169,6 +169,7 @@ if ( PLUGIN ) HPCC_ADD_SUBDIRECTORY (dali/base) HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED") HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED") + HPCC_ADD_SUBDIRECTORY (plugins/wasmembed "WASMEMBED") HPCC_ADD_SUBDIRECTORY (plugins/memcached "MEMCACHED") HPCC_ADD_SUBDIRECTORY (plugins/redis "REDIS") HPCC_ADD_SUBDIRECTORY (plugins/javaembed "JAVAEMBED") diff --git a/cmake_modules/plugins.cmake b/cmake_modules/plugins.cmake index b96a7e7cd09..f87df647507 100644 --- a/cmake_modules/plugins.cmake +++ b/cmake_modules/plugins.cmake @@ -42,6 +42,7 @@ set(PLUGINS_LIST SQLITE3EMBED SQS V8EMBED + WASMEMBED EXAMPLEPLUGIN ) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 109e74b7abb..63de3d1cc68 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -30,6 +30,7 @@ add_subdirectory (proxies) add_subdirectory (sqlite3) add_subdirectory (mysql) add_subdirectory (v8embed) +add_subdirectory (wasmembed) HPCC_ADD_SUBDIRECTORY (py3embed "USE_PYTHON3") HPCC_ADD_SUBDIRECTORY (pyembed "USE_PYTHON2") add_subdirectory (javaembed) diff --git a/plugins/wasmembed/CMakeLists.txt b/plugins/wasmembed/CMakeLists.txt new file mode 100644 index 00000000000..7dff7328253 --- /dev/null +++ b/plugins/wasmembed/CMakeLists.txt @@ -0,0 +1,66 @@ +project(wasmembed) + +if(WASMEMBED) + ADD_PLUGIN(wasmembed) + if(MAKE_WASMEMBED) + + set(CMAKE_CXX_STANDARD 20) + find_path(WASMTIME_CPP_API_INCLUDE_DIRS "wasmtime-cpp-api/wasmtime.hh" + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + if (WIN32) + find_library(WASMTIME_LIB NAMES wasmtime.dll + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + else() + find_library(WASMTIME_LIB NAMES wasmtime + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + endif() + + include_directories( + ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-c-api + ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-cpp-api + ./../../system/include + ./../../system/jlib + ./../../rtl/eclrtl + ./../../rtl/include + + # Following includes are needed for "enginecontext.hpp" + ./../../common/thorhelper + ./../../dali/base + ./../../system/mp + ) + + add_definitions(-D_USRDLL -DWASMEMBED_EXPORTS) + + add_library(wasmembed SHARED + wasmembed.cpp + secure-enclave.cpp + abi.cpp + util.cpp + ) + + target_link_libraries(wasmembed + ${WASMTIME_LIB} + eclrtl + jlib + ) + + install( + TARGETS wasmembed + DESTINATION plugins + CALC_DEPS + ) + + else() + message(WARNING "Cannot build wasmembed plugin") + endif() +endif() + +if(PLATFORM OR CLIENTTOOLS_ONLY) + install( + FILES ${CMAKE_CURRENT_SOURCE_DIR}/wasm.ecllib + DESTINATION plugins + COMPONENT Runtime) +endif() diff --git a/plugins/wasmembed/abi.cpp b/plugins/wasmembed/abi.cpp new file mode 100644 index 00000000000..43b91339e4b --- /dev/null +++ b/plugins/wasmembed/abi.cpp @@ -0,0 +1,269 @@ +/* + See: https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md + https://github.com/WebAssembly/component-model/blob/main/design/mvp/canonical-abi/definitions.py +*/ + +#include "abi.hpp" + +#include "jexcept.hpp" + +auto UTF16_TAG = 1U << 31; + +// +/* canonical despecialize (python) ------------------------------------------------------------- + +def despecialize(t): + match t: + case Tuple(ts) : return Record([ Field(str(i), t) for i,t in enumerate(ts) ]) + case Union(ts) : return Variant([ Case(str(i), t) for i,t in enumerate(ts) ]) + case Enum(labels) : return Variant([ Case(l, None) for l in labels ]) + case Option(t) : return Variant([ Case("none", None), Case("some", t) ]) + case Result(ok, error) : return Variant([ Case("ok", ok), Case("error", error) ]) + case _ : return t + +*/ + +// template +// wasmtime::ValType despecialize(const T &t) +// { +// switch (t.kind()) +// { +// case wasmtime::ValKind::I32: +// case wasmtime::ValKind::I64: +// case wasmtime::ValKind::F32: +// case wasmtime::ValKind::F64: +// case wasmtime::ValKind::V128: +// return t.kind(); +// default: +// return wasmtime::ValType::i32(); +// } +// } + +/* canonical alignment (python) ------------------------------------------------------------- + +def alignment(t): + match despecialize(t): + case Bool() : return 1 + case S8() | U8() : return 1 + case S16() | U16() : return 2 + case S32() | U32() : return 4 + case S64() | U64() : return 8 + case Float32() : return 4 + case Float64() : return 8 + case Char() : return 4 + case String() | List(_) : return 4 + case Record(fields) : return alignment_record(fields) + case Variant(cases) : return alignment_variant(cases) + case Flags(labels) : return alignment_flags(labels) + case Own(_) | Borrow(_) : return 4 + +*/ + +// int alignment(const wasmtime::ValType &t) +// { +// switch (t.kind()) +// { +// case wasmtime::ValKind::I32: +// case wasmtime::ValKind::F32: +// return 4; +// case wasmtime::ValKind::I64: +// case wasmtime::ValKind::F64: +// return 8; +// case wasmtime::ValKind::V128: +// return 16; +// default: +// return 1; +// } +// } + +/* canonical align_to (python) ------------------------------------------------------------- + +def align_to(ptr, alignment): + return math.ceil(ptr / alignment) * alignment + +*/ + +uint32_t align_to(uint32_t ptr, uint32_t alignment) +{ + return (ptr + alignment - 1) & ~(alignment - 1); +} + +bool isAligned(uint32_t ptr, uint32_t alignment) +{ + return (ptr & (alignment - 1)) == 0; +} + +// loading --- + +/* canonical load_int (python) ------------------------------------------------------------- + +def load_int(cx, ptr, nbytes, signed = False): + return int.from_bytes(cx.opts.memory[ptr : ptr+nbytes], 'little', signed=signed) + +*/ + +template +T load_int(const wasmtime::Span &data, uint32_t ptr) +{ + T retVal = 0; + if constexpr (sizeof(T) == 1) + { + retVal = static_cast(data[ptr]); + } + else if constexpr (sizeof(T) == 2) + { + retVal = static_cast((static_cast(data[ptr + 1]) << 8) | + static_cast(data[ptr])); + } + else if constexpr (sizeof(T) == 4) + { + retVal = static_cast((static_cast(data[ptr + 3]) << 24) | + (static_cast(data[ptr + 2]) << 16) | + (static_cast(data[ptr + 1]) << 8) | + static_cast(data[ptr])); + } + else if constexpr (sizeof(T) == 8) + { + retVal = static_cast((static_cast(data[ptr + 7]) << 56) | + (static_cast(data[ptr + 6]) << 48) | + (static_cast(data[ptr + 5]) << 40) | + (static_cast(data[ptr + 4]) << 32) | + (static_cast(data[ptr + 3]) << 24) | + (static_cast(data[ptr + 2]) << 16) | + (static_cast(data[ptr + 1]) << 8) | + static_cast(data[ptr])); + } + return retVal; +} +/* canonical load_string_from_range (python) ------------------------------------------------------------- + +def load_string_from_range(cx, ptr, tagged_code_units): + match cx.opts.string_encoding: + case 'utf8': + alignment = 1 + byte_length = tagged_code_units + encoding = 'utf-8' + case 'utf16': + alignment = 2 + byte_length = 2 * tagged_code_units + encoding = 'utf-16-le' + case 'latin1+utf16': + alignment = 2 + if bool(tagged_code_units & UTF16_TAG): + byte_length = 2 * (tagged_code_units ^ UTF16_TAG) + encoding = 'utf-16-le' + else: + byte_length = tagged_code_units + encoding = 'latin-1' + + trap_if(ptr != align_to(ptr, alignment)) + trap_if(ptr + byte_length > len(cx.opts.memory)) + try: + s = cx.opts.memory[ptr : ptr+byte_length].decode(encoding) + except UnicodeError: + trap() + + return (s, cx.opts.string_encoding, tagged_code_units) + +*/ + +// More: Not currently available from the wasmtime::context object, see https://github.com/bytecodealliance/wasmtime/issues/6719 +static const std::string global_encoding = "utf8"; + +std::tuple load_string_from_range(const wasmtime::Span &data, uint32_t ptr, uint32_t tagged_code_units) +{ + std::string encoding = "utf-8"; + uint32_t byte_length = tagged_code_units; + uint32_t alignment = 1; + if (global_encoding.compare("utf8") == 0) + { + alignment = 1; + byte_length = tagged_code_units; + encoding = "utf-8"; + } + else if (global_encoding.compare("utf16") == 0) + { + alignment = 2; + byte_length = 2 * tagged_code_units; + encoding = "utf-16-le"; + } + else if (global_encoding.compare("latin1+utf16") == 0) + { + alignment = 2; + if (tagged_code_units & UTF16_TAG) + { + byte_length = 2 * (tagged_code_units ^ UTF16_TAG); + encoding = "utf-16-le"; + } + else + { + byte_length = tagged_code_units; + encoding = "latin-1"; + } + } + + if (!isAligned(ptr, alignment)) + { + throw makeStringException(3, "Invalid alignment"); + } + + if (ptr + byte_length > data.size()) + { + throw makeStringException(1, "Out of bounds"); + } + + return std::make_tuple(ptr, encoding, byte_length); +} + +/* canonical load_string (python) ------------------------------------------------------------- + +def load_string(cx, ptr): + begin = load_int(cx, ptr, 4) + tagged_code_units = load_int(cx, ptr + 4, 4) + return load_string_from_range(cx, begin, tagged_code_units) + +*/ +std::tuple load_string(const wasmtime::Span &data, uint32_t ptr) +{ + uint32_t begin = load_int(data, ptr); + uint32_t tagged_code_units = load_int(data, ptr + 4); + return load_string_from_range(data, begin, tagged_code_units); +} + +/* canonical load_list_from_range (python) ------------------------------------------------------------- + +def load_list_from_range(cx, ptr, length, elem_type): + trap_if(ptr != align_to(ptr, alignment(elem_type))) + trap_if(ptr + length * size(elem_type) > len(cx.opts.memory)) + a = [] + for i in range(length): + a.append(load(cx, ptr + i * size(elem_type), elem_type)) + return a + +*/ + +template +std::vector load_list_from_range(const wasmtime::Span &data, uint32_t ptr, uint32_t length) +{ + if (!isAligned(ptr, alignment(T{}))) + throw makeStringException(2, "Pointer is not aligned"); + if (ptr + length * sizeof(T) > data.size()) + throw makeStringException(1, "Out of bounds access"); + std::vector a; + for (uint32_t i = 0; i < length; i++) + { + a.push_back(load(data, ptr + i * sizeof(T))); + } + return a; +} + +/* canonical load_list (python) ------------------------------------------------------------- + +def load_list(cx, ptr, elem_type): + begin = load_int(cx, ptr, 4) + length = load_int(cx, ptr + 4, 4) + return load_list_from_range(cx, begin, length, elem_type) + +*/ + +// Storing --- diff --git a/plugins/wasmembed/abi.hpp b/plugins/wasmembed/abi.hpp new file mode 100644 index 00000000000..38f09fe1020 --- /dev/null +++ b/plugins/wasmembed/abi.hpp @@ -0,0 +1,3 @@ +#include + +std::tuple load_string(const wasmtime::Span &data, uint32_t ptr); diff --git a/plugins/wasmembed/hpcc-platform.wit b/plugins/wasmembed/hpcc-platform.wit new file mode 100644 index 00000000000..9ff9693c540 --- /dev/null +++ b/plugins/wasmembed/hpcc-platform.wit @@ -0,0 +1,25 @@ +/* WebAssembly Component Interface + + See: https://component-model.bytecodealliance.org/design/wit.html + + HPCC-Platform is the "host". + The WebAssembly module is the "guest". + + Memory management rules: + * Guests calling host functions: + - Guests must dispose all params as needed + - Guests must dispose "results" as needed + * Hosts calling guest functions: + - Guests must dispose all params as needed + - Hosts must dispose "results" as needed by + calling cabi_post_XXX for each function call + */ + +package hpcc-systems:hpcc-platform + +world wasmembed { + + import dbglog: func(msg: string) + + // export myfunc(params: string) -> string +} diff --git a/plugins/wasmembed/secure-enclave.cpp b/plugins/wasmembed/secure-enclave.cpp new file mode 100644 index 00000000000..07a76f7d656 --- /dev/null +++ b/plugins/wasmembed/secure-enclave.cpp @@ -0,0 +1,626 @@ +#include "secure-enclave.hpp" + +#include "eclrtl_imp.hpp" +#include "jexcept.hpp" +#include "jiface.hpp" +#include "eclhelper.hpp" +#include "enginecontext.hpp" + +#include "abi.hpp" +#include "util.hpp" + +#include +#include + +// From deftype.hpp in common +#define UNKNOWN_LENGTH 0xFFFFFFF1 + +// #define ENABLE_TRACE +#ifdef ENABLE_TRACE +#define TRACE(format, ...) DBGLOG(format __VA_OPT__(, ) __VA_ARGS__) +#else +#define TRACE(format, ...) \ + do \ + { \ + } while (0) +#endif +class WasmEngine +{ +private: + std::once_flag wasmLoadedFlag; + std::unordered_map wasmModules; + + wasmtime::Module createModule(const std::string &wasmName, const wasmtime::Span &wasm) + { + TRACE("WasmEngine createModule %s", wasmName.c_str()); + try + { + wasmtime::Store store(engine); + wasmtime::WasiConfig wasi; + wasi.inherit_argv(); + wasi.inherit_env(); + wasi.inherit_stdin(); + wasi.inherit_stdout(); + wasi.inherit_stderr(); + store.context().set_wasi(std::move(wasi)).unwrap(); + return wasmtime::Module::compile(engine, wasm).unwrap(); + } + catch (const wasmtime::Error &e) + { + throw makeStringExceptionV(100, "WasmEngine createModule failed: %s", e.message().c_str()); + } + } + + void loadWasmFiles(ICodeContext *codeCtx) + { + TRACE("WasmEngine loadWasmFiles"); + IEngineContext *engine = codeCtx->queryEngineContext(); + if (!engine) + throw makeStringException(100, "Failed to get engine context"); + + StringArray manifestModules; + engine->getManifestFiles("wasm", manifestModules); + + ForEachItemIn(idx, manifestModules) + { + const char *path = manifestModules.item(idx); + TRACE("WasmEngine loadWasmFiles %s", path); + std::vector contents = readWasmBinaryToBuffer(path); + auto module = createModule(path, contents); + std::filesystem::path p(path); + wasmModules.insert(std::make_pair(p.stem(), module)); + } + } + +public: + wasmtime::Engine engine; + + WasmEngine() + { + TRACE("WASM SE WasmEngine"); + } + + ~WasmEngine() + { + TRACE("WASM SE ~WasmEngine"); + } + + void setCodeContext(ICodeContext *codeCtx) + { + TRACE("WASM SE setCodeContext"); + std::call_once(wasmLoadedFlag, &WasmEngine::loadWasmFiles, this, codeCtx); + } + + bool hasModule(const std::string &wasmName) const + { + TRACE("WASM SE hasModule"); + return wasmModules.find(wasmName) != wasmModules.end(); + } + + wasmtime::Module getModule(const std::string &wasmName) const + { + TRACE("WASM SE getModule"); + auto found = wasmModules.find(wasmName); + if (found == wasmModules.end()) + throw makeStringExceptionV(100, "Wasm module not found: %s", wasmName.c_str()); + return found->second; + } +}; +static std::unique_ptr wasmEngine = std::make_unique(); + +class WasmStore +{ +private: + wasmtime::Store store; + + std::unordered_map wasmInstances; + std::unordered_map wasmMems; + std::unordered_map wasmFuncs; + +public: + WasmStore() : store(wasmEngine->engine) + { + TRACE("WASM SE WasmStore"); + } + + ~WasmStore() + { + TRACE("WASM SE ~WasmStore"); + } + + bool hasInstance(const std::string &wasmName) const + { + TRACE("WASM SE hasInstance"); + return wasmInstances.find(wasmName) != wasmInstances.end(); + } + + wasmtime::Instance getInstance(const std::string &wasmName) const + { + TRACE("WASM SE getInstance"); + auto found = wasmInstances.find(wasmName); + if (found == wasmInstances.end()) + throw makeStringExceptionV(100, "Wasm instance not found: %s", wasmName.c_str()); + return found->second; + } + + void registerInstance(const std::string &wasmName) + { + TRACE("WASM SE registerInstance %s", wasmName.c_str()); + if (hasInstance(wasmName)) + { + throw makeStringExceptionV(100, "Wasm instance already registered: %s", wasmName.c_str()); + } + TRACE("WASM SE createInstance %s", wasmName.c_str()); + auto module = wasmEngine->getModule(wasmName); + try + { + wasmtime::Linker linker(wasmEngine->engine); + linker.define_wasi().unwrap(); + + auto callback = [this, wasmName](wasmtime::Caller caller, uint32_t msg, uint32_t msg_len) + { + auto data = this->getData(wasmName); + auto msg_ptr = (char *)&data[msg]; + std::string str(msg_ptr, msg_len); + DBGLOG("from wasm: %s", str.c_str()); + }; + auto host_func = linker.func_wrap("$root", "dbglog", callback).unwrap(); + + auto newInstance = linker.instantiate(store, module).unwrap(); + linker.define_instance(store, "linking2", newInstance).unwrap(); + + for (auto exportItem : module.exports()) + { + auto externType = wasmtime::ExternType::from_export(exportItem); + std::string name(exportItem.name()); + if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported function: %s", name.c_str()); + auto func = std::get(*newInstance.get(store, name)); + wasmFuncs.insert(std::make_pair(wasmName + "." + name, func)); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported memory: %s", name.c_str()); + auto memory = std::get(*newInstance.get(store, name)); + wasmMems.insert(std::make_pair(wasmName + "." + name, memory)); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported table: %s", name.c_str()); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported global: %s", name.c_str()); + } + else + { + TRACE("WASM SE Unknown export type"); + } + } + wasmInstances.insert(std::make_pair(wasmName, newInstance)); + } + catch (const wasmtime::Error &e) + { + throw makeStringExceptionV(100, "WASM SE createInstance: %s", e.message().c_str()); + } + } + + bool hasFunc(const std::string &qualifiedID) const + { + TRACE("WASM SE hasFunc"); + return wasmFuncs.find(qualifiedID) != wasmFuncs.end(); + } + + wasmtime::Func getFunc(const std::string &qualifiedID) const + { + TRACE("WASM SE getFunc"); + auto found = wasmFuncs.find(qualifiedID); + if (found == wasmFuncs.end()) + throw makeStringExceptionV(100, "Wasm function not found: %s", qualifiedID.c_str()); + return found->second; + } + + wasmtime::ValType::ListRef getFuncParams(const std::string &qualifiedID) + { + TRACE("WASM SE getFuncParams"); + auto func = getFunc(qualifiedID); + wasmtime::FuncType funcType = func.type(store.context()); + return funcType->params(); + } + + wasmtime::ValType::ListRef getFuncResults(const std::string &qualifiedID) + { + TRACE("WASM SE getFuncResults"); + auto func = getFunc(qualifiedID); + wasmtime::FuncType funcType = func.type(store.context()); + return funcType->results(); + } + + std::vector call(const std::string &qualifiedID, const std::vector ¶ms) + { + TRACE("WASM SE call"); + auto func = getFunc(qualifiedID); + try + { + auto retVal = func.call(store, params).unwrap(); + return retVal; + } + catch (const wasmtime::Trap &e) + { + throw makeStringExceptionV(100, "WASM SE call: %s", e.message().c_str()); + } + } + + std::vector callRealloc(const std::string &wasmName, const std::vector ¶ms) + { + TRACE("WASM SE callRealloc"); + return call(createQualifiedID(wasmName, "cabi_realloc"), params); + } + + wasmtime::Span getData(const std::string &wasmName) + { + TRACE("WASM SE getData"); + auto found = wasmMems.find(createQualifiedID(wasmName, "memory")); + if (found == wasmMems.end()) + throw makeStringExceptionV(100, "Wasm memory not found: %s", wasmName.c_str()); + return found->second.data(store.context()); + } +}; +thread_local std::unique_ptr wasmStore = std::make_unique(); + +class SecureFunction : public CInterfaceOf +{ + std::string wasmName; + std::string funcName; + std::string qualifiedID; + + std::vector args; + std::vector wasmResults; + +public: + SecureFunction(ICodeContext *codeCtx) + { + TRACE("WASM SE se:constructor"); + wasmEngine->setCodeContext(codeCtx); + } + + virtual ~SecureFunction() + { + TRACE("WASM SE se:destructor"); + + // Garbage Collection --- + // Function results --- + auto gc_func_name = createQualifiedID(wasmName, "cabi_post_" + funcName); + if (wasmStore->hasFunc(gc_func_name)) + { + for (auto &result : wasmResults) + { + wasmStore->call(gc_func_name, {result}); + } + } + } + + // IEmbedFunctionContext --- + void setActivityContext(const IThorActivityContext *activityCtx) + { + } + + virtual IInterface *bindParamWriter(IInterface *esdl, const char *esdlservice, const char *esdltype, const char *name) + { + TRACE("WASM SE paramWriterCommit"); + return NULL; + } + virtual void paramWriterCommit(IInterface *writer) + { + TRACE("WASM SE paramWriterCommit"); + } + virtual void writeResult(IInterface *esdl, const char *esdlservice, const char *esdltype, IInterface *writer) + { + TRACE("WASM SE writeResult"); + } + virtual void bindBooleanParam(const char *name, bool val) + { + TRACE("WASM SE bindBooleanParam %s %i", name, val); + args.push_back(val); + } + virtual void bindDataParam(const char *name, size32_t len, const void *val) + { + TRACE("WASM SE bindDataParam %s %d", name, len); + } + virtual void bindFloatParam(const char *name, float val) + { + TRACE("WASM SE bindFloatParam %s %f", name, val); + args.push_back(val); + } + virtual void bindRealParam(const char *name, double val) + { + TRACE("WASM SE bindRealParam %s %f", name, val); + args.push_back(val); + } + virtual void bindSignedSizeParam(const char *name, int size, __int64 val) + { + TRACE("WASM SE bindSignedSizeParam %s %i %lld", name, size, val); + if (size <= 4) + args.push_back(static_cast(val)); + else + args.push_back(static_cast(val)); + } + virtual void bindSignedParam(const char *name, __int64 val) + { + TRACE("WASM SE bindSignedParam %s %lld", name, val); + args.push_back(static_cast(val)); + } + virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val) + { + TRACE("WASM SE bindUnsignedSizeParam %s %i %llu", name, size, val); + if (size <= 4) + args.push_back(static_cast(val)); + else + args.push_back(static_cast(val)); + } + virtual void bindUnsignedParam(const char *name, unsigned __int64 val) + { + TRACE("WASM SE bindUnsignedParam %s %llu", name, val); + args.push_back(static_cast(val)); + } + virtual void bindStringParam(const char *name, size32_t bytes, const char *val) + { + TRACE("WASM SE bindStringParam %s %d %s", name, bytes, val); + size32_t utfCharCount; + rtlDataAttr utfText; + rtlStrToUtf8X(utfCharCount, utfText.refstr(), bytes, val); + bindUTF8Param(name, utfCharCount, utfText.getstr()); + } + virtual void bindVStringParam(const char *name, const char *val) + { + TRACE("WASM SE bindVStringParam %s %s", name, val); + bindStringParam(name, strlen(val), val); + } + virtual void bindUTF8Param(const char *name, size32_t chars, const char *val) + { + TRACE("WASM SE bindUTF8Param %s %d %s", name, chars, val); + auto bytes = rtlUtf8Size(chars, val); + auto memIdxVar = wasmStore->callRealloc(wasmName, {0, 0, 1, (int32_t)bytes}); + auto memIdx = memIdxVar[0].i32(); + auto mem = wasmStore->getData(wasmName); + memcpy(&mem[memIdx], val, bytes); + args.push_back(memIdx); + args.push_back((int32_t)bytes); + } + virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val) + { + TRACE("WASM SE bindUnicodeParam %s %d", name, chars); + size32_t utfCharCount; + rtlDataAttr utfText; + rtlUnicodeToUtf8X(utfCharCount, utfText.refstr(), chars, val); + bindUTF8Param(name, utfCharCount, utfText.getstr()); + } + + virtual void bindSetParam(const char *name, int elemType, size32_t elemSize, bool isAll, size32_t totalBytes, const void *setData) + { + TRACE("WASM SE bindSetParam %s %d %d %d %d %p", name, elemType, elemSize, isAll, totalBytes, setData); + throw makeStringException(200, "bindSetParam not implemented"); + + type_vals typecode = (type_vals)elemType; + const byte *inData = (const byte *)setData; + const byte *endData = inData + totalBytes; + int numElems; + if (elemSize == UNKNOWN_LENGTH) + { + numElems = 0; + // Will need 2 passes to work out how many elements there are in the set :( + while (inData < endData) + { + int thisSize; + switch (elemType) + { + case type_varstring: + thisSize = strlen((const char *)inData) + 1; + break; + case type_string: + thisSize = *(size32_t *)inData + sizeof(size32_t); + break; + case type_unicode: + thisSize = (*(size32_t *)inData) * sizeof(UChar) + sizeof(size32_t); + break; + case type_utf8: + thisSize = rtlUtf8Size(*(size32_t *)inData, inData + sizeof(size32_t)) + sizeof(size32_t); + break; + default: + rtlFail(0, "wasmembed: Unsupported parameter type"); + break; + } + inData += thisSize; + numElems++; + } + inData = (const byte *)setData; + } + else + numElems = totalBytes / elemSize; + + std::vector memIdxVar; + int32_t memIdx; + + switch (typecode) + { + case type_boolean: + memIdxVar = wasmStore->callRealloc(wasmName, {0, 0, 1, (int32_t)numElems}); + memIdx = memIdxVar[0].i32(); + break; + default: + rtlFail(0, "wasmembed: Unsupported parameter type"); + break; + } + + auto mem = wasmStore->getData(wasmName); + size32_t thisSize = elemSize; + for (int idx = 0; idx < numElems; idx++) + { + switch (typecode) + { + case type_boolean: + mem[memIdx + idx] = *(bool *)inData; + break; + default: + rtlFail(0, "v8embed: Unsupported parameter type"); + break; + } + inData += thisSize; + } + args.push_back(memIdx); + args.push_back(numElems); + } + + virtual void bindRowParam(const char *name, IOutputMetaData &metaVal, const byte *val) override + { + TRACE("WASM SE bindRowParam %s %p", name, val); + throw makeStringException(200, "bindRowParam not implemented"); + } + virtual void bindDatasetParam(const char *name, IOutputMetaData &metaVal, IRowStream *val) + { + TRACE("WASM SE bindDatasetParam %s %p", name, val); + throw makeStringException(200, "bindDatasetParam not implemented"); + } + virtual bool getBooleanResult() + { + TRACE("WASM SE getBooleanResult"); + return wasmResults[0].i32(); + } + virtual void getDataResult(size32_t &len, void *&result) + { + TRACE("WASM SE getDataResult"); + throw makeStringException(200, "getDataResult not implemented"); + } + virtual double getRealResult() + { + TRACE("WASM SE getRealResult"); + if (wasmResults[0].kind() == wasmtime::ValKind::F64) + return wasmResults[0].f64(); + return wasmResults[0].f32(); + } + virtual __int64 getSignedResult() + { + TRACE("WASM SE getSignedResult1 %i", (uint8_t)wasmResults[0].kind()); + if (wasmResults[0].kind() == wasmtime::ValKind::I64) + { + return wasmResults[0].i64(); + } + return static_cast<__int64>(wasmResults[0].i32()); + } + virtual unsigned __int64 getUnsignedResult() + { + TRACE("WASM SE getUnsignedResult"); + if (wasmResults[0].kind() == wasmtime::ValKind::I64) + return wasmResults[0].i64(); + return static_cast(wasmResults[0].i32()); + } + virtual void getStringResult(size32_t &chars, char *&result) + { + TRACE("WASM SE getStringResult %zu", wasmResults.size()); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + uint32_t strPtr; + std::string encoding; + uint32_t bytes; + std::tie(strPtr, encoding, bytes) = load_string(data, ptr); + size32_t codepoints = rtlUtf8Length(bytes, &data[strPtr]); + rtlUtf8ToStrX(chars, result, codepoints, reinterpret_cast(&data[strPtr])); + } + virtual void getUTF8Result(size32_t &chars, char *&result) + { + TRACE("WASM SE getUTF8Result"); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + uint32_t strPtr; + std::string encoding; + uint32_t bytes; + std::tie(strPtr, encoding, bytes) = load_string(data, ptr); + chars = rtlUtf8Length(bytes, &data[strPtr]); + TRACE("WASM SE getUTF8Result %d %d", bytes, chars); + result = (char *)rtlMalloc(bytes); + memcpy(result, &data[strPtr], bytes); + } + virtual void getUnicodeResult(size32_t &chars, UChar *&result) + { + TRACE("WASM SE getUnicodeResult"); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + uint32_t strPtr; + std::string encoding; + uint32_t bytes; + std::tie(strPtr, encoding, bytes) = load_string(data, ptr); + unsigned numchars = rtlUtf8Length(bytes, &data[strPtr]); + rtlUtf8ToUnicodeX(chars, result, numchars, reinterpret_cast(&data[strPtr])); + } + virtual void getSetResult(bool &__isAllResult, size32_t &resultBytes, void *&result, int elemType, size32_t elemSize) + { + TRACE("WASM SE getSetResult %d %d %zu", elemType, elemSize, wasmResults.size()); + auto ptr = wasmResults[0].i32(); + auto data = wasmStore->getData(wasmName); + + throw makeStringException(200, "getSetResult not implemented"); + } + virtual IRowStream *getDatasetResult(IEngineRowAllocator *_resultAllocator) + { + TRACE("WASM SE getDatasetResult"); + throw makeStringException(200, "getDatasetResult not implemented"); + return NULL; + } + virtual byte *getRowResult(IEngineRowAllocator *_resultAllocator) + { + TRACE("WASM SE getRowResult"); + throw makeStringException(200, "getRowResult not implemented"); + return NULL; + } + virtual size32_t getTransformResult(ARowBuilder &builder) + { + TRACE("WASM SE getTransformResult"); + throw makeStringException(200, "getTransformResult not implemented"); + return 0; + } + virtual void loadCompiledScript(size32_t chars, const void *_script) override + { + TRACE("WASM SE loadCompiledScript %p", _script); + throw makeStringException(200, "loadCompiledScript not implemented"); + } + virtual void enter() override + { + TRACE("WASM SE enter"); + } + virtual void reenter(ICodeContext *codeCtx) override + { + TRACE("WASM SE reenter"); + } + virtual void exit() override + { + TRACE("WASM SE exit"); + } + virtual void compileEmbeddedScript(size32_t lenChars, const char *_utf) override + { + TRACE("WASM SE compileEmbeddedScript"); + throw makeStringException(200, "compileEmbeddedScript not supported"); + } + virtual void importFunction(size32_t lenChars, const char *qualifiedName) override + { + TRACE("WASM SE importFunction: %s", qualifiedName); + + qualifiedID = std::string(qualifiedName, lenChars); + std::tie(wasmName, funcName) = splitQualifiedID(qualifiedID); + + if (!wasmStore->hasInstance(wasmName)) + { + wasmStore->registerInstance(wasmName); + } + } + virtual void callFunction() + { + TRACE("WASM SE callFunction %s", qualifiedID.c_str()); + wasmResults = wasmStore->call(qualifiedID, args); + } +}; + +IEmbedFunctionContext *createISecureEnclave(ICodeContext *codeCtx) +{ + return new SecureFunction(codeCtx); +} diff --git a/plugins/wasmembed/secure-enclave.hpp b/plugins/wasmembed/secure-enclave.hpp new file mode 100644 index 00000000000..fe81927e191 --- /dev/null +++ b/plugins/wasmembed/secure-enclave.hpp @@ -0,0 +1,4 @@ +#include "platform.h" +#include "eclrtl.hpp" + +IEmbedFunctionContext *createISecureEnclave(ICodeContext *codeCtx); diff --git a/plugins/wasmembed/test/CMakeLists.txt b/plugins/wasmembed/test/CMakeLists.txt new file mode 100644 index 00000000000..14079ec94ec --- /dev/null +++ b/plugins/wasmembed/test/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.22) + +project(wasmembed) + +set(WASM_PATH "${CMAKE_CURRENT_BINARY_DIR}/bin/${PROJECT_NAME}.wasm") + +add_custom_command( + MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/hpcc-scalar-test.wit + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.c ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.h ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test_component_type.o + COMMAND wit-bindgen c --out-dir ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/hpcc-scalar-test.wit +) +add_custom_target(wit-generate ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.c) + +set(CMAKE_EXECUTABLE_SUFFIX ".wasm") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostartfiles -fno-exceptions --sysroot=/${WASI_SDK_PREFIX}/share/wasi-sysroot -Wl,--no-entry") + +include_directories( + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_executable(wasmembed + main.cpp + ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.c + ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test.h +) + +target_link_libraries(wasmembed + ${CMAKE_CURRENT_BINARY_DIR}/hpcc_scalar_test_component_type.o +) + +install(TARGETS wasmembed + RUNTIME DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/../install-target +) diff --git a/plugins/wasmembed/test/build.sh b/plugins/wasmembed/test/build.sh new file mode 100755 index 00000000000..7f91603130c --- /dev/null +++ b/plugins/wasmembed/test/build.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )" +ROOT_DIR="${SCRIPT_DIR}/../../.." + +echo "SCRIPT_DIR: ${SCRIPT_DIR}" +echo "ROOT_DIR: $ROOT_DIR" + +docker build --progress plain -f "${SCRIPT_DIR}/wasm32-wasi/Dockerfile" \ + -t wasm32-wasi:latest \ + "${SCRIPT_DIR}/." + +CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel -DCMAKE_TOOLCHAIN_FILE=/hpcc-dev/wasi-sdk/share/cmake/wasi-sdk.cmake -DWASI_SDK_PREFIX=/hpcc-dev/wasi-sdk" + +docker run --rm \ + --mount source="${SCRIPT_DIR}",target=/hpcc-dev/wasmembed,type=bind,consistency=cached \ + --mount source="${ROOT_DIR}/testing/regress/ecl",target=/hpcc-dev/install-target,type=bind,consistency=cached \ + wasm32-wasi:latest \ + "rm -rf ./build && \ + cmake -S . -B /hpcc-dev/build ${CMAKE_OPTIONS} && \ + cmake --build /hpcc-dev/build --target install" + +echo "docker run -it --mount source=\"${SCRIPT_DIR}\",target=/hpcc-dev/wasmembed,type=bind,consistency=cached wasm32-wasi:latest bash" diff --git a/plugins/wasmembed/test/hpcc-scalar-test.wit b/plugins/wasmembed/test/hpcc-scalar-test.wit new file mode 100644 index 00000000000..3b632d46ea1 --- /dev/null +++ b/plugins/wasmembed/test/hpcc-scalar-test.wit @@ -0,0 +1,28 @@ +package hpcc-systems:hpcc-platform + +world hpcc-scalar-test { +/* imports --- + + guests dispose all params as needed + guests should dispose "results" as needed +*/ + import dbglog: func(msg: string) + +/* exports --- + + guests dispose all params as needed + hosts call cabi_post_XXX to dispose "results" as needed +*/ + export bool-test: func(a: bool, b: bool) -> bool + export float32-test: func(a: float32, b: float32) -> float32 + export float64-test: func(a: float64, b: float64) -> float64 + export u8-test: func(a: u8, b: u8) -> u8 + export u16-test: func(a: u16, b: u16) -> u16 + export u32-test: func(a: u32, b: u32) -> u32 + export u64-test: func(a: u64, b: u64) -> u64 + export s8-test: func(a: s8, b: s8) -> s8 + export s16-test: func(a: s16, b: s16) -> s16 + export s32-test: func(a: s32, b: s32) -> s32 + export s64-test: func(a: s64, b: s64) -> s64 + export utf8-string-test: func(a: string, b: string) -> string +} diff --git a/plugins/wasmembed/test/main.cpp b/plugins/wasmembed/test/main.cpp new file mode 100644 index 00000000000..c0d7420e5f1 --- /dev/null +++ b/plugins/wasmembed/test/main.cpp @@ -0,0 +1,70 @@ +#include "hpcc_scalar_test.h" + +#include + +void dbglog(const std::string str) +{ + hpcc_scalar_test_string_t msg; + hpcc_scalar_test_string_set(&msg, str.c_str()); + hpcc_scalar_test_dbglog(&msg); +} + +bool hpcc_scalar_test_bool_test(bool a, bool b) +{ + return a && b; +} +float hpcc_scalar_test_float32_test(float a, float b) +{ + return a + b; +} +double hpcc_scalar_test_float64_test(double a, double b) +{ + return a + b; +} +uint8_t hpcc_scalar_test_u8_test(uint8_t a, uint8_t b) +{ + return a + b; +} +uint16_t hpcc_scalar_test_u16_test(uint16_t a, uint16_t b) +{ + return a + b; +} +uint32_t hpcc_scalar_test_u32_test(uint32_t a, uint32_t b) +{ + return a + b; +} +uint64_t hpcc_scalar_test_u64_test(uint64_t a, uint64_t b) +{ + return a + b; +} +int8_t hpcc_scalar_test_s8_test(int8_t a, int8_t b) +{ + return a + b; +} +int16_t hpcc_scalar_test_s16_test(int16_t a, int16_t b) +{ + return a + b; +} +int32_t hpcc_scalar_test_s32_test(int32_t a, int32_t b) +{ + return a + b; +} +int64_t hpcc_scalar_test_s64_test(int64_t a, int64_t b) +{ + return a + b; +} +uint32_t hpcc_scalar_test_char_test(uint32_t a, uint32_t b) +{ + return a + b; +} +static uint32_t tally = 0; +void hpcc_scalar_test_utf8_string_test(hpcc_scalar_test_string_t *a, hpcc_scalar_test_string_t *b, hpcc_scalar_test_string_t *ret) +{ + std::string s1(a->ptr, a->len); + hpcc_scalar_test_string_free(a); + std::string s2(b->ptr, b->len); + hpcc_scalar_test_string_free(b); + std::string r = s1 + s2; + dbglog(std::to_string(++tally) + ": " + r); + hpcc_scalar_test_string_dup(ret, r.c_str()); +} diff --git a/plugins/wasmembed/test/wasm32-wasi/Dockerfile b/plugins/wasmembed/test/wasm32-wasi/Dockerfile new file mode 100644 index 00000000000..5746ac1eb34 --- /dev/null +++ b/plugins/wasmembed/test/wasm32-wasi/Dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:22.04 + +RUN apt-get update && \ + apt-get install -y \ + autoconf \ + autogen \ + automake \ + clang \ + cmake \ + curl \ + libtool \ + lld \ + llvm \ + make \ + ninja-build \ + wget + +RUN curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y + +SHELL ["/bin/bash", "--login", "-c"] + +WORKDIR /hpcc-dev + +ARG WIT_VERSION=0.9.0 +RUN cargo install wasm-tools && \ + cargo install --git https://github.com/bytecodealliance/wit-bindgen --tag wit-bindgen-cli-${WIT_VERSION} wit-bindgen-cli && \ + curl https://wasmtime.dev/install.sh -sSf | bash + +# List of current vertsion can be found in https://github.com/bytecodealliance/wit-bindgen/releases --- +ARG WASI_VERSION=20 +ARG WASI_MINOR_VERSION=0 +ARG WASI_VERSION_FULL=${WASI_VERSION}.${WASI_MINOR_VERSION} +RUN wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_VERSION}/wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz +RUN tar xvf wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz && rm wasi-sdk-${WASI_VERSION_FULL}-linux.tar.gz +RUN mv wasi-sdk-${WASI_VERSION_FULL} wasi-sdk + +WORKDIR /hpcc-dev/wasmembed + +ENTRYPOINT ["/bin/bash", "--login", "-c"] + +CMD ["bash"] diff --git a/plugins/wasmembed/util.cpp b/plugins/wasmembed/util.cpp new file mode 100644 index 00000000000..7f65691c7b4 --- /dev/null +++ b/plugins/wasmembed/util.cpp @@ -0,0 +1,46 @@ +#include "util.hpp" + +#include "platform.h" +#include "jexcept.hpp" +#include "jfile.hpp" + +std::vector readWasmBinaryToBuffer(const char *filename) +{ + Owned file = createIFile(filename); + Owned fileIO = file->open(IFOread); + if (!fileIO) + throw makeStringExceptionV(0, "Failed to open %s", filename); + + MemoryBuffer mb; + size32_t count = read(fileIO, 0, (size32_t)-1, mb); + uint8_t *ptr = (uint8_t *)mb.detach(); + return std::vector(ptr, ptr + count); +} + +std::string extractContentInDoubleQuotes(const std::string &input) +{ + + std::size_t firstQuote = input.find_first_of('"'); + if (firstQuote == std::string::npos) + return ""; + + std::size_t secondQuote = input.find('"', firstQuote + 1); + if (secondQuote == std::string::npos) + return ""; + + return input.substr(firstQuote + 1, secondQuote - firstQuote - 1); +} + +std::pair splitQualifiedID(const std::string &qualifiedName) +{ + std::size_t firstDot = qualifiedName.find_first_of('.'); + if (firstDot == std::string::npos || firstDot == 0 || firstDot == qualifiedName.size() - 1) + throw makeStringExceptionV(3, "Invalid import function '%s', expected format: .", qualifiedName.c_str()); + + return std::make_pair(qualifiedName.substr(0, firstDot), qualifiedName.substr(firstDot + 1)); +} + +std::string createQualifiedID(const std::string &wasmName, const std::string &funcName) +{ + return wasmName + "." + funcName; +} \ No newline at end of file diff --git a/plugins/wasmembed/util.hpp b/plugins/wasmembed/util.hpp new file mode 100644 index 00000000000..22282eb0579 --- /dev/null +++ b/plugins/wasmembed/util.hpp @@ -0,0 +1,7 @@ +#include +#include + +std::vector readWasmBinaryToBuffer(const char *filename); +std::string extractContentInDoubleQuotes(const std::string &input); +std::pair splitQualifiedID(const std::string &qualifiedName); +std::string createQualifiedID(const std::string &wasmName, const std::string &funcName); diff --git a/plugins/wasmembed/wasm.ecllib b/plugins/wasmembed/wasm.ecllib new file mode 100644 index 00000000000..eb5d8313c02 --- /dev/null +++ b/plugins/wasmembed/wasm.ecllib @@ -0,0 +1,10 @@ +EXPORT Language := SERVICE : plugin('wasmembed') + integer getEmbedContext() : cpp, pure, fold, namespace='wasmLanguageHelper', entrypoint='getEmbedContext', prototype='IEmbedContext* getEmbedContext()'; + string syntaxCheck(const varstring funcname, UTF8 body, const varstring argnames, const varstring compileOptions, const varstring persistOptions) : cpp, pure, fold, namespace='wasmLanguageHelper', entrypoint='syntaxCheck'; +END; +EXPORT getEmbedContext := Language.getEmbedContext; +EXPORT syntaxCheck := Language.syntaxCheck; +EXPORT boolean supportsImport := true; +EXPORT boolean supportsScript := true; +EXPORT boolean prebind := false; +EXPORT boolean singletonEmbedContext := false; \ No newline at end of file diff --git a/plugins/wasmembed/wasmembed.cpp b/plugins/wasmembed/wasmembed.cpp new file mode 100644 index 00000000000..a2ce1d35d43 --- /dev/null +++ b/plugins/wasmembed/wasmembed.cpp @@ -0,0 +1,65 @@ +#include "secure-enclave.hpp" + +#include "jexcept.hpp" +#include "hqlplugins.hpp" + +static const char *compatibleVersions[] = { + "WASM Embed Helper 1.0.0", + NULL}; + +static const char *version = "WASM Embed Helper 1.0.0"; + +extern "C" DECL_EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb) +{ + if (pb->size == sizeof(ECLPluginDefinitionBlockEx)) + { + ECLPluginDefinitionBlockEx *pbx = (ECLPluginDefinitionBlockEx *)pb; + pbx->compatibleVersions = compatibleVersions; + } + else if (pb->size != sizeof(ECLPluginDefinitionBlock)) + return false; + pb->magicVersion = PLUGIN_VERSION; + pb->version = version; + pb->moduleName = "wasm"; + pb->ECL = NULL; + pb->flags = PLUGIN_MULTIPLE_VERSIONS; + pb->description = "WASM Embed Helper"; + return true; +} + +namespace wasmLanguageHelper +{ + class WasmEmbedContext : public CInterfaceOf + { + public: + virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options) override + { + return createFunctionContextEx(nullptr, nullptr, flags, options); + } + + virtual IEmbedFunctionContext *createFunctionContextEx(ICodeContext *codeCtx, const IThorActivityContext *activityContext, unsigned flags, const char *options) override + { + return createISecureEnclave(codeCtx); + } + + virtual IEmbedServiceContext *createServiceContext(const char *service, unsigned flags, const char *options) override + { + throwUnexpected(); + return nullptr; + } + } theEmbedContext; + + extern DECL_EXPORT IEmbedContext *getEmbedContext() + { + return LINK(&theEmbedContext); + } + + extern DECL_EXPORT void syntaxCheck(size32_t &__lenResult, char *&__result, const char *funcname, size32_t charsBody, const char *body, const char *argNames, const char *compilerOptions, const char *persistOptions) + { + StringBuffer result; + // MORE - ::syntaxCheck(__lenResult, __result, funcname, charsBody, body, argNames, compilerOptions, persistOptions); + __lenResult = result.length(); + __result = result.detach(); + } + +} // namespace diff --git a/system/jlib/jarray.hpp b/system/jlib/jarray.hpp index a2c1240bb6d..9870c8d9c31 100644 --- a/system/jlib/jarray.hpp +++ b/system/jlib/jarray.hpp @@ -153,7 +153,7 @@ class ArrayOf : public AllocatorOf typedef int (*CompareFunc)(const MEMBER *, const MEMBER *); // Should really be const, as should the original array functions public: - ~ArrayOf() { kill(); } + ~ArrayOf() { kill(); } MEMBER & operator[](size_t pos) { return element((aindex_t)pos); } const MEMBER & operator[](size_t pos) const { return element((aindex_t)pos); } diff --git a/system/jlib/jbuff.hpp b/system/jlib/jbuff.hpp index 5d6392dbec6..010e9dcc66d 100644 --- a/system/jlib/jbuff.hpp +++ b/system/jlib/jbuff.hpp @@ -66,10 +66,10 @@ template class OwnedPtrCustomFree CLASS *ptr = nullptr; public: - OwnedPtrCustomFree() { } - OwnedPtrCustomFree(CLASS *_ptr) : ptr(_ptr) { } - OwnedPtrCustomFree(SELF &&_ptr) { ptr = _ptr.getClear(); } - ~OwnedPtrCustomFree() { safeFree(ptr); } + OwnedPtrCustomFree() { } + OwnedPtrCustomFree(CLASS *_ptr) : ptr(_ptr) { } + OwnedPtrCustomFree(SELF &&_ptr) { ptr = _ptr.getClear(); } + ~OwnedPtrCustomFree() { safeFree(ptr); } void operator = (CLASS * _ptr) { diff --git a/system/jlib/jhash.hpp b/system/jlib/jhash.hpp index c8790fe2e4c..3653abd4c21 100644 --- a/system/jlib/jhash.hpp +++ b/system/jlib/jhash.hpp @@ -468,13 +468,13 @@ template class CMinHashTable } public: - CMinHashTable(unsigned _initialSize = 7) + CMinHashTable(unsigned _initialSize = 7) { htn = _initialSize; n = 0; table = (C **)calloc(sizeof(C *),htn); } - ~CMinHashTable() + ~CMinHashTable() { C **t = table+htn; while (t--!=table) @@ -625,7 +625,7 @@ template class CTimeLimitedCache { public: - CTimeLimitedCache(unsigned timeoutMs=defaultCacheTimeoutMs) + CTimeLimitedCache(unsigned timeoutMs=defaultCacheTimeoutMs) { timeoutPeriodCycles = ((cycle_t)timeoutMs) * queryOneSecCycles() / 1000; } diff --git a/system/jlib/jiface.hpp b/system/jlib/jiface.hpp index 4f3d6b2a744..ca502ab010d 100644 --- a/system/jlib/jiface.hpp +++ b/system/jlib/jiface.hpp @@ -88,8 +88,8 @@ class CSimpleInterfaceOf : public INTERFACE } private: - CSimpleInterfaceOf(const CSimpleInterfaceOf&) = delete; - CSimpleInterfaceOf(CSimpleInterfaceOf &&) = delete; + CSimpleInterfaceOf(const CSimpleInterfaceOf&) = delete; + CSimpleInterfaceOf(CSimpleInterfaceOf &&) = delete; CSimpleInterfaceOf & operator = (const CSimpleInterfaceOf &) = delete; mutable std::atomic xxcount; }; diff --git a/system/jlib/jsuperhash.hpp b/system/jlib/jsuperhash.hpp index 8806ad52993..2cd6216f412 100644 --- a/system/jlib/jsuperhash.hpp +++ b/system/jlib/jsuperhash.hpp @@ -178,9 +178,9 @@ class SimpleHashTableOf : public SuperHashTableOf { typedef SimpleHashTableOf SELF; public: - SimpleHashTableOf(void) : SuperHashTableOf() { } - SimpleHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } - ~SimpleHashTableOf() { SELF::_releaseAll(); } + SimpleHashTableOf(void) : SuperHashTableOf() { } + SimpleHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } + ~SimpleHashTableOf() { SELF::_releaseAll(); } IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(ET, FP); @@ -209,9 +209,9 @@ class OwningSimpleHashTableOf : public SimpleHashTableOf { typedef OwningSimpleHashTableOf SELF; public: - OwningSimpleHashTableOf(void) : SimpleHashTableOf() { } - OwningSimpleHashTableOf(unsigned initsize) : SimpleHashTableOf(initsize) { } - ~OwningSimpleHashTableOf() { SELF::_releaseAll(); } + OwningSimpleHashTableOf(void) : SimpleHashTableOf() { } + OwningSimpleHashTableOf(unsigned initsize) : SimpleHashTableOf(initsize) { } + ~OwningSimpleHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; @@ -288,9 +288,9 @@ class StringSuperHashTableOf : public SuperHashTableOf { typedef StringSuperHashTableOf SELF; public: - StringSuperHashTableOf(void) : SuperHashTableOf() { } - StringSuperHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } - ~StringSuperHashTableOf() { SELF::_releaseAll(); } + StringSuperHashTableOf(void) : SuperHashTableOf() { } + StringSuperHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } + ~StringSuperHashTableOf() { SELF::_releaseAll(); } virtual void onAdd(void *et __attribute__((unused))) { } virtual void onRemove(void *et __attribute__((unused))) { } @@ -318,9 +318,9 @@ class OwningStringSuperHashTableOf : public StringSuperHashTableOf { typedef OwningStringSuperHashTableOf SELF; public: - OwningStringSuperHashTableOf(void) : StringSuperHashTableOf() { } - OwningStringSuperHashTableOf(unsigned initsize) : StringSuperHashTableOf(initsize) { } - ~OwningStringSuperHashTableOf() { SELF::_releaseAll(); } + OwningStringSuperHashTableOf(void) : StringSuperHashTableOf() { } + OwningStringSuperHashTableOf(unsigned initsize) : StringSuperHashTableOf(initsize) { } + ~OwningStringSuperHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; @@ -422,7 +422,7 @@ class ThreadSafeOwningSimpleHashTableOf : public ThreadSafeSimpleHashTableOf SELF; public: - ~ThreadSafeOwningSimpleHashTableOf() { SELF::_releaseAll(); } + ~ThreadSafeOwningSimpleHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; diff --git a/testing/regress/ecl/key/wasmembed.xml b/testing/regress/ecl/key/wasmembed.xml new file mode 100644 index 00000000000..4400c029084 --- /dev/null +++ b/testing/regress/ecl/key/wasmembed.xml @@ -0,0 +1,102 @@ + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + diff --git a/testing/regress/ecl/wasmembed.ecl b/testing/regress/ecl/wasmembed.ecl new file mode 100644 index 00000000000..75d885e777e --- /dev/null +++ b/testing/regress/ecl/wasmembed.ecl @@ -0,0 +1,112 @@ +import wasm; + +boolean boolTest (boolean a, boolean b) := IMPORT(wasm, 'wasmembed.bool-test'); +real4 float32Test (real4 a, real4 b) := IMPORT(wasm, 'wasmembed.float32-test'); +real8 float64Test (real8 a, real8 b) := IMPORT(wasm, 'wasmembed.float64-test'); +unsigned1 u8Test (unsigned1 a, unsigned1 b) := IMPORT(wasm, 'wasmembed.u8-test'); +unsigned2 u16Test (unsigned2 a, unsigned2 b) := IMPORT(wasm, 'wasmembed.u16-test'); +unsigned4 u32Test (unsigned4 a, unsigned4 b) := IMPORT(wasm, 'wasmembed.u32-test'); +unsigned8 u64Test (unsigned8 a, unsigned8 b) := IMPORT(wasm, 'wasmembed.u64-test'); +integer1 s8Test (integer1 a, integer1 b) := IMPORT(wasm, 'wasmembed.s8-test'); +integer2 s16Test (integer2 a, integer2 b) := IMPORT(wasm, 'wasmembed.s16-test'); +integer4 s32Test (integer4 a, integer4 b) := IMPORT(wasm, 'wasmembed.s32-test'); +integer8 s64Test (integer8 a, integer8 b) := IMPORT(wasm, 'wasmembed.s64-test'); +string stringTest (string a, string b) := IMPORT(wasm, 'wasmembed.string-test'); +string12 string5Test (string5 a, string5 b) := IMPORT(wasm, 'wasmembed.string-test'); +varstring varstringTest (varstring a, varstring b) := IMPORT(wasm, 'wasmembed.string-test'); +unicode12 unicode5Test (unicode5 a, unicode5 b) := IMPORT(wasm, 'wasmembed.string-test'); +unicode unicodeTest (unicode a, unicode b) := IMPORT(wasm, 'wasmembed.string-test'); +utf8_12 utf8_5Test (utf8_5 a, utf8_5 b) := IMPORT(wasm, 'wasmembed.string-test'); +utf8 utf8Test (utf8 a, utf8 b) := IMPORT(wasm, 'wasmembed.string-test'); + +// '--- bool ---'; +boolTest(false, false) = (false AND false); +boolTest(false, true) = (false AND true); +boolTest(true, false) = (true AND false); +boolTest(true, true) = (true AND true); +// '--- float ---'; +ROUND(float32Test((real4)1234.1234, (real4)2345.2345), 3) = ROUND((real4)((real4)1234.1234 + (real4)2345.2345), 3); +float64Test(123456789.123456789, 23456789.23456789) = (real8)((real8)123456789.123456789 + (real8)23456789.23456789); +// '--- unsigned ---'; +u8Test(1, 2) = (unsigned1)(1 + 2); +u8Test(254, 1) = (unsigned1)(254 + 1); +u16Test(1, 2) = (unsigned2)(1 + 2); +u16Test(65534, 1) = (unsigned2)(65534 + 1); +u32Test(1, 2) = (unsigned4)(1 + 2); +u32Test(4294967294, 1) = (unsigned4)(4294967294 + 1); +u64Test(1, 2) = (unsigned8)(1 + 2); +u64Test(18446744073709551614, 1) = (unsigned8)(18446744073709551614 + 1); +// '--- signed ---'; +s8Test(1, 2) = (integer1)(1 + 2); +s8Test(126, 1) = (integer1)(126 + 1); +s8Test(-127, -1) = (integer1)(-127 - 1); + +s16Test(1, 2) = (integer2)(1 + 2); +s16Test(32766, 1) = (integer2)(32766 + 1); +s16Test(-32767, -1) = (integer2)(-32767 - 1); + +s32Test(1, 2) = (integer4)(1 + 2); +s32Test(2147483646, 1) = (integer4)(2147483646 + 1); +s32Test(-2147483647, -1) = (integer4)(-2147483647 - 1); + +s64Test(1, 2) = (integer8)(1 + 2); +s64Test(9223372036854775806, 1) = (integer8)(9223372036854775806 + 1); +s64Test(-9223372036854775807, -1) = (integer8)(-9223372036854775807 - 1); +// '--- string ---'; +varstringTest('1234567890', 'abcdefghij') = '1234567890' + 'abcdefghij'; +stringTest('1234567890', 'abcdefghij') = '1234567890' + 'abcdefghij'; +unicodeTest(U'1234567890您好1231231230', U'abcdefghij欢迎光临abcdefghij') = U'1234567890您好1231231230' + U'abcdefghij欢迎光临abcdefghij'; +utf8Test(U8'您好', U8'欢迎光临') = U8'您好' + U8'欢迎光临'; +// '--- string (fixed length) ---'; +string5Test('1234567890', 'abcdefghij') = (string12)((string5)'1234567890' + (string5)'abcdefghij'); +utf8_5Test(U8'您好1234567890', U8'欢迎光临abcdefghij') = (utf8_12)((utf8_5)U8'您好1234567890' + (utf8_5)U8'欢迎光临abcdefghij'); +unicode5Test(U'您好1234567890', U'欢迎光临abcdefghij') = (unicode12)((unicode5)U'您好1234567890' + (unicode5)U'欢迎光临abcdefghij'); +// '--- reentry ---'; +r := RECORD + unsigned1 kind; + string20 word; + unsigned8 doc; + unsigned1 segment; + unsigned8 wpos; + END; +d := dataset('~regress::multi::searchsource', r, THOR); + +r2 := RECORD(r) + unsigned8 newUnsigned; + string newWord; + boolean passed; +END; + +r2 t(r L) := TRANSFORM + SELF.newUnsigned := u64Test(L.doc, L.wpos); + boolean a := SELF.newUnsigned = (unsigned8)(L.doc + L.wpos); + SELF.newWord := stringTest(L.word, L.word); + boolean b := SELF.newWord = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +r2 t2(r L) := TRANSFORM + SELF.newUnsigned := u64Test(L.doc, L.wpos); + boolean a := SELF.newUnsigned = L.doc+ L.wpos; + SELF.newWord := L.word + L.word; + boolean b := SELF.newWord = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +r2 t3(r L) := TRANSFORM + SELF.newUnsigned := L.doc+ L.wpos; + boolean a := SELF.newUnsigned = L.doc+ L.wpos; + SELF.newWord := L.word + L.word; + boolean b := SELF.newWord = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +unsigned sampleSize := 10000; +d2 := project(choosen(d, sampleSize), t(LEFT)); +d3 := project(choosen(d, sampleSize, 5000), t(LEFT)); +d4 := project(choosen(d, sampleSize, 10001), t(LEFT)); +count(d2(passed=false)) = 0 AND count(d3(passed=false)) = 0 AND count(d4(passed=false)) = 0; +// '--- --- ---'; diff --git a/testing/regress/ecl/wasmembed.manifest b/testing/regress/ecl/wasmembed.manifest new file mode 100644 index 00000000000..8a2d68eb5c2 --- /dev/null +++ b/testing/regress/ecl/wasmembed.manifest @@ -0,0 +1,3 @@ + + + diff --git a/testing/regress/ecl/wasmembed.wasm b/testing/regress/ecl/wasmembed.wasm new file mode 100644 index 00000000000..f2f32c33c95 Binary files /dev/null and b/testing/regress/ecl/wasmembed.wasm differ diff --git a/vcpkg.json.in b/vcpkg.json.in index 7396715d0af..199c2b42bde 100644 --- a/vcpkg.json.in +++ b/vcpkg.json.in @@ -181,6 +181,10 @@ "name": "tbb", "platform": "@VCPKG_TBB@" }, + { + "name": "wasmtime-cpp-api", + "platform": "@VCPKG_WASMEMBED@" + }, { "name": "winflexbison", "platform": "windows"