Merge pull request #2138 from demergent-labs/benchmarking

adding first MVP of benchmarks, they will run on tests now, but will …
demergent-labs · Oct 24, 2024 · abfac2a · abfac2a
2 parents 812223c + 88710cc
commit abfac2a
Show file tree

Hide file tree

Showing 184 changed files with 4,363 additions and 8,186 deletions.
diff --git a/.github/actions/set_run_conditions/action.yml b/.github/actions/set_run_conditions/action.yml
@@ -24,9 +24,9 @@ runs:
               # Define conditions using shell variables
               AZLE_IS_MAIN_BRANCH_PUSH=${{ github.ref == 'refs/heads/main' && !contains(github.event.head_commit.message, 'demergent-labs/release--') }}
               AZLE_IS_MAIN_BRANCH_PUSH_FROM_RELEASE_MERGE=${{ github.ref == 'refs/heads/main' && contains(github.event.head_commit.message, 'demergent-labs/release--') }}
-              AZLE_IS_RELEASE_PR=${{ startsWith(github.head_ref, 'release--') }}
-              AZLE_IS_FEATURE_PR=${{ !startsWith(github.head_ref, 'release--') && github.ref != 'refs/heads/main' && github.event.pull_request.draft == false }}
-              AZLE_IS_DRAFT_PR=${{ !startsWith(github.head_ref, 'release--') && github.ref != 'refs/heads/main' && github.event.pull_request.draft == true }}
+              AZLE_IS_RELEASE_BRANCH_PR=${{ startsWith(github.head_ref, 'release--') }}
+              AZLE_IS_FEATURE_BRANCH_PR=${{ !startsWith(github.head_ref, 'release--') && github.ref != 'refs/heads/main' && github.event.pull_request.draft == false }}
+              AZLE_IS_FEATURE_BRANCH_DRAFT_PR=${{ !startsWith(github.head_ref, 'release--') && github.ref != 'refs/heads/main' && github.event.pull_request.draft == true }}
 
               # Set individual outputs
               echo "is_main_branch_push=$AZLE_IS_MAIN_BRANCH_PUSH" >> $GITHUB_OUTPUT

diff --git a/.github/workflows/run_test.yml b/.github/workflows/run_test.yml
@@ -46,7 +46,7 @@ jobs:
                   echo "AZLE_IS_FEATURE_BRANCH_PR=${{ steps.set-conditions.outputs.is_feature_branch_pr }}" >> $GITHUB_ENV
                   echo "AZLE_IS_FEATURE_BRANCH_DRAFT_PR=${{ steps.set-conditions.outputs.is_feature_branch_draft_pr }}" >> $GITHUB_ENV
 
-            - id: check-conditions
+            - name: Print environment variables
               run: |
                   echo "AZLE_IS_MAIN_BRANCH_PUSH: $AZLE_IS_MAIN_BRANCH_PUSH"
                   echo "AZLE_IS_MAIN_BRANCH_PUSH_FROM_RELEASE_MERGE: $AZLE_IS_MAIN_BRANCH_PUSH_FROM_RELEASE_MERGE"

diff --git a/canister_templates/experimental.wasm b/canister_templates/experimental.wasm
diff --git a/canister_templates/stable.wasm b/canister_templates/stable.wasm
diff --git a/examples/basic_bitcoin/test/test.ts b/examples/basic_bitcoin/test/test.ts
@@ -5,7 +5,8 @@ import { runTests } from 'azle/test';
 import { BitcoinDaemon, startBitcoinDaemon } from './bitcoin_daemon';
 import { getP2pkhAddress, getTests, P2PKH_ADDRESS_FORM } from './tests';
 
-const canisterId = getCanisterId('basic_bitcoin');
+const canisterName = 'basic_bitcoin';
+const canisterId = getCanisterId(canisterName);
 
 runTests(() => {
     let bitcoinDaemon: BitcoinDaemon;
@@ -22,4 +23,4 @@ runTests(() => {
         'runs basic bitcoin tests while bitcoin daemon is running',
         getTests(canisterId, getP2pkhAddress, P2PKH_ADDRESS_FORM)
     );
-});
+}, canisterName);
diff --git a/examples/bitcoin_psbt/test/manual.test.ts b/examples/bitcoin_psbt/test/manual.test.ts
@@ -4,10 +4,14 @@ import { getTests } from 'basic_bitcoin/test/tests';
 
 import { getP2wpkhAddress, P2WPKH_ADDRESS_FORM } from './tests';
 
-const canisterId = getCanisterId('bitcoin_psbt');
+const canisterName = 'bitcoin_psbt';
+const canisterId = getCanisterId(canisterName);
 
 // Allows running of the tests without starting and stopping a Bitcoin daemon
 // automatically. That is to say you will need to start and stop the Bitcoin
 // daemon manually. Great for running cli commands after or during the tests to
 // check the state of the test network
-runTests(getTests(canisterId, getP2wpkhAddress, P2WPKH_ADDRESS_FORM));
+runTests(
+    getTests(canisterId, getP2wpkhAddress, P2WPKH_ADDRESS_FORM),
+    canisterName
+);
diff --git a/examples/bitcoin_psbt/test/test.ts b/examples/bitcoin_psbt/test/test.ts
@@ -9,7 +9,8 @@ import { getTests } from 'basic_bitcoin/test/tests';
 
 import { getP2wpkhAddress, P2WPKH_ADDRESS_FORM } from './tests';
 
-const canisterId = getCanisterId('bitcoin_psbt');
+const canisterName = 'bitcoin_psbt';
+const canisterId = getCanisterId(canisterName);
 
 let bitcoinDaemon: BitcoinDaemon;
 
@@ -26,4 +27,4 @@ runTests(() => {
         'runs bitcoin psbt tests while bitcoin daemon is running',
         getTests(canisterId, getP2wpkhAddress, P2WPKH_ADDRESS_FORM)
     );
-});
+}, canisterName);
diff --git a/examples/ckbtc/test/test.ts b/examples/ckbtc/test/test.ts
@@ -13,7 +13,8 @@ import { createActor } from '../wallet/frontend/dfx_generated/wallet_backend';
 // @ts-ignore this path may not exist when these tests are imported into other test projects
 import { _SERVICE } from '../wallet/frontend/dfx_generated/wallet_backend/wallet_backend.did';
 
-let configs = [createConfig(0), createConfig(1)];
+const canisterName = 'wallet_backend';
+const configs = [createConfig(0), createConfig(1)];
 
 type BitcoinDaemon = ChildProcessWithoutNullStreams;
 
@@ -32,7 +33,7 @@ runTests(() => {
         'run ckbtc tests while bitcoin daemon is running',
         getTests(configs)
     );
-});
+}, canisterName);
 
 async function startBitcoinDaemon(): Promise<BitcoinDaemon> {
     if (existsSync(`.bitcoin/regtest`)) {
@@ -62,7 +63,7 @@ async function startBitcoinDaemon(): Promise<BitcoinDaemon> {
 }
 
 function createConfig(id: number): Config {
-    const walletBackendCanisterId = getCanisterId('wallet_backend');
+    const walletBackendCanisterId = getCanisterId(canisterName);
     const identity: any = createIdentity(id);
     const canister = createActor(walletBackendCanisterId, {
         agentOptions: {

diff --git a/examples/hello_world/test/test.ts b/examples/hello_world/test/test.ts
@@ -4,10 +4,11 @@ import { runTests } from 'azle/test';
 import { createActor } from './dfx_generated/hello_world';
 import { getTests } from './tests';
 
-const helloWorldCanister = createActor(getCanisterId('hello_world'), {
+const canisterName = 'hello_world';
+const helloWorldCanister = createActor(getCanisterId(canisterName), {
     agentOptions: {
         host: 'http://127.0.0.1:8000'
     }
 });
 
-runTests(getTests(helloWorldCanister));
+runTests(getTests(helloWorldCanister), canisterName);
diff --git a/examples/hello_world_http_server/benchmarks.json b/examples/hello_world_http_server/benchmarks.json
@@ -0,0 +1,44 @@
+{
+    "backend": {
+        "previous": {
+            "version": "0.25.0",
+            "benchmarks": [
+                {
+                    "instructions": { "__bigint__": "8136265274" },
+                    "method_name": "init",
+                    "timestamp": { "__bigint__": "1729788180680976591" }
+                },
+                {
+                    "instructions": { "__bigint__": "53798727" },
+                    "method_name": "http_request_update",
+                    "timestamp": { "__bigint__": "1729788197889022797" }
+                },
+                {
+                    "instructions": { "__bigint__": "47468265" },
+                    "method_name": "http_request_update",
+                    "timestamp": { "__bigint__": "1729788199085176603" }
+                },
+                {
+                    "instructions": { "__bigint__": "47474297" },
+                    "method_name": "http_request_update",
+                    "timestamp": { "__bigint__": "1729788199623240856" }
+                }
+            ]
+        },
+        "current": {
+            "version": "0.25.0",
+            "benchmarks": [
+                {
+                    "instructions": { "__bigint__": "8135491182" },
+                    "method_name": "init",
+                    "timestamp": { "__bigint__": "1729803431300165877" }
+                },
+                {
+                    "instructions": { "__bigint__": "53810066" },
+                    "method_name": "http_request_update",
+                    "timestamp": { "__bigint__": "1729803449326310198" }
+                }
+            ]
+        }
+    }
+}
diff --git a/examples/hello_world_http_server/benchmarks.md b/examples/hello_world_http_server/benchmarks.md
@@ -0,0 +1,30 @@
+# Benchmarks for backend
+
+## Current benchmarks Azle version: 0.25.0
+
+| Id  | Method Name         | Instructions  | Cycles        | USD           | USD/Million Calls | Change                              |
+| --- | ------------------- | ------------- | ------------- | ------------- | ----------------- | ----------------------------------- |
+| 0   | init                | 8_135_491_182 | 6_454_786_472 | $0.0085827359 | $8_582.73         | <font color="green">-774_092</font> |
+| 1   | http_request_update | 53_810_066    | 22_114_026    | $0.0000294044 | $29.40            | <font color="red">+11_339</font>    |
+
+## Baseline benchmarks Azle version: 0.25.0
+
+| Id  | Method Name         | Instructions  | Cycles        | USD           | USD/Million Calls |
+| --- | ------------------- | ------------- | ------------- | ------------- | ----------------- |
+| 0   | init                | 8_136_265_274 | 6_455_096_109 | $0.0085831476 | $8_583.14         |
+| 1   | http_request_update | 53_798_727    | 22_109_490    | $0.0000293983 | $29.39            |
+| 2   | http_request_update | 47_468_265    | 19_577_306    | $0.0000260314 | $26.03            |
+| 3   | http_request_update | 47_474_297    | 19_579_718    | $0.0000260346 | $26.03            |
+
+---
+
+**Note on calculations:**
+
+-   Cycles are calculated using the formula: base_fee + (per_instruction_fee \* number_of_instructions) + (additional_fee_per_billion \* floor(number_of_instructions / 1_000_000_000))
+-   base_fee: 590_000 cycles
+-   per_instruction_fee: 0.4 cycles
+-   additional_fee_per_billion: 400_000_000 cycles per billion instructions
+-   USD value is derived from the total cycles, where 1 trillion cycles = 1 XDR, and 1 XDR = $1.329670 (as of October 24, 2024)
+
+For the most up-to-date XDR to USD conversion rate, please refer to the [IMF website](https://www.imf.org/external/np/fin/data/rms_sdrv.aspx).
+For the most current fee information, please check the [official documentation](https://internetcomputer.org/docs/current/developer-docs/gas-cost#execution).
diff --git a/examples/hello_world_http_server/test/test.ts b/examples/hello_world_http_server/test/test.ts
@@ -3,6 +3,7 @@ import { runTests } from 'azle/test';
 
 import { getTests } from './tests';
 
-const canisterId = getCanisterId('backend');
+const canisterName = 'backend';
+const canisterId = getCanisterId(canisterName);
 
-runTests(getTests(canisterId));
+runTests(getTests(canisterId), canisterName);
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "azle",
-    "version": "0.24.1",
+    "version": "0.25.0",
     "description": "TypeScript and JavaScript CDK for the Internet Computer",
     "scripts": {
         "typecheck": "tsc --noEmit",

diff --git a/src/build/experimental/commands/compile/get_context.ts b/src/build/experimental/commands/compile/get_context.ts
@@ -30,7 +30,12 @@ export async function getContext(
     const wasmData: WasmData = {
         ...stableContext.wasmData,
         consumer,
-        managementDid
+        managementDid,
+        recordBenchmarks:
+            process.env.npm_lifecycle_event === 'pretest' ||
+            process.env.npm_lifecycle_event === 'test'
+                ? process.env.AZLE_RECORD_BENCHMARKS !== 'false'
+                : process.env.AZLE_RECORD_BENCHMARKS === 'true'
     };
 
     return {

diff --git a/src/build/experimental/commands/compile/javascript.ts b/src/build/experimental/commands/compile/javascript.ts
@@ -60,6 +60,26 @@ export function getPrelude(main: string): string {
             // behave in all async situations
             setTimeout(() => {
                 const canister = Canister.default !== undefined ? Canister.default() : Server(() => globalThis._azleNodeServer)();
+                if (globalThis._azleRecordBenchmarks === true) {
+                    const methodMeta = canister.methodMeta;
+
+                    globalThis._azleCanisterMethodNames = Object.entries(methodMeta).reduce((acc, [key, value]) => {
+                        if (value === undefined) {
+                            return acc;
+                        }
+
+                        if (key === 'queries' || key === 'updates') {
+                            const queriesOrUpdates = value.reduce((innerAcc, method) => {
+                                const indexString = method.index.toString();
+                                return { ...innerAcc, [indexString]: method.name };
+                            }, {});
+                            return { ...acc, ...queriesOrUpdates };
+                        } else {
+                            const indexString = value.index.toString();
+                            return { ...acc, [indexString]: value.name };
+                        }
+                    }, {});
+                }
 
                 const candid = canister.getIdlType([]).accept(new DidVisitor(), {
                     ...getDefaultVisitorData(),

diff --git a/src/build/rust/canister/src/benchmarking.rs b/src/build/rust/canister/src/benchmarking.rs
@@ -0,0 +1,36 @@
+use std::cell::RefCell;
+use std::collections::BTreeMap;
+
+use candid::CandidType;
+use wasmedge_quickjs::{AsObject, Context};
+
+#[derive(CandidType, Debug, Clone)]
+pub struct BenchmarkEntry {
+    pub method_name: String,
+    pub instructions: u64,
+    pub timestamp: u64,
+}
+
+thread_local! {
+    pub static BENCHMARKS_REF_CELL: RefCell<Vec<BenchmarkEntry>> = RefCell::new(Vec::new());
+}
+
+pub fn record_benchmark(context: &mut Context, function_name: &str, instructions: u64) {
+    let timestamp = ic_cdk::api::time();
+
+    let global = context.get_global();
+    let method_names = global.get("_azleCanisterMethodNames");
+    let method_name = method_names
+        .get(function_name)
+        .and_then(|v| Some(v.to_string()?.to_string()))
+        .unwrap_or_else(|| function_name.to_string());
+
+    BENCHMARKS_REF_CELL.with(|benchmarks_ref_cell| {
+        let mut benchmarks = benchmarks_ref_cell.borrow_mut();
+        benchmarks.push(BenchmarkEntry {
+            method_name,
+            instructions,
+            timestamp,
+        });
+    });
+}
diff --git a/src/build/rust/canister/src/execute_method_js.rs b/src/build/rust/canister/src/execute_method_js.rs
@@ -1,22 +1,20 @@
+use crate::{benchmarking::record_benchmark, run_event_loop, RUNTIME, WASM_DATA_REF_CELL};
 use wasmedge_quickjs::AsObject;
 
-use crate::{run_event_loop, RUNTIME};
-
 #[no_mangle]
 #[allow(unused)]
 pub extern "C" fn execute_method_js(function_index: i32, pass_arg_data: i32) {
-    let function_name = &function_index.to_string();
-    let pass_arg_data = if pass_arg_data == 1 { true } else { false };
+    let function_name = function_index.to_string();
+    let pass_arg_data = pass_arg_data == 1;
 
     RUNTIME.with(|runtime| {
         let mut runtime = runtime.borrow_mut();
         let runtime = runtime.as_mut().unwrap();
 
         runtime.run_with_context(|context| {
             let global = context.get_global();
-
             let callbacks = global.get("_azleCallbacks");
-            let method_callback = callbacks.get(function_name).unwrap();
+            let method_callback = callbacks.get(&function_name).unwrap();
 
             let candid_args = if pass_arg_data {
                 ic_cdk::api::call::arg_data_raw()
@@ -28,20 +26,26 @@ pub extern "C" fn execute_method_js(function_index: i32, pass_arg_data: i32) {
                 context.new_array_buffer(&candid_args).into();
 
             let method_callback_function = method_callback.to_function().unwrap();
-
             let result = method_callback_function.call(&[candid_args_js_value]);
 
-            // TODO error handling is mostly done in JS right now
-            // TODO we would really like wasmedge-quickjs to add
-            // TODO good error info to JsException and move error handling
-            // TODO out of our own code
             match &result {
                 wasmedge_quickjs::JsValue::Exception(js_exception) => {
                     js_exception.dump_error();
                     panic!("TODO needs error info");
                 }
                 _ => run_event_loop(context),
             };
+
+            if WASM_DATA_REF_CELL.with(|wasm_data_ref_cell| {
+                wasm_data_ref_cell
+                    .borrow()
+                    .as_ref()
+                    .unwrap()
+                    .record_benchmarks
+            }) {
+                let instructions = ic_cdk::api::performance_counter(1);
+                record_benchmark(context, &function_name, instructions);
+            }
         });
     });
 }
diff --git a/src/build/rust/canister/src/init_and_post_upgrade.rs b/src/build/rust/canister/src/init_and_post_upgrade.rs
@@ -4,6 +4,7 @@ use wasmedge_quickjs::AsObject;
 use crate::{
     execute_method_js, ic, run_event_loop, wasm_binary_manipulation::get_js_code,
     wasm_binary_manipulation::get_wasm_data, EXPERIMENTAL, MEMORY_MANAGER_REF_CELL, RUNTIME,
+    WASM_DATA_REF_CELL,
 };
 
 #[cfg(feature = "experimental")]
@@ -54,6 +55,10 @@ fn initialize(init: bool, function_index: i32, pass_arg_data: i32) {
 
     let wasm_data = get_wasm_data();
 
+    WASM_DATA_REF_CELL.with(|wasm_data_ref_cell| {
+        *wasm_data_ref_cell.borrow_mut() = Some(wasm_data.clone());
+    });
+
     let env_vars: Vec<(&str, &str)> = wasm_data
         .env_vars
         .iter()
@@ -111,6 +116,16 @@ pub fn initialize_js(js: &str, init: bool, function_index: i32, pass_arg_data: i
         // TODO what do we do if there is an error in here?
         context.eval_global_str("globalThis.exports = {};".to_string());
         context.eval_global_str(format!("globalThis._azleExperimental = {EXPERIMENTAL};"));
+        let record_benchmarks = WASM_DATA_REF_CELL.with(|wasm_data_ref_cell| {
+            wasm_data_ref_cell
+                .borrow()
+                .as_ref()
+                .unwrap()
+                .record_benchmarks
+        });
+        context.eval_global_str(format!(
+            "globalThis._azleRecordBenchmarks = {record_benchmarks};"
+        ));
         context.eval_module_str(js.to_string(), "azle_main");
 
         run_event_loop(context);