Merge pull request #2259 from demergent-labs/benchmark--0.25.0-pre-bi…

…furcation Benchmark Results for benchmark--0.25.0-pre-bifurcation
demergent-labs · Oct 30, 2024 · 342dbc0 · 342dbc0
2 parents 3175a45 + adf0b12
commit 342dbc0
Show file tree

Hide file tree

Showing 313 changed files with 18,160 additions and 24,231 deletions.
diff --git a/.github/actions/get_exclude_dirs/action.yml b/.github/actions/get_exclude_dirs/action.yml
@@ -39,17 +39,17 @@ runs:
               ') }}"
 
               SLOW_EXCLUSIONS="${{ format('
-              tests/end_to_end/candid_rpc/functional_syntax/ckbtc
               tests/end_to_end/candid_rpc/class_syntax/bitcoin
-              tests/end_to_end/http_server/large_files
-              tests/end_to_end/http_server/open_value_sharing
               tests/end_to_end/candid_rpc/class_syntax/stable_structures
               tests/end_to_end/candid_rpc/functional_syntax/bitcoin
               tests/end_to_end/candid_rpc/functional_syntax/composite_queries
+              tests/end_to_end/candid_rpc/functional_syntax/ckbtc
               tests/end_to_end/candid_rpc/functional_syntax/cross_canister_calls
               tests/end_to_end/candid_rpc/functional_syntax/management_canister
               tests/end_to_end/candid_rpc/functional_syntax/stable_structures
               tests/end_to_end/http_server/autoreload
+              tests/end_to_end/http_server/large_files
+              tests/end_to_end/http_server/open_value_sharing
               ') }}"
 
               EXCLUDE_DIRS=""

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -86,28 +86,22 @@ jobs:
                           name: 'Property Class',
                           directories: './tests/property/candid_rpc/class_api'
                       }
-                    - {
-                          name: 'Property Functional',
-                          directories: './tests/property/candid_rpc/functional_api'
-                      }
-                    - {
-                          name: 'Property IC API',
-                          directories: './tests/property/ic_api'
-                      }
         uses: ./.github/workflows/benchmark_parallel.yml
         secrets:
             GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
             GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
             LASTMJS_GITHUB_TOKEN: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
         with:
+            base-branch: ${{ needs.create-branch-prefix.outputs.base-branch }}
+            branch-prefix: ${{ needs.create-branch-prefix.outputs.branch-prefix }}
             directories: ${{ matrix.benchmark_group.directories }}
             exclude-dirs: ${{ needs.get-exclude-dirs.outputs.exclude-dirs }}
-            branch-prefix: ${{ needs.create-branch-prefix.outputs.branch-prefix }}
 
     squash-branches:
         needs: [run-benchmarks, create-branch-prefix]
         uses: ./.github/workflows/squash_branches.yml
         with:
+            base-branch: ${{ needs.create-branch-prefix.outputs.base-branch }}
             branch-prefix: ${{ needs.create-branch-prefix.outputs.branch-prefix }}
             commit-message: 'run benchmarks'
         secrets:
@@ -128,7 +122,7 @@ jobs:
                   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
               run: |
                   gh pr create \
-                    --base ${{ github.ref_name }} \
+                    --base main \
                     --head ${{ needs.create-branch-prefix.outputs.base-branch }} \
                     --title "Benchmark Results for ${{ needs.create-branch-prefix.outputs.base-branch }}" \
                     --body "Automated PR for benchmark results"
diff --git a/.github/workflows/benchmark_parallel.yml b/.github/workflows/benchmark_parallel.yml
@@ -2,16 +2,19 @@ name: Parallel Benchmark
 on:
     workflow_call:
         inputs:
+            base-branch:
+                required: true
+                type: string
+            branch-prefix:
+                required: true
+                type: string
             directories:
                 required: true
                 type: string
             exclude-dirs:
                 required: false
                 type: string
                 default: ''
-            branch-prefix:
-                required: true
-                type: string
 
         secrets:
             GPG_SIGNING_KEY:
@@ -29,6 +32,8 @@ jobs:
             test-infos: ${{ steps.get-test-infos.outputs.test-infos }}
         steps:
             - uses: actions/checkout@v4
+              with:
+                  ref: ${{ inputs.base-branch }}
 
             - id: get-test-infos
               uses: ./.github/actions/get_test_infos
@@ -50,7 +55,7 @@ jobs:
         steps:
             - uses: actions/checkout@v4
               with:
-                  ref: ${{ github.ref }}
+                  ref: ${{ inputs.base-branch }}
                   token: ${{ secrets.LASTMJS_GITHUB_TOKEN || github.token }}
 
             - uses: ./.github/actions/setup_node

diff --git a/.github/workflows/release_parallel.yml → .github/workflows/release.yml b/.github/workflows/release_parallel.yml → .github/workflows/release.yml
@@ -16,9 +16,6 @@ jobs:
             release-version: ${{ inputs.release-version }}
             test-infos: ${{ steps.get-test-infos.outputs.test-infos }}
         steps:
-            - name: Print branch name
-              run: echo "${{ github.ref }}"
-
             - uses: actions/checkout@v4
               with:
                   ref: ${{ github.ref }}
@@ -115,6 +112,7 @@ jobs:
         needs: [prepare-release, update-test-files-for-release-commit]
         uses: ./.github/workflows/squash_branches.yml
         with:
+            base-branch: release--${{ needs.prepare-release.outputs.release-version }}
             branch-prefix: 'update--${{ needs.prepare-release.outputs.release-version }}-'
             commit-message: 'Update test files for all tests and examples ${{ needs.prepare-release.outputs.release-version }}'
         secrets:
@@ -159,7 +157,7 @@ jobs:
                   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
               run: |
                   gh pr create \
-                    --base ${{ github.ref_name }} \
+                    --base main \
                     --head release--${{ needs.prepare-release.outputs.release-version }} \
                     --title "Release ${{ needs.prepare-release.outputs.release-version }}" \
                     --body "Automated PR for release ${{ needs.prepare-release.outputs.release-version }}"
diff --git a/.github/workflows/squash_branches.yml b/.github/workflows/squash_branches.yml
@@ -2,6 +2,9 @@ name: Squash Branches
 on:
     workflow_call:
         inputs:
+            base-branch:
+                required: true
+                type: string
             branch-prefix:
                 required: true
                 type: string
@@ -26,7 +29,7 @@ jobs:
         steps:
             - uses: actions/checkout@v4
               with:
-                  ref: ${{ github.ref }}
+                  ref: ${{ inputs.base-branch }}
                   token: ${{ secrets.LASTMJS_GITHUB_TOKEN }}
                   fetch-depth: 0
 

diff --git a/examples/hello_world/benchmarks.json b/examples/hello_world/benchmarks.json
@@ -0,0 +1,15 @@
+{
+    "hello_world": {
+        "previous": { "version": "No previous benchmarks", "benchmarks": [] },
+        "current": {
+            "version": "0.25.0-pre-bifurcation",
+            "benchmarks": [
+                {
+                    "instructions": { "__bigint__": "1300399" },
+                    "method_name": "setMessage",
+                    "timestamp": { "__bigint__": "1730326452954711925" }
+                }
+            ]
+        }
+    }
+}
diff --git a/examples/hello_world/benchmarks.md b/examples/hello_world/benchmarks.md
@@ -0,0 +1,24 @@
+# Benchmarks for hello_world
+
+## Current benchmarks Azle version: 0.25.0-pre-bifurcation
+
+| Id  | Method Name | Instructions | Cycles    | USD           | USD/Million Calls |
+| --- | ----------- | ------------ | --------- | ------------- | ----------------- |
+| 0   | setMessage  | 1_300_399    | 1_110_159 | $0.0000014761 | $1.47             |
+
+## Baseline benchmarks Azle version: No previous benchmarks
+
+No benchmarks reported
+
+---
+
+**Note on calculations:**
+
+-   Cycles are calculated using the formula: base_fee + (per_instruction_fee \* number_of_instructions) + (additional_fee_per_billion \* floor(number_of_instructions / 1_000_000_000))
+-   base_fee: 590_000 cycles
+-   per_instruction_fee: 0.4 cycles
+-   additional_fee_per_billion: 400_000_000 cycles per billion instructions
+-   USD value is derived from the total cycles, where 1 trillion cycles = 1 XDR, and 1 XDR = $1.329670 (as of October 24, 2024)
+
+For the most up-to-date XDR to USD conversion rate, please refer to the [IMF website](https://www.imf.org/external/np/fin/data/rms_sdrv.aspx).
+For the most current fee information, please check the [official documentation](https://internetcomputer.org/docs/current/developer-docs/gas-cost#execution).
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "azle",
-    "version": "0.25.0",
+    "version": "0.25.0-pre-bifurcation",
     "description": "TypeScript and JavaScript CDK for the Internet Computer",
     "scripts": {
         "typecheck": "tsc --noEmit",

diff --git a/test/benchmarks/index.ts b/test/benchmarks/index.ts
@@ -56,7 +56,7 @@ async function updateBenchmarksForCanisters(
             [canisterName]: {
                 previous: shouldUpdatePrevious
                     ? acc[canisterName]?.current ?? {
-                          version,
+                          version: 'No previous benchmarks',
                           benchmarks: []
                       }
                     : acc[canisterName]?.previous,

diff --git a/tests/end_to_end/candid_rpc/class_syntax/async_await/benchmarks.json b/tests/end_to_end/candid_rpc/class_syntax/async_await/benchmarks.json
@@ -4,49 +4,49 @@
             "version": "0.25.0",
             "benchmarks": [
                 {
-                    "instructions": { "__bigint__": "1409248" },
+                    "instructions": { "__bigint__": "1403216" },
                     "method_name": "getRandomnessDirectly",
-                    "timestamp": { "__bigint__": "1729793674136705013" }
+                    "timestamp": { "__bigint__": "1729803403463099043" }
                 },
                 {
-                    "instructions": { "__bigint__": "1326076" },
+                    "instructions": { "__bigint__": "1331416" },
                     "method_name": "getRandomnessIndirectly",
-                    "timestamp": { "__bigint__": "1729793676325719835" }
+                    "timestamp": { "__bigint__": "1729803405328542548" }
                 },
                 {
-                    "instructions": { "__bigint__": "1368544" },
+                    "instructions": { "__bigint__": "1370777" },
                     "method_name": "getRandomnessSuperIndirectly",
-                    "timestamp": { "__bigint__": "1729793678196910035" }
+                    "timestamp": { "__bigint__": "1729803407508425902" }
                 },
                 {
-                    "instructions": { "__bigint__": "1313116" },
+                    "instructions": { "__bigint__": "1315499" },
                     "method_name": "returnPromiseVoid",
-                    "timestamp": { "__bigint__": "1729793680211554557" }
+                    "timestamp": { "__bigint__": "1729803409436362598" }
                 }
             ]
         },
         "current": {
-            "version": "0.25.0",
+            "version": "0.25.0-pre-bifurcation",
             "benchmarks": [
                 {
                     "instructions": { "__bigint__": "1403216" },
                     "method_name": "getRandomnessDirectly",
-                    "timestamp": { "__bigint__": "1729803403463099043" }
+                    "timestamp": { "__bigint__": "1730326449453914519" }
                 },
                 {
-                    "instructions": { "__bigint__": "1331416" },
+                    "instructions": { "__bigint__": "1331300" },
                     "method_name": "getRandomnessIndirectly",
-                    "timestamp": { "__bigint__": "1729803405328542548" }
+                    "timestamp": { "__bigint__": "1730326451554098278" }
                 },
                 {
-                    "instructions": { "__bigint__": "1370777" },
+                    "instructions": { "__bigint__": "1370708" },
                     "method_name": "getRandomnessSuperIndirectly",
-                    "timestamp": { "__bigint__": "1729803407508425902" }
+                    "timestamp": { "__bigint__": "1730326453791902468" }
                 },
                 {
-                    "instructions": { "__bigint__": "1315499" },
+                    "instructions": { "__bigint__": "1315534" },
                     "method_name": "returnPromiseVoid",
-                    "timestamp": { "__bigint__": "1729803409436362598" }
+                    "timestamp": { "__bigint__": "1730326455706568989" }
                 }
             ]
         }

diff --git a/tests/end_to_end/candid_rpc/class_syntax/async_await/benchmarks.md b/tests/end_to_end/candid_rpc/class_syntax/async_await/benchmarks.md
@@ -1,22 +1,22 @@
 # Benchmarks for async_await
 
-## Current benchmarks Azle version: 0.25.0
+## Current benchmarks Azle version: 0.25.0-pre-bifurcation
 
-| Id  | Method Name                  | Instructions | Cycles    | USD           | USD/Million Calls | Change                            |
-| --- | ---------------------------- | ------------ | --------- | ------------- | ----------------- | --------------------------------- |
-| 0   | getRandomnessDirectly        | 1_403_216    | 1_151_286 | $0.0000015308 | $1.53             | <font color="green">-6_032</font> |
-| 1   | getRandomnessIndirectly      | 1_331_416    | 1_122_566 | $0.0000014926 | $1.49             | <font color="red">+5_340</font>   |
-| 2   | getRandomnessSuperIndirectly | 1_370_777    | 1_138_310 | $0.0000015136 | $1.51             | <font color="red">+2_233</font>   |
-| 3   | returnPromiseVoid            | 1_315_499    | 1_116_199 | $0.0000014842 | $1.48             | <font color="red">+2_383</font>   |
+| Id  | Method Name                  | Instructions | Cycles    | USD           | USD/Million Calls | Change                          |
+| --- | ---------------------------- | ------------ | --------- | ------------- | ----------------- | ------------------------------- |
+| 0   | getRandomnessDirectly        | 1_403_216    | 1_151_286 | $0.0000015308 | $1.53             | <font color="red">0</font>      |
+| 1   | getRandomnessIndirectly      | 1_331_300    | 1_122_520 | $0.0000014926 | $1.49             | <font color="green">-116</font> |
+| 2   | getRandomnessSuperIndirectly | 1_370_708    | 1_138_283 | $0.0000015135 | $1.51             | <font color="green">-69</font>  |
+| 3   | returnPromiseVoid            | 1_315_534    | 1_116_213 | $0.0000014842 | $1.48             | <font color="red">+35</font>    |
 
 ## Baseline benchmarks Azle version: 0.25.0
 
 | Id  | Method Name                  | Instructions | Cycles    | USD           | USD/Million Calls |
 | --- | ---------------------------- | ------------ | --------- | ------------- | ----------------- |
-| 0   | getRandomnessDirectly        | 1_409_248    | 1_153_699 | $0.0000015340 | $1.53             |
-| 1   | getRandomnessIndirectly      | 1_326_076    | 1_120_430 | $0.0000014898 | $1.48             |
-| 2   | getRandomnessSuperIndirectly | 1_368_544    | 1_137_417 | $0.0000015124 | $1.51             |
-| 3   | returnPromiseVoid            | 1_313_116    | 1_115_246 | $0.0000014829 | $1.48             |
+| 0   | getRandomnessDirectly        | 1_403_216    | 1_151_286 | $0.0000015308 | $1.53             |
+| 1   | getRandomnessIndirectly      | 1_331_416    | 1_122_566 | $0.0000014926 | $1.49             |
+| 2   | getRandomnessSuperIndirectly | 1_370_777    | 1_138_310 | $0.0000015136 | $1.51             |
+| 3   | returnPromiseVoid            | 1_315_499    | 1_116_199 | $0.0000014842 | $1.48             |
 
 ---
 

diff --git a/tests/end_to_end/candid_rpc/class_syntax/audio_recorder/benchmarks.json b/tests/end_to_end/candid_rpc/class_syntax/audio_recorder/benchmarks.json
@@ -0,0 +1,35 @@
+{
+    "audio_recorder": {
+        "previous": { "version": "No previous benchmarks", "benchmarks": [] },
+        "current": {
+            "version": "0.25.0-pre-bifurcation",
+            "benchmarks": [
+                {
+                    "instructions": { "__bigint__": "11210958" },
+                    "method_name": "createUser",
+                    "timestamp": { "__bigint__": "1730326453422771537" }
+                },
+                {
+                    "instructions": { "__bigint__": "30870605" },
+                    "method_name": "createRecording",
+                    "timestamp": { "__bigint__": "1730326455601170106" }
+                },
+                {
+                    "instructions": { "__bigint__": "43574088" },
+                    "method_name": "deleteRecording",
+                    "timestamp": { "__bigint__": "1730326457536073233" }
+                },
+                {
+                    "instructions": { "__bigint__": "30668934" },
+                    "method_name": "createRecording",
+                    "timestamp": { "__bigint__": "1730326459743405791" }
+                },
+                {
+                    "instructions": { "__bigint__": "29795439" },
+                    "method_name": "deleteUser",
+                    "timestamp": { "__bigint__": "1730326461791915596" }
+                }
+            ]
+        }
+    }
+}
diff --git a/tests/end_to_end/candid_rpc/class_syntax/audio_recorder/benchmarks.md b/tests/end_to_end/candid_rpc/class_syntax/audio_recorder/benchmarks.md
@@ -0,0 +1,28 @@
+# Benchmarks for audio_recorder
+
+## Current benchmarks Azle version: 0.25.0-pre-bifurcation
+
+| Id  | Method Name     | Instructions | Cycles     | USD           | USD/Million Calls |
+| --- | --------------- | ------------ | ---------- | ------------- | ----------------- |
+| 0   | createUser      | 11_210_958   | 5_074_383  | $0.0000067473 | $6.74             |
+| 1   | createRecording | 30_870_605   | 12_938_242 | $0.0000172036 | $17.20            |
+| 2   | deleteRecording | 43_574_088   | 18_019_635 | $0.0000239602 | $23.96            |
+| 3   | createRecording | 30_668_934   | 12_857_573 | $0.0000170963 | $17.09            |
+| 4   | deleteUser      | 29_795_439   | 12_508_175 | $0.0000166317 | $16.63            |
+
+## Baseline benchmarks Azle version: No previous benchmarks
+
+No benchmarks reported
+
+---
+
+**Note on calculations:**
+
+-   Cycles are calculated using the formula: base_fee + (per_instruction_fee \* number_of_instructions) + (additional_fee_per_billion \* floor(number_of_instructions / 1_000_000_000))
+-   base_fee: 590_000 cycles
+-   per_instruction_fee: 0.4 cycles
+-   additional_fee_per_billion: 400_000_000 cycles per billion instructions
+-   USD value is derived from the total cycles, where 1 trillion cycles = 1 XDR, and 1 XDR = $1.329670 (as of October 24, 2024)
+
+For the most up-to-date XDR to USD conversion rate, please refer to the [IMF website](https://www.imf.org/external/np/fin/data/rms_sdrv.aspx).
+For the most current fee information, please check the [official documentation](https://internetcomputer.org/docs/current/developer-docs/gas-cost#execution).