Merge pull request #2495 from demergent-labs/benchmark--0.25.0

Benchmark Results for benchmark--0.25.0
demergent-labs · Jan 14, 2025 · f66c7a2 · f66c7a2
2 parents bca5e00 + 18b58b3
commit f66c7a2
Show file tree

Hide file tree

Showing 348 changed files with 19,175 additions and 8,740 deletions.
diff --git a/benchmarks.json b/benchmarks.json
@@ -38,5 +38,25 @@
             "max": 18422514685,
             "baselineWeightedEfficiencyScore": 118118515.90932944
         }
+    },
+    "0.25.0": {
+        "stable": {
+            "count": 587,
+            "mean": 121252643.32879046,
+            "median": 1138843,
+            "standardDeviation": 1416653240.869716,
+            "min": 187283,
+            "max": 21097096535,
+            "baselineWeightedEfficiencyScore": 12579287.032879045
+        },
+        "experimental": {
+            "count": 596,
+            "mean": 808198534.0637584,
+            "median": 8925503,
+            "standardDeviation": 2674732562.9079432,
+            "min": 88528,
+            "max": 18489430011,
+            "baselineWeightedEfficiencyScore": 83550621.10637584
+        }
     }
 }
diff --git a/benchmarks.md b/benchmarks.md
@@ -1,5 +1,92 @@
 # Benchmarks
 
+## Version `0.25.0`
+
+<table>
+    <tr>
+        <th></th>
+        <th colspan="2">Stable</th>
+        <th colspan="2">Experimental</th>
+    </tr>
+    <tr>
+        <th>Metric</th>
+        <th>Score</th>
+        <th>Change</th>
+        <th>Score</th>
+        <th>Change</th>
+    </tr>
+    <tr>
+        <td>Baseline Weighted Efficiency</td>
+        <td>12_579_287</td>
+        <td><span style="color: red">1.24%</span></td>
+        <td>83_550_621</td>
+        <td><span style="color: green">-20.72%</span></td>
+    </tr>
+    <tr>
+        <td colspan="5">&nbsp;</td>
+    </tr>
+    <tr>
+        <th></th>
+        <th>Count</th>
+        <th></th>
+        <th>Count</th>
+        <th></th>
+    </tr>
+    <tr>
+        <td>Method Executions</td>
+        <td>587</td>
+        <td><span style="color: green">-3.93%</span></td>
+        <td>596</td>
+        <td><span style="color: green">-20.00%</span></td>
+    </tr>
+    <tr>
+        <td colspan="5">&nbsp;</td>
+    </tr>
+    <tr>
+        <th></th>
+        <th>Number of Instructions</th>
+        <th></th>
+        <th>Number of Instructions</th>
+        <th></th>
+    </tr>
+    	<tr>
+        <td>Mean</td>
+        <td>121_252_643</td>
+        <td><span style="color: red">0.84%</span></td>
+        <td>808_198_534</td>
+        <td><span style="color: green">-21.27%</span></td>
+    </tr>
+	<tr>
+        <td>Median</td>
+        <td>1_138_843</td>
+        <td><span style="color: red">12.62%</span></td>
+        <td>8_925_503</td>
+        <td><span style="color: red">0.03%</span></td>
+    </tr>
+	<tr>
+        <td>Standard Deviation</td>
+        <td>1_416_653_240</td>
+        <td><span style="color: red">11.84%</span></td>
+        <td>2_674_732_562</td>
+        <td><span style="color: green">-11.78%</span></td>
+    </tr>
+	<tr>
+        <td>Min</td>
+        <td>187_283</td>
+        <td><span style="color: red">14.37%</span></td>
+        <td>88_528</td>
+        <td><span style="color: red">4.74%</span></td>
+    </tr>
+	<tr>
+        <td>Max</td>
+        <td>21_097_096_535</td>
+        <td><span style="color: red">10.68%</span></td>
+        <td>18_489_430_011</td>
+        <td><span style="color: red">0.34%</span></td>
+    </tr>
+
+</table>
+
 ## Version `0.25.0-dev`
 
 <table>

diff --git a/examples/experimental/demo/hello_world_http_server/benchmarks.json b/examples/experimental/demo/hello_world_http_server/benchmarks.json
@@ -4,39 +4,63 @@
             "version": "0.25.0",
             "benchmarks": [
                 {
-                    "instructions": { "__bigint__": "8136265274" },
+                    "instructions": {
+                        "__bigint__": "8136265274"
+                    },
                     "method_name": "init",
-                    "timestamp": { "__bigint__": "1729788180680976591" }
+                    "timestamp": {
+                        "__bigint__": "1729788180680976591"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "53798727" },
+                    "instructions": {
+                        "__bigint__": "53798727"
+                    },
                     "method_name": "http_request_update",
-                    "timestamp": { "__bigint__": "1729788197889022797" }
+                    "timestamp": {
+                        "__bigint__": "1729788197889022797"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "47468265" },
+                    "instructions": {
+                        "__bigint__": "47468265"
+                    },
                     "method_name": "http_request_update",
-                    "timestamp": { "__bigint__": "1729788199085176603" }
+                    "timestamp": {
+                        "__bigint__": "1729788199085176603"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "47474297" },
+                    "instructions": {
+                        "__bigint__": "47474297"
+                    },
                     "method_name": "http_request_update",
-                    "timestamp": { "__bigint__": "1729788199623240856" }
+                    "timestamp": {
+                        "__bigint__": "1729788199623240856"
+                    }
                 }
             ]
         },
         "current": {
             "version": "0.25.0",
             "benchmarks": [
                 {
-                    "instructions": { "__bigint__": "8135491182" },
+                    "instructions": {
+                        "__bigint__": "8276429035"
+                    },
                     "method_name": "init",
-                    "timestamp": { "__bigint__": "1729803431300165877" }
+                    "timestamp": {
+                        "__bigint__": "1736543842521839580"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "53810066" },
+                    "instructions": {
+                        "__bigint__": "54051962"
+                    },
                     "method_name": "http_request_update",
-                    "timestamp": { "__bigint__": "1729803449326310198" }
+                    "timestamp": {
+                        "__bigint__": "1736543854584260540"
+                    }
                 }
             ]
         }

diff --git a/examples/experimental/demo/hello_world_http_server/benchmarks.md b/examples/experimental/demo/hello_world_http_server/benchmarks.md
@@ -2,10 +2,10 @@
 
 ## Current benchmarks Azle version: 0.25.0
 
-| Id  | Method Name         | Instructions  | Cycles        | USD           | USD/Million Calls | Change                              |
-| --- | ------------------- | ------------- | ------------- | ------------- | ----------------- | ----------------------------------- |
-| 0   | init                | 8_135_491_182 | 6_454_786_472 | $0.0085827359 | $8_582.73         | <font color="green">-774_092</font> |
-| 1   | http_request_update | 53_810_066    | 22_114_026    | $0.0000294044 | $29.40            | <font color="red">+11_339</font>    |
+| Id  | Method Name         | Instructions  | Cycles        | USD           | USD/Million Calls | Change                                |
+| --- | ------------------- | ------------- | ------------- | ------------- | ----------------- | ------------------------------------- |
+| 0   | init                | 8_276_429_035 | 6_511_161_614 | $0.0086576963 | $8_657.69         | <font color="red">+140_163_761</font> |
+| 1   | http_request_update | 54_051_962    | 22_210_784    | $0.0000295330 | $29.53            | <font color="red">+253_235</font>     |
 
 ## Baseline benchmarks Azle version: 0.25.0
 

diff --git a/examples/experimental/test/end_to_end/candid_rpc/async_await/benchmarks.json b/examples/experimental/test/end_to_end/candid_rpc/async_await/benchmarks.json
@@ -1,52 +1,84 @@
 {
     "async_await": {
         "previous": {
-            "version": "0.25.0-alpha",
+            "version": "0.25.0-dev",
             "benchmarks": [
                 {
-                    "instructions": { "__bigint__": "140032707" },
+                    "instructions": {
+                        "__bigint__": "140161753"
+                    },
                     "method_name": "getRandomnessDirectly",
-                    "timestamp": { "__bigint__": "1730326495861346467" }
+                    "timestamp": {
+                        "__bigint__": "1730484665820866992"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "140003248" },
+                    "instructions": {
+                        "__bigint__": "140170791"
+                    },
                     "method_name": "getRandomnessIndirectly",
-                    "timestamp": { "__bigint__": "1730326498152151226" }
+                    "timestamp": {
+                        "__bigint__": "1730484667828941281"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "140014062" },
+                    "instructions": {
+                        "__bigint__": "140168252"
+                    },
                     "method_name": "getRandomnessSuperIndirectly",
-                    "timestamp": { "__bigint__": "1730326500041691621" }
+                    "timestamp": {
+                        "__bigint__": "1730484669841452976"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "140125173" },
+                    "instructions": {
+                        "__bigint__": "140137414"
+                    },
                     "method_name": "returnPromiseVoid",
-                    "timestamp": { "__bigint__": "1730326502147362381" }
+                    "timestamp": {
+                        "__bigint__": "1730484671896040157"
+                    }
                 }
             ]
         },
         "current": {
-            "version": "0.25.0-dev",
+            "version": "0.25.0",
             "benchmarks": [
                 {
-                    "instructions": { "__bigint__": "140161753" },
+                    "instructions": {
+                        "__bigint__": "151172733"
+                    },
                     "method_name": "getRandomnessDirectly",
-                    "timestamp": { "__bigint__": "1730484665820866992" }
+                    "timestamp": {
+                        "__bigint__": "1736543905863426065"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "140170791" },
+                    "instructions": {
+                        "__bigint__": "151341892"
+                    },
                     "method_name": "getRandomnessIndirectly",
-                    "timestamp": { "__bigint__": "1730484667828941281" }
+                    "timestamp": {
+                        "__bigint__": "1736543906409345593"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "140168252" },
+                    "instructions": {
+                        "__bigint__": "151268211"
+                    },
                     "method_name": "getRandomnessSuperIndirectly",
-                    "timestamp": { "__bigint__": "1730484669841452976" }
+                    "timestamp": {
+                        "__bigint__": "1736543907173690343"
+                    }
                 },
                 {
-                    "instructions": { "__bigint__": "140137414" },
+                    "instructions": {
+                        "__bigint__": "151232271"
+                    },
                     "method_name": "returnPromiseVoid",
-                    "timestamp": { "__bigint__": "1730484671896040157" }
+                    "timestamp": {
+                        "__bigint__": "1736543908237266758"
+                    }
                 }
             ]
         }

diff --git a/examples/experimental/test/end_to_end/candid_rpc/async_await/benchmarks.md b/examples/experimental/test/end_to_end/candid_rpc/async_await/benchmarks.md
@@ -1,22 +1,22 @@
 # Benchmarks for async_await
 
-## Current benchmarks Azle version: 0.25.0-dev
+## Current benchmarks Azle version: 0.25.0
 
-| Id  | Method Name                  | Instructions | Cycles     | USD           | USD/Million Calls | Change                            |
-| --- | ---------------------------- | ------------ | ---------- | ------------- | ----------------- | --------------------------------- |
-| 0   | getRandomnessDirectly        | 140_161_753  | 56_654_701 | $0.0000753321 | $75.33            | <font color="red">+129_046</font> |
-| 1   | getRandomnessIndirectly      | 140_170_791  | 56_658_316 | $0.0000753369 | $75.33            | <font color="red">+167_543</font> |
-| 2   | getRandomnessSuperIndirectly | 140_168_252  | 56_657_300 | $0.0000753355 | $75.33            | <font color="red">+154_190</font> |
-| 3   | returnPromiseVoid            | 140_137_414  | 56_644_965 | $0.0000753191 | $75.31            | <font color="red">+12_241</font>  |
+| Id  | Method Name                  | Instructions | Cycles     | USD           | USD/Million Calls | Change                               |
+| --- | ---------------------------- | ------------ | ---------- | ------------- | ----------------- | ------------------------------------ |
+| 0   | getRandomnessDirectly        | 151_172_733  | 61_059_093 | $0.0000811884 | $81.18            | <font color="red">+11_010_980</font> |
+| 1   | getRandomnessIndirectly      | 151_341_892  | 61_126_756 | $0.0000812784 | $81.27            | <font color="red">+11_171_101</font> |
+| 2   | getRandomnessSuperIndirectly | 151_268_211  | 61_097_284 | $0.0000812392 | $81.23            | <font color="red">+11_099_959</font> |
+| 3   | returnPromiseVoid            | 151_232_271  | 61_082_908 | $0.0000812201 | $81.22            | <font color="red">+11_094_857</font> |
 
-## Baseline benchmarks Azle version: 0.25.0-alpha
+## Baseline benchmarks Azle version: 0.25.0-dev
 
 | Id  | Method Name                  | Instructions | Cycles     | USD           | USD/Million Calls |
 | --- | ---------------------------- | ------------ | ---------- | ------------- | ----------------- |
-| 0   | getRandomnessDirectly        | 140_032_707  | 56_603_082 | $0.0000752634 | $75.26            |
-| 1   | getRandomnessIndirectly      | 140_003_248  | 56_591_299 | $0.0000752478 | $75.24            |
-| 2   | getRandomnessSuperIndirectly | 140_014_062  | 56_595_624 | $0.0000752535 | $75.25            |
-| 3   | returnPromiseVoid            | 140_125_173  | 56_640_069 | $0.0000753126 | $75.31            |
+| 0   | getRandomnessDirectly        | 140_161_753  | 56_654_701 | $0.0000753321 | $75.33            |
+| 1   | getRandomnessIndirectly      | 140_170_791  | 56_658_316 | $0.0000753369 | $75.33            |
+| 2   | getRandomnessSuperIndirectly | 140_168_252  | 56_657_300 | $0.0000753355 | $75.33            |
+| 3   | returnPromiseVoid            | 140_137_414  | 56_644_965 | $0.0000753191 | $75.31            |
 
 ---