From 028ff73733e7a961e49bc0191ffa8475c0ca46b0 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Mon, 8 Apr 2024 00:47:16 +0200 Subject: [PATCH 1/9] ci: update sjsonnet for benchmark --- flake.nix | 16 ++++++++-------- nix/jsonnet.nix | 9 ++++++--- nix/sjsonnet.nix | 21 ++++++++++----------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/flake.nix b/flake.nix index 7aba3322..317c5377 100644 --- a/flake.nix +++ b/flake.nix @@ -63,7 +63,7 @@ inherit go-jsonnet sjsonnet jsonnet; jrsonnetVariants = [ { - drv = jrsonnet.overrideAttrs {forBenchmarks = true;}; + drv = jrsonnet.override {forBenchmarks = true;}; name = ""; } ]; @@ -73,7 +73,7 @@ quick = true; jrsonnetVariants = [ { - drv = jrsonnet.overrideAttrs {forBenchmarks = true;}; + drv = jrsonnet.override {forBenchmarks = true;}; name = ""; } ]; @@ -82,15 +82,15 @@ inherit go-jsonnet sjsonnet jsonnet; jrsonnetVariants = [ { - drv = jrsonnet.overrideAttrs {forBenchmarks = true;}; + drv = jrsonnet.override {forBenchmarks = true;}; name = "current"; } { - drv = jrsonnet-nightly.overrideAttrs {forBenchmarks = true;}; + drv = jrsonnet-nightly.override {forBenchmarks = true;}; name = "current-nightly"; } { - drv = jrsonnet-release.overrideAttrs {forBenchmarks = true;}; + drv = jrsonnet-release.override {forBenchmarks = true;}; name = "release"; } ]; @@ -100,15 +100,15 @@ quick = true; jrsonnetVariants = [ { - drv = jrsonnet; + drv = jrsonnet.override {forBenchmarks = true;}; name = "current"; } { - drv = jrsonnet-nightly; + drv = jrsonnet-nightly.override {forBenchmarks = true;}; name = "current-nightly"; } { - drv = jrsonnet-release; + drv = jrsonnet-release.override {forBenchmarks = true;}; name = "release"; } ]; diff --git a/nix/jsonnet.nix b/nix/jsonnet.nix index c3a9ca63..a087ebbc 100644 --- a/nix/jsonnet.nix +++ b/nix/jsonnet.nix @@ -1,5 +1,8 @@ -{ stdenv, lib, jekyll, fetchFromGitHub, makeWrapper }: - +{ + stdenv, + fetchFromGitHub, + makeWrapper, +}: stdenv.mkDerivation rec { pname = "jsonnet"; version = "0.20.0"; @@ -15,7 +18,7 @@ stdenv.mkDerivation rec { "jsonnet" ]; - nativeBuildInputs = [ makeWrapper ]; + nativeBuildInputs = [makeWrapper]; installPhase = '' mkdir -p $out/bin diff --git a/nix/sjsonnet.nix b/nix/sjsonnet.nix index 0cc12eac..bd650070 100644 --- a/nix/sjsonnet.nix +++ b/nix/sjsonnet.nix @@ -1,25 +1,24 @@ # This derivation uses released sjsonnet binary, which most users will use - # However, recommended way of using sjsonnet - is using a client-server model, # for which there is no released binaries: https://github.com/databricks/sjsonnet/issues/51 - # TODO: Somehow build client-server version of sjsonnet, and use it in benchmarks - -{ stdenv, lib, fetchurl, jdk20, makeWrapper }: - +{ + stdenv, + fetchurl, + jdk20, + makeWrapper, +}: stdenv.mkDerivation rec { pname = "sjsonnet"; - version = "0.4.6"; + version = "0.4.9"; src = fetchurl { - url = - "https://github.com/databricks/${pname}/releases/download/${version}/${pname}-${version}.jar"; - hash = "sha256-ivVKMf7W9iLF/KSXYEl6iJeNGEP5jxw2uqPF7pz6rBU="; + url = "https://github.com/databricks/${pname}/releases/download/${version}/${pname}-${version}.jar"; + hash = "sha256-2W5qhe+MFsHQFlbPr7RQO4mx9qRjdfpI1z4qQGcVnYM="; }; unpackPhase = "true"; - buildInputs = - [ jdk20 makeWrapper ]; + buildInputs = [jdk20 makeWrapper]; installPhase = '' mkdir -p $out/bin $out/lib From 9fc25c53db936d05aa1fb8ee288f22b7b825c235 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Mon, 8 Apr 2024 00:50:03 +0200 Subject: [PATCH 2/9] doc: scratch benchmarks results --- docs/benchmarks.md | 284 +++++++++++++++++++++++---------------------- 1 file changed, 148 insertions(+), 136 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 01fae9a6..bff4e4e7 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -9,10 +9,10 @@ Unfortunately, I haven't managed to measure performance of Haskell implementatio
Tested versions -Go: Jsonnet commandline interpreter (Go implementation) v0.19.1 +Go: Jsonnet commandline interpreter (Go implementation) v0.20.0 ``` -Jsonnet commandline interpreter (Go implementation) v0.19.1 +Jsonnet commandline interpreter (Go implementation) v0.20.0 jsonnet {
" >> $out echo >> $out + echo Rust (alternative): >> $out + echo >> $out + echo "\`\`\`" >> $out + rsjsonnet --help 2>> $out || true + echo "\`\`\`" >> $out + echo >> $out + echo "" >> $out + echo >> $out echo >> $out ''} diff --git a/nix/go-jsonnet.nix b/nix/go-jsonnet.nix index 29e26418..d35e43bf 100644 --- a/nix/go-jsonnet.nix +++ b/nix/go-jsonnet.nix @@ -1,6 +1,9 @@ -{ lib, buildGo119Module, fetchFromGitHub, makeWrapper }: - -buildGo119Module rec { +{ + buildGoModule, + fetchFromGitHub, + makeWrapper, +}: +buildGoModule rec { pname = "go-jsonnet"; version = "0.20.0"; @@ -12,7 +15,7 @@ buildGo119Module rec { }; vendorHash = "sha256-j1fTOUpLx34TgzW94A/BctLrg9XoTtb3cBizhVJoEEI="; - buildInputs = [ makeWrapper ]; + buildInputs = [makeWrapper]; postInstall = '' mv $out/bin/jsonnet $out/bin/go-jsonnet @@ -21,5 +24,5 @@ buildGo119Module rec { doCheck = false; - subPackages = [ "cmd/jsonnet" ]; + subPackages = ["cmd/jsonnet"]; } From b4b864c001da6045729ebc9c5fe88caff15a7b15 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Sat, 27 Apr 2024 01:58:58 +0200 Subject: [PATCH 6/9] chore: increase default max-stack for rsjsonnet --- nix/rsjsonnet.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nix/rsjsonnet.nix b/nix/rsjsonnet.nix index 83adcea9..cc19f63a 100644 --- a/nix/rsjsonnet.nix +++ b/nix/rsjsonnet.nix @@ -1,7 +1,7 @@ { fetchFromGitHub, rustPlatform, - lib, + makeWrapper, }: rustPlatform.buildRustPackage rec { pname = "rsjsonnet"; @@ -15,4 +15,10 @@ rustPlatform.buildRustPackage rec { }; cargoHash = "sha256-TsUN9oUu6S1l9oTaR6nET1ZdRvMrR29bkP3VEDre8aE="; + + nativeBuildInputs = [makeWrapper]; + + postInstall = '' + wrapProgram $out/bin/rsjsonnet --add-flags "--max-stack=200000" + ''; } From 13d38c7568683e0094f1c9935e6a1a8080655468 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Sat, 27 Apr 2024 02:20:46 +0200 Subject: [PATCH 7/9] chore: update sjsonnet to v0.4.10 --- nix/sjsonnet.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/sjsonnet.nix b/nix/sjsonnet.nix index 22e3a999..bf7531e0 100644 --- a/nix/sjsonnet.nix +++ b/nix/sjsonnet.nix @@ -11,11 +11,11 @@ }: stdenv.mkDerivation rec { pname = "sjsonnet"; - version = "0.4.9"; + version = "0.4.10"; src = fetchurl { url = "https://github.com/databricks/${pname}/releases/download/${version}/${pname}-${version}.jar"; - hash = "sha256-2W5qhe+MFsHQFlbPr7RQO4mx9qRjdfpI1z4qQGcVnYM="; + hash = "sha256-zKlmh+jXm23x78CKwna8fGHNqziWuPipSbWTPUCADrY="; }; unpackPhase = "true"; From 7f6d52c339d259ee49cd5652eb5bb0d9812f6e4b Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Sat, 27 Apr 2024 02:40:23 +0200 Subject: [PATCH 8/9] chore: update skipped benchmarks list --- nix/benchmarks.nix | 351 +++++++++++++++++++++++++++++++++------------ 1 file changed, 258 insertions(+), 93 deletions(-) diff --git a/nix/benchmarks.nix b/nix/benchmarks.nix index d10160a6..ad42e108 100644 --- a/nix/benchmarks.nix +++ b/nix/benchmarks.nix @@ -1,83 +1,94 @@ -{ lib -, runCommand -, jsonnet-bundler -, cacert -, stdenv -, fetchFromGitHub -, go-jsonnet -, sjsonnet -, jsonnet -, rsjsonnet -, hyperfine -, quick ? false -, jrsonnetVariants +{ + lib, + runCommand, + jsonnet-bundler, + cacert, + stdenv, + fetchFromGitHub, + go-jsonnet, + sjsonnet, + jsonnet, + rsjsonnet, + hyperfine, + quick ? false, + jrsonnetVariants, }: - -with lib; - -let +with lib; let jsonnetBench = fetchFromGitHub { rev = "v0.19.1"; owner = "google"; repo = "jsonnet"; hash = "sha256-q1MNdbyrx4vvN5woe0o90pPqaNtsZjI5RQ7KJt7rOpU="; }; - goJsonnetBench = (fetchFromGitHub { - owner = "google"; - repo = "go-jsonnet"; - rev = "v0.19.1"; - hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI="; - }) + "/builtin-benchmarks"; + goJsonnetBench = + (fetchFromGitHub { + owner = "google"; + repo = "go-jsonnet"; + rev = "v0.19.1"; + hash = "sha256-FgQYnas0qkIedRAA8ApZXLzEylg6PS6+8zzl7j+yOeI="; + }) + + "/builtin-benchmarks"; graalvmBench = fetchFromGitHub { owner = "oracle"; repo = "graal"; rev = "bc305df3fe587960f7635f0185571500e5988475"; hash = "sha256-4EKB1b2o4/qtYQ+nqbbs621OJrtjApsAWEBcw5EjrYc="; }; - kubePrometheusBench = - let - src = fetchFromGitHub { - owner = "prometheus-operator"; - repo = "kube-prometheus"; - rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd"; - hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM="; - }; - in + kubePrometheusBench = let + src = fetchFromGitHub { + owner = "prometheus-operator"; + repo = "kube-prometheus"; + rev = "d3889807798d1697ea0691f10caf1b6a1997a8bd"; + hash = "sha256-TeYWHzoZAmDp2PzT7EH8XRUcvb3tR8Qfxel7o2QBvIM="; + }; + in runCommand "kube-prometheus-vendor" - { - outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw="; - outputHashMode = "recursive"; - buildInputs = [ cacert ]; - } - '' - mkdir -p $out - cp -r ${src}/* $out/ - cd $out - mkdir vendor - ${jsonnet-bundler}/bin/jb install - ''; + { + outputHash = "sha256-AGc0dHlD/Ld7I5b1+gOotzJkYrn+bB1VjISdD5NITtw="; + outputHashMode = "recursive"; + buildInputs = [cacert]; + } + '' + mkdir -p $out + cp -r ${src}/* $out/ + cd $out + mkdir vendor + ${jsonnet-bundler}/bin/jb install + ''; # Removes outsiders from the output # Useful when comparing performance of different jrsonnet releases - skipSlow = if quick then "slow benchmark, but only quick requested" else ""; + skipSlow = + if quick + then "slow benchmark, but only quick requested" + else ""; in -stdenv.mkDerivation { - name = "benchmarks"; - __impure = true; - unpackPhase = "true"; + stdenv.mkDerivation { + name = "benchmarks"; + __impure = true; + unpackPhase = "true"; - buildInputs = [ - sjsonnet - jsonnet - rsjsonnet - go-jsonnet + buildInputs = [ + sjsonnet + jsonnet + rsjsonnet + go-jsonnet - hyperfine - ]; + hyperfine + ]; - installPhase = - let - mkBench = { name, path, omitSource ? false, pathIsGenerator ? false, skipScala ? "", skipCpp ? "", skipGo ? "", vendor ? "" }: '' + installPhase = let + mkBench = { + name, + path, + omitSource ? false, + pathIsGenerator ? false, + skipRustAlternative ? "", + skipScala ? "", + skipCpp ? "", + skipGo ? "", + vendor ? "", + }: '' set -oux temp=$(mktemp -d) @@ -86,6 +97,10 @@ stdenv.mkDerivation { echo >> $out echo "### ${name}" >> $out echo >> $out + ${optionalString (skipRustAlternative != "") '' + echo "> Note: No results for Rust (alternative), ${skipRustAlternative}" >> $out + echo >> $out + ''} ${optionalString (skipGo != "") '' echo "> Note: No results for Go, ${skipGo}" >> $out echo >> $out @@ -116,17 +131,20 @@ stdenv.mkDerivation { path=generated.jsonnet ''} hyperfine -N -w4 -m20 --output=pipe --style=basic --export-markdown result.md \ - ${concatStringsSep " " (forEach jrsonnetVariants (variant: - "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${if variant.name != "" then " (${variant.name})" else ""}\"" - ))} \ - "rsjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}" -n "Rust (alternative, rsjsonnet)" \ + ${concatStringsSep " " (forEach jrsonnetVariants ( + variant: "\"${variant.drv}/bin/jrsonnet $path${optionalString (vendor != "") " -J${vendor}"}\" -n \"Rust${ + if variant.name != "" + then " (${variant.name})" + else "" + }\"" + ))} \ + ${optionalString (skipRustAlternative == "") "\"rsjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Rust (alternative, rsjsonnet)\""} \ ${optionalString (skipGo == "") "\"go-jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Go\""} \ ${optionalString (skipScala == "") "\"sjsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"Scala\""} \ ${optionalString (skipCpp == "") "\"jsonnet $path${optionalString (vendor != "") " -J ${vendor}"}\" -n \"C++\""} cat result.md >> $out ''; - in - '' + in '' touch $out ${optionalString (!quick) '' cat ${./benchmarks.md} >> $out @@ -167,41 +185,188 @@ stdenv.mkDerivation { echo >> $out ''} echo "## Real world" >> $out - ${mkBench {name = "Graalvm CI"; path = "${graalvmBench}/ci.jsonnet"; skipCpp = "takes longer than a hour"; skipGo = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "Kube-prometheus manifests"; vendor = "${kubePrometheusBench}/vendor"; path = "${kubePrometheusBench}/example.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}} + ${mkBench { + name = "Graalvm CI"; + path = "${graalvmBench}/ci.jsonnet"; + skipCpp = "takes longer than a hour"; + skipGo = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "Kube-prometheus manifests"; + vendor = "${kubePrometheusBench}/vendor"; + path = "${kubePrometheusBench}/example.jsonnet"; + skipCpp = skipSlow; + skipGo = skipSlow; + skipScala = skipSlow; + }} echo >> $out echo "## Benchmarks from C++ jsonnet (/perf_tests)" >> $out - ${mkBench {name = "Large string join"; path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet"; skipScala = skipSlow;}} - ${mkBench {name = "Large string template"; omitSource = true; path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; skipGo = "fails with os stack size exhausion"; skipCpp = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "Realistic 1"; path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "Realistic 2"; path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; skipGo = skipSlow; skipCpp = skipSlow; skipScala = skipSlow;}} + ${mkBench { + name = "Large string join"; + path = "${jsonnetBench}/perf_tests/large_string_join.jsonnet"; + skipScala = skipSlow; + }} + ${mkBench { + name = "Large string template"; + omitSource = true; + path = "${jsonnetBench}/perf_tests/large_string_template.jsonnet"; + skipGo = "fails with os stack size exhausion"; + skipCpp = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "Realistic 1"; + path = "${jsonnetBench}/perf_tests/realistic1.jsonnet"; + skipGo = skipSlow; + skipCpp = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "Realistic 2"; + path = "${jsonnetBench}/perf_tests/realistic2.jsonnet"; + skipGo = skipSlow; + skipCpp = skipSlow; + skipScala = skipSlow; + }} echo >> $out echo "## Benchmarks from C++ jsonnet (/benchmarks)" >> $out - ${mkBench {name = "Tail call"; path = "${jsonnetBench}/benchmarks/bench.01.jsonnet"; skipScala = skipSlow;}} - ${mkBench {name = "Inheritance recursion"; path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow;}} - ${mkBench {name = "Simple recursive call"; path = "${jsonnetBench}/benchmarks/bench.03.jsonnet"; skipScala = skipSlow; skipGo = skipSlow;}} - ${mkBench {name = "Foldl string concat"; path = "${jsonnetBench}/benchmarks/bench.04.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "Array sorts"; path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow;}} - ${mkBench {name = "Lazy array"; path = "${jsonnetBench}/benchmarks/bench.07.jsonnet"; skipGo = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "Inheritance function recursion"; path = "${jsonnetBench}/benchmarks/bench.08.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "String strips"; path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "Big object"; path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; pathIsGenerator = true; skipScala = skipSlow;}} + ${mkBench { + name = "Tail call"; + path = "${jsonnetBench}/benchmarks/bench.01.jsonnet"; + skipScala = skipSlow; + }} + ${mkBench { + name = "Inheritance recursion"; + path = "${jsonnetBench}/benchmarks/bench.02.jsonnet"; + skipCpp = skipSlow; + skipGo = skipSlow; + }} + ${mkBench { + name = "Simple recursive call"; + path = "${jsonnetBench}/benchmarks/bench.03.jsonnet"; + skipScala = skipSlow; + skipGo = skipSlow; + }} + ${mkBench { + name = "Foldl string concat"; + path = "${jsonnetBench}/benchmarks/bench.04.jsonnet"; + skipCpp = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "Array sorts"; + path = "${jsonnetBench}/benchmarks/bench.06.jsonnet"; + skipCpp = skipSlow; + # std.assertEqual(reverse(std.range(1, 1000)), sort(std.range(1, 1000), keyF=function(x) -x)) + skipScala = "sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204"; + }} + ${mkBench { + name = "Lazy array"; + path = "${jsonnetBench}/benchmarks/bench.07.jsonnet"; + skipGo = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "Inheritance function recursion"; + path = "${jsonnetBench}/benchmarks/bench.08.jsonnet"; + skipCpp = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "String strips"; + path = "${jsonnetBench}/benchmarks/bench.09.jsonnet"; + skipCpp = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "Big object"; + path = "${jsonnetBench}/benchmarks/gen_big_object.jsonnet"; + pathIsGenerator = true; + skipScala = skipSlow; + }} echo >> $out echo "## Benchmarks from Go jsonnet (builtins)" >> $out - ${mkBench {name = "std.base64"; path = "${goJsonnetBench}/base64.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "std.base64Decode"; path = "${goJsonnetBench}/base64Decode.jsonnet"; skipCpp = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "std.base64DecodeBytes"; path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "std.base64 (byte array)"; path = "${goJsonnetBench}/base64_byte_array.jsonnet"; skipCpp = skipSlow; skipGo = skipSlow; skipScala = skipSlow;}} - ${mkBench {name = "std.foldl"; path = "${goJsonnetBench}/foldl.jsonnet"; skipScala = skipSlow;}} - ${mkBench {name = "std.manifestJsonEx"; path = "${goJsonnetBench}/manifestJsonEx.jsonnet"; skipScala = skipSlow; skipCpp = skipSlow;}} - ${mkBench {name = "std.manifestTomlEx"; path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; skipScala = "std.manifestTomlEx is not implemented"; skipCpp=skipSlow;}} - ${mkBench {name = "std.parseInt"; path = "${goJsonnetBench}/parseInt.jsonnet"; skipScala = skipSlow; skipCpp = skipSlow;}} - ${mkBench {name = "std.reverse"; path = "${goJsonnetBench}/reverse.jsonnet"; skipScala = "std.reverse is not implemented"; skipCpp = skipSlow; skipGo = skipSlow;}} - ${mkBench {name = "std.substr"; path = "${goJsonnetBench}/substr.jsonnet"; skipScala = skipSlow;}} - ${mkBench {name = "Comparsion for array"; path = "${goJsonnetBench}/comparison.jsonnet"; skipScala = "array comparsion is not implemented"; skipCpp = skipSlow;}} - ${mkBench {name = "Comparsion for primitives"; path = "${goJsonnetBench}/comparison2.jsonnet"; skipCpp = "can't run: uses up to 192GB of RAM"; skipGo = skipSlow; skipScala = skipSlow;}} + ${mkBench { + name = "std.base64"; + path = "${goJsonnetBench}/base64.jsonnet"; + skipRustAlternative = skipSlow; + skipCpp = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "std.base64Decode"; + path = "${goJsonnetBench}/base64Decode.jsonnet"; + skipRustAlternative = skipSlow; + skipCpp = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "std.base64DecodeBytes"; + path = "${goJsonnetBench}/base64DecodeBytes.jsonnet"; + skipRustAlternative = skipSlow; + skipCpp = skipSlow; + skipGo = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "std.base64 (byte array)"; + path = "${goJsonnetBench}/base64_byte_array.jsonnet"; + skipRustAlternative = skipSlow; + skipCpp = skipSlow; + skipGo = skipSlow; + skipScala = skipSlow; + }} + ${mkBench { + name = "std.foldl"; + path = "${goJsonnetBench}/foldl.jsonnet"; + skipScala = skipSlow; + }} + ${mkBench { + name = "std.manifestJsonEx"; + path = "${goJsonnetBench}/manifestJsonEx.jsonnet"; + skipScala = skipSlow; + skipCpp = skipSlow; + }} + ${mkBench { + name = "std.manifestTomlEx"; + path = "${goJsonnetBench}/manifestTomlEx.jsonnet"; + skipScala = "std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111"; + skipCpp = skipSlow; + }} + ${mkBench { + name = "std.parseInt"; + path = "${goJsonnetBench}/parseInt.jsonnet"; + skipScala = skipSlow; + skipCpp = skipSlow; + }} + ${mkBench { + name = "std.reverse"; + path = "${goJsonnetBench}/reverse.jsonnet"; + skipCpp = skipSlow; + skipGo = skipSlow; + }} + ${mkBench { + name = "std.substr"; + path = "${goJsonnetBench}/substr.jsonnet"; + skipScala = skipSlow; + }} + ${mkBench { + name = "Comparsion for array"; + path = "${goJsonnetBench}/comparison.jsonnet"; + skipScala = "array comparsion is not implemented"; + skipCpp = skipSlow; + }} + ${mkBench { + name = "Comparsion for primitives"; + path = "${goJsonnetBench}/comparison2.jsonnet"; + skipRustAlternative = skipSlow; + skipCpp = "can't run: uses up to 192GB of RAM"; + skipGo = skipSlow; + skipScala = skipSlow; + }} ''; -} + } From 1c6455b7412b4654beae6045baf73cc6e8150377 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Sat, 27 Apr 2024 16:54:50 +0200 Subject: [PATCH 9/9] chore: rerun benchmarks --- docs/benchmarks.md | 269 +++++++++++++++++++++++++++------------------ nix/benchmarks.nix | 12 +- 2 files changed, 169 insertions(+), 112 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index bff4e4e7..4a0c88c7 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -121,7 +121,7 @@ Scala: ``` Missing argument: file -Expected Signature: Sjsonnet 0.4.9 +Expected Signature: Sjsonnet 0.4.10 usage: sjsonnet [sjsonnet-options] script-file -i --interactive Run Mill in interactive mode, suitable for opening REPLs and taking user input @@ -171,6 +171,37 @@ usage: sjsonnet [sjsonnet-options] script-file +Rust (alternative): + +``` +Usage: rsjsonnet [OPTIONS] + +Arguments: + + +Options: + -e, --exec Treat filename as code + -J, --jpath Specify an additional library search dir (right-most wins) + -o, --output-file Write to the output file rather than stdout + -m, --multi Write multiple files to the directory, list files on stdout + -y, --yaml-stream Write output as a YAML stream of JSON documents + -S, --string Expect a string, manifest as plain text + -s, --max-stack Number of allowed stack frames + -t, --max-trace Max length of stack trace before cropping + -V, --ext-str + --ext-str-file + --ext-code + --ext-code-file + -A, --tla-str + --tla-str-file + --tla-code + --tla-code-file + -h, --help Print help + +``` + + + ## Real world @@ -250,9 +281,10 @@ local verify_ci = (import 'ci-check.libsonnet').verify_ci; | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 119.0 ± 6.1 | 98.2 | 122.8 | 1.00 | -| `Go` | 2033.3 ± 29.6 | 1973.3 | 2088.8 | 17.09 ± 0.91 | -| `Scala` | 801.3 ± 17.5 | 769.1 | 830.1 | 6.73 ± 0.37 | +| `Rust` | 93.6 ± 0.6 | 92.5 | 94.7 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 263.3 ± 1.8 | 259.0 | 268.4 | 2.81 ± 0.03 | +| `Go` | 1204.6 ± 5.0 | 1196.0 | 1215.4 | 12.87 ± 0.10 | +| `Scala` | 720.0 ± 2.9 | 713.4 | 725.3 | 7.70 ± 0.06 | ### Kube-prometheus manifests @@ -303,10 +335,11 @@ local kp = | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 150.1 ± 11.3 | 129.9 | 161.3 | 1.00 | -| `Go` | 3221.2 ± 46.4 | 3137.2 | 3272.6 | 21.46 ± 1.64 | -| `Scala` | 1056.3 ± 24.8 | 1006.7 | 1107.6 | 7.04 ± 0.55 | -| `C++` | 90142.7 ± 371.0 | 89516.1 | 91006.4 | 600.66 ± 45.19 | +| `Rust` | 129.4 ± 1.4 | 127.7 | 132.8 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 197.0 ± 1.9 | 193.6 | 200.7 | 1.52 ± 0.02 | +| `Go` | 2127.7 ± 13.8 | 2105.4 | 2151.5 | 16.44 ± 0.21 | +| `Scala` | 947.9 ± 11.1 | 926.1 | 967.9 | 7.33 ± 0.12 | +| `C++` | 87633.8 ± 569.9 | 86464.2 | 89118.8 | 677.29 ± 8.72 | ## Benchmarks from C++ jsonnet (/perf_tests) @@ -325,10 +358,11 @@ local kp = | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 9.8 ± 1.3 | 5.8 | 11.8 | 1.00 | -| `Go` | 159.5 ± 10.2 | 139.4 | 175.0 | 16.24 ± 2.32 | -| `Scala` | 388.0 ± 11.8 | 359.7 | 405.5 | 39.50 ± 5.19 | -| `C++` | 96.7 ± 8.8 | 77.3 | 104.9 | 9.84 ± 1.54 | +| `Rust` | 5.6 ± 0.1 | 5.4 | 6.2 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 45.3 ± 0.9 | 43.5 | 47.8 | 8.04 ± 0.25 | +| `Go` | 98.2 ± 1.6 | 95.0 | 100.8 | 17.45 ± 0.50 | +| `Scala` | 331.1 ± 5.8 | 322.3 | 343.7 | 58.79 ± 1.73 | +| `C++` | 76.5 ± 0.7 | 75.4 | 78.0 | 13.59 ± 0.34 | ### Large string template @@ -336,9 +370,10 @@ local kp = | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 11.8 ± 1.1 | 7.4 | 14.6 | 1.00 | -| `Scala` | 468.1 ± 9.7 | 452.2 | 494.6 | 39.57 ± 3.82 | -| `C++` | 14893.8 ± 102.4 | 14703.4 | 15074.0 | 1259.12 ± 119.13 | +| `Rust` | 6.7 ± 0.1 | 6.4 | 7.1 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 7.1 ± 0.2 | 6.8 | 7.6 | 1.05 ± 0.03 | +| `Scala` | 392.7 ± 2.7 | 388.2 | 399.8 | 58.29 ± 1.19 | +| `C++` | 14376.2 ± 35.8 | 14317.5 | 14448.6 | 2133.86 ± 41.53 | ### Realistic 1 @@ -597,10 +632,11 @@ local long_list = [ | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 23.6 ± 2.3 | 13.6 | 29.5 | 1.00 | -| `Go` | 11709.4 ± 73.0 | 11566.7 | 11825.3 | 496.43 ± 49.03 | -| `Scala` | 446.3 ± 8.9 | 428.7 | 467.4 | 18.92 ± 1.90 | -| `C++` | 24687.7 ± 144.7 | 24488.2 | 24951.9 | 1046.65 ± 103.36 | +| `Rust` | 12.6 ± 0.1 | 12.3 | 12.9 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 49.7 ± 1.0 | 47.3 | 53.0 | 3.96 ± 0.09 | +| `Go` | 6858.1 ± 34.9 | 6815.9 | 6951.8 | 546.32 ± 6.87 | +| `Scala` | 382.3 ± 3.9 | 371.7 | 387.1 | 30.46 ± 0.47 | +| `C++` | 24472.7 ± 155.7 | 24230.3 | 24799.3 | 1949.51 ± 25.63 | ### Realistic 2 @@ -719,10 +755,11 @@ function( | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 283.0 ± 10.0 | 264.4 | 298.1 | 1.00 | -| `Go` | 10224.6 ± 83.7 | 10015.8 | 10346.7 | 36.13 ± 1.32 | -| `Scala` | 767.9 ± 22.3 | 736.1 | 805.2 | 2.71 ± 0.12 | -| `C++` | 28183.8 ± 403.2 | 27302.6 | 28819.9 | 99.59 ± 3.81 | +| `Rust` | 259.9 ± 4.4 | 250.9 | 266.7 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 550.2 ± 16.6 | 531.9 | 600.7 | 2.12 ± 0.07 | +| `Go` | 6732.4 ± 51.4 | 6660.2 | 6858.6 | 25.90 ± 0.48 | +| `Scala` | 675.4 ± 10.9 | 667.3 | 720.0 | 2.60 ± 0.06 | +| `C++` | 26414.5 ± 259.9 | 25948.9 | 26934.4 | 101.63 ± 1.99 | ## Benchmarks from C++ jsonnet (/benchmarks) @@ -760,10 +797,11 @@ sum(300) | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 3.0 ± 0.4 | 1.8 | 4.0 | 1.00 | -| `Go` | 8.1 ± 1.4 | 5.0 | 10.3 | 2.70 ± 0.59 | -| `Scala` | 346.8 ± 11.1 | 320.1 | 360.6 | 114.96 ± 15.61 | -| `C++` | 51.7 ± 10.2 | 32.3 | 61.0 | 17.12 ± 4.07 | +| `Rust` | 1.8 ± 0.1 | 1.6 | 2.6 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 3.1 ± 0.1 | 2.8 | 3.4 | 1.69 ± 0.12 | +| `Go` | 5.3 ± 0.2 | 4.9 | 5.9 | 2.90 ± 0.20 | +| `Scala` | 287.4 ± 1.2 | 285.4 | 288.9 | 156.55 ± 9.92 | +| `C++` | 32.1 ± 0.3 | 31.6 | 33.0 | 17.49 ± 1.12 | ### Inheritance recursion @@ -800,10 +838,11 @@ local Fib = { | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 388.9 ± 11.9 | 367.2 | 408.5 | 1.00 | -| `Go` | 1566.1 ± 20.3 | 1532.8 | 1605.8 | 4.03 ± 0.13 | -| `Scala` | 515.8 ± 15.4 | 481.6 | 540.9 | 1.33 ± 0.06 | -| `C++` | 2719.1 ± 22.9 | 2660.2 | 2754.3 | 6.99 ± 0.22 | +| `Rust` | 376.1 ± 5.2 | 364.3 | 383.7 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 496.7 ± 13.0 | 470.6 | 506.8 | 1.32 ± 0.04 | +| `Go` | 873.9 ± 7.1 | 859.3 | 888.0 | 2.32 ± 0.04 | +| `Scala` | 420.8 ± 6.3 | 413.9 | 437.1 | 1.12 ± 0.02 | +| `C++` | 2647.5 ± 14.2 | 2630.9 | 2675.6 | 7.04 ± 0.10 | ### Simple recursive call @@ -840,10 +879,11 @@ fibonacci(25) | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 105.0 ± 8.6 | 83.9 | 119.9 | 1.00 | -| `Go` | 666.5 ± 14.3 | 640.1 | 691.4 | 6.35 ± 0.54 | -| `Scala` | 398.7 ± 11.8 | 361.5 | 419.8 | 3.80 ± 0.33 | -| `C++` | 217.0 ± 7.4 | 198.5 | 223.3 | 2.07 ± 0.18 | +| `Rust` | 81.3 ± 0.3 | 80.9 | 82.4 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 202.3 ± 12.0 | 183.1 | 237.6 | 2.49 ± 0.15 | +| `Go` | 349.4 ± 1.7 | 347.2 | 354.2 | 4.30 ± 0.03 | +| `Scala` | 322.5 ± 3.0 | 318.7 | 330.7 | 3.97 ± 0.04 | +| `C++` | 193.5 ± 0.9 | 191.4 | 194.8 | 2.38 ± 0.01 | ### Foldl string concat @@ -858,14 +898,15 @@ std.foldl(function(e, res) e + res, std.makeArray(20000, function(i) 'aaaaa'), ' | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 16.0 ± 2.8 | 9.2 | 24.2 | 1.00 | -| `Go` | 84.7 ± 6.1 | 68.5 | 98.1 | 5.28 ± 0.98 | -| `Scala` | 591.5 ± 16.2 | 567.5 | 622.1 | 36.86 ± 6.41 | -| `C++` | 901.2 ± 14.6 | 868.4 | 929.5 | 56.16 ± 9.69 | +| `Rust` | 8.9 ± 0.1 | 8.6 | 9.3 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 303.1 ± 3.8 | 297.6 | 308.1 | 34.06 ± 0.68 | +| `Go` | 50.9 ± 0.8 | 48.3 | 52.2 | 5.72 ± 0.13 | +| `Scala` | 550.6 ± 5.5 | 542.8 | 563.7 | 61.86 ± 1.13 | +| `C++` | 902.6 ± 4.8 | 891.4 | 912.1 | 101.41 ± 1.65 | ### Array sorts -> Note: No results for Scala, std.reverse is not implemented +> Note: No results for Scala, sjsonnet doesn't support keyF in std.sort: https://github.com/databricks/sjsonnet/issues/204
Source @@ -888,9 +929,10 @@ true | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 5.4 ± 0.7 | 3.1 | 7.5 | 1.00 | -| `Go` | 20.7 ± 2.6 | 12.6 | 24.7 | 3.84 ± 0.66 | -| `C++` | 3826.6 ± 47.9 | 3765.1 | 3946.1 | 707.87 ± 86.40 | +| `Rust` | 3.2 ± 0.2 | 2.9 | 4.1 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 14.4 ± 0.6 | 13.1 | 16.2 | 4.46 ± 0.28 | +| `Go` | 12.1 ± 0.2 | 11.4 | 12.8 | 3.74 ± 0.20 | +| `C++` | 3637.1 ± 24.9 | 3557.4 | 3664.7 | 1128.64 ± 56.81 | ### Lazy array @@ -910,10 +952,11 @@ slowId[15](42) | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 74.7 ± 6.4 | 53.8 | 84.2 | 1.00 | -| `Go` | 665.7 ± 20.8 | 609.9 | 692.5 | 8.91 ± 0.81 | -| `Scala` | 380.4 ± 9.8 | 356.6 | 397.8 | 5.09 ± 0.45 | -| `C++` | 206.7 ± 6.0 | 184.6 | 214.9 | 2.77 ± 0.25 | +| `Rust` | 54.1 ± 1.1 | 52.3 | 56.3 | 1.30 ± 0.03 | +| `Rust (alternative, rsjsonnet)` | 41.7 ± 0.7 | 39.6 | 43.8 | 1.00 | +| `Go` | 596.5 ± 5.2 | 585.1 | 606.9 | 14.30 ± 0.29 | +| `Scala` | 306.6 ± 1.6 | 303.5 | 310.4 | 7.35 ± 0.14 | +| `C++` | 184.5 ± 1.8 | 182.0 | 187.9 | 4.42 ± 0.09 | ### Inheritance function recursion @@ -938,10 +981,11 @@ fib(25) | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 2.7 ± 0.3 | 1.6 | 3.4 | 1.00 | -| `Go` | 3.9 ± 0.7 | 2.4 | 5.1 | 1.48 ± 0.31 | -| `Scala` | 354.0 ± 8.2 | 341.5 | 367.9 | 133.42 ± 15.65 | -| `C++` | 1070.3 ± 8.8 | 1047.1 | 1087.9 | 403.37 ± 46.49 | +| `Rust` | 1.6 ± 0.1 | 1.5 | 2.5 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 2.9 ± 0.1 | 2.7 | 3.7 | 1.78 ± 0.13 | +| `Go` | 2.4 ± 0.1 | 2.1 | 2.9 | 1.48 ± 0.12 | +| `Scala` | 293.2 ± 1.5 | 289.9 | 296.1 | 178.20 ± 12.35 | +| `C++` | 999.1 ± 9.2 | 974.4 | 1010.9 | 607.23 ± 42.34 | ### String strips @@ -963,10 +1007,11 @@ true | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 16.1 ± 1.8 | 9.5 | 19.2 | 1.00 | -| `Go` | 18.9 ± 2.4 | 11.1 | 23.3 | 1.18 ± 0.20 | -| `Scala` | 357.1 ± 11.4 | 327.2 | 378.3 | 22.24 ± 2.54 | -| `C++` | 38984.2 ± 160.9 | 38481.9 | 39276.8 | 2428.07 ± 266.80 | +| `Rust` | 8.6 ± 0.2 | 8.4 | 9.5 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 18.4 ± 0.4 | 17.6 | 20.8 | 2.13 ± 0.06 | +| `Go` | 11.2 ± 0.2 | 10.4 | 11.9 | 1.29 ± 0.04 | +| `Scala` | 294.9 ± 2.0 | 292.2 | 301.9 | 34.10 ± 0.65 | +| `C++` | 37345.3 ± 208.2 | 36567.2 | 37689.0 | 4318.10 ± 80.13 | ### Big object @@ -999,10 +1044,11 @@ local objectBody = std.join('', objContents); | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 3.5 ± 0.5 | 2.1 | 4.4 | 1.00 | -| `Go` | 6.9 ± 1.2 | 4.0 | 8.8 | 2.00 ± 0.43 | -| `Scala` | 342.5 ± 11.1 | 315.1 | 365.0 | 99.05 ± 13.39 | -| `C++` | 42.1 ± 11.4 | 28.4 | 55.3 | 12.18 ± 3.67 | +| `Rust` | 2.2 ± 0.1 | 1.9 | 3.0 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 3.2 ± 0.1 | 2.9 | 3.5 | 1.46 ± 0.10 | +| `Go` | 4.1 ± 0.1 | 3.7 | 4.6 | 1.85 ± 0.13 | +| `Scala` | 292.3 ± 2.9 | 289.3 | 304.0 | 132.11 ± 8.64 | +| `C++` | 28.3 ± 0.3 | 27.7 | 29.1 | 12.80 ± 0.84 | ## Benchmarks from Go jsonnet (builtins) @@ -1022,10 +1068,11 @@ local objectBody = std.join('', objContents); | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 4.2 ± 0.6 | 2.6 | 5.9 | 1.00 | -| `Go` | 27.1 ± 3.2 | 16.4 | 32.6 | 6.44 ± 1.17 | -| `Scala` | 377.4 ± 9.2 | 360.2 | 394.0 | 89.59 ± 12.70 | -| `C++` | 15045.5 ± 112.9 | 14763.1 | 15288.9 | 3571.12 ± 499.61 | +| `Rust` | 2.7 ± 0.1 | 2.4 | 3.4 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 2195.9 ± 48.5 | 2098.1 | 2229.6 | 804.07 ± 39.58 | +| `Go` | 17.0 ± 0.3 | 16.1 | 17.7 | 6.22 ± 0.29 | +| `Scala` | 313.8 ± 1.2 | 311.3 | 316.1 | 114.91 ± 5.07 | +| `C++` | 14621.6 ± 110.0 | 14473.9 | 14774.9 | 5354.05 ± 238.94 | ### std.base64Decode @@ -1043,10 +1090,11 @@ local objectBody = std.join('', objContents); | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 4.1 ± 0.6 | 2.5 | 5.5 | 1.00 | -| `Go` | 25.1 ± 2.9 | 16.1 | 29.4 | 6.18 ± 1.10 | -| `Scala` | 377.5 ± 10.4 | 358.5 | 397.1 | 93.09 ± 13.03 | -| `C++` | 10304.6 ± 62.8 | 10154.3 | 10389.1 | 2540.95 ± 349.00 | +| `Rust` | 2.6 ± 0.1 | 2.3 | 3.1 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 7889.2 ± 74.7 | 7711.6 | 8051.1 | 3045.22 ± 138.95 | +| `Go` | 15.2 ± 0.3 | 14.6 | 16.5 | 5.85 ± 0.28 | +| `Scala` | 313.0 ± 2.5 | 309.5 | 321.2 | 120.80 ± 5.47 | +| `C++` | 9930.9 ± 26.5 | 9895.0 | 9997.9 | 3833.33 ± 171.41 | ### std.base64DecodeBytes @@ -1064,10 +1112,11 @@ local objectBody = std.join('', objContents); | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 54.8 ± 10.4 | 34.7 | 68.3 | 1.00 | -| `Go` | 422.7 ± 17.7 | 387.7 | 447.7 | 7.72 ± 1.50 | -| `Scala` | 425.6 ± 12.4 | 392.0 | 444.2 | 7.77 ± 1.49 | -| `C++` | 10028.3 ± 42.1 | 9937.1 | 10106.0 | 183.13 ± 34.75 | +| `Rust` | 34.8 ± 0.5 | 33.4 | 36.0 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 8116.2 ± 37.2 | 8049.1 | 8176.2 | 233.52 ± 3.24 | +| `Go` | 271.1 ± 2.3 | 267.1 | 275.7 | 7.80 ± 0.12 | +| `Scala` | 357.2 ± 2.3 | 353.5 | 363.4 | 10.28 ± 0.15 | +| `C++` | 9653.7 ± 22.5 | 9621.4 | 9694.3 | 277.75 ± 3.69 | ### std.base64 (byte array) @@ -1085,10 +1134,11 @@ local objectBody = std.join('', objContents); | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 49.8 ± 10.1 | 30.5 | 58.2 | 1.00 | -| `Go` | 268.9 ± 5.2 | 250.5 | 274.8 | 5.40 ± 1.10 | -| `Scala` | 427.9 ± 10.0 | 413.1 | 448.0 | 8.59 ± 1.75 | -| `C++` | 13958.9 ± 151.1 | 13822.2 | 14478.0 | 280.37 ± 56.84 | +| `Rust` | 30.3 ± 0.4 | 29.1 | 31.6 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 1460.8 ± 47.6 | 1375.2 | 1568.3 | 48.26 ± 1.72 | +| `Go` | 156.5 ± 1.9 | 152.5 | 159.4 | 5.17 ± 0.10 | +| `Scala` | 355.9 ± 2.9 | 351.4 | 365.6 | 11.76 ± 0.20 | +| `C++` | 13362.4 ± 88.3 | 13028.4 | 13473.8 | 441.41 ± 7.01 | ### std.foldl @@ -1105,10 +1155,11 @@ std.foldl(function(acc, value) acc + value, input, '') | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 9.1 ± 1.4 | 5.2 | 10.7 | 1.00 | -| `Go` | 44.6 ± 4.0 | 36.0 | 51.2 | 4.92 ± 0.86 | -| `Scala` | 463.9 ± 13.9 | 428.4 | 486.6 | 51.22 ± 7.86 | -| `C++` | 274.5 ± 9.3 | 255.0 | 286.9 | 30.31 ± 4.68 | +| `Rust` | 5.3 ± 0.2 | 5.0 | 6.2 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 81.5 ± 1.6 | 78.9 | 83.8 | 15.38 ± 0.55 | +| `Go` | 26.6 ± 0.5 | 25.0 | 27.5 | 5.02 ± 0.18 | +| `Scala` | 405.4 ± 3.0 | 400.9 | 413.6 | 76.52 ± 2.34 | +| `C++` | 265.3 ± 2.7 | 260.6 | 271.5 | 50.08 ± 1.57 | ### std.manifestJsonEx @@ -1168,14 +1219,15 @@ std.foldl(function(acc, value) acc + value, input, '') | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 2.7 ± 0.3 | 1.6 | 3.4 | 1.00 | -| `Go` | 5.2 ± 0.9 | 2.9 | 6.5 | 1.93 ± 0.40 | -| `Scala` | 350.7 ± 12.8 | 315.6 | 370.8 | 130.60 ± 15.89 | -| `C++` | 124.6 ± 8.5 | 105.9 | 133.3 | 46.40 ± 6.25 | +| `Rust` | 1.7 ± 0.1 | 1.4 | 2.7 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 2.9 ± 0.1 | 2.7 | 3.3 | 1.73 ± 0.13 | +| `Go` | 3.2 ± 0.1 | 2.7 | 3.8 | 1.92 ± 0.15 | +| `Scala` | 294.7 ± 1.7 | 291.6 | 299.0 | 175.84 ± 12.30 | +| `C++` | 101.4 ± 0.8 | 99.3 | 102.9 | 60.48 ± 4.24 | ### std.manifestTomlEx -> Note: No results for Scala, std.manifestTomlEx is not implemented +> Note: No results for Scala, std.manifestTomlEx is not implemented: https://github.com/databricks/sjsonnet/issues/111
Source @@ -1234,9 +1286,10 @@ std.foldl(function(acc, value) acc + value, input, '') | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 2.8 ± 0.3 | 1.7 | 3.7 | 1.00 | -| `Go` | 5.4 ± 0.8 | 2.9 | 6.9 | 1.94 ± 0.37 | -| `C++` | 1118.3 ± 21.4 | 1094.0 | 1190.0 | 405.19 ± 44.55 | +| `Rust` | 1.7 ± 0.1 | 1.4 | 2.7 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 7.8 ± 0.2 | 7.3 | 8.5 | 4.65 ± 0.36 | +| `Go` | 3.2 ± 0.1 | 2.9 | 3.7 | 1.90 ± 0.16 | +| `C++` | 1046.1 ± 4.9 | 1039.7 | 1053.9 | 622.36 ± 44.25 | ### std.parseInt @@ -1254,15 +1307,14 @@ std.foldl(function(acc, value) acc + value, input, '') | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 2.8 ± 0.3 | 1.7 | 3.7 | 1.00 | -| `Go` | 4.8 ± 0.7 | 2.7 | 6.2 | 1.72 ± 0.33 | -| `Scala` | 357.2 ± 11.3 | 334.5 | 377.2 | 128.46 ± 14.87 | -| `C++` | 126.7 ± 6.8 | 105.8 | 134.7 | 45.57 ± 5.63 | +| `Rust` | 1.7 ± 0.1 | 1.4 | 2.6 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 2.9 ± 0.1 | 2.6 | 3.4 | 1.71 ± 0.16 | +| `Go` | 2.8 ± 0.1 | 2.4 | 3.3 | 1.64 ± 0.16 | +| `Scala` | 292.7 ± 1.7 | 289.2 | 295.8 | 170.68 ± 14.76 | +| `C++` | 101.0 ± 0.7 | 99.0 | 102.0 | 58.88 ± 5.10 | ### std.reverse -> Note: No results for Scala, std.reverse is not implemented -
Source @@ -1277,9 +1329,11 @@ std.foldl(function(acc, value) acc + value, input, '') | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 63.1 ± 11.3 | 46.0 | 77.6 | 1.00 | -| `Go` | 487.1 ± 31.1 | 441.0 | 553.5 | 7.72 ± 1.47 | -| `C++` | 712.8 ± 11.2 | 693.0 | 742.5 | 11.30 ± 2.03 | +| `Rust` | 45.3 ± 0.5 | 44.0 | 46.5 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 784.1 ± 20.4 | 741.5 | 802.0 | 17.31 ± 0.49 | +| `Go` | 327.4 ± 2.1 | 323.8 | 331.4 | 7.23 ± 0.10 | +| `Scala` | 367.0 ± 2.4 | 361.1 | 370.2 | 8.10 ± 0.11 | +| `C++` | 659.3 ± 5.4 | 644.6 | 666.0 | 14.55 ± 0.21 | ### std.substr @@ -1297,10 +1351,11 @@ std.foldl(function(acc, value) acc + value, input, '') | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 3.6 ± 0.3 | 2.7 | 4.5 | 1.00 | -| `Go` | 11.0 ± 1.6 | 6.9 | 13.5 | 3.09 ± 0.52 | -| `Scala` | 359.7 ± 9.8 | 345.8 | 374.6 | 100.48 ± 8.47 | -| `C++` | 49.4 ± 10.0 | 29.3 | 62.7 | 13.80 ± 3.00 | +| `Rust` | 2.2 ± 0.2 | 1.9 | 3.2 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 3.2 ± 0.1 | 3.1 | 3.6 | 1.49 ± 0.11 | +| `Go` | 6.8 ± 0.2 | 6.3 | 7.4 | 3.12 ± 0.24 | +| `Scala` | 303.6 ± 3.6 | 300.5 | 318.2 | 139.43 ± 9.76 | +| `C++` | 29.1 ± 0.4 | 28.3 | 30.2 | 13.36 ± 0.94 | ### Comparsion for array @@ -1318,9 +1373,10 @@ long_array + [1] < long_array + [2] | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 45.0 ± 10.3 | 26.6 | 55.2 | 1.00 | -| `Go` | 160.8 ± 17.0 | 116.2 | 182.1 | 3.57 ± 0.90 | -| `C++` | 135271.6 ± 10230.4 | 130120.3 | 172035.4 | 3002.97 ± 720.97 | +| `Rust` | 25.7 ± 0.2 | 25.0 | 26.0 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 228.1 ± 7.9 | 214.0 | 245.1 | 8.89 ± 0.31 | +| `Go` | 150.4 ± 10.4 | 135.9 | 169.4 | 5.86 ± 0.41 | +| `C++` | 125755.8 ± 989.6 | 123095.8 | 127475.4 | 4901.85 ± 49.12 | ### Comparsion for primitives @@ -1337,6 +1393,7 @@ long_array + [1] < long_array + [2] | Command | Mean [ms] | Min [ms] | Max [ms] | Relative | |:---|---:|---:|---:|---:| -| `Rust` | 414.7 ± 10.1 | 394.7 | 437.9 | 1.00 | -| `Go` | 2087.0 ± 36.5 | 2030.6 | 2148.9 | 5.03 ± 0.15 | -| `Scala` | 535.1 ± 9.9 | 514.5 | 556.0 | 1.29 ± 0.04 | +| `Rust` | 385.9 ± 4.2 | 372.4 | 391.5 | 1.00 | +| `Rust (alternative, rsjsonnet)` | 1287.4 ± 27.0 | 1244.2 | 1335.1 | 3.34 ± 0.08 | +| `Go` | 1817.1 ± 18.6 | 1779.7 | 1842.9 | 4.71 ± 0.07 | +| `Scala` | 453.1 ± 3.9 | 448.1 | 462.2 | 1.17 ± 0.02 | diff --git a/nix/benchmarks.nix b/nix/benchmarks.nix index ad42e108..699d288e 100644 --- a/nix/benchmarks.nix +++ b/nix/benchmarks.nix @@ -89,11 +89,6 @@ in skipGo ? "", vendor ? "", }: '' - set -oux - - temp=$(mktemp -d) - cd $temp - echo >> $out echo "### ${name}" >> $out echo >> $out @@ -145,6 +140,11 @@ in cat result.md >> $out ''; in '' + set -oux + + temp=$(mktemp -d) + cd $temp + touch $out ${optionalString (!quick) '' cat ${./benchmarks.md} >> $out @@ -173,7 +173,7 @@ in echo >> $out echo "
" >> $out echo >> $out - echo Rust (alternative): >> $out + echo "Rust (alternative):" >> $out echo >> $out echo "\`\`\`" >> $out rsjsonnet --help 2>> $out || true