From dc646567107cf5eaa5eb804ba6c22465a4f9429b Mon Sep 17 00:00:00 2001 From: Siddharth Bhat Date: Mon, 1 Apr 2024 20:47:03 +0100 Subject: [PATCH] feat: compute temci --- .../run-2024-04-01---17-10-tcg40/.gitignore | 1 + .../outputs/noreuse.speedcenter.bench.yaml | 600 ++++++++++++++++++ .../outputs/reuse.speedcenter.bench.yaml | 600 ++++++++++++++++++ .../outputs/temci-report.txt | 243 +++++++ .../speedcenter-worker.sh | 33 +- 5 files changed, 1464 insertions(+), 13 deletions(-) create mode 100644 1-runs/run-2024-04-01---17-10-tcg40/outputs/noreuse.speedcenter.bench.yaml create mode 100644 1-runs/run-2024-04-01---17-10-tcg40/outputs/reuse.speedcenter.bench.yaml create mode 100644 1-runs/run-2024-04-01---17-10-tcg40/outputs/temci-report.txt diff --git a/1-runs/run-2024-04-01---17-10-tcg40/.gitignore b/1-runs/run-2024-04-01---17-10-tcg40/.gitignore index 6488e2b86141..41a6b8df7d7e 100644 --- a/1-runs/run-2024-04-01---17-10-tcg40/.gitignore +++ b/1-runs/run-2024-04-01---17-10-tcg40/.gitignore @@ -1,3 +1,4 @@ builds/ +builds/ builds-speedcenter/ diff --git a/1-runs/run-2024-04-01---17-10-tcg40/outputs/noreuse.speedcenter.bench.yaml b/1-runs/run-2024-04-01---17-10-tcg40/outputs/noreuse.speedcenter.bench.yaml new file mode 100644 index 000000000000..4512c373b6af --- /dev/null +++ b/1-runs/run-2024-04-01---17-10-tcg40/outputs/noreuse.speedcenter.bench.yaml @@ -0,0 +1,600 @@ +- attributes: + __description: binarytrees + description: binarytrees + tags: + - fast + - suite + data: + __ov-time: + - 18.669635772705078 + - 17.017271757125854 + - 17.48976755142212 + - 17.364582777023315 + - 16.66629648208618 + - 17.61938738822937 + - 17.55997133255005 + - 18.334264755249023 + - 16.952329397201538 + - 17.701575756072998 + maxrss: + - 179456 + - 179584 + - 179712 + - 179584 + - 179712 + - 179712 + - 179584 + - 179712 + - 179584 + - 179584 + wall-clock: + - 18.601809765 + - 16.983050754 + - 17.451730381 + - 17.317890976 + - 16.619900294 + - 17.571445744 + - 17.522161768 + - 18.2920834 + - 16.911957422 + - 17.620908368 +- attributes: + __description: binarytrees.st + description: binarytrees.st + tags: + - fast + - suite + data: + __ov-time: + - 8.57723355293274 + - 8.84149980545044 + - 8.135608196258545 + - 7.918590545654297 + - 7.9795451164245605 + - 8.16500735282898 + - 8.171606302261353 + - 8.879554748535156 + - 8.047658205032349 + - 8.125501871109009 + maxrss: + - 124812 + - 125068 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + wall-clock: + - 8.547783699 + - 8.807361987 + - 8.112682313 + - 7.890720928 + - 7.957641857 + - 8.13736774 + - 8.14505892 + - 8.850172975 + - 8.013188436 + - 8.099375015 +- attributes: + __description: const_fold + description: const_fold + tags: + - fast + - suite + data: + __ov-time: + - 4.515593528747559 + - 4.268319129943848 + - 4.181943893432617 + - 4.263890504837036 + - 4.246982574462891 + - 4.207585334777832 + - 4.368007183074951 + - 4.204219341278076 + - 4.205229043960571 + - 4.248189210891724 + maxrss: + - 1828480 + - 1828608 + - 1828480 + - 1828736 + - 1828608 + - 1828736 + - 1828608 + - 1828480 + - 1828608 + - 1828608 + wall-clock: + - 4.48492891 + - 4.225169256 + - 4.158962416 + - 4.236422762 + - 4.218845806 + - 4.182863754 + - 4.343208079 + - 4.179667125 + - 4.180043576 + - 4.215309781 +- attributes: + __description: deriv + description: deriv + tags: + - fast + - suite + data: + __ov-time: + - 1.7428219318389893 + - 1.5460469722747803 + - 1.5592319965362549 + - 1.5986943244934082 + - 1.5308115482330322 + - 1.555021047592163 + - 1.5809605121612549 + - 1.558642864227295 + - 1.5959141254425049 + - 1.5479545593261719 + maxrss: + - 441728 + - 441856 + - 441856 + - 441984 + - 442112 + - 441856 + - 441984 + - 442112 + - 441984 + - 441984 + wall-clock: + - 1.693710917 + - 1.522684948 + - 1.535724002 + - 1.562243642 + - 1.507693053 + - 1.516058479 + - 1.542600738 + - 1.534002186 + - 1.544716085 + - 1.519947782 +- attributes: + __description: liasolver + description: liasolver + tags: + - fast + - suite + data: + __ov-time: + - 2.4147393703460693 + - 0.5514078140258789 + - 0.559441328048706 + - 0.669954776763916 + - 0.5586857795715332 + - 0.6122326850891113 + - 0.5858306884765625 + - 0.5898227691650391 + - 0.5732893943786621 + - 0.5890285968780518 + maxrss: + - 124812 + - 125068 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + wall-clock: + - 2.389799553 + - 0.528655766 + - 0.532493549 + - 0.638018483 + - 0.534090204 + - 0.55119337 + - 0.562622661 + - 0.566127829 + - 0.551354398 + - 0.558869188 +- attributes: + __description: parser + description: parser + tags: + - fast + - suite + data: + __ov-time: + - 8.535115480422974 + - 6.354198217391968 + - 6.2784833908081055 + - 7.934902906417847 + - 6.263048887252808 + - 6.533065557479858 + - 6.523089647293091 + - 6.4253833293914795 + - 6.202568769454956 + - 6.015015363693237 + maxrss: + - 124812 + - 125068 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + wall-clock: + - 8.511658829 + - 6.32550018 + - 6.252354044 + - 7.907860488 + - 6.226621618 + - 6.51114483 + - 6.498620764 + - 6.396782263 + - 6.181295554 + - 5.990979419 +- attributes: + __description: qsort + description: qsort + tags: + - fast + - suite + data: + __ov-time: + - 3.317399263381958 + - 3.135924816131592 + - 3.117540121078491 + - 3.4067211151123047 + - 3.1671924591064453 + - 3.031217575073242 + - 3.612485647201538 + - 3.1504671573638916 + - 3.1397969722747803 + - 2.9963552951812744 + maxrss: + - 124812 + - 125068 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + wall-clock: + - 3.288446115 + - 3.108885508 + - 3.070581708 + - 3.372042426 + - 3.13658666 + - 3.007407735 + - 3.587006094 + - 3.127292059 + - 3.096338494 + - 2.966097765 +- attributes: + __description: rbmap + description: rbmap + tags: + - fast + - suite + data: + __ov-time: + - 1.0505144596099854 + - 0.9571702480316162 + - 0.9313404560089111 + - 0.922461748123169 + - 0.9385075569152832 + - 0.9374275207519531 + - 1.1067585945129395 + - 0.9132881164550781 + - 0.9708516597747803 + - 0.9491796493530273 + maxrss: + - 124812 + - 125068 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + wall-clock: + - 1.026484797 + - 0.927260032 + - 0.903845406 + - 0.894168904 + - 0.912828414 + - 0.901382006 + - 1.069492143 + - 0.885205414 + - 0.92012631 + - 0.91141856 +- attributes: + __description: rbmap_1 + description: rbmap_1 + tags: + - fast + - suite + data: + __ov-time: + - 4.826166152954102 + - 4.729350566864014 + - 4.589245319366455 + - 4.545488119125366 + - 4.593832731246948 + - 4.572415113449097 + - 5.141026735305786 + - 4.585630416870117 + - 4.545737028121948 + - 4.790630340576172 + maxrss: + - 2371200 + - 2371456 + - 2371456 + - 2371584 + - 2371456 + - 2371328 + - 2371456 + - 2371456 + - 2371584 + - 2371456 + wall-clock: + - 4.802098963 + - 4.704650864 + - 4.562820255 + - 4.522216642 + - 4.572211709 + - 4.547194885 + - 5.112293868 + - 4.563469003 + - 4.515382703 + - 4.743070629 +- attributes: + __description: rbmap_10 + description: rbmap_10 + tags: + - fast + - suite + data: + __ov-time: + - 1.4346542358398438 + - 1.4388916492462158 + - 1.4067938327789307 + - 1.4160192012786865 + - 1.411107063293457 + - 1.3830864429473877 + - 1.5490832328796387 + - 1.4031956195831299 + - 1.4239723682403564 + - 1.4049954414367676 + maxrss: + - 322048 + - 321792 + - 321792 + - 321920 + - 322048 + - 321792 + - 321920 + - 321920 + - 321920 + - 321920 + wall-clock: + - 1.404389989 + - 1.409202651 + - 1.377067309 + - 1.37001227 + - 1.387435967 + - 1.356205788 + - 1.523891833 + - 1.380568127 + - 1.399297827 + - 1.379083631 +- attributes: + __description: rbmap_fbip + description: rbmap_fbip + tags: + - fast + - suite + data: + __ov-time: + - 2.5770862102508545 + - 2.6572067737579346 + - 2.3889222145080566 + - 2.40480899810791 + - 2.343416452407837 + - 2.406348943710327 + - 2.567289352416992 + - 2.3670246601104736 + - 2.3221595287323 + - 2.339367389678955 + maxrss: + - 124812 + - 125068 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + wall-clock: + - 2.54361863 + - 2.634626614 + - 2.364111817 + - 2.346210957 + - 2.321874833 + - 2.375171117 + - 2.543404614 + - 2.344522635 + - 2.299620431 + - 2.311503679 +- attributes: + __description: rbmap_library + description: rbmap_library + tags: + - fast + - suite + data: + __ov-time: + - 2.5746910572052 + - 0.9376511573791504 + - 0.882436990737915 + - 0.9239380359649658 + - 0.8876967430114746 + - 0.9010052680969238 + - 0.8776917457580566 + - 0.8844945430755615 + - 0.8831386566162109 + - 0.8681070804595947 + maxrss: + - 141568 + - 143104 + - 143104 + - 143616 + - 143104 + - 143232 + - 142848 + - 143232 + - 143616 + - 143104 + wall-clock: + - 2.544390275 + - 0.914495977 + - 0.861128064 + - 0.898153748 + - 0.864203123 + - 0.866586963 + - 0.855127334 + - 0.860276415 + - 0.859457763 + - 0.845855002 +- attributes: + __description: reduceMatch + description: reduceMatch + tags: + - fast + - suite + data: + __ov-time: + - 5.2773754596710205 + - 4.878661394119263 + - 5.007373809814453 + - 4.988689422607422 + - 5.08323335647583 + - 5.6726813316345215 + - 4.853859186172485 + - 5.010188817977905 + - 4.977128505706787 + - 4.876942873001099 + maxrss: + - 523608 + - 523464 + - 523356 + - 523308 + - 523612 + - 523480 + - 523336 + - 523592 + - 523336 + - 523844 + wall-clock: + - 5.242941256 + - 4.855367366 + - 4.985722106 + - 4.957825613 + - 5.060164361 + - 5.64525074 + - 4.825728017 + - 4.983451282 + - 4.955473292 + - 4.855079622 +- attributes: + __description: unionfind + description: unionfind + tags: + - fast + - suite + data: + __ov-time: + - 3.9517345428466797 + - 3.8693182468414307 + - 3.838263511657715 + - 4.0278520584106445 + - 3.818434238433838 + - 4.48534893989563 + - 3.9726717472076416 + - 3.794212818145752 + - 3.8597679138183594 + - 3.858513832092285 + maxrss: + - 124812 + - 125068 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + - 125452 + wall-clock: + - 3.923303875 + - 3.842982499 + - 3.814143036 + - 3.730670685 + - 3.791237109 + - 4.455727956 + - 3.945541099 + - 3.769729436 + - 3.828915711 + - 3.83202426 +- attributes: + __description: workspaceSymbols + description: workspaceSymbols + tags: + - fast + - suite + data: + __ov-time: + - 17.169426441192627 + - 17.108542442321777 + - 17.401740789413452 + - 17.150473833084106 + - 18.231353044509888 + - 17.647992372512817 + - 17.87350344657898 + - 17.213216066360474 + - 17.69843029975891 + - 17.67744493484497 + maxrss: + - 501064 + - 500808 + - 500808 + - 500296 + - 500804 + - 501060 + - 500548 + - 500804 + - 501060 + - 500808 + wall-clock: + - 17.144924194 + - 17.08456657 + - 17.379319222 + - 17.120523428 + - 18.20775143 + - 17.617012174 + - 17.848220488 + - 17.185384645 + - 17.66818111 + - 17.6534527 diff --git a/1-runs/run-2024-04-01---17-10-tcg40/outputs/reuse.speedcenter.bench.yaml b/1-runs/run-2024-04-01---17-10-tcg40/outputs/reuse.speedcenter.bench.yaml new file mode 100644 index 000000000000..69cf56d45859 --- /dev/null +++ b/1-runs/run-2024-04-01---17-10-tcg40/outputs/reuse.speedcenter.bench.yaml @@ -0,0 +1,600 @@ +- attributes: + __description: binarytrees + description: binarytrees + tags: + - fast + - suite + data: + __ov-time: + - 16.855204582214355 + - 17.41478180885315 + - 17.44905185699463 + - 18.872823476791382 + - 18.60306143760681 + - 18.644100189208984 + - 17.848021507263184 + - 17.234209299087524 + - 19.612183809280396 + - 17.522130012512207 + maxrss: + - 179584 + - 179584 + - 179584 + - 179584 + - 179712 + - 179712 + - 179584 + - 179456 + - 179712 + - 179584 + wall-clock: + - 16.828079155 + - 17.386723386 + - 17.374363272 + - 18.835609366 + - 18.556661326 + - 18.605553175 + - 17.792626403 + - 17.190701561 + - 19.558539079 + - 17.485105288 +- attributes: + __description: binarytrees.st + description: binarytrees.st + tags: + - fast + - suite + data: + __ov-time: + - 8.128392457962036 + - 8.044058084487915 + - 8.18216872215271 + - 8.186760187149048 + - 7.916617155075073 + - 8.955979824066162 + - 8.3144211769104 + - 8.63399052619934 + - 8.120436429977417 + - 8.322941780090332 + maxrss: + - 124692 + - 124948 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + wall-clock: + - 8.102859416 + - 8.011745104 + - 8.14992298 + - 8.148995821 + - 7.88414416 + - 8.930821509 + - 8.288844971 + - 8.608654839 + - 8.088899768 + - 8.290782223 +- attributes: + __description: const_fold + description: const_fold + tags: + - fast + - suite + data: + __ov-time: + - 3.7704451084136963 + - 3.621281147003174 + - 3.674027442932129 + - 3.642462730407715 + - 3.707956075668335 + - 3.9242305755615234 + - 3.844841241836548 + - 3.8924007415771484 + - 3.7064900398254395 + - 3.6258199214935303 + maxrss: + - 1828480 + - 1828352 + - 1828608 + - 1828608 + - 1828480 + - 1828480 + - 1828480 + - 1828480 + - 1828480 + - 1828608 + wall-clock: + - 3.746879525 + - 3.561498737 + - 3.647428861 + - 3.617388463 + - 3.651365815 + - 3.896961353 + - 3.758137069 + - 3.867121514 + - 3.626533365 + - 3.593084806 +- attributes: + __description: deriv + description: deriv + tags: + - fast + - suite + data: + __ov-time: + - 1.758004903793335 + - 1.596761703491211 + - 1.6106235980987549 + - 1.6203038692474365 + - 1.6058533191680908 + - 1.6174826622009277 + - 1.6206164360046387 + - 1.7303543090820312 + - 1.7303836345672607 + - 1.615342140197754 + maxrss: + - 441728 + - 441856 + - 441856 + - 441984 + - 441984 + - 441856 + - 441856 + - 441984 + - 441728 + - 441984 + wall-clock: + - 1.735574406 + - 1.575606922 + - 1.576273956 + - 1.595736206 + - 1.582790747 + - 1.594906192 + - 1.597504626 + - 1.706796329 + - 1.70137654 + - 1.590615219 +- attributes: + __description: liasolver + description: liasolver + tags: + - fast + - suite + data: + __ov-time: + - 2.4692161083221436 + - 0.5830347537994385 + - 0.5899002552032471 + - 0.590080976486206 + - 0.5687785148620605 + - 0.5664575099945068 + - 0.5949153900146484 + - 0.6372668743133545 + - 0.7249743938446045 + - 0.573549747467041 + maxrss: + - 124692 + - 124948 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + wall-clock: + - 2.422351798 + - 0.547970948 + - 0.56698248 + - 0.558129102 + - 0.546325665 + - 0.544878481 + - 0.573346129 + - 0.613566566 + - 0.696191918 + - 0.55004538 +- attributes: + __description: parser + description: parser + tags: + - fast + - suite + data: + __ov-time: + - 7.666473150253296 + - 6.358872175216675 + - 6.588859796524048 + - 6.338989973068237 + - 6.419281721115112 + - 6.302893161773682 + - 6.45180869102478 + - 6.494197607040405 + - 7.698554754257202 + - 6.341972827911377 + maxrss: + - 124692 + - 124948 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + wall-clock: + - 7.644354953 + - 6.330438984 + - 6.563100136 + - 6.312180519 + - 6.394199709 + - 6.277006895 + - 6.426805835 + - 6.469884265 + - 7.588118621 + - 6.315699593 +- attributes: + __description: qsort + description: qsort + tags: + - fast + - suite + data: + __ov-time: + - 3.1947052478790283 + - 3.1945176124572754 + - 3.175844430923462 + - 3.2883236408233643 + - 3.1432595252990723 + - 3.2347891330718994 + - 3.2393388748168945 + - 3.351576805114746 + - 3.424471855163574 + - 3.1358463764190674 + maxrss: + - 124692 + - 124948 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + wall-clock: + - 3.16901424 + - 3.171215533 + - 3.152069305 + - 3.25869722 + - 3.115650375 + - 3.200452841 + - 3.20050711 + - 3.323517761 + - 3.397871687 + - 3.07870422 +- attributes: + __description: rbmap + description: rbmap + tags: + - fast + - suite + data: + __ov-time: + - 1.0787708759307861 + - 0.9966549873352051 + - 1.0047719478607178 + - 0.9662439823150635 + - 1.0137755870819092 + - 0.9999206066131592 + - 1.0572872161865234 + - 1.032524824142456 + - 0.9881141185760498 + - 0.9827101230621338 + maxrss: + - 124692 + - 124948 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + wall-clock: + - 1.052706259 + - 0.967995884 + - 0.979145751 + - 0.942160542 + - 0.990490755 + - 0.966892384 + - 1.027320674 + - 1.003517074 + - 0.959862471 + - 0.958315979 +- attributes: + __description: rbmap_1 + description: rbmap_1 + tags: + - fast + - suite + data: + __ov-time: + - 4.54656982421875 + - 4.431166172027588 + - 4.500474691390991 + - 4.6129608154296875 + - 4.5613932609558105 + - 4.962039232254028 + - 4.615298271179199 + - 4.615195274353027 + - 4.50944972038269 + - 4.543499708175659 + maxrss: + - 2371200 + - 2371328 + - 2371328 + - 2371456 + - 2371328 + - 2371328 + - 2371200 + - 2371328 + - 2371200 + - 2371328 + wall-clock: + - 4.50064704 + - 4.40877606 + - 4.477644084 + - 4.59104639 + - 4.539252716 + - 4.934255062 + - 4.529585061 + - 4.588665482 + - 4.485792039 + - 4.513755282 +- attributes: + __description: rbmap_10 + description: rbmap_10 + tags: + - fast + - suite + data: + __ov-time: + - 1.4634530544281006 + - 1.4870057106018066 + - 1.4394795894622803 + - 1.4702544212341309 + - 1.4966156482696533 + - 1.572812795639038 + - 1.4282724857330322 + - 1.5114328861236572 + - 1.4519810676574707 + - 1.4936270713806152 + maxrss: + - 321792 + - 321792 + - 321664 + - 321792 + - 321792 + - 321792 + - 321792 + - 321792 + - 321792 + - 321664 + wall-clock: + - 1.436259481 + - 1.462496019 + - 1.414450241 + - 1.439795226 + - 1.46767275 + - 1.541561319 + - 1.403467532 + - 1.483855584 + - 1.42632414 + - 1.466634689 +- attributes: + __description: rbmap_fbip + description: rbmap_fbip + tags: + - fast + - suite + data: + __ov-time: + - 2.4816484451293945 + - 2.408409833908081 + - 2.3691205978393555 + - 2.3154852390289307 + - 2.416867971420288 + - 2.816767692565918 + - 2.4117648601531982 + - 2.409359931945801 + - 2.395930290222168 + - 2.3507862091064453 + maxrss: + - 124692 + - 124948 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + wall-clock: + - 2.449233448 + - 2.383105166 + - 2.345195612 + - 2.292105831 + - 2.393912343 + - 2.780921919 + - 2.388508083 + - 2.386228104 + - 2.370784632 + - 2.325324151 +- attributes: + __description: rbmap_library + description: rbmap_library + tags: + - fast + - suite + data: + __ov-time: + - 2.5818231105804443 + - 0.9777231216430664 + - 0.9613795280456543 + - 0.9871921539306641 + - 1.0089073181152344 + - 1.1995761394500732 + - 0.98175048828125 + - 1.0531964302062988 + - 0.9997427463531494 + - 1.0118408203125 + maxrss: + - 140032 + - 141312 + - 141312 + - 141312 + - 141952 + - 140928 + - 141952 + - 141312 + - 141312 + - 141056 + wall-clock: + - 2.560132159 + - 0.952873222 + - 0.938281618 + - 0.960421824 + - 0.986294385 + - 1.116922909 + - 0.937501113 + - 1.030316068 + - 0.947813458 + - 0.988428273 +- attributes: + __description: reduceMatch + description: reduceMatch + tags: + - fast + - suite + data: + __ov-time: + - 5.155909061431885 + - 4.9919939041137695 + - 5.011842489242554 + - 5.124230623245239 + - 5.090702295303345 + - 5.6274354457855225 + - 5.931148290634155 + - 5.276548147201538 + - 5.249046087265015 + - 5.167018413543701 + maxrss: + - 518980 + - 519496 + - 518884 + - 518868 + - 518984 + - 518800 + - 519276 + - 518856 + - 519140 + - 518980 + wall-clock: + - 5.130045349 + - 4.967751359 + - 4.987839547 + - 5.098229667 + - 5.059988276 + - 5.599987854 + - 5.907321135 + - 5.252619838 + - 5.22200057 + - 5.142983138 +- attributes: + __description: unionfind + description: unionfind + tags: + - fast + - suite + data: + __ov-time: + - 4.065519332885742 + - 3.8762423992156982 + - 3.9212543964385986 + - 3.9724268913269043 + - 3.9292044639587402 + - 4.298598766326904 + - 4.634092807769775 + - 3.9907913208007812 + - 4.172258138656616 + - 4.045690298080444 + maxrss: + - 124692 + - 124948 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + - 125076 + wall-clock: + - 4.026504364 + - 3.847987254 + - 3.893522802 + - 3.942391873 + - 3.908036354 + - 4.273081825 + - 4.604377536 + - 3.961133861 + - 4.05590611 + - 4.022050275 +- attributes: + __description: workspaceSymbols + description: workspaceSymbols + tags: + - fast + - suite + data: + __ov-time: + - 17.157243728637695 + - 17.25099539756775 + - 17.176440477371216 + - 17.12825632095337 + - 17.220181226730347 + - 18.455241680145264 + - 17.94197654724121 + - 18.011901140213013 + - 17.197839975357056 + - 17.60256028175354 + maxrss: + - 496708 + - 496708 + - 496968 + - 497092 + - 496836 + - 496356 + - 496200 + - 496456 + - 497096 + - 496968 + wall-clock: + - 17.135292947 + - 17.224365725 + - 17.148518038 + - 17.103257038 + - 17.115423512 + - 18.43072937 + - 17.811687299 + - 17.985466874 + - 17.171190478 + - 17.578200059 diff --git a/1-runs/run-2024-04-01---17-10-tcg40/outputs/temci-report.txt b/1-runs/run-2024-04-01---17-10-tcg40/outputs/temci-report.txt new file mode 100644 index 000000000000..c7ea50ee13f2 --- /dev/null +++ b/1-runs/run-2024-04-01---17-10-tcg40/outputs/temci-report.txt @@ -0,0 +1,243 @@ +Report for binarytrees +binarytrees [0] ( 10 single benchmarks) + maxrss mean = 179.62240k, deviation = 0.04563% + wall-clock mean = 17.4(89), deviation = 3.27642% + +binarytrees [1] ( 10 single benchmarks) + maxrss mean = 179.60960k, deviation = 0.04276% + wall-clock mean = 17.9(61), deviation = 4.62235% + +Equal program blocks + binarytrees [0] ⟷ binarytrees [1] + maxrss confidence = 74%, speed up = 0.01% + wall-clock confidence = 18%, speed up = -2.70% + +Report for binarytrees.st +binarytrees.st [0] ( 10 single benchmarks) + maxrss mean = 125.34960k, deviation = 0.16965% + wall-clock mean = 8.2(56), deviation = 4.00809% + +binarytrees.st [1] ( 10 single benchmarks) + maxrss mean = 125.02480k, deviation = 0.09383% + wall-clock mean = 8.2(51), deviation = 3.54147% + +Equal program blocks + binarytrees.st [0] ⟷ binarytrees.st [1] + wall-clock confidence = 97%, speed up = 0.07% + +Unequal program blocks + binarytrees.st [0] ⟷ binarytrees.st [1] + maxrss confidence = 100%, speed up = 0.26% + +Report for const_fold +const_fold [0] ( 10 single benchmarks) + maxrss mean = 1.82860M, deviation = 0.00490% + wall-clock mean = 4.2(43), deviation = 2.22283% + +const_fold [1] ( 10 single benchmarks) + maxrss mean = 1.82851M, deviation = 0.00420% + wall-clock mean = 3.6(97), deviation = 2.96524% + +Unequal program blocks + const_fold [0] ⟷ const_fold [1] + maxrss confidence = 96%, speed up = 0.00% + wall-clock confidence = 100%, speed up = 12.87% + +Report for deriv +deriv [0] ( 10 single benchmarks) + maxrss mean = 441.94560k, deviation = 0.02607% + wall-clock mean = 1.5(48), deviation = 3.28734% + +deriv [1] ( 10 single benchmarks) + maxrss mean = 441.88160k, deviation = 0.02168% + wall-clock mean = 1.6(26), deviation = 3.64121% + +Equal program blocks + deriv [0] ⟷ deriv [1] + maxrss confidence = 22%, speed up = 0.01% + +Unequal program blocks + deriv [0] ⟷ deriv [1] + wall-clock confidence = 99%, speed up = -5.02% + +Report for liasolver +liasolver [0] ( 10 single benchmarks) + maxrss mean = 125.34960k, deviation = 0.16965% + wall-clock mean = 74(1).(323)m, deviation = 74.22953% + +liasolver [1] ( 10 single benchmarks) + maxrss mean = 125.02480k, deviation = 0.09383% + wall-clock mean = 76(1).(979)m, deviation = 72.86571% + +Equal program blocks + liasolver [0] ⟷ liasolver [1] + wall-clock confidence = 94%, speed up = -2.79% + +Unequal program blocks + liasolver [0] ⟷ liasolver [1] + maxrss confidence = 100%, speed up = 0.26% + +Report for parser +parser [0] ( 10 single benchmarks) + maxrss mean = 125.34960k, deviation = 0.16965% + wall-clock mean = 6.(680), deviation = 11.82660% + +parser [1] ( 10 single benchmarks) + maxrss mean = 125.02480k, deviation = 0.09383% + wall-clock mean = 6.(632), deviation = 7.52088% + +Equal program blocks + parser [0] ⟷ parser [1] + wall-clock confidence = 88%, speed up = 0.72% + +Unequal program blocks + parser [0] ⟷ parser [1] + maxrss confidence = 100%, speed up = 0.26% + +Report for qsort +qsort [0] ( 10 single benchmarks) + maxrss mean = 125.34960k, deviation = 0.16965% + wall-clock mean = 3.(176), deviation = 5.61797% + +qsort [1] ( 10 single benchmarks) + maxrss mean = 125.02480k, deviation = 0.09383% + wall-clock mean = 3.2(07), deviation = 2.84632% + +Equal program blocks + qsort [0] ⟷ qsort [1] + wall-clock confidence = 65%, speed up = -0.97% + +Unequal program blocks + qsort [0] ⟷ qsort [1] + maxrss confidence = 100%, speed up = 0.26% + +Report for rbmap +rbmap [0] ( 10 single benchmarks) + maxrss mean = 125.34960k, deviation = 0.16965% + wall-clock mean = (935).(221)m, deviation = 6.23790% + +rbmap [1] ( 10 single benchmarks) + maxrss mean = 125.02480k, deviation = 0.09383% + wall-clock mean = 9(84).(841)m, deviation = 3.30251% + +Unequal program blocks + rbmap [0] ⟷ rbmap [1] + maxrss confidence = 100%, speed up = 0.26% + wall-clock confidence = 96%, speed up = -5.31% + +Report for rbmap_1 +rbmap_1 [0] ( 10 single benchmarks) + maxrss mean = 2.37144M, deviation = 0.00448% + wall-clock mean = 4.6(65), deviation = 3.79006% + +rbmap_1 [1] ( 10 single benchmarks) + maxrss mean = 2.37130M, deviation = 0.00324% + wall-clock mean = 4.5(57), deviation = 2.97617% + +Equal program blocks + rbmap_1 [0] ⟷ rbmap_1 [1] + wall-clock confidence = 16%, speed up = 2.31% + +Unequal program blocks + rbmap_1 [0] ⟷ rbmap_1 [1] + maxrss confidence = 100%, speed up = 0.01% + +Report for rbmap_10 +rbmap_10 [0] ( 10 single benchmarks) + maxrss mean = 321.90720k, deviation = 0.02783% + wall-clock mean = 1.3(99), deviation = 3.17753% + +rbmap_10 [1] ( 10 single benchmarks) + maxrss mean = 321.76640k, deviation = 0.01591% + wall-clock mean = 1.4(54), deviation = 2.60729% + +Unequal program blocks + rbmap_10 [0] ⟷ rbmap_10 [1] + maxrss confidence = 100%, speed up = 0.04% + wall-clock confidence = 99%, speed up = -3.97% + +Report for rbmap_fbip +rbmap_fbip [0] ( 10 single benchmarks) + maxrss mean = 125.34960k, deviation = 0.16965% + wall-clock mean = 2.4(08), deviation = 4.68665% + +rbmap_fbip [1] ( 10 single benchmarks) + maxrss mean = 125.02480k, deviation = 0.09383% + wall-clock mean = 2.(412), deviation = 5.37342% + +Equal program blocks + rbmap_fbip [0] ⟷ rbmap_fbip [1] + wall-clock confidence = 96%, speed up = -0.13% + +Unequal program blocks + rbmap_fbip [0] ⟷ rbmap_fbip [1] + maxrss confidence = 100%, speed up = 0.26% + +Report for rbmap_library +rbmap_library [0] ( 10 single benchmarks) + maxrss mean = 143.05280k, deviation = 0.38004% + wall-clock mean = 1.(037), deviation = 48.49357% + +rbmap_library [1] ( 10 single benchmarks) + maxrss mean = 141.24800k, deviation = 0.36305% + wall-clock mean = 1.(142), deviation = 41.64851% + +Equal program blocks + rbmap_library [0] ⟷ rbmap_library [1] + wall-clock confidence = 65%, speed up = -10.12% + +Unequal program blocks + rbmap_library [0] ⟷ rbmap_library [1] + maxrss confidence = 100%, speed up = 1.26% + +Report for reduceMatch +reduceMatch [0] ( 10 single benchmarks) + maxrss mean = 523.49360k, deviation = 0.03102% + wall-clock mean = 5.0(37), deviation = 4.62246% + +reduceMatch [1] ( 10 single benchmarks) + maxrss mean = 519.02640k, deviation = 0.03984% + wall-clock mean = 5.(237), deviation = 5.35693% + +Unequal program blocks + reduceMatch [0] ⟷ reduceMatch [1] + maxrss confidence = 100%, speed up = 0.85% + +Uncertain program blocks + reduceMatch [0] ⟷ reduceMatch [1] + wall-clock confidence = 12%, speed up = -3.97% + +Report for unionfind +unionfind [0] ( 10 single benchmarks) + maxrss mean = 125.34960k, deviation = 0.16965% + wall-clock mean = 3.(893), deviation = 5.06435% + +unionfind [1] ( 10 single benchmarks) + maxrss mean = 125.02480k, deviation = 0.09383% + wall-clock mean = 4.(053), deviation = 5.30991% + +Unequal program blocks + unionfind [0] ⟷ unionfind [1] + maxrss confidence = 100%, speed up = 0.26% + +Uncertain program blocks + unionfind [0] ⟷ unionfind [1] + wall-clock confidence = 12%, speed up = -4.11% + +Report for workspaceSymbols +workspaceSymbols [0] ( 10 single benchmarks) + maxrss mean = 500.80600k, deviation = 0.04563% + wall-clock mean = 17.4(91), deviation = 2.01575% + +workspaceSymbols [1] ( 10 single benchmarks) + maxrss mean = 496.73880k, deviation = 0.05991% + wall-clock mean = 17.4(70), deviation = 2.52270% + +Equal program blocks + workspaceSymbols [0] ⟷ workspaceSymbols [1] + wall-clock confidence = 91%, speed up = 0.12% + +Unequal program blocks + workspaceSymbols [0] ⟷ workspaceSymbols [1] + maxrss confidence = 100%, speed up = 0.81% + diff --git a/1-runs/run-2024-04-01---17-10-tcg40/speedcenter-worker.sh b/1-runs/run-2024-04-01---17-10-tcg40/speedcenter-worker.sh index 0245457d751b..94d58cf556f9 100644 --- a/1-runs/run-2024-04-01---17-10-tcg40/speedcenter-worker.sh +++ b/1-runs/run-2024-04-01---17-10-tcg40/speedcenter-worker.sh @@ -24,7 +24,6 @@ else fi echo "time: $TIME" $TIME -v echo "time" -# rm -i -rf builds-speedcenter/ COMMITS=("$COMMIT_TO_BENCH" "2024-borrowing-benchmarking-baseline-v4") KINDS=("reuse" "noreuse") @@ -55,13 +54,13 @@ run_benchmark_for_kind() { # TODO: elan does not like '---' in folder name? elan toolchain link "$LEAN_TOOLCHAIN" "$EXPERIMENTDIR/builds-speedcenter/$kind/build/release/stage2" - cd "$EXPERIMENTDIR/builds-speedcenter/$kind/tests/bench/" + cd "$EXPERIMENTDIR/builds-speedcenter/$kind/tests/bench/" || exit 1 elan override set "$LEAN_TOOLCHAIN" # set override for temci mkdir -p "$EXPERIMENTDIR/outputs/" for benchmark in "${BENCHMARKS[@]}"; do RESEARCH_LEAN_RUNTIME_ALLOCATOR_LOG=./log.txt ./test_single.sh "${benchmark}" # run benchmark, write result to CSV file. - while read -r line; do echo "$benchmark,$line"; done < log.txt >> $outfile_temp + while read -r line; do echo "$benchmark,$line"; done < log.txt >> "$outfile_temp" done; mv "$outfile_temp" "$outfile" fi @@ -69,30 +68,33 @@ run_benchmark_for_kind() { run_build_for_kind() { local kind="$1" - mkdir -p ${EXPERIMENTDIR}/builds-speedcenter + mkdir -p "${EXPERIMENTDIR}/builds-speedcenter" if [ ! -d "${EXPERIMENTDIR}/builds-speedcenter/${KINDS[i]}" ]; then git clone --depth 1 git@github.com:opencompl/lean4.git --branch "${COMMITS[i]}" "$EXPERIMENTDIR/builds-speedcenter/${KINDS[i]}" fi # build mkdir -p "$EXPERIMENTDIR/builds-speedcenter/$kind/build/release/" - cd "$EXPERIMENTDIR/builds-speedcenter/$kind/build/release/" + cd "$EXPERIMENTDIR/builds-speedcenter/$kind/build/release/" || exit 1 # build stage2, with ccache, since we are only interested in benching the microbenchmarks - cmake ../../ \ - -DCCACHE=ON \ - -DRUNTIME_STATS=ON \ - -DCMAKE_BUILD_TYPE=Release - make -j30 stage2 + if [ ! -f "${EXPERIMENTDIR}/builds-speedcenter/$kind/build/release/stage2/bin/lean" ]; then + cmake ../../ \ + -DCCACHE=ON \ + -DRUNTIME_STATS=ON \ + -DCMAKE_BUILD_TYPE=Release + make -j30 stage2 + fi } run_ctest_for_kind() { # run ctest to make sure our toolchain is legit. local kind="$1" - mkdir -p $EXPERIMENTDIR/outputs/ + mkdir -p "$EXPERIMENTDIR/outputs/" cd "$EXPERIMENTDIR/builds-speedcenter/$kind/build/release/stage2" && \ (ctest -E handleLocking -j32 --output-on-failure 2>&1 | tee "$EXPERIMENTDIR/outputs/ctest-speedcenter-$kind-stage2.txt") } run_temci_for_kind() { + local kind="$1" local outfile="$EXPERIMENTDIR/outputs/${KINDS[i]}.speedcenter.bench.yaml" local outfile_temp="$outfile.temp" rm "$outfile_temp" || true @@ -104,12 +106,17 @@ run_temci_for_kind() { # TODO: elan does not like '---' in folder name? elan toolchain link "$LEAN_TOOLCHAIN" "$EXPERIMENTDIR/builds-speedcenter/$kind/build/release/stage2" - cd "$EXPERIMENTDIR/builds-speedcenter/$kind/tests/bench/" + cd "$EXPERIMENTDIR/builds-speedcenter/$kind/tests/bench/" || exit 1 elan override set "$LEAN_TOOLCHAIN" # set override for temci temci exec --config speedcenter.yaml --out "$outfile_temp" --included_blocks suite # run temci mkdir -p "$EXPERIMENTDIR/outputs/" mv "$outfile_temp" "$outfile" fi + local temci_report_outfile="$EXPERIMENTDIR/outputs/temci-report.txt" + if [ ! -f "${temci_report_outfile}" ]; then + temci report "$EXPERIMENTDIR/outputs${KINDS[0]}.speedcenter.bench.yaml" \ + "$EXPERIMENTDIR/outputs${KINDS[1]}.speedcenter.bench.yaml" > "$temci_report_outfile" + fi } run() { @@ -117,7 +124,7 @@ run() { curl -d "Start[MICROBENCHMARK-RUNTIME-ALLOCATOR-LOG-${KINDS[i]}]. run:$EXPERIMENTDIR. machine:$(uname -a)." ntfy.sh/xISSztEV8EoOchM2 mkdir -p builds-speedcenter # clone - run_build_for_kind ${KINDS[i]} + run_build_for_kind "${KINDS[i]}" run_benchmark_for_kind "${KINDS[i]}" run_temci_for_kind "${KINDS[i]}" # TODO: add run_temci