From d09fab5d8e682a278f4654892330ccc4f890b354 Mon Sep 17 00:00:00 2001 From: Niccolo-Ajroldi Date: Fri, 11 Oct 2024 17:37:11 +0200 Subject: [PATCH 1/5] fixed :bug in summary_df --- scoring/score_submissions.py | 14 ++++++++++++-- .../performance_profile_by_score.pdf | Bin 0 -> 18888 bytes 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 scoring/scoring_results/performance_profile_by_score.pdf diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py index 02ad82fc0..4ce1dee16 100644 --- a/scoring/score_submissions.py +++ b/scoring/score_submissions.py @@ -88,8 +88,18 @@ def get_summary_df(workload, workload_df, include_test_split=False): summary_df['time to best eval on val (s)'] = workload_df.apply( lambda x: x['accumulated_submission_time'][x['index best eval on val']], axis=1) - summary_df['time to target on val (s)'] = summary_df.apply( - lambda x: x['time to best eval on val (s)'] + workload_df['val target reached'] = workload_df[validation_metric].apply( + lambda x: target_op(x, validation_target)).apply(np.any) + workload_df['index to target on val'] = workload_df.apply( + lambda x: np.argmax(target_op(x[validation_metric], validation_target)) + if x['val target reached'] else np.nan, + axis=1) + summary_df['time to target on val (s)'] = workload_df.apply( + lambda x: x['accumulated_submission_time'][int(x['index to target on val'])] + if x['val target reached'] else np.inf, + axis=1) + summary_df['step to target on val'] = workload_df.apply( + lambda x: x['global_step'][int(x['index to target on val'])] if x['val target reached'] else np.inf, axis=1) diff --git a/scoring/scoring_results/performance_profile_by_score.pdf b/scoring/scoring_results/performance_profile_by_score.pdf new file mode 100644 index 0000000000000000000000000000000000000000..4d654f2ba0920a39821072a085332dd8344835a6 GIT binary patch literal 18888 zcmch92RxNu_;^~lRzya|jVOhCuj{%-L{|2WWbZw%y&_RYWQ(lq6_TAT*(*_^sH`#~ zq2+&G^!=9ne&0U-&;RrH{hr?QKJRnRdCqg5b)NSZh)Wy7Z!$@*cwBKe7`i1w6n#*(OU=z zO+`aX6C)fPyL~I+hLcjo8Q@IdJlg;z102r8(H4$@Zea*ja|2@wTQfNC&QnJ_BUKX| z{0bOWToRCC;)a7Gq-_8R7j~{hKuDu95j-?U1VH!;Pt+4#5+1l+- zxfKBm-@U(&wtxH*gufP4|Gl#M)OiNIf^SP_f1G1@3I466{kF8vH@S%3$D z%)d*eo|FwSvJv~}U0}ecN4CkBOtw}>GDz3F#4vtpd}U;*XiT$u;{|*hT*0cAjEO7V74G6N2DPN`A=Pz! ztZOATG>K;iiR@I8?6K%+Hbb}*Ku7jWH zzTD`2!oj{2>9g*!&UWzW$K`s5^|_g$_UHDWjn`)<*FSiDA6%NVUU&EYXj$p@edR$^ zNJ!L!Y1fal)^W2Asb>mn7GhqXc6&fJ&Dm+Gbi+KhY^C2A8%QH0^Zfmc(v!Y{PcPn| zr*Xv|Uvo)tvokt_cVDShc`0JDue7{%f_0uB5B7wwAo<&2AAu!Aw7-k3zldJI2;M2owEtThbNsrsES)g9Y5w)PIUw=Af zd_KMIcv#I3{iXXW52($iN4}p=`Mhb@)3fR7;qqyH(t9o$H})oL&S&7sr3>9>Kv6?Z3vyEs%3#BJEatP+#5z zVTwoCqwK~vMk|Ux8N1=a_09}GRhq(VGc zbaL_=$(aNr?sJnt(Mv_dZKvMrEIfbhAD;a*GI4@r`b-)NFTE6>by-o?iLCwEHk#CY zXRhU4hgF#{(&fdfVEH5{NuttdZ#m`BoULtH3O@5mw^qldjd!0BvFXG0Z%PmO<|7I5 zDbq6Px-FeJPSbUmUP?rlpADLCEI39bHYC87I}!hu>x*^}WwE&2+jHK5Rrd&l(gnIc zK2|OZct;~ln}9f^8tg8UL-&%IMpr+u-QBJ@sg=7;0ZVBuY5H)8ijp(P{8C8njl7Am z_@QT-$`1osuDDP+%V9%+Y-g8jCndzRQ^Upm%A&@zKv{Yg0kDvQ-n%u zt-YYUEwHftq4lK@zjRAHPu*r&sUejdG23`t;UuL+(}+}R3xjHKr7Nk)OpbF2UQeIF zH}a^CgXCYmKCB}1<*w$2edi>#2o$q0lzF~bP;)7|D4_Vk0;6)xi+eg)UM{u!amK^* z2Riz_yJFjJElb4BSEEiTpQp#0ug+Fo3_V~pcDW*{%IOF_Y{2t<>{o6>>gXCB*Bf|L zq5aRDg97}^KGcG4&PNwN1--sdYKl#NCp1CXzhOSiz(ATqE{LhI{cvBoG5&!L_A}++ zk8_kY3-Kjqo7>GF)_SNF$K0(t_VH+WbR~MYcs2CM6bTB6-bfhU$xNnu(l-gj?0u* z&xj++#&wGBv*&vI!@E_7(?kg!USn%3Z@{vO(Zj)ZIr?c3AmSnuOrdr_@xG+FA>qgiIffUb%AZ?PR4&nir{5OPFhgt7ft}#bw5g z;L>vfDrY;5xMJJ1Il`+n8CRY!(}cVu*Z6tQ5ZMX=P}C5Pl=kHvgzbB%%_Gb5bB_(( zyIPmyx6yu3Kl}~A0{Yh6>f5R0+!Vkx>treT{oaTJbbTe6uw_{e9&6_Re3_WK6YZx7 z0r-@50RToDcr;%Fkp!Xm%1b8Dt_&We#FyS+M1%@mo zi2OuCH*!D#htypFfDwLRyWee$f1A7kjqS;WkKXvFoA~SDRTRGhcBm0wA9v~Q@`UM) zdff{Wx)iXKHYa3ouT|OPi^1Tl)}pEJbrbgv=N=Qb%sDf`QWOxrUh}o`Q+GN6lM%VF zZBpGP_9sWZHUjO|=>c|fGKKKegQfn8_c9QZ$ThaP|H*+cUzp2bsExJgOJ$2XlZF$h z7Hn1HD(Bu%Xn1Q+4zy(k=Izw;!ezBr9d07LUuK5ad~%2rUWj|q=B#Q^-Z7j_wDvOQ zhdlLy%~Qlg1FrMclDF}M;8N40_W_H2L~FLv>+uNcg|XZv@AOzMJ<)7HwccveBZ>86 z6ITN4$m=B5Z6(%)zbTNIh+wM&!|N@NZY?d4k&Qy%W>?v!3hm~#Q(pq@s&@as*jXF6 z_-OIz`bex4i`te>)-fLG+|rlJjQ{8pxZhD<2NOuYpy$*09XlGihXVfWV23+kvO1p~ zhz?KYy)SyOqw7$I8`kcjt;;v8{!}ZVos3|B9mV^pR!`4&O8kG-&DHx6v#%dKQhjq& zr*dt0?hTjTdrs;F@BSt!x7jy&`m=BD{8Gjr-=*_iAX+Q>O1&1Jf5qu0R5}Z*0)`5! zT6W5v_kY5mXLkTD;mwpw$#qbddjG~JQVH35HK;o zIMlDCXqs{~N$p{WDuH?cyA(&X7E84Dk3_}_VuZinrDNJipo5H0%JuM%)J|3laqM2! zx}AVeG86a8&p*WRD{<8%nj*TnG3g>;=Q*m#ayB=0Q zF4KPW*)7w}&%MEfUs)f7v>oShS>>`#M%R4wA^Q0Elx(p* znqhJXXRk3hv3xV=OG!DWjg{8qke%MF^#;wIg)94V7sXrkUDpJM8^`U0kUVV(0Hj&-`{ z1w6byedta`4YXF_YnKa>Q5&_Sb>q{Oh+xb~$e6G$O>y;SEeXl^DgoA@a?nZc=NQ*RcL@y>0a&rpm2MEcON_@UA=H0 zC%wL$)2kKOn1AM~mS{E;`}l!|C@pj2wV;G7r;d5M#pjN`jni%}pJ(RD@ao@rUuv3V z%ZR}jMq_MSNtr(5C=xoxm65OwvY!cXY(sv z8nUJAZZQ_7FnG6oS8bv-!hL(LdjfMw6hE9Ydw%qlcN?Exoo@Yj#gNZTorM5(=pcU` zW4=GPmVZ#VtH7htAD>rQggsmylR4^sXF^h@y@E0!J3INEoXFO;A4&!AAknCwiIS~+ z0R+4Pk_tZ&q3p=+oB$1fK5;Fr^79$zNrTllVs3YLrTGix5Y z<;rVT*67J-w?f_NsZteiAt7HXy{L(@{Y(~&DcY0Uc2RoNla_Lk?32y?Vd+JLZQ4b; zJy`W?{%9v_!i_=w1qYM`BU_OK@4aES^jFYHrvwGBydA68@jK-%!4cL-ae6d>En#BB zb8spn1%o?v6YU+?En%qeSOmjONRfaeL9<@2{CHAcdZt`B&RBMRt z)St5U>1`*Lv5%R}y@3jC*$JF&o+IT!8dB)-`=~lZ zYtYNO*+vq`xHZ|aHes|gZv??flbAc_+!U9dmz()Md4chfv@TcuGRP4o?3o%wQ1XQ- z<7}jQ+-vE4QOR5t^0s`*vYuA+j0mw0g6z(uq*S?`!|_K4&q$cSH;+ce({~+sPmw)} zDS0d&W|m%?%&F@3u+XqV)Jz2%LNWPvLi~(`%ds-5kRo=v#Kq$Jhy*R(!3QI#v~Pyx zE~Rma1NwG0&9f#NuchwPhP32z=CcP)%%42taJ0}6S*>Vl;`1e_q>8R{(0kJBL=0Dw z!24o~t`klDRsw-$IuENC^{yUsYyWczK&J2yMds$^{_iD_SC+PIMiSS8B@o%?a5QtR zlEA}$)3=-WZrLGgdk&Qceds{q5>7JbcA<=^2!xQD>cHD<(J~C93j6Q z8*Cv;2eYqEG&b~jXh^gFBvDc-;StTlrQ+7liH{fOJv!W1IJ8PC`?jZ*x$llQYr&vD z3%oyCy+U%xb4>ix5vQ`;;)RB+mEneRDQC>-r25-5iyQlqdB!jHVCSz{+@Dr~=P#^+ zkTSpRK@{=(?=TCLKVuQq7j@Ph%e0&b z=NCnbPw{k8Zhsl?Ny_1S;-n_%&weYN|6$M}w_}wWiUFlJ*5L2S-qmRZQnI@sE@MTh z-==d(&bHbi)H$==?=9s<6P~}*vgjRSU#5LCr4}>yN&HELfMI22A^#(fM72~Q)|n$z znS%@NRg71XTR!l5i*j}Ov<;kIAoKX)JLKsVHcXcC{1WjVoP#pXyFAKY3L-yb3qAnO z>Ek5}jK6_jy#1|yjMfq>ZD231?9|7it|s%&;6$_Bl+s3Ver<-f57*~?b;nOzN`8N4 za*&rbu7&GkG>vTeOO31R$`k^LFWPd~-qD2-r%xGXanpal$^g@ zN!&B>lwKl+0d;!3OQ+w9;u>ghYLZ-9U3Uj=wcjA1CBllUQV$$-+e~ZBG2Re+6n5Wd z^0ub(J@xx{#rvW>c0zA6 zcC(X*|G)KGNLkkQ5Q<=Qno>Jer3=ySo|RV}@47jU?y}P0J2n#7qMSgXTBlD@L}9ud zrmQ4gf0&tWjBNQEynyoUu^hr%ow|-B>cM{BrMg~i7 zT#k9AhOB0`@VA&JIl|QaQ94x}%aXSJ3<~_dWi#*e>(8hAo~nI}z;=^vFi$c>`UHP2 zYjzhblp88&cF;-AJeTjtZaYRisD@AKUFcLKFg;oQK>fW@$MhH1+Swl~hn+UGl=t8u z^uA{o2hn`I|II;+is-GAD1w^dv+E5qej*pg2lM-Nr#6Ql-klR8)YxC{S5Rd=V)^yW zpe4h5;;-)&i%fiOeTPX@naGD+qp?8;dMDG{qQkpm)>aDF-P3hWo3gV!cDM4a@Gzi#EW`a&BSszl0CCK|G_5tW^my+GEInsEN4o3i4eM(m zWIHT|fr}&BJrd2{d-t)hxH{^Ex=xJqi)M+Xh*>(IZqQ$~6g;_bGLoS;TJ#lX^X-a5 zw@7C69j>3?Kh&q-NT^`O5I7Da3lKf9PxIDvK)9g83}@2qj>3B);~t|Q(oAAi#7~+^ zE_hkZjai%AjQQ9>AX<^tDH-MwX2Nv28+Ung;abhc2H8?#y6GNE3B9Y^wUj7+^nWj< ze1xpy0ThAwn!q8c*JMtN*($nxYp=3ycQD#zy}GL9Uv6kv+R_%58#ycWOkb=2RcX0A zB0?oALS5;z+mbJ>P^`PEL&U5EtIXr_)e;^>2_c95mP=)dQgjup?9~?K7y(!Olf$(% zs}jlRR-7H|Zq_rBb9d~Xn)U`k-kv(ObVq#!{wU z>0-y0J_NH|GtZ#xdp<&!EYcNP_EPaB@A3LUpTg-c-|_hiWPA7E1oWD67bmzee<9S6 zyWftuzKHCbFMWlU=okC4X~ljBJ#zwXq)Ne_&5D{&l?`PkjbZ!}No6hO8f!DooNO!#}G=wEBl%rhGw!@9|h){%cb9Q?x zQvb**#}x1D!%p49_?mp4IP8ybw$OW&119_5>*_y^#k=DXBD@MKoAE51Pv^}cIGUrX zgV0!GkKTjkJ+>_9zgc`C72p$25RdkX7u+UQ9H zy<=jooVv8@#=u!lR?~JnmZbXQF9($w9P>$D3O3L$S2qz;8r|sS(=LC8cfWJlDR7Zj zCp|IPvhu0%SbX~7)Z_bjDvDmS8nCut=B`EN8HMx^?yHeu50htqH=1CG$IclHUirif_d|_LPyf~*VzT~rPi{%ZDfOco1zV;1Lr2b?Eg-hbi02Ye z=S#G%EN)HCepsogc%)&1wgT_WC8;PD00q# z``HOk!)yiK`?ln~+DxmzpHfs7vyOLz{tel+W18M}`W`EW-_7xrmoAP?hMfo#R<3+< zlrjE2&A!6zy zPjBqV_!^VLi7@Rw7Hf~a5$|6Fl4uN438_?!yo5yas3Uois>q5mwTq~?+&mQr(N0w& z{_u8sE|#ORr-^bHPbM7Dh;5E(j5)lIqnLF5Pa@EJ><&=;e`78pL~peq2@Wp$98zrP z)Bh@Tx$twu_f2Oyo;zJ9W@V0K$?Z>{Y}e?CjyZGQNq_lkJ+s^bs+&=>TKa?ARr$4I z3%!F^B+3u*w_Z5p8B!SC7j0Gf?Di4YxYuvo1@7Z@vb~K>rBEoQE;yLYGSiZvG>;q3 zi_j`wSi-Y;^u3*H--gKLT6X?ci%!|Jsmd549r6=(q~5W7Wh-R}sQ4Zn7m#JL7 zPB7UvmI&CS$SSJn?RqGTgS5$+*B1~Mcnmo9Iu!jvYb!dW3GEd z<0$UG$nVI9%aR^AK`geG#4H8(bxctnlJMM|KZ!IPd3}l=-)JoLfCOV8u`L6aHP77V zSDGq5tTMSbzWA@*j&7$e2+VyiYJBJ9N%)kLI(-I>w7}cccQP}{Vz0ZhJB@_&U(i9Mg0^tUm9Y~ z9^~v1QF8PD#a>$?veUU0PAtBrmPQd!rxfnwSbFixE7OzTJ{(HMqs6g~l;4m(kir?I z9aA^8RP_Y;BQQV8J;F1RljVM+Mo0C7ujMEQZ$jxZ{jsjDyM}4EF7_>4T3T@+wOw&P zBi=)V!JmO0uzY^TO5 zy^{hknepS%%mdN>s==k7lw#jME#1IKn60opIwzrPR1$n%_l?u=xWK#v+vTD&H_}DP zhM7gxdz54B@7j3huO8<4l6-_Hl%YJZWxDns{VFtz^SHR_TbODwAQsBK-%7qRB=}JH4A@UHK60Pl6zsf8 zUwEKmX(Wfw;buSeA%P#!Eb{uLg+zhDz3d|+g(p1G-Dh<|{b(&q@)o*3wl2ljhNG

TnT!wpbsQ|{xeyN zQR=j51!*=<80XTy_g0?C=O`Z;Z(2#S9@>B7wnVU+d=xwXyL(&-Y_nW?48rQ?-ZC-< zHMmYnU&yA9u{!8sDa9OndR&u}Oz$c^bJ7wzk)r(Lg|Hk7nZroeV%_G`B*t1oX>Gc< zo>@|LTdO}aBO}K;`71oJu%h&1#8^xaI)=UKX^CG$c1mi!K3m5kF?WGzlP{>K)ezHv zcnNtKiNUg!iwjTf7rS!0t;PZGYI`OMNYN)zvjeXCl5$~Fo@6}mWS z`8z+t8!w*sEm2n`+CyIU2t~b+5FnS!Qniw(r#iTE|aPedKe@>f}x9x#`b0FK}z<9lMKr`C5mL zX%E`=2;8`_f3bk-k+M*dPJB)H5JF$%Tf%_5%E8MSYp+}a9s4N>8q7lINZYCmpB%a3 z_~LXs=H%ErS+e8oysC9r_a`;dvaMy}-K`f)a?g?0s5o9Qm5z5i+4;tvet(ln(Ft(ZboQYaHY`5P-@Q9rwDe(!@+gB`+`Y)2 z<^5ysg5OFP4qnt!9Z9qFo}SwW_cyaluFfuIHhv|}WL91x0kVJV{c9hF%zW7!auXgc zt_OWPfP*oWh0{L6U#Qe=Tfrw#w{qTtt$T!6DDJ-R;&_wn}w zRpzJn$l>zr5vBc=c*rYlZ|bI@9In;Max!}`t}r5jr$AnXU30@9=v_HjxDpLhwg(_?a1w<0Smt~*ox!reuf>& z31!A84MV|LW)@?)I)a$O>4TdW+p2%`xG?A?&OUIvkaqD5zpFCl8FOp}&1A(q9AC(C zu`gu0T2PW2V-P~D;q{&YC5U@T@M^X0YJ|LFJ928E>xCiZ2_vg7k6KbXk3JFap0`Y3 z9?1F_WplhdUrKm_O8TR)?>;1l=fqO40PPahueeXX7C^uu8NSoElb zIAc>g^6YX+V3_WpC!+K#9L_) zKk%ns@az%%p?Ury&=msa1|$i@*7&b;A+N56%&*8PhcXR~0-XnVFR|AT_3o{wUF8IA4bW6nh58U;r)R5K$x zJtBL?d7B+cN1YqQbQIKV2&wCHpYo*0#u!hLy4eKEv0a@GLv&j?$U5R&hNjCfy^D)P zwzOjRhLnnvctl%EFPqy%l^X76ARnshJ;zU>gU{Ng7_ETi=%9GawDP%}u9g;_XJr+f zF5bKrbN0tQKPDPSG@424`sgRZ@t#1wN|DfB|iQ+(W-D{ijW`rG>QDO^9T z5$TuD(=jMaHyg1W(rsF7S2`<2SoHW=Q54nnm$y?$m?)*@nB<8zGKHi&zbEadV!ovH za=%9lbI%veM*RjS%SLuW*12}`n+W=#*7qDS&Y7HQu|mV^b#+C(dPZcvANE+EJ;GWv z`Y(JHMx`^10YzL>1H%+wU9EVZ&}v8~y*V6lhgyH1!zb$5|rz} zk1{B~J#=G6g0VI@*X8TO9y(?lD$^WxOSK-Jkm#08Wv(CmZ!g!&hOAIY&AzLp9E=um zC0Y<{ouPTMVpB=@&gZeA=h!uxFrLtn=8gvxzPhMS8$BfmYVqJY!ntBE?wRIsswMly zzJ)2Hs%lb|mqqI$I!;Yh4RPGGJ-KlsW_b3;Bko=euIbbp(DGoRZBtT|ivE1rNALQ9 zgIEDc`VsPMBJ|flQVBnIQw1_$s)Vp;@5CGZn8$C_e>F8M4AHZ)X*V6x5H(8 z3vWt9IGG5aARdHA6I!w>EpTpLPd7imxsTDudym{2&mQ4DitoRpEHrPK67p&@rC1Tq zK8C3SDtj#WUa>6pFBDQfqK4!Eidf83vGEXT@k!t5D)>c*=k+ABVv}kfSseZ0Eo%(_dD}hT`dH2HwE8?3&~BqqxMJn)XRq^y|<8*h}Ie2exT* zgYq1L4(A*t#`1A)W|OSE=xg<(;%==OA4sMg`xTf2Q^$$Y$(|q1=4&FG;7juH9&51D zUDYlTe-=uW#lbdaE|O3~rYB*QR3D`?8)?T~bJS0P@;;(XTK|JAg z)zgHM1Dq)uS9)w-t}-%9I;CE z!1kHPZ!X+gc~ZMBdf`RF;E(>SX+z4$x|uuQSWi?dd14XXp8}TOkYq9*)|x(0 zz>sR(&RClLX3&S+cv+EN>bQkcz*o8Xrr{S(J z_5u z=%Tdpr-+SB=hJ3={3lOxBrv}#YZ2%A(R8eX3PGDrMDCZO?DAynq{h*N>vs+tSV@Zp z50}}KP>Ed@{Z2lRiGSbVV^SsVexqvcGgm`4ZoZ_WPQo-UXN;5avg-;7S>_@ysrKO3S})xSk%72QVyS{DkA5C~JE9%S+1rZ8MY=|!K{x8L zRN^@zBWcXzthJzj%ImW~?Nx@V?*Zm-o4UxNx|g(LKSdh39zmQ;kqotrBLBhomM-Pv zk3-~DD)M`9YL6Hm#rIb_{4=A{M4cn18=>uoMrq=%M)aDYEb+6TMe%2{<^9Uj zU5{Qp+mBD*?JggkpP)J*6N48nCjhBcXieK2aSO zBh^EyaH-+G0l5>c5kx&b*O8*j{Mt{5BaBHe=#XkaSr_qm(v3d{#_SOt^Z$iemcJzk zHg;Q?ENT(U0q{ah^hVlCUsiQ3{gPzs=>z!G<2(n8h%LMba_0?R9yD9ID=0B>y?-`i z$TvDIRbk+nX2!t;0}srndr3OjC~k?8p{Nccsk*`0^cB1)yc27CP`X#ZiTaCxQ$C!Z zv{{5e!c(DmQGtF~a3h_4j}yqzTHyoIl5N?l+)q1LzFlqhd^uT2b$Ar4#UHwdd!kBHwZ_8uk-L`<{4b)_=^mEO@D0dYUpdvdilgrp~v_ z1MyWNe{QIVy-{4hz)52Nin);^@llEW-nNI(6{1H23LmB#;f0Lk_avvYA8>bxU1ze`R)?(1~|c?zGQY9Jjx?$?eA3LzmKlVr;%(r z(7(~)IU$Nk;(QS;!@R*4qUI^klbz0pxF1Q$V)&3vfpQdg%q-mi{9C&mh?c9y( zdo?Or4Haow%KJ+PYjnH|t3(BLZ{g*ia=FA3J{TJR*|JHaX6XgbJ5SXcG_r@UC_Pv# zA-ggVmAN_mTIRg(6Didne{u))YuL4YrgZC|DB^;FlgUm^Q%d5J+*xrGOM^?!swR#W zrck`+=!6q9H*f^oM}(ZgPMsTrfgzwq;wDZ;ju!SfJ4ZP9)NShwr>e6dZtKV@Gzj3n zf`JV<_<7-vzPCHLuzlK-6ALmxoKQf-jY9ImQ7G`8rVbb#3?XlTbF|nx5z2)`LPiDr z_fHKv7z&6}0tZ#WSj1N>2f^MU?U;QYW+INKVVI64{GIRZ>GM+2yV0UTinM;O5o#&87G7kB_in8Ohk z;1nxRBP$rf8ji4mBW%IIfEYN!0geEpIf2nZe{h5g9N`KMvx1zzJ0K8zzX}eoA~Ym=%A1Rd+;=LX;8KqEjm zoZKK)3(hUuU)*L85{A(HSp|oN!Et5_BT-v3YZE{?LKSCXa|x=*8@O%F2uuj<-2Xk` zyfr^)I#=MlNaX*E0{g%12MzW?JVv|9>m6pU=@~ENF+_>I3QrDj3ifg#;aAxw+w3GzO04#Q@QR>VPTyJa8;O zKb#NC4G6%1@19X`EKn(4ULMehn+H4q$RJnF3$|O(6^lW`dBKOyyu4r(kQ0GA1&Dlb zEckYs7fg>ANHUlTY>N~yr7aS`=+M+K;MFNF$kt&XJm?Bu{-A*tLv>!TeMEx+&_IoO zfj&cZexOzW4%iO*-oin`d9YwgU}+(CLTxb6Y`{x=44@dQV?Y8EVh1!nbcM#>S~iG< zNI)0FS}-OCT({V=Mc593TM`6Kj|TuyD5wiifu{6x<+qxHrWh0o1`3!0=n6uHIQ(yA zi$E{|KK`wiTSS3L|CEED>$P*;V%ASOwtC_LZ07|Wg{X%F;h*jyEC>iK6jY#lULZ+Z zQ%3Q@p!=<75L-b9{Ji}C6i5=bga8tRT?GsPRvz#Vk_u2@J6A|5ek**y?)(%ANFaVI zfUvE5AQ+Hv>=f9RT!73WFE3ymkb%7l|BhThGVyEKk_kvIw#$wz@Bp@g_1{&t*Lqtn zAldj?Ai3CwTFYn*{*H7EcDLg z9DuxRKX8O^rF_AlPVlW9C?Ev#SYRc$8t!<#odI!ho}B@=9Ubt7TUe0q0~Tbf?hH9b zP=o#I+rX}8>!Aw}-tC$zP^ayhJJ8?lng=jkJ2l9!Vzx*9*SbRwfBGs(6nraI`PG?0 z_&=YEqJH)3|NNi@LQE7OA&kI6Lv8~h%Kbm1ae?&WzliAn5fOOWf6;jUM>HfC@ZSF- zV*VFI5Lp03Bo`l`@fYv*OKkQpQ+^TYe}T|i>~P{=AjIXL?Adnp+rJWmkhL&|qMWUt z<4}PDDlt1};F`BMv~&0c^UK%>G4M+mP%L*E1YP#lb~tMbL%0i?3(LjB0XN6t?41M< z2%CQ%aoIVVvBMA`_%e1j0za(rYb$$WQ@Ejlkrfzarw4$!)xCwCtvK`xB&Wp%xPenb zAwjSQozUSui{w6qM4n>*g)eT7CZ;f8ig{tcBJcbG3yA^RVG92T2E~2QAKZ2ah5}v> z$lxv*3X;a%Fz{A%H;e}f#C#VX52VYxVUWcC1_L+1IPHd^A)WsnkB0}ddkiF!`;T#v zJUo!b|JD`-?BDKjQ5b-@8;^${QrzF$@?yaz_IDT<132N`Ff3$sert=wfbXyVfPuz; zz_1_)-qn^D2^__47|)-4;e{fSU3ge-=pYyL_s{ww!TSC|GZu1YyYcwJ{&W|NkLM40 zLGocBI|lvzI|d&g1~AxFC#5BmrIfS-nZ$8Y_ikdSlS4dVe0 zcQ=d=@@T)e1AfLDk25sMV!O$oWP3?y9 z@a^U)h|8dN4ZrsXK26`vKgdS>3FH2g79I=;BzE=3!wbGP+yw)o_6rQ>XkcM&;<)vT zOR5$gCcx%^@WIXwGL~BoPTJNKI4(fJ_Abi_XW)q2GEd+wDjy$=l~qDf683)pAng{6 literal 0 HcmV?d00001 From 57656d89363ffaf3cd22ebdd3fad2ae802f5ffa9 Mon Sep 17 00:00:00 2001 From: Niccolo-Ajroldi Date: Fri, 11 Oct 2024 17:40:03 +0200 Subject: [PATCH 2/5] fixed :bug in summary_df --- scoring/score_submissions.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py index 4ce1dee16..4cb640515 100644 --- a/scoring/score_submissions.py +++ b/scoring/score_submissions.py @@ -98,10 +98,6 @@ def get_summary_df(workload, workload_df, include_test_split=False): lambda x: x['accumulated_submission_time'][int(x['index to target on val'])] if x['val target reached'] else np.inf, axis=1) - summary_df['step to target on val'] = workload_df.apply( - lambda x: x['global_step'][int(x['index to target on val'])] - if x['val target reached'] else np.inf, - axis=1) # test metrics if include_test_split: From 223b90b976a459eb1a018d968ae6175fce24530f Mon Sep 17 00:00:00 2001 From: Niccolo-Ajroldi Date: Fri, 11 Oct 2024 18:20:01 +0200 Subject: [PATCH 3/5] fixed :bug in summary_df --- scoring/score_submissions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py index 4cb640515..79195a063 100644 --- a/scoring/score_submissions.py +++ b/scoring/score_submissions.py @@ -95,7 +95,8 @@ def get_summary_df(workload, workload_df, include_test_split=False): if x['val target reached'] else np.nan, axis=1) summary_df['time to target on val (s)'] = workload_df.apply( - lambda x: x['accumulated_submission_time'][int(x['index to target on val'])] + lambda x: x['accumulated_submission_time'][int(x[ + 'index to target on val'])] if x['val target reached'] else np.inf, axis=1) From 076c1a7600eccdd4613daf101fccb806fb06e623 Mon Sep 17 00:00:00 2001 From: Niccolo-Ajroldi Date: Fri, 11 Oct 2024 18:21:20 +0200 Subject: [PATCH 4/5] fixed :bug in summary_df --- scoring/score_submissions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scoring/score_submissions.py b/scoring/score_submissions.py index 79195a063..1fb39d193 100644 --- a/scoring/score_submissions.py +++ b/scoring/score_submissions.py @@ -96,8 +96,7 @@ def get_summary_df(workload, workload_df, include_test_split=False): axis=1) summary_df['time to target on val (s)'] = workload_df.apply( lambda x: x['accumulated_submission_time'][int(x[ - 'index to target on val'])] - if x['val target reached'] else np.inf, + 'index to target on val'])] if x['val target reached'] else np.inf, axis=1) # test metrics From d0fa8889a893e2624c4b56e1527cc51ed11dfcc6 Mon Sep 17 00:00:00 2001 From: Niccolo-Ajroldi Date: Sat, 12 Oct 2024 18:07:30 +0200 Subject: [PATCH 5/5] removed pdf --- .../performance_profile_by_score.pdf | Bin 18888 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 scoring/scoring_results/performance_profile_by_score.pdf diff --git a/scoring/scoring_results/performance_profile_by_score.pdf b/scoring/scoring_results/performance_profile_by_score.pdf deleted file mode 100644 index 4d654f2ba0920a39821072a085332dd8344835a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18888 zcmch92RxNu_;^~lRzya|jVOhCuj{%-L{|2WWbZw%y&_RYWQ(lq6_TAT*(*_^sH`#~ zq2+&G^!=9ne&0U-&;RrH{hr?QKJRnRdCqg5b)NSZh)Wy7Z!$@*cwBKe7`i1w6n#*(OU=z zO+`aX6C)fPyL~I+hLcjo8Q@IdJlg;z102r8(H4$@Zea*ja|2@wTQfNC&QnJ_BUKX| z{0bOWToRCC;)a7Gq-_8R7j~{hKuDu95j-?U1VH!;Pt+4#5+1l+- zxfKBm-@U(&wtxH*gufP4|Gl#M)OiNIf^SP_f1G1@3I466{kF8vH@S%3$D z%)d*eo|FwSvJv~}U0}ecN4CkBOtw}>GDz3F#4vtpd}U;*XiT$u;{|*hT*0cAjEO7V74G6N2DPN`A=Pz! ztZOATG>K;iiR@I8?6K%+Hbb}*Ku7jWH zzTD`2!oj{2>9g*!&UWzW$K`s5^|_g$_UHDWjn`)<*FSiDA6%NVUU&EYXj$p@edR$^ zNJ!L!Y1fal)^W2Asb>mn7GhqXc6&fJ&Dm+Gbi+KhY^C2A8%QH0^Zfmc(v!Y{PcPn| zr*Xv|Uvo)tvokt_cVDShc`0JDue7{%f_0uB5B7wwAo<&2AAu!Aw7-k3zldJI2;M2owEtThbNsrsES)g9Y5w)PIUw=Af zd_KMIcv#I3{iXXW52($iN4}p=`Mhb@)3fR7;qqyH(t9o$H})oL&S&7sr3>9>Kv6?Z3vyEs%3#BJEatP+#5z zVTwoCqwK~vMk|Ux8N1=a_09}GRhq(VGc zbaL_=$(aNr?sJnt(Mv_dZKvMrEIfbhAD;a*GI4@r`b-)NFTE6>by-o?iLCwEHk#CY zXRhU4hgF#{(&fdfVEH5{NuttdZ#m`BoULtH3O@5mw^qldjd!0BvFXG0Z%PmO<|7I5 zDbq6Px-FeJPSbUmUP?rlpADLCEI39bHYC87I}!hu>x*^}WwE&2+jHK5Rrd&l(gnIc zK2|OZct;~ln}9f^8tg8UL-&%IMpr+u-QBJ@sg=7;0ZVBuY5H)8ijp(P{8C8njl7Am z_@QT-$`1osuDDP+%V9%+Y-g8jCndzRQ^Upm%A&@zKv{Yg0kDvQ-n%u zt-YYUEwHftq4lK@zjRAHPu*r&sUejdG23`t;UuL+(}+}R3xjHKr7Nk)OpbF2UQeIF zH}a^CgXCYmKCB}1<*w$2edi>#2o$q0lzF~bP;)7|D4_Vk0;6)xi+eg)UM{u!amK^* z2Riz_yJFjJElb4BSEEiTpQp#0ug+Fo3_V~pcDW*{%IOF_Y{2t<>{o6>>gXCB*Bf|L zq5aRDg97}^KGcG4&PNwN1--sdYKl#NCp1CXzhOSiz(ATqE{LhI{cvBoG5&!L_A}++ zk8_kY3-Kjqo7>GF)_SNF$K0(t_VH+WbR~MYcs2CM6bTB6-bfhU$xNnu(l-gj?0u* z&xj++#&wGBv*&vI!@E_7(?kg!USn%3Z@{vO(Zj)ZIr?c3AmSnuOrdr_@xG+FA>qgiIffUb%AZ?PR4&nir{5OPFhgt7ft}#bw5g z;L>vfDrY;5xMJJ1Il`+n8CRY!(}cVu*Z6tQ5ZMX=P}C5Pl=kHvgzbB%%_Gb5bB_(( zyIPmyx6yu3Kl}~A0{Yh6>f5R0+!Vkx>treT{oaTJbbTe6uw_{e9&6_Re3_WK6YZx7 z0r-@50RToDcr;%Fkp!Xm%1b8Dt_&We#FyS+M1%@mo zi2OuCH*!D#htypFfDwLRyWee$f1A7kjqS;WkKXvFoA~SDRTRGhcBm0wA9v~Q@`UM) zdff{Wx)iXKHYa3ouT|OPi^1Tl)}pEJbrbgv=N=Qb%sDf`QWOxrUh}o`Q+GN6lM%VF zZBpGP_9sWZHUjO|=>c|fGKKKegQfn8_c9QZ$ThaP|H*+cUzp2bsExJgOJ$2XlZF$h z7Hn1HD(Bu%Xn1Q+4zy(k=Izw;!ezBr9d07LUuK5ad~%2rUWj|q=B#Q^-Z7j_wDvOQ zhdlLy%~Qlg1FrMclDF}M;8N40_W_H2L~FLv>+uNcg|XZv@AOzMJ<)7HwccveBZ>86 z6ITN4$m=B5Z6(%)zbTNIh+wM&!|N@NZY?d4k&Qy%W>?v!3hm~#Q(pq@s&@as*jXF6 z_-OIz`bex4i`te>)-fLG+|rlJjQ{8pxZhD<2NOuYpy$*09XlGihXVfWV23+kvO1p~ zhz?KYy)SyOqw7$I8`kcjt;;v8{!}ZVos3|B9mV^pR!`4&O8kG-&DHx6v#%dKQhjq& zr*dt0?hTjTdrs;F@BSt!x7jy&`m=BD{8Gjr-=*_iAX+Q>O1&1Jf5qu0R5}Z*0)`5! zT6W5v_kY5mXLkTD;mwpw$#qbddjG~JQVH35HK;o zIMlDCXqs{~N$p{WDuH?cyA(&X7E84Dk3_}_VuZinrDNJipo5H0%JuM%)J|3laqM2! zx}AVeG86a8&p*WRD{<8%nj*TnG3g>;=Q*m#ayB=0Q zF4KPW*)7w}&%MEfUs)f7v>oShS>>`#M%R4wA^Q0Elx(p* znqhJXXRk3hv3xV=OG!DWjg{8qke%MF^#;wIg)94V7sXrkUDpJM8^`U0kUVV(0Hj&-`{ z1w6byedta`4YXF_YnKa>Q5&_Sb>q{Oh+xb~$e6G$O>y;SEeXl^DgoA@a?nZc=NQ*RcL@y>0a&rpm2MEcON_@UA=H0 zC%wL$)2kKOn1AM~mS{E;`}l!|C@pj2wV;G7r;d5M#pjN`jni%}pJ(RD@ao@rUuv3V z%ZR}jMq_MSNtr(5C=xoxm65OwvY!cXY(sv z8nUJAZZQ_7FnG6oS8bv-!hL(LdjfMw6hE9Ydw%qlcN?Exoo@Yj#gNZTorM5(=pcU` zW4=GPmVZ#VtH7htAD>rQggsmylR4^sXF^h@y@E0!J3INEoXFO;A4&!AAknCwiIS~+ z0R+4Pk_tZ&q3p=+oB$1fK5;Fr^79$zNrTllVs3YLrTGix5Y z<;rVT*67J-w?f_NsZteiAt7HXy{L(@{Y(~&DcY0Uc2RoNla_Lk?32y?Vd+JLZQ4b; zJy`W?{%9v_!i_=w1qYM`BU_OK@4aES^jFYHrvwGBydA68@jK-%!4cL-ae6d>En#BB zb8spn1%o?v6YU+?En%qeSOmjONRfaeL9<@2{CHAcdZt`B&RBMRt z)St5U>1`*Lv5%R}y@3jC*$JF&o+IT!8dB)-`=~lZ zYtYNO*+vq`xHZ|aHes|gZv??flbAc_+!U9dmz()Md4chfv@TcuGRP4o?3o%wQ1XQ- z<7}jQ+-vE4QOR5t^0s`*vYuA+j0mw0g6z(uq*S?`!|_K4&q$cSH;+ce({~+sPmw)} zDS0d&W|m%?%&F@3u+XqV)Jz2%LNWPvLi~(`%ds-5kRo=v#Kq$Jhy*R(!3QI#v~Pyx zE~Rma1NwG0&9f#NuchwPhP32z=CcP)%%42taJ0}6S*>Vl;`1e_q>8R{(0kJBL=0Dw z!24o~t`klDRsw-$IuENC^{yUsYyWczK&J2yMds$^{_iD_SC+PIMiSS8B@o%?a5QtR zlEA}$)3=-WZrLGgdk&Qceds{q5>7JbcA<=^2!xQD>cHD<(J~C93j6Q z8*Cv;2eYqEG&b~jXh^gFBvDc-;StTlrQ+7liH{fOJv!W1IJ8PC`?jZ*x$llQYr&vD z3%oyCy+U%xb4>ix5vQ`;;)RB+mEneRDQC>-r25-5iyQlqdB!jHVCSz{+@Dr~=P#^+ zkTSpRK@{=(?=TCLKVuQq7j@Ph%e0&b z=NCnbPw{k8Zhsl?Ny_1S;-n_%&weYN|6$M}w_}wWiUFlJ*5L2S-qmRZQnI@sE@MTh z-==d(&bHbi)H$==?=9s<6P~}*vgjRSU#5LCr4}>yN&HELfMI22A^#(fM72~Q)|n$z znS%@NRg71XTR!l5i*j}Ov<;kIAoKX)JLKsVHcXcC{1WjVoP#pXyFAKY3L-yb3qAnO z>Ek5}jK6_jy#1|yjMfq>ZD231?9|7it|s%&;6$_Bl+s3Ver<-f57*~?b;nOzN`8N4 za*&rbu7&GkG>vTeOO31R$`k^LFWPd~-qD2-r%xGXanpal$^g@ zN!&B>lwKl+0d;!3OQ+w9;u>ghYLZ-9U3Uj=wcjA1CBllUQV$$-+e~ZBG2Re+6n5Wd z^0ub(J@xx{#rvW>c0zA6 zcC(X*|G)KGNLkkQ5Q<=Qno>Jer3=ySo|RV}@47jU?y}P0J2n#7qMSgXTBlD@L}9ud zrmQ4gf0&tWjBNQEynyoUu^hr%ow|-B>cM{BrMg~i7 zT#k9AhOB0`@VA&JIl|QaQ94x}%aXSJ3<~_dWi#*e>(8hAo~nI}z;=^vFi$c>`UHP2 zYjzhblp88&cF;-AJeTjtZaYRisD@AKUFcLKFg;oQK>fW@$MhH1+Swl~hn+UGl=t8u z^uA{o2hn`I|II;+is-GAD1w^dv+E5qej*pg2lM-Nr#6Ql-klR8)YxC{S5Rd=V)^yW zpe4h5;;-)&i%fiOeTPX@naGD+qp?8;dMDG{qQkpm)>aDF-P3hWo3gV!cDM4a@Gzi#EW`a&BSszl0CCK|G_5tW^my+GEInsEN4o3i4eM(m zWIHT|fr}&BJrd2{d-t)hxH{^Ex=xJqi)M+Xh*>(IZqQ$~6g;_bGLoS;TJ#lX^X-a5 zw@7C69j>3?Kh&q-NT^`O5I7Da3lKf9PxIDvK)9g83}@2qj>3B);~t|Q(oAAi#7~+^ zE_hkZjai%AjQQ9>AX<^tDH-MwX2Nv28+Ung;abhc2H8?#y6GNE3B9Y^wUj7+^nWj< ze1xpy0ThAwn!q8c*JMtN*($nxYp=3ycQD#zy}GL9Uv6kv+R_%58#ycWOkb=2RcX0A zB0?oALS5;z+mbJ>P^`PEL&U5EtIXr_)e;^>2_c95mP=)dQgjup?9~?K7y(!Olf$(% zs}jlRR-7H|Zq_rBb9d~Xn)U`k-kv(ObVq#!{wU z>0-y0J_NH|GtZ#xdp<&!EYcNP_EPaB@A3LUpTg-c-|_hiWPA7E1oWD67bmzee<9S6 zyWftuzKHCbFMWlU=okC4X~ljBJ#zwXq)Ne_&5D{&l?`PkjbZ!}No6hO8f!DooNO!#}G=wEBl%rhGw!@9|h){%cb9Q?x zQvb**#}x1D!%p49_?mp4IP8ybw$OW&119_5>*_y^#k=DXBD@MKoAE51Pv^}cIGUrX zgV0!GkKTjkJ+>_9zgc`C72p$25RdkX7u+UQ9H zy<=jooVv8@#=u!lR?~JnmZbXQF9($w9P>$D3O3L$S2qz;8r|sS(=LC8cfWJlDR7Zj zCp|IPvhu0%SbX~7)Z_bjDvDmS8nCut=B`EN8HMx^?yHeu50htqH=1CG$IclHUirif_d|_LPyf~*VzT~rPi{%ZDfOco1zV;1Lr2b?Eg-hbi02Ye z=S#G%EN)HCepsogc%)&1wgT_WC8;PD00q# z``HOk!)yiK`?ln~+DxmzpHfs7vyOLz{tel+W18M}`W`EW-_7xrmoAP?hMfo#R<3+< zlrjE2&A!6zy zPjBqV_!^VLi7@Rw7Hf~a5$|6Fl4uN438_?!yo5yas3Uois>q5mwTq~?+&mQr(N0w& z{_u8sE|#ORr-^bHPbM7Dh;5E(j5)lIqnLF5Pa@EJ><&=;e`78pL~peq2@Wp$98zrP z)Bh@Tx$twu_f2Oyo;zJ9W@V0K$?Z>{Y}e?CjyZGQNq_lkJ+s^bs+&=>TKa?ARr$4I z3%!F^B+3u*w_Z5p8B!SC7j0Gf?Di4YxYuvo1@7Z@vb~K>rBEoQE;yLYGSiZvG>;q3 zi_j`wSi-Y;^u3*H--gKLT6X?ci%!|Jsmd549r6=(q~5W7Wh-R}sQ4Zn7m#JL7 zPB7UvmI&CS$SSJn?RqGTgS5$+*B1~Mcnmo9Iu!jvYb!dW3GEd z<0$UG$nVI9%aR^AK`geG#4H8(bxctnlJMM|KZ!IPd3}l=-)JoLfCOV8u`L6aHP77V zSDGq5tTMSbzWA@*j&7$e2+VyiYJBJ9N%)kLI(-I>w7}cccQP}{Vz0ZhJB@_&U(i9Mg0^tUm9Y~ z9^~v1QF8PD#a>$?veUU0PAtBrmPQd!rxfnwSbFixE7OzTJ{(HMqs6g~l;4m(kir?I z9aA^8RP_Y;BQQV8J;F1RljVM+Mo0C7ujMEQZ$jxZ{jsjDyM}4EF7_>4T3T@+wOw&P zBi=)V!JmO0uzY^TO5 zy^{hknepS%%mdN>s==k7lw#jME#1IKn60opIwzrPR1$n%_l?u=xWK#v+vTD&H_}DP zhM7gxdz54B@7j3huO8<4l6-_Hl%YJZWxDns{VFtz^SHR_TbODwAQsBK-%7qRB=}JH4A@UHK60Pl6zsf8 zUwEKmX(Wfw;buSeA%P#!Eb{uLg+zhDz3d|+g(p1G-Dh<|{b(&q@)o*3wl2ljhNG

TnT!wpbsQ|{xeyN zQR=j51!*=<80XTy_g0?C=O`Z;Z(2#S9@>B7wnVU+d=xwXyL(&-Y_nW?48rQ?-ZC-< zHMmYnU&yA9u{!8sDa9OndR&u}Oz$c^bJ7wzk)r(Lg|Hk7nZroeV%_G`B*t1oX>Gc< zo>@|LTdO}aBO}K;`71oJu%h&1#8^xaI)=UKX^CG$c1mi!K3m5kF?WGzlP{>K)ezHv zcnNtKiNUg!iwjTf7rS!0t;PZGYI`OMNYN)zvjeXCl5$~Fo@6}mWS z`8z+t8!w*sEm2n`+CyIU2t~b+5FnS!Qniw(r#iTE|aPedKe@>f}x9x#`b0FK}z<9lMKr`C5mL zX%E`=2;8`_f3bk-k+M*dPJB)H5JF$%Tf%_5%E8MSYp+}a9s4N>8q7lINZYCmpB%a3 z_~LXs=H%ErS+e8oysC9r_a`;dvaMy}-K`f)a?g?0s5o9Qm5z5i+4;tvet(ln(Ft(ZboQYaHY`5P-@Q9rwDe(!@+gB`+`Y)2 z<^5ysg5OFP4qnt!9Z9qFo}SwW_cyaluFfuIHhv|}WL91x0kVJV{c9hF%zW7!auXgc zt_OWPfP*oWh0{L6U#Qe=Tfrw#w{qTtt$T!6DDJ-R;&_wn}w zRpzJn$l>zr5vBc=c*rYlZ|bI@9In;Max!}`t}r5jr$AnXU30@9=v_HjxDpLhwg(_?a1w<0Smt~*ox!reuf>& z31!A84MV|LW)@?)I)a$O>4TdW+p2%`xG?A?&OUIvkaqD5zpFCl8FOp}&1A(q9AC(C zu`gu0T2PW2V-P~D;q{&YC5U@T@M^X0YJ|LFJ928E>xCiZ2_vg7k6KbXk3JFap0`Y3 z9?1F_WplhdUrKm_O8TR)?>;1l=fqO40PPahueeXX7C^uu8NSoElb zIAc>g^6YX+V3_WpC!+K#9L_) zKk%ns@az%%p?Ury&=msa1|$i@*7&b;A+N56%&*8PhcXR~0-XnVFR|AT_3o{wUF8IA4bW6nh58U;r)R5K$x zJtBL?d7B+cN1YqQbQIKV2&wCHpYo*0#u!hLy4eKEv0a@GLv&j?$U5R&hNjCfy^D)P zwzOjRhLnnvctl%EFPqy%l^X76ARnshJ;zU>gU{Ng7_ETi=%9GawDP%}u9g;_XJr+f zF5bKrbN0tQKPDPSG@424`sgRZ@t#1wN|DfB|iQ+(W-D{ijW`rG>QDO^9T z5$TuD(=jMaHyg1W(rsF7S2`<2SoHW=Q54nnm$y?$m?)*@nB<8zGKHi&zbEadV!ovH za=%9lbI%veM*RjS%SLuW*12}`n+W=#*7qDS&Y7HQu|mV^b#+C(dPZcvANE+EJ;GWv z`Y(JHMx`^10YzL>1H%+wU9EVZ&}v8~y*V6lhgyH1!zb$5|rz} zk1{B~J#=G6g0VI@*X8TO9y(?lD$^WxOSK-Jkm#08Wv(CmZ!g!&hOAIY&AzLp9E=um zC0Y<{ouPTMVpB=@&gZeA=h!uxFrLtn=8gvxzPhMS8$BfmYVqJY!ntBE?wRIsswMly zzJ)2Hs%lb|mqqI$I!;Yh4RPGGJ-KlsW_b3;Bko=euIbbp(DGoRZBtT|ivE1rNALQ9 zgIEDc`VsPMBJ|flQVBnIQw1_$s)Vp;@5CGZn8$C_e>F8M4AHZ)X*V6x5H(8 z3vWt9IGG5aARdHA6I!w>EpTpLPd7imxsTDudym{2&mQ4DitoRpEHrPK67p&@rC1Tq zK8C3SDtj#WUa>6pFBDQfqK4!Eidf83vGEXT@k!t5D)>c*=k+ABVv}kfSseZ0Eo%(_dD}hT`dH2HwE8?3&~BqqxMJn)XRq^y|<8*h}Ie2exT* zgYq1L4(A*t#`1A)W|OSE=xg<(;%==OA4sMg`xTf2Q^$$Y$(|q1=4&FG;7juH9&51D zUDYlTe-=uW#lbdaE|O3~rYB*QR3D`?8)?T~bJS0P@;;(XTK|JAg z)zgHM1Dq)uS9)w-t}-%9I;CE z!1kHPZ!X+gc~ZMBdf`RF;E(>SX+z4$x|uuQSWi?dd14XXp8}TOkYq9*)|x(0 zz>sR(&RClLX3&S+cv+EN>bQkcz*o8Xrr{S(J z_5u z=%Tdpr-+SB=hJ3={3lOxBrv}#YZ2%A(R8eX3PGDrMDCZO?DAynq{h*N>vs+tSV@Zp z50}}KP>Ed@{Z2lRiGSbVV^SsVexqvcGgm`4ZoZ_WPQo-UXN;5avg-;7S>_@ysrKO3S})xSk%72QVyS{DkA5C~JE9%S+1rZ8MY=|!K{x8L zRN^@zBWcXzthJzj%ImW~?Nx@V?*Zm-o4UxNx|g(LKSdh39zmQ;kqotrBLBhomM-Pv zk3-~DD)M`9YL6Hm#rIb_{4=A{M4cn18=>uoMrq=%M)aDYEb+6TMe%2{<^9Uj zU5{Qp+mBD*?JggkpP)J*6N48nCjhBcXieK2aSO zBh^EyaH-+G0l5>c5kx&b*O8*j{Mt{5BaBHe=#XkaSr_qm(v3d{#_SOt^Z$iemcJzk zHg;Q?ENT(U0q{ah^hVlCUsiQ3{gPzs=>z!G<2(n8h%LMba_0?R9yD9ID=0B>y?-`i z$TvDIRbk+nX2!t;0}srndr3OjC~k?8p{Nccsk*`0^cB1)yc27CP`X#ZiTaCxQ$C!Z zv{{5e!c(DmQGtF~a3h_4j}yqzTHyoIl5N?l+)q1LzFlqhd^uT2b$Ar4#UHwdd!kBHwZ_8uk-L`<{4b)_=^mEO@D0dYUpdvdilgrp~v_ z1MyWNe{QIVy-{4hz)52Nin);^@llEW-nNI(6{1H23LmB#;f0Lk_avvYA8>bxU1ze`R)?(1~|c?zGQY9Jjx?$?eA3LzmKlVr;%(r z(7(~)IU$Nk;(QS;!@R*4qUI^klbz0pxF1Q$V)&3vfpQdg%q-mi{9C&mh?c9y( zdo?Or4Haow%KJ+PYjnH|t3(BLZ{g*ia=FA3J{TJR*|JHaX6XgbJ5SXcG_r@UC_Pv# zA-ggVmAN_mTIRg(6Didne{u))YuL4YrgZC|DB^;FlgUm^Q%d5J+*xrGOM^?!swR#W zrck`+=!6q9H*f^oM}(ZgPMsTrfgzwq;wDZ;ju!SfJ4ZP9)NShwr>e6dZtKV@Gzj3n zf`JV<_<7-vzPCHLuzlK-6ALmxoKQf-jY9ImQ7G`8rVbb#3?XlTbF|nx5z2)`LPiDr z_fHKv7z&6}0tZ#WSj1N>2f^MU?U;QYW+INKVVI64{GIRZ>GM+2yV0UTinM;O5o#&87G7kB_in8Ohk z;1nxRBP$rf8ji4mBW%IIfEYN!0geEpIf2nZe{h5g9N`KMvx1zzJ0K8zzX}eoA~Ym=%A1Rd+;=LX;8KqEjm zoZKK)3(hUuU)*L85{A(HSp|oN!Et5_BT-v3YZE{?LKSCXa|x=*8@O%F2uuj<-2Xk` zyfr^)I#=MlNaX*E0{g%12MzW?JVv|9>m6pU=@~ENF+_>I3QrDj3ifg#;aAxw+w3GzO04#Q@QR>VPTyJa8;O zKb#NC4G6%1@19X`EKn(4ULMehn+H4q$RJnF3$|O(6^lW`dBKOyyu4r(kQ0GA1&Dlb zEckYs7fg>ANHUlTY>N~yr7aS`=+M+K;MFNF$kt&XJm?Bu{-A*tLv>!TeMEx+&_IoO zfj&cZexOzW4%iO*-oin`d9YwgU}+(CLTxb6Y`{x=44@dQV?Y8EVh1!nbcM#>S~iG< zNI)0FS}-OCT({V=Mc593TM`6Kj|TuyD5wiifu{6x<+qxHrWh0o1`3!0=n6uHIQ(yA zi$E{|KK`wiTSS3L|CEED>$P*;V%ASOwtC_LZ07|Wg{X%F;h*jyEC>iK6jY#lULZ+Z zQ%3Q@p!=<75L-b9{Ji}C6i5=bga8tRT?GsPRvz#Vk_u2@J6A|5ek**y?)(%ANFaVI zfUvE5AQ+Hv>=f9RT!73WFE3ymkb%7l|BhThGVyEKk_kvIw#$wz@Bp@g_1{&t*Lqtn zAldj?Ai3CwTFYn*{*H7EcDLg z9DuxRKX8O^rF_AlPVlW9C?Ev#SYRc$8t!<#odI!ho}B@=9Ubt7TUe0q0~Tbf?hH9b zP=o#I+rX}8>!Aw}-tC$zP^ayhJJ8?lng=jkJ2l9!Vzx*9*SbRwfBGs(6nraI`PG?0 z_&=YEqJH)3|NNi@LQE7OA&kI6Lv8~h%Kbm1ae?&WzliAn5fOOWf6;jUM>HfC@ZSF- zV*VFI5Lp03Bo`l`@fYv*OKkQpQ+^TYe}T|i>~P{=AjIXL?Adnp+rJWmkhL&|qMWUt z<4}PDDlt1};F`BMv~&0c^UK%>G4M+mP%L*E1YP#lb~tMbL%0i?3(LjB0XN6t?41M< z2%CQ%aoIVVvBMA`_%e1j0za(rYb$$WQ@Ejlkrfzarw4$!)xCwCtvK`xB&Wp%xPenb zAwjSQozUSui{w6qM4n>*g)eT7CZ;f8ig{tcBJcbG3yA^RVG92T2E~2QAKZ2ah5}v> z$lxv*3X;a%Fz{A%H;e}f#C#VX52VYxVUWcC1_L+1IPHd^A)WsnkB0}ddkiF!`;T#v zJUo!b|JD`-?BDKjQ5b-@8;^${QrzF$@?yaz_IDT<132N`Ff3$sert=wfbXyVfPuz; zz_1_)-qn^D2^__47|)-4;e{fSU3ge-=pYyL_s{ww!TSC|GZu1YyYcwJ{&W|NkLM40 zLGocBI|lvzI|d&g1~AxFC#5BmrIfS-nZ$8Y_ikdSlS4dVe0 zcQ=d=@@T)e1AfLDk25sMV!O$oWP3?y9 z@a^U)h|8dN4ZrsXK26`vKgdS>3FH2g79I=;BzE=3!wbGP+yw)o_6rQ>XkcM&;<)vT zOR5$gCcx%^@WIXwGL~BoPTJNKI4(fJ_Abi_XW)q2GEd+wDjy$=l~qDf683)pAng{6