From 1944d6519054956b85c626ec28903c4e919258ac Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Wed, 25 Mar 2026 13:14:15 +0200 Subject: [PATCH 01/11] Changed project name to smlptech --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b5ef6b6b..b0f803ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires = [ build-backend = "setuptools.build_meta" [project] -name = "smlp" +name = "smlptech" version = "1.0.1" description = "SMLP - The Symbolic Machine Learning Prover" requires-python = "==3.11.*" From de5828ed34adf2f92cc861cf642091686488b581 Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Wed, 25 Mar 2026 13:49:37 +0200 Subject: [PATCH 02/11] Changed pip package name to smlptech --- ...nux_2_24_x86_64.manylinux_2_28_x86_64.whl} | Bin 12763762 -> 12764488 bytes scripts/docker/Makefile | 2 +- scripts/docker/repair_wheel.py | 2 +- scripts/venv/run_dora | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename scripts/dist/{smlp-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl => smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl} (99%) diff --git a/scripts/dist/smlp-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl b/scripts/dist/smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl similarity index 99% rename from scripts/dist/smlp-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl rename to scripts/dist/smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl index 508afec3e295ae436187e990a8fd87248a8f87c0..2a1c06f05db4848ef0f829abc60355a863981ce5 100644 GIT binary patch delta 96950 zcmYhh18}BI6E?cBlZ~_C#~ z&WX{~(b=5w|DOo^zlnOUNosEa{}G6ZzJ0^|zj=Sf)8GmI$FS~%s0ylq{svZr^Nsoc zW~9?M^;0$AA<%)rTx4qrflO}3j3qKU%Qk>GAw-S~vN5NK(warH)Q19&5MjTs_Nub{yk@s4>OX1U_9Qz0 zyIouB8C`Y$NLzir|8;h9YTAA*(_6KjF5~7Qzg1_@<^6b;TDe@ze4nJu`*OJY<`7qT zPSCEF`s{?|WAhi%Z9S3(Km??(`>V#6gYOy_z^12jPW`XHT zFm@*4R-0V4$%2;6zIk}T!dA?l!6rMCla*#H)zc^7ML6-2gnba5L-6{UD<-IB{ZO+ntG7Tz3SYgUP;QRNp{Iu%B@eh7VWXw-vdi`HUDjSR82)T{ zDpF!V5qHYuS?)qSQx|YGbv4pm2o$lO#P-{%ZSp7BnyOurF^gi>w=Nvjsc!2+6SMiD zVJJ9sT03|ZE!=JrtrqZiZy!m%!Y;ey`>^YQ3S$_x#dUpRjev9947}?ZK`CR4vPYYi zDbMl&bDn~X#N7?)(*16=<{0@tc^j!x8@$rmmy$X8^O0F=WLaT`MYGQRPEgfa=dSvNNZgpu`;baBDGUF`Y? zx*;^wopos0y$UI*^QXZiY7&vt z`x9~bo^UQ8^AE$(w-#oeTU6$R2TGsYZcX88wQX6ssf;6k-7Tje@2DKim9b8 z)eYqbZYSKklG>WQFX%~tM}x-#0HVaG}TMl+dmntX-?nG z&5hElSyUy!EKPOQ8;)=+@4(513Ok^}M=<3=wXhKeo8rvsvr+vj#YfRtz`$udw&YW| zL?0@09SXSLu03dPXsycijL4jJIO&J#l~ukTct_i*A_#bQg}%7aN9fy;Yc7~E*k_`j zOQaR+KxJG6c9qw`&{(xtH~Axt=V+^fzZa^?-Td2zxRU92X{Ri^Qo^tzo~ zGZnx;rZ%P@>?EjBcO%=5i@_MgR5vZH&)eB=T#wFDr9Yxc{cu;TEqd^AX21J3G%ZC` zHr_qPFU_fji?~@NfNnO$($h{{CZ$A5HA>vSoltgEMD12B9Xd0SU#VHqmE_v zq2F+8?VXzKZJRxoWB-E6PD+DvP_t;q&O&<+YgU1WR#J#mT1c%!ptlzO*^UutE(qu| zr)v%xl<`N_nNQ03An!b3p@)3`Ks?thO3m93+2T$kK4FvVIis)VAGhz3giC2IK~4Rk z&j?QteT8i=D(6i5x{`A2uZ#os@9wmCGQED%!*mG4o8CLSg*>BX;cz_7i;)1?~ z<({*l&eWQEVnwCRV$xhxN@C5P*%)B+dz~EF{0qIJxVLPm7*We|_=8sG50u0E5o!rZ zW_0SH_}W~>H|TP9(q=Mj$+utGxIR0J4kRi!?D)r18awps7Y@`b#{bg8DovX5TG`ar zsNFE@RkN;3q0QY2yCSz2Q(E+e$1d$yM}_50)F4?UL>ia(CMEn^@vwBbF}#3g2*m(g z?93=6iuxxk_^wQLQHnWk0i3)`*MTb4!Ls{}PT~((@mS-<+Cte$nagES^jR?UDpBzq z?4`6zN_PXb39R$Dr2}!(Pl#!#tTW~vpQb;D#fU?6A7TXERV}lmiD@X<`5ag(Gw)bF zK2qfCvvTuqqUE6ko+2=${5*iJCHbXz_VwFQ_sZPE@KHZGz}@`jT93Fu|0}pu^-7j^ z3O>GWujxKDg!PUFG4yFNcv?UttZ*LU?U5oY5DWh|OU-@3c7|XS#me_<#Ydc(OOiF- zAEWfvo%_R*L8m3IQdHWn%-O#YP^RPlrb~{(Tj^rQOD+~!Hak~v!jl3RzTDF9$cJ!S zZkosw33oy=F(DL@#CZOq1DhJiZlM{2>B(dv+tAfMV7WDX)MD=^;SbcPiS~!jHsQ{8 zDUJB0N~rz&5z&pQ2n4=dr{TgYs4|s~306#_=-x6ZGSaw%7ktv9UJ|GI)jy^6gchm4 z3o;x4WI&RpP?Fu@=pcZGD3GnQ;yfr58JI#R)$AWg<22BVRD6y7u5%r)C-&aehxCmz zXT=Dq9#!u1o0H}M+6>RBjQ5N15-sTat}E1tA6Cp zU(bN)9nV++!1TqG%|AyaPZ`4PlO`P(Xq0{@#BDB8$xjR90f6d)&S0eLBx3&`v1QLQ zmvvtsQo%qWGvJj!4I*Mv!|oYQ_!KB`9fjqp8srCA8`HrO#pM^Z)g$G zGyfgp=yUarS{05Nu8lb1v_fPj7)Ca^bZu1Af7M zJo~$UL?3_;Y}RHy-!tMC+{YfW4+Dv5hLHf>e>=gOc)XoJJoU4S=y|9><>?tvGB*9p z`>_u^T>Q71x!TpUF5)d`H^0bnyday$OUry?%CkAC`_QM+86HeN^JhL78PKuUfd#Q^eoFjxXXqJ`ho3Zk6w^ife*%n%`*9fX+S8@GI%VyM*W$O6l!?jn;kP z*cKnfaRc^pe6b)*WjGtun=29r;uzFCu|$fk`nRJZDWbES_{>R#o*@2-h{I$wyAyB% z{HwDI;gGk@GQ}gZdFE)8d?Uh&`#M{g80lo=d^wt8Kk*O^bo%i6R$v-&ARQ~vVrsqC z_vLYq?t`4sX4iu%Iv$W4ySb*p+G^*NNzeA2l-QgF_82@((E2t?>;e-VY%A7rxmY%C zcf1DE-PL%sUJ?&(zWK{@ONJq+OD_F>8V80Z?st@riW-XKhU7Bzf@y&o^%AKL}A!q za*D7&&8Sagr&TcEaK0H8wOa^B5tz{6G~IpHA5JBA|0VGK8#q2rJO1-k$ff!Q(APW| zCu-mGqdnN(DAvBxLuzKAr+G;YNx#W?l$kP~uff8scv9o@HdykaXOBi#dn&Uzt3=f9 z`of&oHnhfjZ8uR~Ha%@E4cW`>rT+|hP~{V~(;V4nGLjT}pcmVx+LS z1L}iT*M0ZoG`D{ILjji$c+Y?UpyTj6zZ`lhe)wseAJX?PhV@yh)9G_qyeA}sQ_>T# zw_mgg#}UI^zkqVE;tAVa&5$N@Of3|(+U#zwU^p(=?TjWKK90C}W}4VXn<{#P?-lAl z);aul{hQm9bL)Pvlv^;4+K%?Fmq5(vEpV7IEmkW~*{;5X)kAMcakF^>vtzR18#cq7 z&yvl;B|N^Ymdkuvf9*i#91f#dVSb!Hhsn9o(x93H&FKr^+(zmuUk)5C6*^6vxyQ87Mt!ZiM*9yfU>otcMIvtzzMYQ2c{2z-5l5@gaUE3Qugx0 zT~Rv$5Fo_X<6eHngHEB3^j-jJ^E``hC*ZiH*B%nY(dyCLB|ESK@#XjW4)#nG_=dAd zDG>&}tNMDmv-k!6{sjSIZ*@Frnc8&(1JzJ`0(&n&7N{nr4ZwqRU}`0xPX4#GGXM6N0`XscNB<;m{?xsZAkiF0T``|@1)#A7J!ujAQ zI0d@B@eMfo5PJ26eGU)GhIFW>*i^_P*yZhO>uu{(53E{#zhauY$M6)u-%Q-KIRWw9 z;GHytzpD1-0T$&|*xPu0Q{XFd0~WzM@Y9kz8|8v9?`^wBb0DZ)#^RZ1wo^ig8Gk46 z4=m`O90892#`)u+T}i~l$Ua^ZGxXggh@}!w+;_qO$MoK}U%LwVJ3twnB%8?(KSSit zv4#3*Nfe+ zC+SsF^^0-W5WxYd6_4$gL&&l2$s5m&BUB6FCc*9%qz86uPf!QmCjaggd*hH_7U*s7 zMHv0}_O~5EXb^1c2-}`w|GnzNhxN@D^wkfinIIqJ&HG)0!aYXSFZCNk2oO&zC%g3r zdhQN5(1Nb z*wT9suwQ=#AJ^AM_D?yZY&b-kTbH(97MY9#jRcDSto!Kz}-(-tR36g1_TBx$U0yu?J+sJK%CL zeoP^raRv!S3vdPs;{8U^!)x{0JzDq^x@ai&VcL6q>hj$;P$0a|ab?k(na?9AIw<+X z4A zFQ*2uGuR6t+1{QQf&UK7#z`~q4WZSxPv7z+p8dujqP1z)0o;SNRkybW!UL~$qpzmy z*VVVLy4|aiYf#|Vx7P4pe()>i*AKXxJa7=39{~M+-1kAS>F}24VznTiN3_=UPvW{B z@AzUVQ;%!NukQYtm(Uv3u+3^^fT0sOT3(~fadCM8V|C2&f0fqkxp(xx1z{WTJ$KM{ z3!WuqZxebbiLagA20!tho`*9M3ce=t#v$=PKVjl=h;8k#p}ZY+oQ^R!y%OH($PI0OPgFefF1K*U?XZ6X(+uQ{0hi-n(7UIm*|u`d?au^`YZ-UzZ8> zzQ;}5b&m;1@xeI3vHV=gu_9+v*w(s+)e z&&HRD?1FB0tR+`ArJlzIF#<@>d)w~x;AmB#?Bj4JTl1G=_|B^Jb*~r5*XcCxDo*fg z*Vu4n@^zv$cMm3Br(3$*Emn}&|8ewcb-XKX2lSZg_?z>A)WgZU_yG9$xRSoW#dCSc zzqK~5Yw>t@I{SUixaHzE3cMvXyGdTR@3_r`8@xPH1NH*TTP~9;mI7{*r65a3E|UiV zN%n4!$HA4dtGm}_r0i{vr4oe~*Rg);^q5n$xb7)|6_2$;!{7W*J-l0gD;x&p&%1TJ zCRtbH^gciD=lGke34z;3;+d_M(}}4ntTBZ;y)L)?wiP4@iBP)7qNTxhE+$1k*Vpw2vpcg`Z-NJNXHAhE@r4I zeA<0z(lWliv7>`LtIIAQ@KAW{CpyrHeuHi*OIEi02dEQIz-kS5^gs4DdE@z?zr76i zY_(Ir4{KiaW=RtAdc2N##PeN`mQpTXdEa%$9nA0_?G;^@`yXPofL_TXM|E$f5{m?W zI6J;|d*J_d1RmS6&w{E(;NJZ2?xbm_?R%=dHz6E<|1EGBIrnc^W<5MSs(Z}Rk9ZaS{2wm(NqkK=WJ{C=5sqg{b<*bLWg zd|7Z&*u2gkhMMq+|9!XEF3$OE`0TtcNOAF3+JOsr0CXpaS}ja=XUY`unf;%qT(0Z< zZ=;_oA)xu6qnu=41jjwRUpG$~`Mh5b71tX?Bt(Jem=d&#D>h>xx zy{mI&LEj~p$M2&E@LYa!ANhfBrQ@=8f&SI%URNr6fv4Ev`Jns7=i3wh+p*?RuVsLp zlFs+Nwzq5oL@O6uhxM?Y_Vb7lUYB(|+kZlUso}W> zMi;+^40rDGgcWPwt>()8h^j7hbU0M-eJ?XTOwU=UW5eflP1Ph>o7@cE_oW@dwx}K7 zqwJPQB8OJXlPpxt`|I4hyW^HxL7NMG#vEJ&l6UL*$?LLbciq*e-xD5%pbOI{!ahf* z=FiibA3JXT9H4n?(rx`_tN!sFr1s10fhRu8M_Fveo%Tg`dl1q!Um7mE^$k$O`03u; zEh}i9{pzBYTisyROfKzFKn_?#v$-I{dmd%xv%MIQmK+k!Yn zkl@Eg9b-GK?`L56mHF3x@+?A570B;tyl=+ut`&8q)algMW}Wdbf|~T8@`AoZ4Y=USHJ2CVJ^}i+^`qQrN-& z=x?dZ!F>{3dwT9+XuF5<+K$}LCB~tnP{D2A z{?G=E8Nq{#lLSe_PO}GoR)e>vHkj^?w^c6~!gtSuYE6$UzY}HFD%VQc z1usaH9Nrfw@m+d6rb8dsolc|7+grLv227?0g%!aC@(yj3O;1}uj_b<)m)s71>*4HcFuo`RrLzmXuMuN2^1b~=nQ3*n zOh^e)~TD1AiOV$>tiyH&Z*(!JufF*@ozNM=vs(Ut zS2O&QzE7_c9tO^Djx# z*#`bzdjlYVulacY`dVMpWgM(l`pesDGbZ}e^RXu>)6rvdhbQiXvuSZU^)e` zd)`Ko;)^*IeQZZnVP2LQ@%`=r=+2v?Ly+KY{2XvNV(9+;>&kY@cABgD>TX2mIsDUS zgO<^&WGb>}5^EmC<5IwD-!c1|*QD3{{tD>2N*R)b$@4XS{R$&&vwHbF?&kVrHa@|m zyli(5x2}2fb(H`G)9?4y8G?G#P#93y*FzdFquBHC7xdISK=RkChHW>e2!w48Iu|`v z4#1ba_tWZeH(`hOdqP!5ZQH}G58;OES#Y!i5p((Z9etCn0k7jZ@reH_>lG17y!Y$L zYrLa(jqPV^4}seyGku@5$2I!KWlfLR(*t?Duch$NX?(H6m+LeSsBnB?=B&H+QG0~+ zAgSo8qi*kD=`y^|8Q-hqS5u+foVQ7vN=%8r z&-$j{#zS|c|A~iI0(L23IiUZSY3L0AgZR&QH!;>c%X{R$g_#9?AYF&!ZD%80ME{t$$1`x;4lB4bNYtV=s$4sh69|T6n;w1B)vn=gVY)JzoNRY z#Xk7U1^pIZmMACcUEVA-|eoAfhq^0qdP(#(G2 z>%&HPK)#lj;qlg47!DHBuQ?6n&Cz@78mVuwt?Tal4(+SAI5oB^Wemngpy%o zRwePgOA`-L{v%stCh2W48eEvqU|>;!IF<$u!em|dWT-?OfbNc#ysnDgLRCU?ncP)do04$x$OZL+YLP%`r&KBpru}rXT#Tt?c}A{u90)XA z1bAObKJW)s-qzsRIcdCN>=+8VstNV5Ii(;mvlCmr(X_^b7e9qUsUlW<^bYF){G1g>%~X`;HC6H7&1ZfzW%RNll5Zd14~Tb+BtO-- ze56JhETTnd0^6;qGQZi2&zIcG)r9a(9Mxc%R8UFu+93Jub8XK4<3y#e3EQTqlO3&+ zigYZ&ksTm|6Xjn+W@Du|mvYf4PvfzKN|N#{g?lB>EX3cXHex1%P6{~&Mkq2${G3@Z zQ(R(v!Xb#gAmIq+;}0H1A?AX$Jk%X$|NNN8em?J>&hD~V5_3XYTj%7>Y>?=}ybot7i z9FdAt>t9LH(u*Rf;4m+eeCh3Gt9{zjU`&Twmal7#hPMjw{88!!erJ$1kA!?DG)Wl~ z-8oGNE_FX2kCOP6rPf~ynP7sxOtugwOZ6)jUOC7!4faaLf|!$}`4T+6gX;dY&!|PsaR{$Y?Ojnp5@`+%l>_za;=)?V3I~@m z5>zxBS)c}bPN^xn7yi(b8~7vCG%?3_WrY7-c!jhkv@u@$%~?+hmNXYuxm?e5oIcEG zF|r;SVnOH1SNb-DEEtdbI8G#K%*o(P%!t{G_^INCJKiNAew}TV7?Kw$8hvVfTxuxg zLPTm12&;Dyjx0)|tYN^U77TxmIQm&?O-%1&+3qZ2?k0+h@glD%f|wxlW$T1JgziIc3Q^GK9eO4v?@n#dEsYBOUhM%jrC!QP`om6S2s`ourub7b^s&g zwSS#RE#BVNO7Fteq^?DlXHV){AGWGOc;xh z*4hIkC`lf-xfD|;l}BB!6wBDdLshX+KvAn#ZB%$gfa0s^C3w8ZPeJ@|q!?+0{`t0K zOd+DFKa=}plU%AWRp5D9d%ZAVcKhogkc4Z7-|+s_Cd04kQE}j2o_~^BsJBO!$cVg$ z;#^H-`w$DxR2EU)UXFRvq~RqZe0DMWO@pDD+-ix%6bMjL#f$8l9h!!tQj{8V1DRNN zh+`_p(kxS?f2HX3j0(7nz?B>!Y9$DeC*yjtGg%VA7Gshq^RFy^sly4PfrE8}Xm0-$ zhaNzz=l%L;*=rUZ5zE*ukkxo$WdhAWpA_dQ_mof%br_<~Qq4BEKCC%*pe|3Yv(zgh zMEXeQ?3Zl2m!G&=^fv@u0xAmQI}q#w&mZZjLh+Z3h9_UQ!C}0GHdr593+!Y2Y|L(E zTqHh2mad}qgXGrJk|xok$p$UeCAqaT%A8`X(My~LSB&>K%Ipt)>bJX9GS`Ja8uegu zK}aU=F=RtJG1tiiOIbUnf4R`qEdHH?MYk&HiPul7Q;HE&xkd-HDrraB=>v@quK|s! z>|j+T&?dr2o17-gtP>gD02xav1FSrHu2{B6cv+MhLjn7lnfKaAqhq@JL8+Z_kX&5-yd(M4y+&948CpvgRx)Dd+}~ zK_WIDdRBW;jUTBht|SsPrr4Ep%* zdih+xNQjx6@iA1RUqj!yf)`V*-xv6#kKWZO=#xvKz&X;Jt0CdOWt%R0$H|>@r!2de z>7XpjNgauTO4;;d#KYF@t?p7)d$O>{hE`0y>`OzV=uyX3y8!uy-A3(yHvUwIsKJr- z5$fp4eCw!#ftma{e9}%Y$wIim2S*BhD=3X-dt*}ZaT+q0FUA6;v5IP>Pxc!MWB4h? zV&0I@y~da31T`64^04Y0t_uoW4$2vT?@ zTZQk)GnwlTK_EHL5~%xb-gn{hGneV@H_OL`$b@^tHR`>YUSh_q!^_(RPvV1o_kpY2 z`Zsl%`!+?h9ZK0G&g|}@q@=QxJF)c)Uor9ovjLnF{UE5BG`0%Zr=KymFLbe22sz1fCC}l8Dh>tkC zx^)OY`NTl;qJC`rRSmMD;2-@5w)2WYvE4s9{m+C|F>?GYY%Pe8NHW+u3%>mSgk7uB zip(q?+Yfp#Ml{64o-{cOV#$-Lyok`G#YmF;p$IWgmr3&L)Rhreu1!cQX@oxc@16$z zOBCwa>I4+-L|G$BNj5kjs4-8D5q;~U_YtSyT5HY7u2A5JuVbLf&l5I{Ikw2jV0ZPH zI+h^_=GPJNmg9n)mt4Go0IC*=ne^LS^U3*Q~{gTT}Aas+HTzTu^KV?FgsW zeE@)M&PFn>8WPD{IBa~|g#Wl55!ndoRbO~G82eP{hBQWYeW$BlLa-VjB|w*#RZ9v& zV@P%?|7aed(4-cjzg`qMH^@`>z$7X=ux~i!dy#7ytVHthSPJXn#c6C%LNqeyR5kvW z95sKK*gk|Fd%#~D;4Uxx_8r%JjN?T7lHtEHiXZ*OmAaLU41ao^lFro6K*+?*dEW!? zekD~qB9~c!W+O?|wJ!-Zw6FgUE(q882luuH;9a--7vR;JciU-Gjx04m!=I}H(~XhQ zGm%`aO=k%9HPwC=@#?l-)w&a=TROS;ZL-lU_qYG{7qLsEPz# znSD}PV-%7*f}Il8_Y%QS<6!*-GtlK+whitFcM<0k8J( z{JUxneT4LPozcVl!YdM{eAEBoU2>-rSF2_@&C9p;Jc1L{*cL&`a z`se$)J<5LwJ<>B64pjbJysio|#2N@MW2#(Ch>FywF6`Njl+bm8FQx`=IMvuO77ijm z?=l8dHX?JQA4c|>nWd1V&ElwR;+*G${UJN_F*@evsN||z=%c=pTG^`@T=Cjxhmy4{ z%0KTd4xKqvn+h^Ff5@Rws??hycu#g~#bqZLxmDie#td~wE<45lU%A@ZZ<+JyipHzo<4MmK7k)r0 zUjnIa9C{|vFO{rIrR`j;q zFvfb&$l)%O2B$w+udts{p$GS}qA!a$joa(s2$Nql7oS|tyX?p;K4P61bFb+qk5;TT zLec1fi?E+#|s&&}XINSE|P|_UM4t_2b zxA+G$z^l?Q&jG+;xC+Fx%UUKH5mHaTRpeS`x0LK;mDCGnhptS#q-tc^Sfy5kX=IM8 zdj0w$)}1b`p)av!*IrAlc1~%DeJlu2kJ{Br(Y0(*rt=Q|dI9Y--Lt?;LchtXnk} zo+pzmHO539EocRb*0ScTnvdAB5Kt@eR(Y4VH(I-*QFj}VSD^`%t^>ruwx8?|{C+3L zgI`)7>>2)7=NDqZ!pXzEoExHI+Kol+3F%~~YaE`-?R(NDc?6MDf9jm;cF}yIRf|Fp zovx6Ic?OJ^W>Fy>EaZSIVhfykOQpihqK5==C?$#BrqJlS9AKtiQR1e;*c43y5Yj!N-=W#+&3KdZ@k9)z0Il9zg6oVsil9OWT=C5sv+q%CeJ|K_5<-P zm_m^aTcAAq@4o5X}NT;yCU$80H+#Qg#jaZ)(n z#D8eBBxo~Ok9;dpYJ7?LjOk=$H2+~Idbnv>^QS1C;AhaGUZdhflU0ihzhb6x-yfTr zh3Zu;R1l0rp}#%aZvn}LnX2%M_3b4^v6M4X7n+S?C?~8|D5aMuYe?q^Y78q*{=pPZ zpsI9OgcU)8Z^H{etHnovCNWh`ZBb3@P|ns3MtM)Ys#EXz3Y&YuGDD+qbQ~)OyRcoG zxH21k_L@)UO)}JS;!&}A$l(d<@1yjz+`-P2k89~a7I%@vQ&SVl#{Bp8`unPU&$7v- z$}Qh|2D)Y}Jt+1hd`wf5MIz zmiAM0LeruW4b_cuu}I+41-zqt{IEl_MS?{RX_#WW=-^P3`^rpF6Ck`aO$}NYPpR}m zp=qK5VKJZRbxK?hcw6f3APNrFa`0H?{e`l-@H=!O{reT>o@#U|+$==Z+?sF;L~8jQ z8C7)y}zPvkU1-y&k1-uQh z3mK{jOf`DLJ67c(8jGC?DDyXHl^ekj?a9Kg{)Vjoz`JYwd9c=J6?s|^D#J!zbZ%J4 zm1Bt0+bqLu^n}LOq*2ru|J;HLuUp*okYNW0#{d?4a1@R`ppvhvgl@66?W7;h1^eIr`)kjGG%OaeU2(ruksT(n@0nj~4K-@+QhKq8P<0#`y3s|Tc9ouxs zD#ubDYBn~>^7*UU4nZvfIxb*M%X&igM3_bX1@tO0XY-7WkrB?ym~+C`eKb_6iBHlH zubMfUEfMU6m7}UUr27j~Ii138(EOeWV@ldrOW}}-iP!^Wt+S@jWThXd2X#q>PeQD( zje`~bLW=|xE(eMx%ma!6;mbPRcQC2ZI(q+}e$;5=dOt>yxfe+ zSXT4;vEr5^&24MKSg2glg*5I~4K!MweKdb4!YV2T(6Cw}G1}cBDTLy^%cBoi(q-4Z zyt|5>aV;W~!^$~h=G)6x_RLD{KR?(Z8&toffHW6*?q;XXmStdGCT!R@Uodf!REz)~a&w~;HmlGk zsAOs_L2%CCTAN##+fb?ucb+sZb=ybGEdu|LO`*K^f7_706F+XGXsQYo*q1Uxh{wSx z^Ss0lFCx@$xk-h*+dG%EB|vM*AEMKA%{jQ-9%?o~oy3yvv8H0YhU;50^Zi&>$jE-Z z5n=6wM%Pb;ugnEscbXPAQS<`Z-6Rn$yuxLQ$_C@`BkPDjai!bhYQAJ2n>q5y;xTFO zETX#{6S@$A-o-d~klSsc3a>DUC|+`2IJ&<8^$43ITcDhl$Qu^AtE2xGWBP21P*x>Q zK715FfhmSBMUx*{)eSX&k~vrdNr=1bLIL+pWT1SE4Xv}EN@@FNDj8Ocs2(}z^WQ57 zKjIg%NggnNNZ`dBTEjyzA_kcL@z=(FTv{p{q6Oy}Aa`4Y1=6CRVKvHkzN?D`?+CH7!xnti zWDu8xnHeL^$)j;1$%6&w!i745qtT%PG{&Pk17=nyGO1~``3<_CK*w_ndtmFA+ zNQm$1zs1Vh(Mk)uH5xxeoQr#&lGJJ2VUWkZm=tGME2;Bkb$f+q?DTmk*OQBIwB=1A ze~mTbr?fsNpkh1fi6}fxh#0O9n>d+cbzc5R(B7}5Ln6H#S-9~M^WL13DyaOBY&GoBn{8fb;j zF|McMw5Np79IA5v@IAM!9eZT0bew~pyx>7rA{a>h$1=Uyf^6wX*HXN6G|?Eg@kjH5 z;{vne6zT_ebit9XRWpM?_RxX_a9&6vl5BTB(XI|JmU4TamUCwQ;IRgWE3A||J&s)~?P8fMny{&&78d~~Nv?r2;j~Oa{M$T&A3aE; zqci4OcS!botSc?A(we#(#aL){J5_1LuVvxa96C2pS$aVv4@YZiEmL z<47l56HxfMI8|{VXA0^U_6B%|p~?Q{`93iRn6VV6$PvRuUD4by)&SHBmQa{_)2GUSH{H zz8<>$7o$P4RN8zL?my;uBXJ>m?#h_YYI>+D?X}LBN1eCE=W2vKFrGc-*AQ=Nr;FP#sDSf2Zsl>wGuq_7>(WSI-wGyq3*-k7dmwP@3jb^{p#HCOHww^ZJ zS^n49ul0YgYw#O0=l&%@AzsyU&Ky~7VhQY7nOUyY*w8TAh4#7nn0RB>JANh3B)T(X`zReOfoWI%# zEervwBo<4mN`>x&f$W@yyZ-1_YY5&D$9s#hc!Z8tXdxDs;+QX=95X+Ds++grtBA@$ zmQ>FIW77!Rgj#Agdu-Mk0;0rs8lm$7Xx2j%V;0C;6(j^^Na%`9R2&;WY;xSy!?G@|N|0y=P&SJ{lRgCSz|M>%pL^{qNkjC2qW z>PzP`XWH)Sn)AKmoEF3-g(F&pCU(s+MG>v-=3?s0Q|KxV+;Tu=S0RbI?&%AO{;%pq zSvT(i5x$t|lHHAGxdbKJ*ku?e4KVkbQQoR-Kt4_8_pb`GUN_evByN)( z3I4?DlT^_e@hOB|pRjFB231>*G2 zfLbAU8mZa?L?{wZ;kfkV7Z4judSRq2By-p+!#(*){4WkNl-qFg!kU_y09vNteuU*z z)?7FLVb74np=>1{H;z|nAWh08Zonx`3eSWfeB3#uzP~e&LZjF|`oTEAnIxY&ANdn8 zKIu+s`Nc%9Uo?!wQdf6ON6ajkrS3BcsQFG_2H$9YWG!q#%px8z8HtQJLaUt_nso$A z*1@46b3&`z_ zKKymSFfVm7_p@yOsJCbRM@NB_9Dyt_LT38tlPq1Ph&kS-P`W<%ufSNh(K2xj$X*ms z^mT!5`eXI6vFrtf=Jv7E8+usLz#c97$cpgNXkTmu&8JCzNiJ=HTTRA_w49U)Z966> zddor1hO>;aycMe^)XVp~|{aoueEV)r^9{N;A02**YXW@nIw)S3SSDUXE^t z)j_QIPxi$5sW^c1Ff};;1_O1@oE!SG%3mRSp>*oIJ+N|!8_N= z7cPW@l15#kn^0LI9Pt^pOcDUKctcu7n=9j;p;sjhVUH%dQG>cv%H7^WXqc!K5C8l0 zjl@T!qBth13I&)T3y0EA#i1M}eG)`LzgUZhF}RpprxrMnXTFMT6%A zrK33Na7==}E7y!2Elbh?6&>%0#=E_OMa2fzkTbBKbfeW|gnBEx4MUFsbr*~Caa6Y^ zHTL&qL!J|OQ-Wt;b-2eUVsf40wn7kE34-ae*2JD&Eum0iYCm!ad_U1~LdYnv1w-mp zH+x-UiL(He3|>qESykm2^}1CjC*(5)7EwatQ$>Dq0eTZbQ)_ILAvp7*CoYMvyyDJb zxRmoT$Pmt$d^_k^0bA0cf#Rtwngy7JNh0f;b7k;YO23Db=8t|4>Y;C+L@aFO%>O@B zol|foPSEa?jcsgf+qP}n*f`lZZ|r2_H@0ot*x0shCuh(9t5fxzs=0V>W-e#Cr@Mbo zLb*`N{CjV{afj4oxDy)#2_ByY2o<+EnNPBpsj#c1>Vj6^9yS2(UIr5=A&kLJZ4$r= zm?FVUm6#Q$i|k@zG**^=_^ItIPXVF)M{iz~5REEX?1)AMP@=^W!IlxvN>ksB@b@fXh zHjAbNz7zry(1bf0B7|tt{a06yMN%+`Qv_GWpm=#+YYn55GV7f#IMIo~Xu@I6D?Eyb zbo3hk7Nx4eVFmw+QiG$jts{xDi#y@S;*?boV>N3Zi=$@FQ<0Doc5yHaku3TXCvYz$ zK4H)-Uha!c?8{(NS086r`!dzHYz?1=(ME|j)^TYo=Z3csl=QiO zyXDo8^Qeg;y8ll4p6=wQrTC3jI(`}PE>$|DVN4YK>gkjz1x_6R%FtT1NyMjj+Qr@k zLT~}NeKvhND)I@peI8`N2V(RwQX0Wx?kD|?mHo*g-FGc65fSL}y?6hjAT|<{T>f+_ z;ND>-eixlnKkMh(d{DP$Zi^6K?3LSTGVdNU<;dJ?jn`YLQ?S3Kl z%6gzj+(Gy*3roar)nWhDpNv==w%(OtPz2Cy{rJcyh=#p$tYYZ&)B8 zBAg%~4F7u*@Tm=w4Ul5xTiGo)n@1%ZD>xPJa~yS1;arEmI(4`wJ5bO{Yi*Z6d(B`c zTg2F?XlJ%;okSwx77|o06t@-=^ybgo3q>Y`_}QB$2wUApGWdfg`sqq?Mv(6SkxB*q z)VtwBONH?GoE@N(#>>ike>9o#yw&M>Z}F+w)IxB<4Gx}D4t zBDey!pZwn_{ZEmD6sj@&BbjTy&ih062%USSj1P6=$4G)P{(JVrAr1LyID8L~k6{koW!MOaMaGh{DWmV-$>jB>~JiEL~)Cygb$YtzL;L49CUNf6e zOkJ2*{F`o7*WR8ickDX({Hc~?i=Fs2P{sFzbG?p(_bUbu9x=kc>!3G%rhLTCE z7Ms#r9lyAEjFTArFVQ=S?5W_CFv&nA&ZtE~yk+&yzyLB{{s~*;M|w3YP%EYPm*lN? zT%UzSJqs*KHb4CV9pc~kVO-D+h`kL^TsP=^!XA8cS7|s-&_@Sh@yHzDaRVStSlK8Z z61%HV)AJx*Ju?Nud?5Ug;v+w498XknLSbeox{g)xZVvLVFEYu*l7ovSQQLDXq@`1p z7d%)nVF2`3;oy^xqX6*YypnL;A!C_Xbo;+d>ULB4_A+x?+x@Tal@@G!;>0 zQTKE~9X`um>)2wDbzl^p&FddFv*T_op$USqlVk-;pBUg8jN6Yw!;2c(w1k#nuXnAk zc$v28$xE`UgGwG`Ffk(mTI9~tL_li)o{t=BXxhd z6BM$Mer)vw=J?#ql;GF!a%mDH9$rdY5xV?yDzJ;{No~hD^3pW0L}GFd$H?0GDeC?i zSdA?4iO2@QSPM3?O!4tCXdID$9U}jT3tePM2|pg|ut@~ck)gl}*Qwo@c6J$qoO>%k zQe+JM{Thrc@+9S!gnlp$c+P;lCZOx~rr{yo6iY6JPs)gq&r5^b@W3<`cMw4)!bN#; z!3q_Fg|eop(x?qFYN^<_Qn!*JBXa1&EM7*#pcojCxO2GA>tSX_j6?65o@EE!jJZjkT zgtJ$j9?wlz-EZO2x9H3%R=GUpINJSbZFa}el=Pc%E*aWsuh-KIy&@T=v@vwCa&%<& z`|VF6F|yQX!Kt9jAg2}ium_mt|KRlh1GDq^nAGv7h+xgJw+BI7n^v;f8&o`hFn-M?U3L1q+907w@zr6w526CZFOT2X2 z%gR0j8IPRjn%Y6n4l;f*!UlS%WJvdiB!DE$v^C21Y04q0$>v>IY)0BU z;%irvin?p5)VeGyd-zdOi%lhFS?AXlk*QCV>ik-*Ezw$%c&UESH#v-qPEbb7$6JKc zfM@{NIAEk52Hr#Wh6HxZD2dr<)VB}>gAq(@+c*&Tts?da6)=_` zo|ck14Br}|U9Xfur-s&T3fjrm6B0TDYjvbcd(;!pm7>WvC;20P6Q}nytJWk_* z5_VoA4qh#DfP6lH{QYVG10RHCuW*w)B9l=a6zfIn$L=F-ddZ{zOi9=Z6Epdi^O)DdWXVT?UtXlJZWG zN!`W~LStJbqxn!Mio*-64nwyRuAp+Pd zi=&KY!S5J4{BK15x>i-!!?9ENA^`|(CmMncce((fMS)V49Ep4s11h;O^QH=@U2(eU z@E412$xMgGv<^2&ccuyazcWx(Ke_urY~O$iygcOq;}w}f5YxGKuhMkqZt&dLzV{1@ z!9SHUkjpuTZ>+9@!~~8`;=Zh_+Uk(gT832gyo1*aQ-`_-J{kEnb<{^V_jqnuq0WVS z=&}yJTk5!~jZ(lpu?2OgPS$33j$OIg8m1NJ5!7A`B8`ACal5;g2yGL^^w`!+e==wDAZmxu;|8LT zzd0dOJ=-41)n;Dg?JcRg@&;h{hPzv*oSB>2mhRam_*Edyb6yn!{5mv+l7!2Rdb}VI$(d(> zH-cMVTf=Yv?se~J&+XT~At?Fhgy%%a9>Iz0X{L7&h4TXDRl4ZyuENv!wV+Yc%-1)TS zrx-=SKOh4;9d~c^#4_N*T8{dkYx8m(ce8|+usXe;zABKWEk+cvyA!py#}s`4jQn?9 zW-kU)Zw6b@NTVjsrTWglpsh8uapSv%#&K4W{lLg}V|lEwy6st;%?naJ=P`7+m_x-K-3!7D1h0|d@QRl86-F3>Z`sksVHt2=nm!V--Mfc z-0-WU9%=p-9BJ~za1#<0!d_K zUNei=npd^Z?#=scg0%4|r?cpIh@M3nMItwMW5psLYmIzsjjZ9pxtz-vfNy(o%484-Mur{@8%l<8f%h- zR?wf@$UYasDJ8C^Y=%zcxV_#db)cSRSw`c3T_Ao#(YupA?@Rm}>cYil)!Wxz_7Hyt z9cbb`U&|`VT?Y2VsrlDSv|i^?SvaBHkjK#45AP zgVkGg%TE#mA`A32ZyhV9hzKua?C*Bxhx|@lw}kzz$DIuT85-Sp3XSNd{e;UIE_XFO z&Y#K2{<{~WJiK!;=6SKcg}xujIX>3|*FPsISr~mrtiGlMdw8-UhNCt2S2gz~TsC%4 zwHe}#QO$n_dDCN-k$#~IMC*^zBA!FrJ?|tYK|-B19tx_r4kL_IQmA0I9p z_T~(zqp@;a)Sf-7v~i1gP!BCpsXXa6iWvyUGpMauO-YMcvbnJDT3kA-z8j#Cz+-wOevuYtuSx@`f>>*bQvE zjgrSMZ7b5{+c2^J>u@uF!D9N*e7A0q@J%R{>fxp+gP&TFJ__KPY2ndnR>v_A?(IH9 z!hjRuD?`yNz^91@r2kY=wGBXC>_J>IU)g)5vRpf?<5NX{;uTf(*DJR{5O!&BD;hIZaZ*$Gtc3U z!ZV7QiX;6O6W_E)Bh%(q6_TDxDsL5eE0-oBPh47^&nI4CVj8@%S-STl!Ri zEDqsmJR)riq#oTz4#m8Ntak*`WIuj2xRO%*?s80899XHVT{vNgTG3NUGs^DCk(V0| zu)N7`8B`X#nS;!Of+@c%)(I#rq$7e-bIILyeTd?{fkPYKqH;JPp9&*nG(A=>Y>x1+0BDvV@^ zok)a`vO6611Qk5tOsA0x!Eb+_lDrWNK%*#@qBOSBC8IX}wUKXZ)x*RP4S$V`ag>xA z4&0!mB_V9RkM}s|wi#GVn?$uE01b2=6R;z!qZ;&GU{ZHdpyK`j)!|oAmpH^zEf~Sd zsFF_E|JQDz3O;zlwVQ_0#am+hn@Rr_xmm&x1mS z-}DuVC!$!gJ%CNL3@-H@sW5x~i$8CexmONE(G9)V+w?m0I@YqSk#_&URSjs9dwM_F ztr~^U(eAYHm~tCF8^V9S&LD+e!&~xA(;;=X9Uo=m_UZN=?ra`x(|tYebuE0Vd4Y6| zd|tm?gGT}>(Fi>VF1|;mDK=BKQWl{O5-N2g`U6PzcAD z6Ap~nQ`wabP>}A8M|@2az7umX{4Gyrc=X&@P$O7d4SXU3P-lk6FNfK<7xWyPzzSO# zs5`lsfkc-!AXN2)HTQ9q?iSExxih;xTZh~0vZ-i!j`ZCA#OHEuu4w&1W_o<#Y}Dpj zm$6Iz{dd*wmE0erW#47M@tjLr=+T2&?4e7h`)ZxfL+E)5Eq`Ry=Sut5Ox;`l>b07lj z6Z%oT+!1`0=j4s?FeJ5LwXyplUHBSgk@aNek1j#DU;<#}v%o&+3x}=0-7M1+N_-xd zazY6M#r0iHwOMHizTtJ%;U2kR!0fgICUMBx1YA4x(zpZquE%76fhDcu?K33olMs?6 zFG5j-k|0C@J5Lp`I#VXH^YWrCWMJz{E_Y?mltfIif4}QF=Gvc4WysvM6eEJYb|go@ z;gXELNRD@j#QaKnAmSu2%=q{xZ|@j-pYkH0DF(RNLEzM((c=kd`Rt3(liXg5A~F_$eo8^g zFNgYvGaRVYES?UFAYP!eUl22m=kjyn&skufF9_I1PV5IUcWzLLH+9+j`z3<%DO{mN zN&8#wjK1&_LGBAYO?x9EF2bis=9A;E9Jgxqvk>+*u=A28^bR7F0u{97jRLVJa7;J# zgPJ1Q)!H6VZZ-gy(~kaL(X#uvjY46@yd;Xf{uOR(h_9E?*>S^^bPsVA%l|;^@@IB* z)0D0LIkhR8uKhzLt)~0b@W* zEEdz2V;!`e2ZW6;rIdi`JHOn`Iv$25-)m=x4}>wGem)Ylfdumf0kkcu1s=*kCP9|H zo;C1~ReK>V&w+;Hm3cYx2T>gyyEFzT=f>~JwSHfGgUE8Yv*!qgphUD#`rkYhLlGLN zh~;Q%yDo0$ zYf*FtDQ@tLMJcPbbml~qL(%h8h+CWc9HGG5p2`Oyy_Qvk(~yLeRPhYKGw%XRtJ9!{6?cSm6da(Z>9BQK z7{nEvT$xS0x=A(Wyy@!+b?|4M_g(?qN!box0||@0wWho#nN7Ll`QQ!Gg}>vzGfKim zSwRzD!>?s^obXB8v$}%vy0?G-YO9kfvA3giD!VfSo!ds!9TZCv0?A&igPFYIwg>e8 zD?R`T<{+v5FVGt(JqP(4@&8h|<)0f$r{O_BKpB9$w$RwX!#PMaKm;LG6MDw)tKGBM zl7gtLSYG>!h%5tMZX_rYVi*MRdT=VqvB#e6s~?3zs7c938)*{YDes>l5Q?jtmqQK> zwrSyd5v%Gn&h$VPHOpTHcV_e!+v1E03ag(wG|fqAMmG*)w3CtKsu`8h3JM3sbdJIy z(VDa?Y9wZWQW8OKfVRd^s5ms%vW`WhwpLMv7S?S(RefS2*}hC2FNl}+fovNOukj5q zl03>ZO%{IyjlBMFp>m&4Kq6U4jhO`ln5;F9bdlM`;5-Ylj$Sqz8(c1?e(WTo=h63} ztDpL^cc<>d|Gj+pcqs0}+>_(YjI!|Y`0@7qUIBE!W#q&H05M~;Alvs?koXukCIMa1 zhFhI}ZjbZpocHe)8Ot|wwAs{zPv;pqdzzZdz}w4=2~EwOm)E%}_q<)DJI2Z4iQ?b2%tWKUUD-t>hog|M;|sHX=Qk z+Au5jZ3bbLbV&(OzeWbb$SPFRYElj0Fv)Fj>+?-m98xq1Z#{2sd5h1;TMb8DLD$Wz zw0pY=;)qqE@j$jEw&XVgARG0`gfYA=Axv_U}Gjhb6T9lc6u zG?f(XUNi*hxR=7X7eW0o^7hHk7uCfmb@hd!{;)>mhd*l4HQtV@->>&hxa@0!yt!Z1)uE7aZI~E}jT<7)? zY>H$`-?`~5`@Bl61E?O);Cd~iyJ35}@d^OEhCb$T>>Iv?FjNvZb)jiL1S2o&5XfRA zXm1F1vAbw>KE*sd;yF_jC~A1^zmXUiXyz3F=@FA!mK3Y@wwKY`ws{Qmlpl6#v^yE1 zkL>F39>rSJHZg6%9wH9-CcJAR{1jpRxH@ruwDBkTYi<2l7N%lfei7U%pqvt|DMa3S zQ8G-~0qkWTqV~OoduKb!{R}7PbqP9AK5Pyb>>|`*4#A>fIKYu6xg`n+(dKd3?8pZI zF*X-{aOmaFvq)R*1AMHl>sZgb+~V&pqa~1`T!66}8ALF12!}XBs9PV`$Fgw}+tWVq zu$B0|J9qcAH4@Rra)k=-lt)k$%%+GPf(@!j=~mo6x>-lQj(*za2A_?0(czbPzSGdg zUR0s6dL%fmMQ>|pV1o|Wb|V zmo5Z99UXD!;m3(jJbqXtByaP{-Fh1CfYFlHzgeoPK>7&nV&?2OfyXytPb_3Sis z3KvR0Y9Hc~EBQjIhTF`|PiNn#uZtZv{WoOtE>JG{A91hEqtWvP&js0Sr=`CDqVY{s zKi6%R;=`ON)C$T`%8J8>hP!m0&B5tIS>F3YWas-8}m>Y{s zLC-}ZsaOxnb*RowL?=Me*TQ^3V!GTyZ@3kv>FB{s--|Se=PhHutN;sObu@&5C>cRd z$*341f|;*pux!P3{PT{34XNXRa;=GwK}h&cPhyuudLb5M>!ziX7*quxX%$Txn?JUW zZ>qO4CmvSb&W&--W!{*q%+X;C&uxYlj*obx^;lQAQ7d7y{a5n)bXV$Lq1KDl%@$5r zyp6eG(acwRI^f56vg?Go+5E$ZI0fEe>}oVRRW47=8m;VKY}C7hpl+Lh7tn6jB8Iic zrD4JuTfAumc@Lr4zt0BQ4Fp-$=i*uV;~S!cg6?EDk$*@Y<%9C`Q}ttz(NouWB!Mz{ zBuN5+S88X{gu9L6;3Ij{!nPN(G!BMqmTaUro$KW;r2$rE_~w;^Iqa>~i{2RlcrsZx zP*ti@r5_q9hB70Sh`4y9jI=2sY66{2ZG&<3|?%+BIgjn zUK~nn-+6=hzEE~5u2VkboK~kN&xstt0TlijHfk(WP-!g!3(MSuL-htilBOJItwcb3 zBhCt9_$W`T6InUt%AX1K8Rbmw8~$kD&3(dPc=Y)Rq=;M{9+)FQHnlXglSLrx84q%< z#@GYqRBawj^Gr-p-vav|+hE~w#{}1D5^inhRFgMZWpa)gsPvyRWZ{^EJ#fV%d3O#E zDAfXAfZKl;VC${F%`Nxmwo(k#r0T*ufs+qDzIj2;*S5+TzN}hYWZr&2cJ<0i<$317 zbJwxVL4H2`mi#}OarE}swBn%!E%i9Z?JIdqbY1YvUd=g*;?a{JY?|V);C8#8u7twT z4G=vnNI%b5t=rv(>JVIk9_e>%DP6nJ4u459z=F9ga@|f`cgy_-*|S*&W=!x$SCaQi z`y&7ZV*&#VBgy-W1AO{Eaz3YfBsJ)G$Q7j{CXKci~4!Ct(4dO|wm zj<*g=5^u4HaLM)zjVaVe4d&>C zLUf_FS3Cdl-$lNR;0N7TPMKA=2&uNA2!M4KB^m$n-24@mW0m4jo%aQf1yECm4d)7K6}p7T9XcAU zfMvW>Qb`r#+VB=VR--4g5|QEb=mnLId3U2uf(Ug@gV%64Za^J|T<|Zj4OvWr6nn!- zW*qJIJPmbv7CxU6|2mSGtcCq5;xmR4qE&JwXQ9agcz;dF5gwv$40KFv{sgZcG~?%W z_g}PY-FBI}SQc)(zBK;|f-*gC0>W^tCD5{gw*6n;2N6Yvxb8=xe7Wz0%!*Mv3LbLtac zS#hNiI9L&qXLdxX93;kT3ZK9tV(hF=C?``#LzyzNzn;;4$OSUVr_9ItI0UD`H?M-! zEAdJ`{SAv~n>tzj#aFe3xXLX;oh{pCI;KMgxPA_7jiI8>oqR@5Y;05B8Q?SY-`To}Lb~#4`0<)M#C?Y^&GmH% zZHJfcg0t+{Ep{?g=$3Uwo-Zx03}7$X$b-bgv5iJZvb6qfL{FD^M1Sr1Kdp&1Jvgx`khj zUM4FJuRTCk7!|!oEAa6eisGuN-)t&^5e{(bgL(e2gzPS=}&rFQZzOg;~Z z`XeLun<%hK0@%ueRJL_VaqMjNBt2o80VJ{va9dr>QVu1gUo6FIL7b-Yd|dQn(2v#j z{y_gy@XrbMiNN6rtsSFn)+Ts9k@05TO$)0e=cqwsi4Cy!nCXCF_Vopsj>PL)6ue8j zV29nTskU{N=QgV~RFgnJaU4J9n&|Q1ljuCZ^SG3>uUR?r7v^A)A4Z;=0&68(=+kqkzL9~&45NCgxK2*LlfR<%rQL285jXM9At2kD~ppUPr^vJm)B zK{`PdApfT-1E6B?|5JQ9P{(NhDT)Rt6YT#~)CANl0)NgQMpn|D`jor1SS9I@&#yKQD4)<_6)}NtDT~W_cnNMp=~hkZE2@um6Ts9o#j8 z6p4W5A^V+|XTrm>&*k4AMCR|P3t8e9Q?Adh&kz3=JmSL82q!S?`Yw$D*Z1Xj^#q6E zLA!uPjppiw0-+!4>)TMkn=6nLU=xp})7Mi`X|LX-G+D1DZ_qgktbiq`tar|4hy7|v z&#*l7YFpbWa5m34Y?uz=*#0Ugleu?a?DVKBtthg|71jVI7TKTsEKzy(ENFAv`RQ3U zWh=VrYd&f%Zp&q8tJ_|+uKnFH=Pi4*tID9Lk(qag@V(RK)z@&*ClX!&IO6baK@^N# zxncQxH&?hTo8Q|zJj+RZX)kND=26PGlQxC(Dkx#jt1CA(=X6g;x-HqTO67{LHM^w% zRRhg%dL*>~d6dGW`K(qC`1~+95uJe+SRa2&>^G0>)#P;**KNF1)tQ399t~RD7xC08 z%Q5~W3mqR`YH_~_Z?$U!u$YE*NYOp2LOr=@ms~y)Lj`eYKdlT{s6g8Fu+p+Y@~@ZN z5O@rJsWliFH%Rn#a2N*IAUQ1M)~i?4=VNRC*jcwzKfc^V9UYiO&KQoAY*ovBicI@~ zcI5KA0(bCU02^E(4Sqp;;-r{XU(uMZ0@flcUEN$)vo|BG;|VbxpjL!AZd|`vd2dp^ z)=?Sq`?=jdj>r6s!yi^FrA&e}vU$FaJk%z2w0T6ViykynLwt#ZzoBy0w6{|B@$2KW ztz2W&x~US#ZjBiDV+xF5;xBS;7+hOWZpNd0!I&7G>r!Q3WJF8Zl|$UsyjJj zY|xwx273U?LEEFr2Id2)s!ChwM!NqS?Vs<~U^5I;uJ);R_YAm|20akE!5?&|6|<6= z1u~eEA`@dOB}{J+l53f~{7R#V6x3gNQG0$)tl|C#Qy0cmJ_I#QssbC@j&wn&cr}Kc zpcbCx&-Hvlz;gRmTu486!(YPhM1$zKov7z)0h&|;Srv#yHHe3hI(9paDXXOvoYEi7 zftd3bKkqA1o!k!X3rS_ctd&I?d?x(mK>3XHK-?xqQ%UMEsu(y-XL*U>@*3rSEwRmj z-lTi#Gqc+@iLhW-iHgo;PP0?{EaI25%NaR^5m_SB0l2}NzMQg_sm9t2td~4{!R`~} zjJQxmlrb|xC7fjCB@(e3en_cf)i%d9*~J=Cz5IO?kuS-$zI5dQe`S>Tz!Q9N>xZMP z%xh`!KNZUgz;V^-Z)=q-uC5s|GLbX z-)S8h0nFc@zMc+^AX#FS=ouV#yL(NT|jt6=wl@ z3&F%*o5#nZEsVdjUgq6LW6>+-9JAfx*|W)hCM%E_Cj}v6`qd^xb#285*_SbF4t}$* z1z=#cj%_2kHK3Ty*qZPT|CoKnjX#xJRkuQ5*sSE6s#alo=*SATvYL$X8$rT3IRK5M z;vNtFtg_0e>13#2bT!GXRg}PEZK(Y=6H|oN<)u`=IyNQaG{&AxZ;e6)Czi5v!DF-r ze5mf_>$v9&S0hu-hdRi^AycNyu8$Hd8{nT}Mysc0HVX#6*sdSoa?J%keSGqV(YScO zT-DZL2m9o%ya58&MWNK}#YpcwT-pIEOSu58p}gYVCw796%Q(O8#2nvY;6*akKI-Fk z{dAFVa!pF8rz`s_UO89wsVddC_8EDhQOCd-IjL!TT@khLV9-zUBXTow46^|d(Y7nJwVnp+^)g+*vat-v!E`(~XlDp%R^^)?(a~D5gluOX z>Etpj3S5;JzK1~ACXV}an__t-v`x^9Vc%yIK=5(u*-TC1TM=`o|GlCC;eb{lbYaT} zR5V_J!7Q{%|A0)D=uf}DXE18Ay1BP;dt z8X^6)D$d7>2ZEvy?oS2Ln_7AFqCas~kgO{;S~>Nquo&JWFOv_N>P9~|d#)r$qI88E zeC;VMqF1)!d$tvL))uDajP@M*b*faXdo< zJt#B-kF7;xvady=Qc#u11%$JMN8(<9(u5oT@b8+P+%_fo%HCVuE^tD#d7BO>#4c$C zm-E~``~@?`qQoL&YgDgmkc`@(@;ebfJn|ldg9gHzGbS(LDhBPO-N`mEH&Lub;PE5y zOCm2DX_t2_Qn0E=N+dIx66Ocyo|`0M=HfV4Ea|k;j_M9@wf6dx7ofd7>(kD@-4``Y-TyZiKBw;qWg z-YcR5rLu4y;u@og$Dmc%mt>#N`{=ikU_a`e)Z-{xf= zW;|^k^|z!*v`8C_PQ|eY|VH-0kC+l7JrN7lbBKbdBTy% zjgr2SsbD=I_oSoWXQr4PK4O()6}m&=Gbn_Yga=k%J4SR(Rg`B37I%@uHo6 zxgCK+tdf2nprXSF8iIG2yi5h0+epICQH4=T`Jjj1SA)(w?1HsqhahYkiawsCv4#5t zm1ZZqXxKXM0tm^nB-gMlv^-g%S{mhfwM3Q?>{*$$Dr_D;4K>{$h>bf=M{&{MWTkOt zggVB)H!pTJf^&RRjRHiBaC*Q9Dv6(5RTV)zaA%EK5LI1CGdT^Mtx0CK5-^4?V{|Ah zQ&*wi#sywsb+)R3=*kVBkI#eP7L{U1`E_pq2$N+ zp$)pH$EC|uo0)AU?lDawP8-&{7$$U7B6R~L;8vbcdokUHncc3;}@mt6WojD0< z$&W~&pc|cf$dFfJJ#sAEz4zw zrVKApiRYCcd$FvAKmd7gL}yY*Pm$`pAlj&y2zd2svk7H}FwLzTOsGx-1aL=mLt|2A z8qxT%$@phts231}U}5GW_O?u<~OO|++n19VZWZ=d3zh~B+(-0zef(@s2-W6v+3h=IgvSxxK;Srpx3KZM}rKW)B`p0@O;ppAe{TS9nkTc zT@9-BN(eseSDsebxG^^>jb@}I69D{KoP;Au@VR!9MFEx3(J`h$kA!YMq8}$(ii1`@ zoAdSTVe)v&5nXtPgUfk+;O4i78-FCM}eZf!c7`>G5W`6)G+7mSo5p=CM;Mp#fwYG5x{3}6$K2TI6g zz{;?aJ1G8)PHq8G&y<`k7x#aJJ79Ru+l_RuzUhV3x+; zU+-4hN$CkQry_Zl!E`|^!gNRsz~_!LCVTQ9*2Vc$K)42Nw~Nr8007M-JZSn8`;>D0b1E<1h)ApZD*I( zaH|GPW`vzf$yYf!5J)g)=jJ5IZ&uMz_-aid^wYT=^(mr4N)l`^hYn|&);bkQ{`+M1 zpJZFKml7Wn%X-IE1T0bSnvis0n2Ad96@su9O$s;=cB7xB;ll!+pr6|-^JKuYxp2{j zud+Fq#G@tryWYIK2PeK0cj$j3zuTrfK*K;}mV#h0#B96(Ok zBdnVW!?LL2{9zo+Lx^L+l2Bg!!|NfD0YNmYomHa)w5BO*kr(G7a~Db|7Yr5@tzM+YErwf}qK_4Vu&x7pE7L)ZL618rJ!Q zMy7e58_TRO3dYmm8UONCGvjQZxi3%=#QV(L=p62{)jU!Sd42fs0fSK3?|ZP z>_8|pG9}D3r13mFj1>_cHsTY3bC&|d3@5d51!~7nA13`&e}%IwJdo;+`DY7R$u38u zT$_VfRH9s-=<|LxKhvk95aG*rIpV~A!o1i5`S^3o zH0O`h7C<2F%f*|DE9Isx5XPVdjFZy}T2U4uwR*1^25m1GVPyxT{qh9wmr_t!?NVzd zxOLNR@Slh&vYLY*jlBROCXrB{TWArc8ntLmqOg8xs0)D$TDM9^C!}}Kb!8EpZ?xwau3!DfV~SwjM_?3xa)|MFYH#v>64;Z$wv^G2?HtFN_OwjB zd)>fSUC(})wHf4GN#dO;e{Tv+g6AHDae_U~n#^L-cHZ=#{?PuzHIv0%?9x2=`#(9D2kmMbLLPD3htP%3Qt#&OuNJN zQ>6~kPzmoOBn10spk%m|ZdY>`pg!%N+7!JaRjAE&)t9WgPCLoFa1zoLs-GU}2LK3O z)Y9G_4Ya=5pD+T1;c$Eci5@e5=vkCD?TB!iIO{#o*ra$o#pzjLgL_?bI$;|HQ-nsvUm8Y>4LJ6>ep1rxprYg@M-ow%N4P}SpzFQ4&r?jC+y}w7h zX_rDZ-BTbr-ebI0FwDvM3Wb~ zb9Zz7ZElS$%&nh<^91uG`W7pYL6@=N7c6PXvy8h-PPy2wTp{lrX4`SlRtbo`BO|NL zPp?kttErX_OJvwcy318C8fy|+w{hSY^UF{Z%%K!XCNve9;S-#yOW@EG?f;8mQ5=gU z9#XWIrTONxcn_JI&UqWJC(@%R^e!+H$QAquC!UC*gg+LIfmI=qtgY6H!$i!&j69Dh z_rn*NhD8^`UOc8I7HKvOUkY#@OX)phof<&gvYkJE-;YmsA!3m(xwI8&=BvmflG~b`MZ$NdaEM*kYI2 z`qzywStXH*-B-lTY#)P={L$(ejNl(Dz`SL_?jXQ{Vo+qT8v`dQT;U33`X-zG#D=Mb z()m`c6|s8}?t8tQhU3Y*gsZ>OtBl&R-IYRLX7+b@LNkdUobuX6BfPD3N{V#-4?!Kr zerKW_?mFz85(KAhbNJf4AIPEOk#(We$lABCv2X zQNJtpd(rJC`-;%9)il1|bd<70ZpNg%&{6-6gu;a=bXuSW?vm!cb26p|dP!5QJ0a~2 zV_H3A))6%bcJ1Jc^xO(E`ygO733y7wp$pElhkMnGK`@BLFbA;h7`fOiqLPHkE2Cz8 zAz-ZYA&VHt_xOUywy|J@$qlsP=uVZQM?Tq_64 z7{hkJ=)j~Qr)fyyerbPYwHLWNnGC%#Q3D9$o^ea%-R8Y&zb*sRL7j)9yw}cA_n;JT zlx3*Lw-_xNVE}x%+r3UJ)9GJ^ml`wc0WcLs`c$5Zl{Ttk?wKW}!?Rh;4@^48oLX`F z=+mJNvhr27eu}hsq$x9M!^Ncj@>y~`E-Raw&N|$xb~YX_AGlUDU$j8UbV-Rn zh`4Q{c?AxMh>`A~RBZx{6r65jzrm+Zy_Q1$fTYW7AOdJc>`h#I@6-B2elqtd#>sfU z@fCUChF`G^Ex5SX;=hQtnWFZs??mlxylCW~>IaAB*>$hj8Zxp4c z28k?Bf56)?Vcd6)KiW?BeU(A~3U9JXd_xc)dGDT}wo&?z(ZkYAR8XfIY+mAmGO4jk#_~(@iP!YPNRg-tHhKF(>EexO;;K>DThZ zGA0|7fNsn<{RG29wrO0FC6~I|p0ge1LZ%>Ne_>K8z0x6P;>v|T^zzHq=7&1?`JSx()Md6g3P~PIIuORzvsSP#OvhFO)`7`l)a4fhE8|U@_drS z1_B?5NnbCNxL9a+1iwA--R~}({(El%Sm7Nw=@qpn0+pp1uO~A3&b?RYhiHYy&l6T@ z{2X-dof9IQD25~a?V3jL@^ykUb27-#e@3f%k@IStx3Bt8`T2gV^BfHP%g6?Vf#~7w zH!BItgdV_l|3&16fls^63UEAlk7Ua3g+^R?zZdPqKT>Ke+RsU zZ_~W;JWKyVt(5#gWwbnRt8#GE_MbjWWYs!Ye`OP)lh%l&DC@k-OyKV*h|9BG1YR8OOQ1JA-%9|e z7oU2Vk9QL7#e3uJ(7(SC=RKThzQ;>OlJU@g5T2U2^rIl+^QJ#8+K0vOUjN(ayVEx* zF4Ir=(QTrc!c2KtE%U4}rQGKzp)qE(Z&+hZqsks;Yx2NDUy7de{eYu!tH(Y zgMPKAf5PFqgwJK^0$BZ!)EZmPvQhl1d3KU3K|-HOJcWv5%FXsf|sK z=E4XW#nHx%<_L^D59nwU??)UEqkuyE9pJeXw|jSh1N}=bM!Xzvh&eLG1&|{5L7Ms3 z-9276?u;#r=-wQ9iCN9Zf9fA_Q_uYl2sb7d-S+B?DBMZ_ml_nG4jo*|Z{*pRA!&GZ zbI_W832~3jrUU#B0#EMDbi)KXLgeF#SEb8%KXurNg+{53wM}@yu@fy_cg@irmWVbw zeiQp+_EVl`?PUUW?GEPCzs%B{o&J!v;HcFs7KbL8z4o$ zZiKK!pu3(~Sy}n6qwLM=uYNfGrkJa$QO7S9s%nd6p*APklXX77QpbOsJo@A)JH0BJ zY*~~lgTL#1(`NNDYgXlY^2vWE%T2wa?^&^0*YIjyZlWYxcI)n+HF{$G3`OKjVsw2cp&b~9`1*+pHK zbhH`iF)cUwye;a=R(h@<@au(|7puH9FTdiS{I1=ti|W$6`o6+S`p19U#(iJs)go^) z_;0;1&#LWey~9V<`V#rJ-WmThrdU%XW?h--MXZi`j%_Z&ww&TfkKDr?&O zq9}`Y2cWHBhIvu7(0sLk|99E8Q5$IS5-L|-fx3dZDp;~=y=}+%s@>FOHhfpr7bB`( zZ(DekUFVx3zbI9cHPC-;1M9z-d@?WdrXi5O;t{EnPcrx{)H0jRimGU5vtgsk<%pl- z2Y*gxFoRiHUtX$BHqBnvmCAwwjX{AIFjy#KQDQg>O0F!1=%Uozk-(g&3So?2+Tu-A zxN%r##^RntrP~#tTcc(G;JKQWYPEv(oMzASvQf6E{Q~{wu%3Sx3fgVzY?*JeVWr?- zTHze<6W6s=BV3tfak<^(jFzyqaES2TENmOwa(frl|erx zY~yUZUgWK6Mq|K&ti6F@)SFe_HYcJ*MAd9j%-gK3VYt~wA;v;;@B-1AH$I_TlP#8s zQuF4T3Qc@~;naU;Jl+|#Kh1vpsi(+NFPSZpfN4{g*NR~X(|0d;g}!?B9RC02^a-@| zX!7Xu5cBZ;v)6B4eDmrhm-t^|BUX0M(< zfAj1V$_*Y3l6|~_l0V@4s_rcCPXBB6^xvL6{kprD>u>h*$+y(22d8Ao zXFt6DuBYUa*RQ{O^}~yApPW9MJ^$_rRC@LD%?T3u?s1rMv)A7}d-~!Hd^-WE2<89f zPpPpm6rO)jjW?&?K~MksUnf9H5xgn~S~dXWJAL)ltL)QHKSd%~!0IZX>Ubz9+*Vnc z|9$t6%FX*8iD>ooO6^<&P~09i_H1z{$$M&dqnb4oQsM<2g3G!r>l;ahL_avr>?AzO z=Fnb2#OyH+K3^;f9%c>H3#uGnZi)qUia$1ab%`ahEdUf8@Y}pBcU-WnFW^LX9Rldv z{6v2ovm14ZN4qOQaFj#B5#2yFT(q1xqGApZ-@x>sJ1hr_v;i=wvbZcR5X?wr=k>M% zZs=IO&sS@)c1N^CFljR_U_eA?fG99GAbnd#rYk&HVB&mS?*Jf6Kp8mWsrSBa;Ml~e zQmKtHz2p_tZLjhcUzpV`syUQs6yfY9Ule~^JZ+%mMc96mUdDA(ly#NrjQaqH!1+$> z=J3lAZ6WNcy#lPcs>?-_o${b^LKe}q{lGBk=!2GVAg!uYLkB1}l$V$F2KLn|IXsQ) zqCe;k5DPf58DUS>Z0A=zfFZC#+ZD2Yi(;c_N8_x%n2ypI{OtXNFzT31ELzZH5S+XTcfh23 z6b~6RK-qlN*kjo{atC_p1}E8%gZ$!RGdQu2#?Y5q=D@TLPNFjvULwUBoGi;a2b78q z^1+oV*U%bqFJ_%eRb2v?3%~Lz1Ac!1m?PvXcoF!iWsjkU&*Eln^N0ppth`1Z!Sd7> zIpP~G8UoGsutGWl09xhmvkOG*CL3g*b+>+|XAWTb>3lF@RtiOqVFZcM=%=v{Q;@J= z%gpNKvQe#stp6enUyo<7P==m+VEmNY73j@9Fx1Vn51y|#vS`*vgNu4LSMPu2eD#y` zM49X9ncI?)m2KyaWnaAZbMrq%4t$UF7q3`1`;AIlCt~!02O)xBIV|RryLa5+LyI zT0iw*;+!|7@YWX#8*wn zx`zz#Vbgn~R>iyqS`5T|jmr;|tSlPZ*kT~r&@!JREwbh7H3)@txYL=gnKRf7Ng6O< z77xTN;#*g6I)*%=0BRQ{IS7YBNW{oQlKRZ`Cy~j%yY>LxBMM1KTYY~~7sRu{o2z<@ z1hix709qQt0{9x-U#wNgnzfoEce6lxTW>F~5Ju_4Yryhs_5e0)uC)>mzRVoxu%-{P za7rU6F*qMYn^3bkGiLE6*z}VG$O4)09-svv@{i=9LUmO^8~}(EC{~arY>iqm3))d1 z$Tmr_WwnK^z}+FLsl|VnJ#BOWH~>HgY+*lb>i5J<%&b;XLKq*;axBz^m`Ta@ZtylE zV6n0xO^=Zf5G#R)EiS8-LdNkznVrq#K+J?V_2w=>>zq#q_{@N3K(por60Hb!_@?K8 zh-G|tL;$hsrlDG^o7tXGn5zUB5MWMGe6eGw9CC6b#uO2=>wJH{1vzszzf$vea?<)m z7q)mRn7d(#)}O&VK`2F%jsn>n1$62;qahu6}GrgUN8<;mL=hFwaZ$I?29MrMgCO6ZrfG@U~d)9*naG zq#=eco&7y9MDu?E4HbVjyXpZ-tAcw-p})64{S=v~)y(LEJl+VzGyA-UJ4&gn4rGzu z7!}me3dzJIAZv+&qcKk;trDb4h86Aw)1(``Z?F@=vyi3Hd{N{ji~lg?LZP}r&ufKP3EQIGHl!9ub<9a-N7I_|V-}n9Cfio;s`{pS zxY6bgS%8WGzgi^0;QKFnjPFe!t#0mg8;|KZgN;rSTdM2 zl$$MKl{N@3e+oW5B(2q_!HiRY(ko$78{E%i}$Q6WG%Q=5}H>PDgt#SQhk$<_vYJg zUVF;-w}C83|dsS)#P%aE-q>1j*Q#>GLSdL z3>i0X72M-JgR@KEY9wZEoiAih!PBTCX?qXK%9#UYs5cACa^jIsR`Ps)RVc(qA?8bS zz|9pJf1(pYpKce>y=l=_2FMKz?j@>1=+*y$U%=0{J3B{@2{UEgQj3`Zvxy3jF8WZQ zv99$EmoUEVrs}K$T;VL)|3Ct1S)dWf?g1RWh9tfz7C#`&8MOmOpz?`(CFf_Zze=1j?7->zk2H$aP4xvduWV)R}qZ*$@ zy|L?I%mWj^0uK3dhU1#bDrdIF`Pl&84$gxLMb%fKs6+(~+aZJG*fucA9%e(^y?8EB z<@@^sEP_j~)9(UcJkvjyH>wP3{vIl7cNtE`%Qhhi}rY(PK( ze})3_d}NxvCAcTq)9q%X$QTN(VaF(WnL+7Bwb^Z~N5_5#O?Fk^Wb?9a6#Mm%e;qXL z1_x-vhJgv1&!l=sH%vQqyW){qr5V{@Yc|%J62>m;ks@est`^%3+Z*$;okZwk;P(5+ zBZ)lP!f5P;HH(byz(O|jZ`%S*|0GBQe<+aqas@1$bW1{~tw*b2^ux(|G5HE6@q9zD z*#*Cy3FQRU|7XDGo$IG$bZ}1vKo?6S%WIVN@Q{WUVdirXtIvKI@1T1AzG$YOJ7$WS z<6#LGE+I)j*G2}!1BJ+hI3~|;X_BDUphL37Dmd=54InZaBvEBU_H~$_^f`Iae;PP7 z?AKOZ7n_=OKfFDOuf5n&=fjcpxFM}3>50tdW2=W~yHxqLvfcthpD%~3bybA%>--XD zN45~MjW8-;ZQe%pQZ!HV6B?8%m?h4wFixm>rD?`H^gF<9g?GF=OB%m*+NrX= zY)G06SXZcS$L)+fBX$+k_zI{^fBXeg!Xdtb^Io*C4I1QRvdhjMjXyspgl1nu?9}$I zHHD}~92*M=)FD+$Q61xEVvEMett@T;TrslR3;_!Sj^c;j!D9O^gZN=0Nj#oJX29ce zyQ-Q=TO-~NgZ5;bXHfJU1-tr&zIp|{AJF;?F?7nvp2xo&1Uu4c3{=HbcVvXF>qhMO}1{k?Qp(CmX4>b1ehGo9GSmd2b0^{vHY>v~dWSdYoNo=Nqs0Nzp14 zgZx<+%j`na@oYM!?|2Bre=80c+NkW=`?(@T<8biGZbzfLtw~F3oVI8mnpdU|E+O|X zI%C4`wI5!TCM$#i@DQcMzx1}PZ6bD)QZ{m1)kr5>o_{>TL&_7 z<}I5#D~JY>dlT6y^Hvz-syZ8@2lM0nmoVY) zR&}v0Sq`QZJHda(+4`!WUC9-YwEa<>w(Ol@w`9UugWh4I+yH)q2FgdqD{5=M85cSI zlWiEN+brhTZjTaje^OM-dN>dh#--l0V~0mGCppiyk7nf~q*@tT;XPgY_R(mJf6GUs zk;7>%9+r=?Kf`zvj}gsaq0Wx0HI4;?E%5Jo7M9(1myKxw`tuHTu@N58^$qM0+ElL4 z77B3@6e`#2xCEnx3jna{en8Bz9vGsJsv0^h0VOY0+h~SCe=Tf7tZ_gOW1qMjupH=` z)3+M4?X0vb;{NGc8~VG|Kjlhkg1ALW$WV#t;P4c&ZiK+y0vi1B53nf5u!?>ws)Tk* zZ0ESIp^_Uasi6`Z+C~jEDz1A|b?pNawl zfDBN27-7FSsM3ein8=WSNOTQ$k}noRbBv9$zET#yMiJ_5h^Y*<>5-}%2d)Q2yK(4x zV0I9Q@n9UcL-AA21~y9V7$VLIRir3_@6tMW9+Nh}YJ=twT8wDtLDd_;Agmk{wu27? zYGg2qe;U98?eqW&dT>YSZpx?|{MfTH3583v1u!HiCXL_6P{hyf(3Vh0>1?T!M$_T+ znc$iqnb1ZdjQ{J(* zyJ>O|Zu;@6-|uiJP*IKt_f%+w>_snyP3l3Ff3xklo6yRUkN7|-4atr9%(ay$qOG39 z2Q6}$N$&JtV5qic-HlBM9Va47%I)Q_A-YwgT!YHR4b+{pcEmo;-#cTS1sMp%|5$Pe zhL!z(<&X%UV3(3h!H%Dup`4#cc4IPy~6)M*LOCcsZtPa6Hs? zfAQUCgONR36O1AB<~T#xhQnh7GyFmR#~ByWJjF+hl>zjPUIIOQYV)Rw=DPO>8)gHx-Mfj3kA6<*nRXgYYc5h1IT zTByaBUw-*XVw14Pp57!TvqXIMlJZiY%f?sLs%#BhNHKeoy}<)V@Vi#uI@IJ~H?etq8`gC4OQ6h1&A(^Crk>MEyw~I*k^T5#%Kd+&!vvWWV zk^)6AIeQ=RdL19b-n}m9G3R@^e^S}HQQJj*?37Q)G49^Ni|uAz!-2eB0)f9mQGcO* z1*IqOYbJE2d$I`&)fFqwSu3uC;!UrG?A5mSO&?HNI629%iJ3{9zSICSsWc^fn$@w3 z90%wNo^LLej^u94) zc#X|NY$Kr~)=T8c&{XeWRfC%j@igAy&jCf3unV*&jCO;_=kqPJu}f$qd~w}w{d(`# z9~u%d&c5kT=cJPL++m))e{6PG)Xq3!+l;C$fVtgQzaJ%HIAmXycrW_FunQ22<~fte ze?&cD-=1O@N?a5i{)J2Rbl7H!#998k^-kA23$vjo1Dg(nAp&q2J5zflO%31Q0ey0# zoQjh!!JqNz_mBB=HS^v4rDK|vHRsN z@YBOYcv`6!ocN4Mh)I+bO0sqAN$9DibY3p~POJsLghiuwutStdbS9PB zxWj3Avj5@re~Btl+gJr1^Z96g76ij-&Vc768G96SJDQQ**+CG@m+Ri;~% zr~andf9td3&(F^WmTUuI=WI#~q0^3!-UFH$B)nCBR;vc=xj{matyhXhhzeF5vb52? zk|J*^m4|WSwka(OOz^ycqaJxrg{oGa{}r9V#d@H5$3=y3L3T**A95fuWe7OY*I*HrnngldX5t5g{R4jI!ue9oD{;|kR zf?z($j!|RJhO6bo#b|fhRXizBmD~@FRfK#D<;ROjCRAcH@2~}=ZDV1qmv$ke)z|D1CEOeMVFqU6T zy~-aN;qW+a?Y9)-KowAZMybB{aQ+q+e|!jzbYgeT4nNgPKvsSloh^QdDf-ulpP&Jz z?Sz~f1}#aH@IKZHv(h)MC92Qm4u!AXCf1|Ii5GI`v{e^)A@ zBMaS1Zl@#LMw+oOd7GhD2QG4?}idWw)*~x7E|im z43xa>6=#;z`Dw*Ll;~Cy4C@xDrDeXI!=xernM7Z%5NXo%`rP2fPzsgygEyy7{_@SU z8LrhNW`O8!4{ie>tSLB5`LgCQP#LtFm~fbd)XBz#OQF9=jV7V@>M_ zGm0gq`%BpQdR}f9N<{t!c`<@D-593COSWw7nT>4;xsx`32Pdy>VK+FTgfTtES%L8PXa!54CVUlB$u1+qG6A ztxGs>8k2XT#k3qO^KkZ>)cyf#A}<|5sNr;U8`|#R2Bq^(%{g_BILU@1lZ!?BOaO8v z`r8j4ypzLXm@lm}WxLA#e?2d7~>7 z!xhcQ7o9j`OXz-Wmz|9pSo`f4I)$8lmC(b>Pimu_#qDBxlH^j|YCn;nX5xfQn2?s* z5(!4k$AU)E#ZE{@3D$6&{^kp9M`bsZlkiEaz%wbteNdCsR&v%|VBOviSmCU>N3d@1 z4ukDF4b~58vJ0%+fAt|}h%}f65@tHw=~g4Xm7x+phnpKoL2?kAnP6UiK`Z`Y5=xg9 z{CVU~h99ChpZ2JRdfW6aKQy1xI(^{rOqYU*6lEB!!)bqXz8=kEp#&APGTlxHnr(C- z)Gza_*{v?>vY2O@x61ga1HO8d?NaFw5XcUw4O zOOoVTSg2?`Eu@)WL)ZgV<|?d1XB*6i^C=h13C~rfP)oV4^;LdbmE=ry!ZzFFpfQbd zzn9y515{1f#FV**&J<_Z7M!|1eUG8-%mZNu#ki!a{$@Hw{hF?rpz#b-J`lGc4MsEU z58{iYSYH|~f14>6<9bpb@Jo93q@8*`%%3No(7n3Gz`@t)6Z2bke>t*8O6p9B_B%cl z`Rh8C#M$pq1mBxt;wXE@(O#q_LXG#0V(pWW8A$_W)JQ;YU$aq~?nJs)n+;yyO81tK z4~Z;{p0g^)oBHspBD$O)$=M6@l5?<$e4_xyWkDH7e_7-&?NxDYlo^4JxEb^~86A=yxTUf90#qJG#FFQEc9zri!u}v!;b^H{m^|j6O+e16WztYeY>-iLCF}-DZE5fEC(~ zTQKz^kMAvQt0JNAEv>l;`*VyDk*AN(*2ntrM1_GBJ#n0$0Z{t#2%ArO7j-os55&W> zd-EB1=6M}SXQNMShvM$s#FcH?#kSSCZ}HkDe}xy#wu*yXgFL&d;_^x>NvXxWWI~~P z!3?j(a;YduHt=@G&6?DUI)|;F36kC!{jAXz0(C*2h;DvvkOc1ZjhWPn?j{%%(g95B z!dGXI&RAx`wlia<(RS7CkTY_ru{#TyoX<8-HD_-glWpC5aI`Niw1+5%xB;1#?a}Q$ ze-H^x5BJyc5&bYXphLZi_V@xBzdHWqD8RM$1Mb!CV(vKUU4_mNCw#w)MJE&v2wQ-7 z0H?XwuGZSmCD#Fo&+T=C#I)Vi8^>=TWk-7F8wk2mN>j2QKczI~#14RFk}D^ZN^2Ji z2aU7h4#z83UpIJsYeYWa5R|o8#Ob|KAYLW6o-k@lTKbts>yV2YC%lU;p=&bBe{kk? zaBha#eFmhx7{Ya9xv9&}b1mg9gpTjwz#a{;D}zXk(c#z7Bk?i(x<}+eJ#v}UVSD=t zXi6Cl4qy>k@IH2T0Q~ zS;)d2HEe1JS#~JVAifgmv%%zTe*^7|v%$?^gzCOP`-*QTNU4XG{{ByTY{0qUO)LxG zHm8HXw$S9fY^K$E_U~_Ay?j>9q4*}wr?n_hAVla~6=+0XsT`@l_EF(0WSGwm?-+44 z%lhG>O4KYWs%bpapXy)_1fjU!!g!M4Z}MBGOB=9PV0ZIXU0pWj9tV7sYq~5$yFuWQ zwW!U^tn&jDE;iXEg$je{Ds^r8H0Dd`@(rmr%y+URN+tBhGAmmwGk zAb&%yFC^J$5$C;`T_XqSgq6Ln5{E?^IDSJvZTVsv;^0F%*2(oBAav}LGa?bgmjrGj zR@m*=NvDlIR4YA$D3x&5mO5)8#iP!2leHkvR3^U3&5;B=GPi3-4-pK{XUHSNH~SP9 z5{BC7L=oO|?cXYF6?zn+Sp_xAnt9-*77e@Otq&bc zqe4~9$~?|IS|GC31WM5~`x4d%;O*!!)ZOY^#@s1RA(Yz-y7kx)afuz6Tf@Jc@1b{L zSpukLTclPoYaNEP#>{tCu&7V(k%|*E`!PSdq+!D=C)G!gn3S&$j{jJ!g zKd`lzG>OR=o1vW{93rm>idE>(RsmJG|mND#BeHDIF{~8kL9k9Oxkm z7X@BhYaC!alHOxp$N(N_TfR_q^q?0_04rb->`_B*DZ1RTcOb>+pK`8Z{Gl^&Y=hPH2?#eV=1+pc@$x7)s{fv4I3vjH<&_9>|{oWHW1 z_dd1c=B~;|GaI8Ka{Oe>JKFp@fohAH?K`NqqAp@F%8l>#0u{B``c6g%0IIDe(Jv+sWMp~$$2iP#CO^OAid(^5* zlXuUyLu|E~nRI<&hbTH`X>u((1E@+Z#vV|F=Zf`{aT#k^dx#6^onw}7)ZQtZc;BXO zw#P*pQ)}aFBNeqzDRroq4F%P0$A3+Iyv#R}Ut*^mWVYNr^bb*wOtJ|=|KYs3&N!mh zC}$(4jiRCYLY<&79rOo9H!dv*I@i&5L~qT-l2mj;-(p|e4)XhvbB8-vzco~8g@$23A8dt$qOyBBiyEKky?|d!B^$dHDJ2t&s7k}gInmW2p zcc78O?Nr*=*F{E^iOQo;2)Ki$?16k-RQ7RNiGrb|MV0WycB4MKF7TMYzCIfePl+uu z=mWa=9(<78{Kt{NE*)FzU&!+n5up_w(XcH!s%RGRv_BU(7yFv#~h5peLrD`)nQrB zu?i1jI(T6;G=niqQd7IlyOUd)0ou9AWZ|nQ4iZI zylcWV{N~9i8(tpjD{EKKi}ctfs=!yi^EtxqrFasV-tY`=t{*Z>zz34nhEf>0cl451 z;lgO(Hd|y;E;BTuI)8gn_j#qSKpV#+Ou5)}IpcVEY|XyfVnWXB)k(XBr)S?+uuNxY zHUakU{9G7mu4B7NuY*)GAqU?fr_!Fvlrhr(8j6j`Bi3OMlpj0#gRPtr+>-6EFWb5l<5onh{EeK zvKCLOzZREC^rcx1{4)&3<_$>57PiO!x^7saE}c~dt38mtQ@dH2Unta|43-!W?FX-R zc>lgm7}ISJVL}2yo=!-5iWY6~*9!l<@zX?Mh8TYF$^>ikzTg)wd5nbW*ay@o5WIOa zr;kt%7T5WFTYt2>J3~X){~bUj+^-%O&iBT-P1Td*Pv(bhLf^Mj;_&LoEO$ zb+0R%8MI@r4HyF>|GIZY$ZBso-; zsIv3fk3WsG#OdBaSW^yc3Gyn=*6t)&LDC29a4JmTk$=6PS-Vu|&5aV8YMr9r^J(J% z&^?f+NGo)q?c$OBCba@!U$cqfTGV%Om@(f)QClkv%7Hnt^UVg?NcFdEUK$4<*;1k- zVbXBNQ8rBF$T19<9fnTJ=MrVmCQ9hyjyQCP^8L7$b(|_D(OY8tn@iV$HYZQBV{+JZSBkB62e_XLDlv;86JRJ{o zVn`IZUkWY4iI`PKV^Xhh67jsjfOHsCS5s0KaX?cT7bR?q_#vH)+L1(t*b|YKUi}Hw z%sBT-Z~Bf-jmX}0hOe}`iG_qGaef>(ra+NE!hZ}itvl$BqL}-Mb4QbY^$EO60>^w@ zZfzTLPjtX;QfN-CMC?$-n~lbc+s$lIT&QSqI=kg&bx1dzA&J2gbUTlf?zRtEG@;ny zv5;NM4J&3#ycU;6D;a7r*riIQDG4u@eo zJb!Kw1`}?FhuS=06eDSF&oZs7gRl+vd4k(G^e$KM?8y0l=gp=?-*MyY?QaxUE;+oz zox{_}vYcSTmgf3^NkFLq_w_yut>_jnKg`>HADnOHaen|b&$I-5{^7p(=0gp}WGGmK z#im}*7Ta}M%+beM`z=ia1vz$2Y4A+^;D0-%*3$uo`44(~?Ox;ng7F&5RzLCoLB(8e zf(Ta4-A9^;Sjzdiaz+R*>2%@r~Xq z(T7_nkQ{H5z-fP;rnFhUBgY0x1JQs4xO}pKp2-dfkxNT_6mx1)Tf_NeUJnvUz<=n- zon{HHp-0o4)G<5Tj~~nFn}p;6iL{79d zJg=_{6BvWyaXPIWp;j_8noa#02I~UydYPhF{ZFzqNa8?>6q+THN#m$wGC=~#>j?x* zs>n@FBY%ywk@uJ`@?O$J-bPNoB~~7|tevQF%hfy(97&L->q^1buPRo(BRSFD zwuK~oo3>FR?ri(R5_{e`nP=#ikr-7VZXBf|cu{w}*v{D~6i2&yvA*OOXn#^qVfwE^ zhNSicFeLNmoT2?cN}{8mdNBHB66JVZ-a3GloAuiGS(4^NIdh^cIX-%QSjwEk&6Dws zR=@3(Ic|y{m^(*b0@I&vC32EZ8F+QBz$P9NrxZmYAsP_&O#>1!Q2)PX^YJHOE{X~k z-7kb1Wpn}0D|1N^i+?Gb^?wWPM(vz~(Jac2%{qeVs!(XIHo=~i7n)RPUoec>*Co&b z2rDE-ySB|&>sANgY6G~=FPSa3C^@;v$!4T87>FV1)WA(%k#hvz0#34`^vsqCB`(^3tf@p1nIK zKg;C0eid43Z^&OyuMuWm;1n5{{h@AcQabCM03fsxb((xVme8&^@tg~I)ixy&3`?P? z95ampwOL-QIEtA2zRoKq{u*_`4_G1O=-$^s>}#~eMCfaFjf2=werr`-vhRx7 z!6d~qp>J7pYPvl>POe>H*G}}{VigE?$oq3g`_EV>bTHwCx};wTDkM!dbYBmL5xA>q(1R9JYg@ky z)w&oE+&y~(ygO~1eD_XxA6bHa>s5op;Oj&XqSXmnTIT5nPc#6=SwdfYqHjC%bfPEO z*FR;6b1=HuO@DL47MkN9f8zhVtJ}117<4k54JL2v!aLLCV`PwrY9yQ_&%=w|WT=c= zG8*6BmFUU4C3%Bb5DdcKycoBRL|Zm+E}djc^Sv^oOWbW=hL>4zZ>(V~4henpA=)1( z=HVgZEV(hr(;WlQBey-9$VlgiACe!C$-Oj6Eu_~^8-IHQA8;ne=WuikzqL*z$Ur4M zO{mD%X|){XjG5B#yMbhm?rtycdzz4Q0qj@S?lE7qb`9?A7;iqFr2>hLGulM4S$2pHNvqYK_~e|ZO*!bQigD*gMbr5c8?Q_p@S->2AG7{q7Mc>5km zZIWe=C_X{Mv?zA(<784k>V$V7GrOi&-4!e3LNk``8a9^MWI(JbRnW$`pH}L9+t1T_ zjemYUqg1kIzD2mji)&}JU#3K%*E@9dI?{=?`8VGmNvV)HhyE3c0PF{g=?3Y(ndK7G z>bv%W8AaDMU=owJc#mYg(n)%POBnQ8R%w)nmBbR4hLi=}QS&#HevmJ_v^hqo7e|(; zAG{35C|_96AClAJyJD^Lh>-NnuE(JXlYcjS@tzk}!{HRM?|=VEr&|}ehO;U0gfcmi zpvBZ_=}1#a4m~aW8WDnP_iehPQD2?v{9$5c)WTh~omrMYB_(_5t(=s_jo=={AZvTG zYU1`B|8wLrP7$4)WKSBp^a5oZe`HfV+vCrPd8{noMJ2RBQE6b zxM{TOe(KC#92`}nnc$bS0X?nYe}9V0#~0NoIsgF|k(Rlw^y5#Nmk8>|pC*%u3>g9` z_3emna&IWLV_L@VkeZ3VvkK0!-?8k~>rl9|LXZ19m$%#7+uM^R?Oxi0k728aUMQbT z0nbV$U}bTa`nfWT)P+g5^I!Bvd-K6Hv{%0)LW(3b15PT(8Vp>O_pHDmI)B{K^(IU* z(ieZ;;->n(i|M@)g4Bg&2hq*+o$DIs4!_+M^)9*hfj4TwYIJV+PxoEDNFz?~?&rmO z|6IJU&8+F-TjM|ws_{8CBfY`=qO3r`zLKTskcAT-nBmkRiyz&DJ0DYRb(8rH(7j~F z^5+3xN1xvW2}T8r20d%(NPqKVB!Wfb=tOBTKtB$^XC4eK^$*UjX$1VG_cPR1n2%Fa zI*ixP0NIboQJc;qU2eNRZtG)|!@D8*i>Pw&6RC5cdsbNPSLc|Iuk^C!fb9Ae%qcHkVKZC>OC=s9ep`Av4cDOV1`hxQM5df zC8Jvcv?5@KW#b-J&L?#N$>Z6)40SVhp+yVoVN^XA)cUi-~s zE{nufq{qj}!Qck<61pjI>$mR1_{N-MKMvA zo5Y0Gdd}cS6#5?3rl%Jb!hp4-<@jpRM!uYRZR$e!R@lyHP(o zr6HIa^XA%B2p#K%zxz}mLZT3RzRMdw{X!1?ppg3!O4~N~%2c4HUnoe%G7a`pEGi$N z?H==^VA^9o$P09!4Kf*J%<|c4Kf^tM_mptJ{xF<9K|rR4Ps$IH|7TIbrf#CRAJ}iI zvVXPR6VIPM?MqLotN1CdL|HS5oFRiaqg#_IymcuEEb`Wk$)D~$N!6q%EGbv850A7Y zV}19Ix_B$z4)MQfG9pNbBOhLLi`?>Zx5E(JPVbR_qwU?Xw?&*m5~C7wruG>%i#p#K z6z;-FU#dF@PhbOhn?#FHCNO^A{_%2+uzx}*?)4y6@`axb$t5V0swGZ+=30J?9!DEI z?t*##-cS;}UAwX-RwoSx4U)K(78EtpStW024>?0?4<~eWBbrxm1{3SdLj7Xn(>j z`fyiK>UJnSC%N=D)XvJb(|Rmu?3}ZbP2l#k-9!`7+bj`Dq+R@e0B)^Iv>PQ3QNQu< zQR-x~`d5o8bECH%;+nPIg+&cH-XNQUklB6d#^9b4*%8U6%_MtNqnat|7Girsry_dT zLE_U#T0-OX0otwu^;}&Ej{?|~ihuK$wVR=Fd9n~5O~`3!=+Qi{L=b!}6z#zx;s#ow zA19AgVX%$9sKc0|7PVrpV)Y(SMAM8zNY%H7-iXwvLB>}%s+e3ksr^8bb&FI|)W|QH zNb1Acwtyf@VvLst;#EKH1{(4q13B2;RD(1QFcTJ5rvyQd)ZucY)JP}WM1S{Cy3u!R zQ*+W|Bn0TRN2xeP+E;pQc`Q^%E}1$n**5N2c2v5%cV%4{ zZR4ESPAax-+qP}9q7$cL+qPX%#kP}*ZCkhQbMKe8wfz&;ZmZ2T$LL+q;_rQj*!7^F zlv{iLOoCvdm|JvJbzMVbpH}*N9$J}`;=%JJU>YUq0icao<{KW8GiU(9fhaCv<~dR% zbw`7l3n&Z}rb9iZbzCe-Hbh1=EnVO}5b-9h{q-1mwO*nT1sh zJ0`ogTc=U7+-ei2W03!CzygnvCf#y-0gFpq&lGI7r)JY;u*D_N$S+g1>{W)XjPreD zF4b=_WwW>TqDbO2m`z5111-`QPA-D|rt|!>ASWCC0o!DWHR2!U>-!6sGWi3R#9UVwd(~cvVP3Ba)FP3-&8pg zGt+_%w4e=&W1aX%Ie%Bo6Vu>KmT&O1`amBmp+mV2;7*W@1RC^BHG>bFO9^cxI=o8- zKA|9@Op?9Cb}dBN6m6ZCCa|Lyf&TbfebUq9E-3H&={xWT z{}@)6-OJoQIpc|23B;lw1+hZ`Ok8=vAT?$rqUvE~mgLaF_)bCX}LO!jf zrOcW$ySXFvwR+#~ik)|)yCLw`cXx1LRHK$6hOQcWT3y+T|JI#%4c=>m&fOklPewiK ziwtnR1o-a}>}3AX;;YwkCO0<~u3V$3FNEEm-?8ddTpa`H2cH(Y_JR_xpySW(EPkUK zy4T35&f1LxLB7Z&pUvHleDa%VsFgqBXF~Un^TB z>Hodn{L1r9O6sMFRmUy#g)5LVQM=mtTo<;Wu3p#VfqtHHV#Vh#Xx#96(}ouy7o5$L z-I3I%M$P4y3&cI|yTk;!tT`iq_U`fSYcdDjh9g`$#lv z53BE1l@WDN{ra!*f! z)xIWz)Q+^^{$WZ&3|gyY6dDEPN}E080cTFH5%Ovj^wKB3X+^N))a()qSYFbl`Zqw> zE?E6~m)0R>iLAs7=;xebLny|gVkE;cupCNls$0vn3>f}bu2x@|Qg5M8t*+LIP*~&L zL9f?Gmjjo^2sL3^KisU}7e-+!-tZlL$oL8r^v2wiH;@&Ir-QvWn-S9|&jZ}S z$=B-=kO|n_bbV)|@_*h-16?V;0CZsekyZNm3kp2=lbcnfpDK3IrK<<5o8bsjs@ zBU5)UD0mtkY6VE#5%U#oIvYe8zl`cT3_PCSuB_s+Qwv!G1wmYp|I+dB%javj4kqRG zTENVN-vrl$R!y|R8&-EmXwmAApU}_4Rg!Jv6NkQ_uu44;`U;J~z}e91IfQ%;+Fy^*<~uN2_o6QJ`cK%N~U<;M|16AWXiYWDu3Wi+wL0HO_o-5B9SGMa~l- z01uo-^eFOivNE&hHD*#}HKB<3_p#%vNz`mXL%=S8Zai%OcvZO!SLy!t@!E7;I*ou$ zX9Rukg02L5vpD4x4Ldtd@T2j=twY~Yf|$8B{>Pl47vDz>WSAcMRKvM5-aFWc@++LH`>r#NK$o?Iy>T&Fslc%FJ%`m>w|_8C*E=Bg1UxXZ8@z= zSOCF>#j@fvG1iR^ha;wTAWO_w2}*fy18Lo(x^kaY2&msj;TG7|AZ?nUGJqCTH_cWV zzz2B5re2Z#v+J5xrwsT3g=G}CjNg;CpbP+_M`i9>wL%&loJe`xc*8BTc3Di}rdg-} zc!1l3;?sBtmd7<}r-7SykPJ1g2n>2nFleM5PgJBvvAj_SF4DDP`U|;C4N@E6_&%nj zd}wrqh9DGk&$y_ui{!u7F!Ls2H2i3OYVy*FV5i2fTfSNkj9LXIsu=RUf2~Dt?qGtA z3Lbh~xB&+O+jkACUA2#8CI5g8nh~zz|8Ci})zyT= zBQA;!j5)*iq|9<$ORDry(DY#cF{bM)b0@?l_ur{&~4k#Thda z1%2*^MIwM}CD^t5WjdnzB$%)}4|4CKYmdjz5;5mf!5pLr*VcAW_=8PlNob~bt9U>$fY@fA1RI*wka7a;@YLK?W*H4Td;1+^||O?}}p}qK7D7bBaKv3&}JIwc}rC z?!AA?fG>GaE$^-ON6#r(K}9i627{KlqFDC9f%L{^)>rQ&u4o6wwsGAu(l|JhkKXR+ zN`|Qf#6~Uz^!Bk(3)v}G!~at5Gf)YN8$Q__gC!b0jMcZh7qK~!6Wh*@p9jK*qRYrg z$(x7A=@$HhF)zT=NHs!xz@RH9`VOba#*a;@_q2VRhK znQo@&P7iy$f1$+Br`a!y=*h5zzB)JF1|50Bgo=z3YCPk=u8|ZkM)7|?g3TMF<3KqK zXmv6rpt}4+nFe<$rU0eJ6#|=Q`zXi0un!^r5f3J6qNB#~EIOJ;DNW2&sKE6u!h*`0 zo(+rR10O48hfmBAn;80w{XatHy!}9ip)SNk3A|H5 zEoK$Bs+mlj$ZjD_ZN>LZE5h?nhy}BwG|5xI9)~XCQ*!F~LoJDcgf? zg~DqFpu76&LoRe8FAUsN+ZS*ZI>5eS6F)MHbE%O?)LC(;O{f4#Z_$5XS+&YJN#V#A zk{@61sl)R()dqUv)&S>h)pBoPSn56yD@eNF$bgA$6QEahx_gJ3KS;lgdJ=R&;;_gB z%H&nHiQiXHu;>VAeb6uHsd!!d7A+%~H)LOGHv$t25(kp)sn#tu`7StkCh~8oD`TvY zneCvkK;jz{?0L62iUelA)Gs%&5x^zIJZ-<=A52urIfSq$x}O?SikB%?6H4=>xBW`- zLOmn}PQc9Y8uV7t@7b21(zIn+S_lM(TGgh0QbE&iuGg!>rZmfovZSUa<6?f8Cr?p+ zMCzmPrUF$g%%|##ja@n1HxorGt^vbq?Ktt611RoDZ(zT)U`DZ=y7?l>6) zBcXeML9qeMylZthmu{U+N#JYJ4ALdGG4t6VGYB1;!N!rkq?`~$a+uqS+KwTas5Rn? z^_^}&ZqL5P>`tJ{!R>?s@WxfKY$)K^L+Af6STG>fxH5njrE)DN3j%SjuXO;$zP7I7 z!1L-_9lQ5`qV(%v47$#V@;<$m`jJa)E_})@sSwCKmdXQ)WbnsXTTIU1h_j^~buqWa zwZ5|IT*S`)Tt7d^D6A7Nv$;^2hS5c+ap(L~i2XuRDFg_El-DX0Vdk)%w`!LoE{FtZ zI$jS}Zj8o6*8ftI!iZ{;iJZXgkqo6P13P4kSno7=9BLBZm=-5W~+r7egqYe#Yz%M;bCQQ^w|v@R$eAgtK~BA)wHLqmK0z7Yr6>e-GNsVx~f` zNlHUSYYRS41XT=uA=0%ApLOygc_egKSxC_rAgm2z|b2WAIORL+}>O~#7o^qQ(Sh_l0S_nP3S{V6Gj}`IqG(B6A!osC_oU2(26ii|X)Hs`oDlDsWGvJI7u0 z)i>QQIe$2R-zVB;unUG8D=?~7R4JFz!ceN&w?QzK`EThTrW}0J#hbrmI49hUsXXd| zdc)PxkbQHIAhBd+l8>!}-Bg=iN|@zaN*U`5q++Jw%Oty3&-JH;>lYHHVx7%X)^GJ6 zt<|G}JVKf~=D`MXJjQ{k*FYA-8m|~LKnhCGcucexs2JlV*Wjwv%%fMl9Tq`umw$UNb+>wl1!rsY^iN zgQ)Ufr(EFZEa#b1IIv(bL%=_6qRyl8-)!nEZd*i6HWt9?VaH%CikEucYO7L4oWEJEbk4O0Eo1@1n#c1!W-yPvu6|?pkhv8a?E58p+gb6p8cBoQ8xM1wfPlBe^AkR z%t%-QlcP$bQ3PG$Scwo6Qs@_)#3F;8Urtz$mP^jCUk;JaA+A=tQ}Q!&h3q8$~)$?k}0c65MmsD)f{ZuhzzSnWQcigIRjemdwIB zgB<3vJN!((&^pLMN-6yAE<_u8C32sYib4{9n@=>duf6j3o0G*2-BHkKSIfK+h%0F%rd00YD znYW#fBlgI{;h|O9&3P)Byr%V|=8xkWUL_f8h=->U_aM z1}KZ6nc2|Zv=U_MUpn$Mstc%1<5mWyoIRr980#IvC-4!3s!;V~W;a+j@S|J_EUAcx z=0L8`$}PWhY(*P5veIBQ#h05T_&|+3VH>Hz@ov=017;Ovt9xEqniCyCv7!D2jFzz< z%EWHv(O6PLcv$Uj!-&FiBj*yFaWp(}Qf1^icc^2l9MZ*e>ObQ|i$=!aedLzLbs4F+ zT1g-1oz;RAI>PqldBxwd_iv!Q!;|nMc7b!81b!qOR8NFeR{~imIW-TqF8dcv3PmHL zk#xz=@brxq`=MM~hKYR=pVHN6e#s)nBy1=+D7MOJd6Ip13M1^DY@h@@u$=w3JArQ8 zx7GTb^DJ|jk=y_z7*%eFjD|lcia`pzkxyu(%_B;P!qhk$w|c|SfNmq{Yv!8aQJ@gc zZq8vSAX$Hus%Z2>0IgG^4X6JgUcTx`9}aEhMhD2g(4z5ggR_sV!RA7#v*%DVsttX4qEHg6|XTC^^Th;);-4oP=HCyC`{=O9F+y zn#JWAsS7E(+$XkaMD>R;3efw#4&U-RGu3ly*C&nZy&Q%$k=l@F5BV8IVkTWI7>IX{ zS(`G1I^sm~R{u^^B4L3WcbY!LpAcajEHKPxR&sM7&T1o(+OUvENcrtTGIR&>XrB7y4uwB62+_By2oBI#)+cAS9I z0!dx8E1-bc$?M)l8MT^6V!PQZWAgh(f;VE<^O_QqCY&eT^TZ3mY#h8&FxxBZFJ>kv zTjJcE9RlX`U%k_+8{mbZ!{cm^;c`zJ71b+3GP-f5#WS-`;`CX%a0KWSRz`klSU)=z z<}vPUWFdAOuce&-P!8St+HcT;kXIu$l-C@RnS>u4eESn7SPuid@4KE?l4g`5caMrG zC~s@08`Nn3k}99W3LXn)D#r<9RdVU>{4{uzDhTMpr{(ci=D=JVaXV2x)V3)MYIXvY zI=CFB)y~f`%8je?wXIXoPuUDiZOiz;?9#C|`bKR60&$gTH#zOehuLVtHEbN;vs}Y^ zikzde_7TY#Aq^RbWZhHlhg|fbo{BRcR@@DwE6kZX*23p|TOYU)yqNXgxed0jN~l7ADoZxj9r*3DB}$@BJt3TX+ZPi_k0f0KyG%mqlCh4 zL*UWm_b7U7v~pQuQYxeR;^Zdy%ALXN2eVf@Tsr<<)-@>OQwhtUdpM8aYXPT74@3?JO)-$W9GK z3O5x#u>>s1b=cRDoch_(F>R3j@tm~qx;6&Kk+>G2Tt1S>xkt)*qWlPHiZk>3G&w%K zNj>|b{L46>oh2!)_fpI`&mAF3VdAJQN>kAR~viCF;3jjONJn+matNm(rJI-k)UBZJ- zMH=+|LYHl8IP`M^;>XAn|F!FT;fle~&^Dld~QL3cA8u|$5y*-7#j zEpw|PR$FA$FljhS(k6uDHu)3I?Skw|eD&ev>QVf5@KkbG?X1d*g}Pe;!O@CmQrblI zG0ARw5^5muM)>snQf)i!%%&a`Pgw%fIq5gUofOp*+b#3N=p~&ChgFX9R%7T&@D8r| zsW6iavz{wnjlwL82PukGPxuHinJ5ze#}CvnC#i!#KhLGSe;s49iMIWWdVtVVnwX~zYY@2&duhBT0#m29))Dj+SY5Tl>aKth8Hz6Zop!k$u?=f z@G0^C`LTC;nxQTS?>P#!S9XxNYGG`7*5|H%ZuTOD{G1G&jbqNvNcPQd5mRK=hGBI~ z=&t9jF-nJA6t}RnP{hmngas4uK~JW_NK?^TQA%%@wPk)QZ&TxiFw>2T2th3b~zRK>o zL(KM9I-02U0{(*o+{yd5nj9|B6QKkG!y1?sf^c?c#gYrtgN@nE-0(9!hnpS*#2j)0 z3LR1mkDFxUT}M8+poW4--Dy&bsR^4Zy34t6ZaPPPKbg4kgWv|QyG#1*Bn@E-H6GfA zY~l5ove0T!4?OI+?<)=Jr>whb2C9Pz9;`t6Hyn4d< zaBbCA=!HJK2+utDo_;4#n%SHoFDpjG66b5Xn%Zou1h15E08S8 z;K_V4Z6ChBOcczkKVrrvfDhpa5YBMlotj_j9#RSGBPqRP!sJE#NH@~|nOlXf({P); zYG$%Hh*Nq$t8X{ZnqHRqhN2{hHKYT1g^Dg_l}C^_8{#jy#eohtAvNnJag?cvzowM4iqvgw<1i;i9~`!mrORzQINzKw>yp# zC*ied-le*$p;Y^$ZqYcoAKE3u(3P_7ViptrpFkRz1r~%-I%|gor)3b3^AXDbD}jR` z)S!-6Zat(dvjLsgQBg}RRxBFE0~O1BK2#|x$dWuK-eB*|MgLGXRk;(pPpncs1EU|? zLiKmcpjgrXk~0R>u;~bxY>xjDzu_EUQEj4+VHvMSlu*vtGH21~x=RqIduDT`K+A|X ze7=FvHXu6t*2s=vI9EGQZt20Nyp>FM{pVG}4;f0kq~*@s6XMajdw1K0cO4oF`S2p++ciWF}l= zy}=eDOCV$OB!CZ4^;GSF*>DhVb?L47E=(omg$9f(l@F#n<|U&kG%>cW2N%-z9ZWSV zS~*%NbSbF(BDu8AF}Nyy(LX`0YuN3D(}X+#=s-G>nGZIB>(?QsG9`rdV(166bkRho zG5bV#gh=1T)y;i_A~S@0)am3w1w$2PT{lH9cAuQd?cr-ntOKMTez+eGl*bE0+k#-y zdu@&iu>^~3v29wx!9OW2N3fa}*49jL7cN3;+tRayk?l|f(5d<`0V4W$i-(FdaA_Zz zlR(k766KT4sX*|3FVmKKrY>qP9VHSQ3Mz!=*>Y6-zC&_s1D8HuD67sQd8W(*RII)W z+sKqqJ?odWL34^f^k%SGCy<>IM)t+=0mdaZ@lxEBQf=HWoOl%7N`nAI#n)*ggD8v4 z+XA(k91S{prax>BXaZqC1IpX6AvZ8X8(=HuE7yutms5lrIs?5yA%`Sx2Gcv-3=fuu zOm#60H_Tnb(CrX1&cjlY|`gk9NLvoLe)yo52h$`xp?{C2F%ja$0*YnTS;zCt&0}H$WwYtxT+rMlac*QT`;7llFs@L@2Nt`>}#poOfU!}7a!s;#Ft~YDbLjA zF%ij?j750!*zPAIG_kjcfQ|iW zG`Di+?PMe{3Eepvn0(GOxhLsHM`@LoV>pa0An^u4Ap(wy#9-<|KunupzAUOX|x$ea1 zYH#Ocn09jw4QNyU3819=FC7Cc0l;zp2gElRM}=Jt00F@k0t3PQ4~XAZzXc!#Mc+R^%3^4J1m#gMEm?jzhvU2U3Yd4z2&xK})oY@o zfNJBs*+KB_b%p+U5PmFZmgo1R&7qAk>zqnj1R`jj^m7a38H9Yh->ipT11`r<039 zp{tAo*J6RUfn%@8mI~Bnw>-* z*hHYMV>WZ4j5qd}WuzJJ{abQJr^W06D8L~3(^~cb8lYim@cRHl(9|@#eE=gUFimeC zKn@J1PbZXbXp8dgVTeX{?v8EjEm01?QsN&!=E`}AJ)LI}Q*YY>32UFUR{AUU6!_$= zjqT40XsG5C(r(^>@tq_QRZcHy&Y(x9;~NW0Y#)t}YH^eJ8LC4lJiiAI&0(6GARJ%3 zcL}yKVNQ((os%fhm~m?byAHqDRa#PFp$UX~c#x5bVlgq8m=0AmYM7ZI35EnnH8Npa z^wvf#vBO)PM4W)`P3*-!!HKDP1STFFhosu8?`Y_g;w{|a+%+bi!X9s4&38GY;*o2~ zMnM(BJdnO~ds4i&0HR(WmAGc?>wnd3_?wxpL@Verq99#qUKViWELaQE8HSsxcLUA+ z(yh|9RK_%i>dsw4S-I&`AzAq!OhfVo(a*pIX2Tst8G{$(8tNFZ>64|0f=_*s}91cp$!u4Tqwa*5ejvW)*GM}uI?d$8@IV7 z*0;4dkz-KT22NhzcL*D%Q~g%2Vli-W@9Pfr{(hX-+c~HR9*!Q3ofVBeo~||aNki}C zmwSwGK@VjVxA6?+ulu38WAxim_cSpRC8%5y;v}WTOmTJPpM?0f(ngYYZY+ zX>F#lwq&?XUT;iJCA9Y2v-Vcc3ySre8Z@9g9s+AsE*s3nB%)=!KNt1+!vhh7!6n(H z`lsVKd3u6&R=Ko@7Y!I63tSsET?JdeO+P8s3K~-k=f!1fFs}q)ZpW18EYSi}S zgVVVbzpyj^WnHLItRhuY33t$^FP3bj5>u_C=-tCD6zEoitLA1MuB0@x;jxCl)hk#- zIc4<|!A>dWY-#ON^7;7i6c*y!@5IP$i{a?stvq{xF-e%aFR(PK0Q_|g^1wb#_2|F4 zb~;e53i&8$a}0I;N6)|@mVuBx$u-sPaZg|%`8)cOwx~;c%j)7tUN*oyAViP_4lgb9 zHF_C*wJo->j0vgNQn31Ou?{K12iFkccD3T}_Oyf5^hn^_r*S^n`Yn z_+I#z|BU}$9&qMM1=u=^@cF!L`}(K5H+$s~7DN}>EtcfL;I`r2b4U5(*>d>AQfQ1L z_fH>gR?kaH)*P_c_B0Rc=DN4_0m(c)k4%Y zOD1Y4ye_THcUeQr5JeXZ`Ms9s1C#8bj6}*is5TpQ(3PkLSzAaYOYsr^c;zmSl zHL^lTW;hBifbMC}M3+UTUvg7`?pnf`auFpAFE=$@rd0zwHb+Eq7KL!b5}jpOAA3zl zVwdIFRTBmFEAeARY25&GLb9Q!CQ?tKDjnuI)LNhjU)g{f!m?au=2;@Nitq9?uch1Y%i9^XvMK|Z;wg;M86?2`1j6Z*sM;1Tv)f6<=Nb~??LD|OEU z)FqG2H{I+^lwN_)oVwjDSv*22OtK6g4;+NS31>+uEY0%-kPOt0ldRKebL^S9Dvt;s zvoe|chw-t?(LhbY?dd1zX{>f3{0^iSj>s6vG+Z&s)x}6+xAVa|5^BoD9x#O-h(SlT z<<@AFJK*dIVwE_bAP}mFK_;YE8Yv>hd^`-WJ5ZVT;o-f}T3OvUZ^xe65Vp0lBJZLtNaSKFqf=kl!qHiaSry!G(2R zMgOhw%U1nuV$ip1!0wj^tFoW9-$B0aj4{$==2)d(t43UyM1ZRRc*T3VvH7u7i1bP%3DF4o?L~hWv&G zDvfd=?Gn>g_7bFwo~+`4^4!@0+8{-=Y2u?N{tGW;c<9j7V+z)GgwSmMCP+oyh&_EHfmQx_aflmCkmSJ;Ed<2_<)ArHc*azuF zpC-T7uT>!{N;L4`ZB2N(B9WkliHet zZVa3o)?!iOyUVvkdJTNA=L~kL#3g1{u!~6c6{4^=)IO=Z3EDQ~G43r@l0$7+quBPB^*5I@9_! zwuWo*lH|&cVBHqW>B>IlMBe9E5JntQ*a0Na?Br$Y=DIiy;Px}p<`?tln0Q;hXJ`=jc)1JL#Vv-szng@JEkwEbpZ)c;P2KVoF)Xk4J1 z=92#)^Kmb3W3%+@Dr_~xC4gzsMM#6QO<$YD(bZ z6?OM58~7qpmIcHt_tevmKVB;SVw0e){l0jatC{7sbNBp*4joF!qu~k2L@#8`RIs^L zC;Vixj!>JyB=VfAiyT?Yke%2VC;sABEf=id7tfb%+prjN+F^k9HhC)U7z>q*F5YM1OH*sorg^yhYKL75Y2^bH6=L z{P!B9tD*3iE*cNqS4%>+yTy~7;!HHp4=Jn>}Fo0C@TEf*QOunKCJ1|Mb@*@J0; zcwfuT#OpEpG0&&K1Ktgx9+%ebT+JbY3f8CGn^ss?-C}!-Hqp($`~TeTX^oeFOkm`o z>A>#wm#B8hyW(hG1_&P($T?}Y>HPJ@2q zY9xq0R+Xbd`K|$#VS zZ_zFEm8b>p+{X%T$aQ!*Sw1ci*+}AgE$pM#;u-I6xXxGFulr``fOb*BMW9*31wmpS zu2cfl&?M}lJUpeEQxCTY_V$jT;>kYR$MG)#e~-ock(J{J8lT8E^D!Tt})PIh7D!d*3OC2=PVg zHnDViCkJ)PtsU_k-|L%FTCK^H3g?}?=t0DtwrB4rA?^|;N%+u*!_=Rm?zD4oze+8m z;0>v(4rcq^37!#T&IF*(=gquQG_-g6Xw zzug~VfF*_e)T?LR4?1_5_*rSvBC|~=C0u3}cvRrjEWD>ro|E0dz4nDmwP!8GJU*7l z$a2`XLCUx$y=6upuIXagB4u}oPIL0Mu!pLY=q0_X#inzqXir3tq2}K`zr*&+hDA2g zLm7~33xeHM*?@tmnT9Ek{jHmtBjoWH*w>DPs}##a5d5G_7p1i~U{IKU+0K)QU6l{i zMM0CLaNuPmJhIG9T(LYgFr?}02{y5?`&oC&6@=P(#A0DY>X4{N`}v>`KOWX0ly&J4 zTCQnXd0XEl8!GLKh#GbLB}#?|PmE~oA2@n9nA}K$a)9LdAPN+(U=x{4^w5|;a(bd7 z6}Kh7#UHSphPpz) zZvO_;FAtxJ9BS;QxB-9Ctjn1rnzhw_2y98%D6c;}vM zh#6`uIvHm-Y1Vj8!i6gd;wDLxlG&Wqtg$0)^N4HQM{{heII|{BFe#ew{i=-v0q=-y z(~IH0e!4}tT?pCzS1j1r0BlZIQFk-blqL9!Py$E4Aa|YnjFauRd*l8~a7m2l;Eat3VDB=O~VN<_NF;&w!*M zf$!~!;`5;j)Z5>gXDsf@v>OC$ohdf?PXSX3eY&#>{az0H$fz02U=;X(L}HH57>^)4Ok9KG0 z;kqo6hV{%Y8;lKF2DGN%CQ^1nr(O@vQ@O`oNhk{Q=`YTjn1+luMmc`qyo=Ti-l)tz zZlK~Ta)Y-vpnHbQ*JlTe=8pr3of%k7Ptw4r7-9@xJ!77T_U%N)t-q?>`04I0bUi1_ zuU-2Pg|`Ym(*G(Oif^$Vq=PO@YK!eH+!3_sN6J!;xCU{E~JCBx~2DYcn-@DQ3lgJW#QV%j`G1y?cT1)xDWU*^CL21OU`u?6o%lKNTcV#%yo@( zjr0|wFf7Q4{1#ouDRTM9yT2KT)f}t}tFQzt9vai~dMQ*%2ybu4zKyp;Si$10KBAUK`(tB4&8C1+7HSOH@W_?yK6K|rW-|fBMwNk6 zY7$?Fy@QZ3nA^N26(T@)&Q48^>z0Q>>P z9Er!JSG)uC15J<;k0;WCPfw3I7k~P%>Gud<@GWv@p~)0 z!W(@iHy_&tYe>^umLRWKe9xM{`)Ok@$xV0GbvFIF<;c?6=fQ&@jTKwh=o>xBPTN~B7+JgVOk&yP&WqXCsAQg~g&R$_JCuSE zfZh`2fRgdn1DOZB_auD0Y1!St!dtgv8PALRAdD`XlB?}}JE~N0 zV^=DSrk&x;CaAMSgNc(PdD%jnzTnPU`N`{|0I|f4TPjUj{MX0IrE)YMpZ2rE`#p;*~v4i_B_3c)X z8J-kDKyfNi#3Ufhd)m0f+iE!p#v+KHp5j#pKi1^|Njp<+S*ngN!Vs#b z6r6D+;n8Vs=OA~5OEXecbLflm0`H%?&E??(hMU_%^qgP$wF7^!Xm(9)!Eyjn%xa?< z8CrB3`l>EC+M(R>>u4cRmGOiAfP%Aq(I54}PWQ+GR|vc} zkr$nRaeoy2-D}RnuL{}lgN_c6EX<-NM^kC5Ap}<)=4jz=tvXi#cj1JN0ZW6=7Ls!u ze=Y3<8oI_3os~G)=Ri9iIe|;^{uv->P)LW6W_QCSF^|=&;n9oWI74shl&N?@$R+3| zwCGJ>05yb?yuL1-Q7u-YdWv-U4&%9)=*qr7V{JRQK#|})D5r*K)A_Ow(dNi)6cwa3 zw+uGw`g7=LL5VK-Kw5J!%Df{&2_j-1-WbOLl$a1!1g+i!vC31=_e3Yc+}Tm-& zCX|jXO0D|0{V|r8{9&7!^24h2cbnZ7KHS!KDWoMYYhP_UsE(JOKnRWfpCeArmFF3y z4p4nCKabf|3j&YkbJh%M^Iz?8)X*T~AycFW3sm_MbhPUsA0B9reF4FUc(F#eiF*4& zz%7JR8Eu_Xzb_J|k!yu$mu>mk;@5tZLc*LLC##pjuFfnzl?V<*tGQPC3iY= zO}WnZzVrE{k8w-Wwx3f`LnMS7*QB%1yPlGaB9kLVR%K~C#1u~cM9JEO7M@|TwmPTK?Tb8DrbDNJ_DP-F zHEqFGSWH6{Ca2?NFx%i`An&4~WkrTvg2N)t;6HD=hQaL1a+;nPfDjboMib2UX^(@O zDbiXz-ozVxp4U(k#D1l-&#m>|bT@4(tN5m*BSTrUngQ5Au^c*8zYe0mq)+zH%uU{7 zdSr)Evl;2OPmR}T?sM5yh2t;43AB2hnV<7#(iyVG=v>z7Nv^He0TjgMAPs8B7~9~* z^%S!i$_JyA$(+eO#U-umTPGl93k}MuwlO$~tKz1*%mjbeiW(gOu%@-82XfLF;Wr4n zHR3FlNhuG=TU^-YjK|y>cl(}*^5SJU0_^|R2(iLV33n`9Pt`L;Lqfv93YEb_M7-98 z$@^t4<*I%ai?9yFqRZibq67<_K#QDmyt`h*L$$dQabS4ZoYs+6_S9mO&XW7YIPO2C zK|`F>TjSn1lgf64Px$f8HW>D+6JE#TwR(4S85KjyiN_5s+6;mLS|@Mo;N z{|8DywZAE5z}X|vewatYF`t^pcGO(di57+2z+;86=^DeUjc+~(8y8+cb$gr#FWI0# zPnr}{qCnqG=s+W9S53eaT#E`QGFx-Ouv3`&cg!u9RNZMgFRmDb^@(km) zWCC*JEaQ-WPUZFT$*OJ|I|r-_a>0P9E;)31t;+D3!X&ne%Ev8CLXNy+I}O{Lptm>{ zZ8E(ly+clDA3Q$d=SA@R8+tqs zn}5U2=V9|BeqQ_hqeYJ|W>6#vh6HfOc1$rTEp)IhA{y6s>LfR8pH1MmX|C4%|3L*myz+JG3`Zo|=| zF$!}d6)a&8lzIimfPdFfB`dIF7W!luBMFDvno*!_u(G{-Bwem=_GVriCJc=tWo<21 z;qg{0!KNB5=~Z;`(9|4#qZND~UG($8waRC_6V@5H!}EI1rPa*6sMh;=#nXift6Qj1 zS@$~}!I(&@4eW&cSdpci8k7xHjq)AQ&0e-qzc#LlwNtBtQh$`e{=VuTzEG9ThEcGE zFGo%=`4}n2LSZEGiLa8_^kjh_k$okK|6vYt<5t{BnY()9-sqRQtop~;5)=mPp*LhR zw(AVtmCKp#-XkVy$Ou4#_n>2U4Ao*RFw@gR5WT!#=^T(ho4Ta_xhY!3=JvT6iUHXj zr2G>TalJWdI)4@Pv5^~%qn>oZ*P(69$`gpc0Vh5 zz(MhI#BgYL*)&AoAteOxutjafTwJ(LA!VPP-) zDy=e!yfA$XO)+K8^o09B9&Hb$p&89R`@186@4quOfyuT0ya?oqx0m*%Tyq7dW;EKN zKuuJ{c1jPb^%NSrF0AKHwxN)=w4X`UQHcvkqG2h^LD&A7%Eaut^tWLez)57gRT#QK_jF@( z!+%H#cx}mN=29x*D9JLUgd=n8>2->b1J(*pwN^N@X)TgMifmjI?O`Fgns98&ve(UI z-OZ+Wt31t@65XVDNhR+}pmtf#=P*k2KzAa#Y{s1ov0;y*$;RHTj?c5~J;D#822&h0 zC|UYy0>-^is58k}6_t~^h?=RnH3T^ynt#2CkerOIZDQ%B zCpBs*z-O}3*&pzN4f;6RKM1lijO7qLqA%|5LIFlwnAg#9dBQoxKRe>Ra`g767ca&? zzkdGW<*U~(p0oOR7rLbMVAId;dh5YP6;PKW0Ui6v+Xrar=A;mml+EAuOat?~$$uQA z_)@aXG$5hyEA5hVEIM3pL>y&)M-Yd8o+Et6g0=6vZueM`l6)2RF-{x@_7`JL9>EUO z*aiRF#NJaLEspVplP>bct~@eJdXgAK)Dk1K8a&Mz+sVj2>3Cxa2ZtqQ5JuAU5ql6> z?sK9&<}^C)vqa|Xg0_!DG=xtn#(!yYUe2d5i(`y)V7!Km2oW#t^P$)0lMm^Vf+=jD z50ibSeb9)72oDBHN|b@bkE$_#9Sh&^ZwUmM;Sm&mI?=w^8uJG}PuW(EL9?a?6%d5J z;Z#4@lsx6sP{_s0+1l66-Qq@{BnRMe)IGpJ^lJCdfLxdtNglueEeW^@aHccef{UJ9-R!~%fqh^zC1X1 z_~--!0uK^1Zw{QZa)l9-`+o;UM%kpkWGG_qF5FPfVL<7>){eL%bs@a(@>buQ7id<-xOy5U=5{GWa=WcfrC)FX7Y&@x0HOqd@2`@~ghQ zmf)Z4Qc&Dy6b-8#iWFR~>eKwR#2YuwqX!Qjj2?V0V2|e~_baTWvalTuAI5#90B(bG zyVbQ3_2|d{xa*6ZTga}b_M~pt@vikanfod}5PBFBsbh5}Mt>sV9xq&o|20M{-s00jbAHa+|jC29!uOVS~zXEp1r zvm^wXu|}`~!kBsby?eb(+lYz*`&QwwNaY4p1}B4iy?;SZk2n|wWw!ZyTKM>@$&gCe zt$RtO_hmB*%t%df7$0RcF%>rqsK>cWK4SNumY89r6)!X2RHaC3jp6A@;4qrv1}T1& z6i9^LryuNPKdwr&kyIICJw@Gzo%=I^Ue1jHjFz}Mo1pm`6L5E1!Wwtp%;SidH{mkBDJT+I;>;}-Z7(#%FAL#@rxSNU( zrj3ZPCi|Q%W;TFSMuWC@(F^@iS-h>sqwQaxPR(^o12W|TZwrG1ld515c*%}zbv9}) zMp{?y&IYcs=`a}d(1yukbT)goWzS{e(6jC*o`1M{HS)SaMuqY6MWdH?o%1u`K@-XD zUHq0+RY*Y&%IJd$eZ=FzUEuIVZ)I;sz!8f+{O`QJ6n^w(eqPt_f-+TG}(|?Cm zJn0#G;)BswU%FqeuK(cxd?^6F`16Bw`yse)I(nwqu9~f7}qCm=W%X%>iNK zO_Igy;6DuMO=kwm?xf$sT{{g?dhH&gw+Q~ke_tK^>CvOF9zOVDr$vyMlTU`Gw12x5 zo)5O+d2*t5C^n>W$OZ*PfF8l_FD&2k)&y=G{sD~kDP`1SHEQ>Z+n8#MwnFhu_v_|D zAg8m~1yw2%7YGvmZi6A+#`7rPT!GchxVA~hbis_d206SKuk``StUOywUaK1P)J6FZ z)>gcWeG0Y;uQxsIByrP;WJ=z4!hdOUD6WjCoK}7U_;MEf)_rr|;1c^?pf2+l9P>`|aa08*%u zW5(3Q>AX1926+yz!dn9hCnK1Xd(70)Jjecu3>LZ(h5}dQZ^ZgB%?HZq>zOuMdABv?;FHVkQcg zh30C!UM;HXk!wrh_*Bk>M|G7~Tt7Q@~dr<)YXNyO0N@2jO7j zHHLqQb@8zW9(8de8pH^CgWD!NCiA)}`iZ-EQ%%I<9g`-4Vk~LT;D00uD&DsF*2@-x z9ub^Of>Ly=J>PoV*GoPl1_dn`iv0!~G+Cc0IkSMVH8IXw@zNxdkGTmhvNMc7EmrIt z>1qTk*ICNJJFb_u+!-eUmo%JR@fnE-w&Qxqi9F4#hE_5=<rB16F`n=k`#(7u^kYTNhi-%H1;ef;lK^SMo|4$WY_`7fuQ8zstWc3!sLJTzI&O^4 zh}xFf#ErToeVDjpUvnN!JLJ}RU2)3+5>)1WjiEyU|6pQ0l*q9EZ< zes&{exwcvmDM3~P(!osvzi)l9t6Ds8SwwQ4b~!mIzjTL7OTp2j>?v<$#x2K{w4*BI z($6Qic>Gb;C0ikik3*3K*TkvWtY}E=4YJ?TYTJ9JI(Sj>R6d137j= zH`4REcX*7GIIqW(xqMzP?FV`I{w5uf>niO(M@JNK9Vt~ID#zI6iP4(z;lo54@V)r8 zdI=JJ^M9LMVMgI|^wAJXy@dq35k60LLI>vQHuQm6y(xX3>_(qm5K7{+oj%E%<7gs0Ify{%AQ`$L( z3dfLN^`kes#_&YOP(@=9CXg{axzQM^lkRhrdaYZ~nilr#;Pe8loW_Z5n#FPs+Q4bz z&NNri8f9bS>2iC`n~ove%l@rUwY{^uv|ZelC~9|kZ9?4W!d+{J( zOY2k46V*(wo!!7SOjfCLvyXjrytbZ$W$cgD0hf8@w3yMU0Y2ulmeQ1v>Xp*kQks;& zG_NJjZ7vnsv9=2ZN89DVg+RD|0{OQpQfc+@&tx6$$iNY#sxW`XnV1rA`rb(0PJe6~ z5sdR+o@^LAXFAO1odpSys6z{Hke-wsw(yTP6vk-lfWn!vA=t^wo`f-!~cZe{e|SWKW{cnPgFp}k&(%OBjxa!W`n*Wh%LvPdoyVTJM14SMM`I` z1gv|RW+&>|B7t{q_+<0YSxMNiaDV#%WN4udvXEAb4Wp&~uV{q9!-y_n7=#gX#a`g; z;&c(4F2xrCH(&nV2~a0LiQdlY1t&IHEwM#1*-;nHY zDyf{GwnGvuZW#t8MM?}o<&IITV{@rQ%EI8p#8y(~w(*0p%eTg_rB(KFXn!vOl;b^J zzJ$KYrl_5}^-t>ya_glP1>>`LKX6!#`uL>20zqH+mv$extgPY{#R_T_|_Ktf5 zrf}y#3wnolsbBw9dK4rC&WttAc&GWQpdFWXQ|p(u_(#JU;C=W{`;fJ_#54;@``>tl z5Kv+~`A=SjAFl=bgtUw=OO)7M`;`17L-=KkTqS6@B+dN6>Ok1{j+*yvIA#b7W< zO*6nT%s>qvefi}VUp#u0{pqWN2VZ~vCw%S6F#9I18j?MDGKd++@W6it>~~S6x`dEn zrdEH5~xX1vaLEEakD)S$U#7EHvP?Azn@f^C>VcpjelakBaGcfU1+D1 zc{Y_T@0sKnG%cH7ymAETPN>|C&0Nc!i2HJDthC0ej`4Bp#_USdbjIKuHW=qfbW8;lW%L-7=?DB|$dw7~pI+yR}zY|rS#n^$;G$ku;d z-O1~9LnjhiY!M15E`QK;&DT!SZk`cMbkqablAcRKR~Co=>rpp7)Pgcvx#3uG8A$0{ zuq($Lu5JK(c=6@38iWos=5KR__XKJ;MJNQZ+bZ97mUBIkx+=PzUE67WUGg|k3N{l} zWZtH#o~EuB=FB%gjL-nl9}H44qlkq7bz|^zrgWvxNz}^Zb${_3>!;z8po&S@1C}q6 zdo{W6M86{&`AwIe&fCnTli`a)L#eFK7vFR_A8$D=oH4IJ z%PnWm2MM;tHxd50oXs{dHktd)&^EHVJL~KOJqcWoB6_3Oi!Hfq=;GOd zy(s%VQ(q4T*?)iYem}^Lo@_yBhk1fC>~8aPuxXxZs9ADdFSpQ3^uqD+++}g zct>#)1=GF*S9}aKws8~TgVvOdhPb|(fuTQ+n=&c(wdXSj_}IyC*RhO?gZz&r4bFNC@Gd%^Y#CtN>-K?;GBqx!#kPG}PQ){^&Hk-VKoZ!Q2L+Q*E4%%y+0T zd6kd7ynp#B+kS8-8OCQGWyh{#r}U7j=3whQ{88?HouPl{UR|dn_k67S-&*MIdrmDW zw}+yKn_03SWErG!fda*`T0@$Co&V#_(I#4BTdJAYY-TeV|w9r;1%w_|C2g1I9OqK4hV zHGDmMTNVvB^F8uiVrpf;wJj1$i(GNUD8`;QY~}JiZ_IIxb81muP+HrOA5bZa(8QSp zKKP*_s{^hvfY=ZOwx^pRfkq&cMT$cGJ;d#11UvTz z1AkFD2$K`t@*rq3fzK%o_tGXl3=y)rh}A%d5IcFy>^52lbi=$xhBKf@Nw#{w-5o8_ zx4Xx&EHa7a44(zl>~kkF(|XbGVAhCRUOw1?Ztj+$$nesBUyt4(6<(m7ZI0b^WfP9|X083GTb1YJ}cFddPltj%+ zHv`3N6ocZcwQf*irk1RaqTLBz23Ev#H-#t2rQmJDkDZ(HtBknQQY;ImGGy%CAsWJM zfXFzi^A;OE$MtM%bo0dKrw>@R&VPwttP^w{X<^H-M9SbVH&X_0u2tb?Lf#;m)+R9R zp*B~`Ih~ArWUwZKc5Hh|4k`8Y-VQp-FN2c}+T1Vcw4T2plHm;PX9ww1LTZ*a^vTC4 zOES`-5~E9Y<>i(16~b=w5NzOV!rEYToBG~wyZio>x;n}rOrjdlaFDPmnSVdbj=2p@ zM6uIOIWZN*Q$PONKN#Rme&^2^uNq=Y##td=H{vw>Uv{uaZ=$SD{BJ4I-RFn0oW0q0 zxF^mNiM4rC{xgedgDo^U7FQA`I$?CJa`#HN3AGCc{QCd|H!Q3EmbZsD9$hzAY?54Z zu}o=aY5PLQFDVK{%1wdy?SFWoe^E|tT;h6x&fo#O&i?UlZ#%W> zp;15eu1f;zehmXTZfwCPw~#&mAzk9>i&>pG#?MU@is&G$jdthg~3 z5JBT@ypyN_;IV6=FQiyGr;BVK&WQ(N2eUF^<-1~cArRh*1?R23$~*r0_}=6OExhdH zb>Q8cMM)$orOxcgaT^)Vyza)(PdCKBkIGU+B+7u@eGSQkynn@_aKMj)o7HX(`LOv> zbEKUZZ@$yqVD$uyo1VwPP1jIDUZ>a2PTdn5d3A9*82Qd#_Pr!TP-T5>JfolagFL*6 z4+hCt*JVAwuIfdZ&#eVCyTf@DeuSS$4}{}I-ztSN3zQ&cjLO=0OzUQ+>f`^!is!1= zXOz$=q*zFap?^$3vqMNI-nR*GQ-{aq6^T=qn0hc+tAzF3R2^eGctp(Kl*ioOR*e** zbY1O}^9n9jOmJ2gGx*Q4zARRzSvi&TiDI4VNT7(U>$(_VAUT5>HRt)V!1@fE#rO-8 zn|%9-w08sj+~QO!UoO%KohiygEfAy|5Z2^9EhGc#=bxKoqmd&I z9J?hFhJWG>NZF_R3wW7KjvYOMWGN^^lqmZq!k^AYQj%4r`YglS|65W%+xz_Uz-;5u zr^9ArgQ}U69}YbhyLZrWy1or-gv;K|JxY(nX|aPhk_~mE9n>TR+haNY|DC#{|ERWZ ze0yX*;XLz3x~D|u?Cx)_ffY9I0q9-4VF~+>gnv}x<$=5yt8NY{Z;RAIoAwt*P2D9K z&_o-YDvg`B2X*9_70A7z?jv0s5mV(#BbTCA_Yv-b-Km1*PDkbK4c`qi_xdkMuE`|E zK}(ZiGE@$KOAzr8Dfix{ru#r;hqc0wqw=`?CXE@KUA_c`AG$GNCOPRu61PgS4vh_K zF@JFUC@-)S+_G+(@)SNa>)EWl;<&2bxB<r4ti=N z=RS}?@zHd%KC7*6;ZjNt(^muU>~!?Xvwycm)zqumyuQr-2Q(38=bWD60Q`pIwo6XH z`BLKj;M>z5U*QSw?PYC!x;ekhOv9q!by;0H+2SF*dN9hq%daW9`wM|^n7xEvs$#uD z7xtp1lZPo)$W5&S`Gf?9ipOHt-F$L>XD{%B?<2&z8kl%Pfjm?NK%p5qCBVt^qJON{ z4fkhq-%yPd5DRxCOOr(cm8qs^+y;kzf`D94NC?>e^ni#Yc&|kfj%de`R6;V64eF~5q&ZwV!OXa_Qj>27jR>>H)-s+$<7N0yG*uXQ-e^t1d;Kh^e4nO~_ucO#q}5ocp?@r*N4*L| zT%c92E1`4H;ZfH_j0F0Kwj$in$!dvupY3A{eiCp0Y0zWfyJ2S(tVZtwn5?S`WF;)I z*RNd)HkRGIBbb{`Sa*|xkmFS;9rRnB=$p81N>}|nOUy)r$0n1Dk*${nX3%E6>a@&N zWoN<%MYPg+Kz%ZjAw(nWN`GaD27*irOrOs`Q=N0A{)BH1sq3*|Cf^V@HD#`OxU4zy zA*eYJ(ILK$=`e(N9fe@fyG|JaLv}c!C5`>mQ0|l z>w^y5m{<{Z1jhAwvkNq5dP+@`y{hrC84N;}aJdH6`HsvJijgA$$Z~}48yctgzJ47Q zWek^upEO!`sgIhJ$A2l1T3U8$?I($udsjrD#NYAHgOOKi!JK*-O)X{l39d7sRJDXR zR{1q4@1ZO%NPjmTT3&AgofT)U0fCu3=qnajL>I5VMR>q3J?4GNCJzmmimL>e(|Th5 z%?{*Y5~ifYxJ2H3PHRx_>E6Q5rZNT0{KJf#S7;~}mf1;z^M9gII|f53hycncNSP8N zA*Ba6b*+dMBVI9s(9_-LqLQa3Hzhqyrr8(p;g%-*R$|26V;ywwkOZ!Ccl20l=}lJs%2N$-^BuBqP+*EQB~Z=@u#sSR{dg4wT% z67S|T2`kK`X*(M9A`#$l(i8ODPSE9fK`@~cZ6p+fOv7r5-N}rq^aB=Wfmd{WmS7Vi z!?pjmI)6wDO<6VpWvP|#63SYSH`Bxiq79A7fI1$NuKEgJCv#(B;FXT=}WlYu~%`GPqY7ExNI2^VFT(Lgg;jk1 zXr1)9QFBv=(up`M!V-U{mN*UH9ETl6V*kc zapl5tx1yIH^=t2K^x8^C`gZO49lO2xOFD^lZtASh$}aXC&vJn?4tK7M4|1@i34i30 z(fz^kUDx2Q?<{6d#bQ6vugh=!!Q+Iho=@xKd6Q4&@$u4}5?eSrHcRrrNZdKg8PRJL zjyB3{(=BYBDaj{I54oUsvMv5)FXr^44xQNi@uhh+Ev`JToDn_hccYR@A$M6={nOJ6 z;QS%!7mf6^J1(1Kgwm~=MkHr2Fas0nPcz`5$F~IUONh}vtooT@l#NF-MO7K2`?%^vp1F$ z9ɄvY#U7EgF93$W!Ge$>(CokMppsQ|DF!@f};vdh}tSlMokk3?jU4u9|eU?X__ zdZ;b{Q&)86vfo^QINvJkNWqc2bWU|wbY$kf>t^$=(}l}G!QNn;q`LG0*|H|WNf;$V=6`Ms9~6G#i#t#>t!s#m8n>x9U`wedCB4iSBn$CkvMZv) zdhT)5pp)@TM}gac5-btpDUr$#KECZdY3{|&I!xy~Sl&0heRIQm$^ws+^eyn6(-8~` zt>+t9eHUYUOy`}A@3*zS(|9*o->HFK2Et<5eY7T+y4Th#q^{gC&3}2=slx2rPCh;d z|9|@Elh5ZN*Cm*9Pi_C~Yexm`q;_3knQ)rM!1J(_!z&&8c~e#JS#~)mV(~#=AOxv) zFs&DPSy7n@9up=Mv`1rnG-QvzqLpK>F6HM;Y&wvgtE;b@{ERmD*hAV>js%8Eqz_i` z7x}xQk!C?FG1!e1hkv`00XIy>l$sl+hSAMvX4n{DR1I4MOBojAR$3_)ObcP%q2P4RK_T z!U`--8^cim{(soq&{05yNw@8j)au(y8xS#rH@5bhe&N~Zg9^Vwqx87KoKVPKq zgoSLIw!kF2z4_Gy6N`He@5w|!#)~M8b)x_Mox1<{wZaonlG+HnqXGb4Kv zhlAAY4$eoSG4|vX>V<4O#a)VqkBjsvUP)$T~xNZ3g zSE~R)XTtAgm2`H)_bTHye&Hi!Q_@v+weQ~&8<`pO1*%-ZEl3sBb~fpoZ5H!+tg;M7 zI|}&E@2OtGD!A;d`7I?T5U+YMOmB!6|IIp~(c-KZyW2yZfz0d6T+UDuMv za*{OHMP||YE{4#LcxjpK4MW{vcw_&p*h~~YxtBKBd89V+?(UR+7{Uod-m%uD@mRkr zB)m%j0NGu3P`IlMe`Kej{gd)>s4z%49>jJ)>&x^v^sXwKaoA6=V$U$eg_Qzw%v=C zrNUs#M&G8{+c$(NhE87?J`=k7xM=5)Be&$kp6Saokl0d(eASI5bT@IPq8c3 z7hM!@7|8qj#hbS;o`1;DrSDHL@=a+3HzioK*He}N*rmGbF!tvJtzs|6fEs*Jc4sep z{lnWACQP}k>Un*3&G`-5%Nkw+xqmb*o5{MtrGe>D2oK7P!NFCpu`J8H{0AHQaCN+C z;op2Um1mUm!EXge8O*c*Cs#~Yj?Rzj4RAeRBOwdC3Z$6Hj_?Mv!>l+>!A&wFEPvr;XF%uK zdh_h*cQ2lg-@f?i`#0k^KfiqW>fiA)!X&S8_n7~(Tw#K-)@2FgCX#lAGT($9TA!E* z8mLTlb%Ccxc5ZQ*O*r$7l<)^MC#Z0>wl}AH_Y$j68l`@DL$Fg=gi%BG^6a_>dWFn&ixf9?J&#D zggYwlUVe+xxUHLzav>7#l2CklahzhRGwMg!kDumf*qO*EJy5BOL&MbLq&rc2Yqyux zAaNFE71>#qEHeSkytcrQlVVC-A{*s;-Zu84Z(Z%n>qZr0git@OTYuScfp9Au#z0>% zKAC`kJe(mo;EB}x0I;2b+xrW z*zM^05%{=4b-CZsDeijN=6(}HI6mA*>v9*&lbv98dezySZdQ00RSMOptFfC4(5dF@ zMU`C{)!81ZTr<3*zJH(fvEmZj?&|^>yCy zfMc%)L$l^^EQ_+@orjfZ9yZGb`WbstKNGb^u!EAX7A%)E5r6SqCcUhTv&*EWo4Xc7 zNXxe48I2BA2t4_ki#<++G+X7Ypo2EQZd^Y4s?O$hL(+=O(T(q0C{l?@xC&HXfAWUW z^b9RYhIaG=Rj>xFkcMWU{+T98X5}@YYO-V1YL}~lG6M44@C%dzD=ZB1V6cO;sp~0S ztRuk0ebogxqnF>snwt!H4Z_ z%(cQ+v4x!k!DwtS(yLj$ddRioZ66#=LNBQ!k!A9uh)A5WRVWp_fI49J3rO^E&Tq3$ zC#-!S#-C_)8fC{TONag4T26_9ifk@opK`XU-E()kcT8GNW$_9Vb7HL$-GBD2`5v`g zn(`vj(v}-hE%&zM1r0z}l}l7AO?B?l81N*YoEO%k)=v)M!^WF|RZE$K$Do;@IUtpF zR4l|>T6O0`bqJC>=Bl3K6&dhy%ZkGk*5=gZr$AfibSlooxVGO_wYnW7HN*O%E7+*e zXwP_rJI|$Fff#o%(Fg)}7k_Y+3Plnis?p3i)B|VO zVgM~yjLeX1{Me^)xPo6~XX1e*W&on~+akyEm`C|M9z5 zFaK{g%0-Y}!?!=ZcrkwQ`q>Z9U%mcUido>i#o1mDD^A0I(|?G(e*NNY$9rduy$E7%V96>9lA@c zn4B2Ai}HzDp~iMv*Ynt3(mOBaOX%gzdA*)bGcK9#CpYB^W>QELE8S-XC04MCc?p{< zvN8|~^o^YJ&3|w?mIipOC2MvsyBGJ4VL~jDYw?yCvx3%YtXHw#*R(QQQ5 zEI0+VO4RXfy60n|V)onnlir9VD`+>H2s(K0ldmVqc&>Th(sLI4(miKBYu|3pc(aCS zQOSY$m@PL7_M%315^K&hKGBp^vX;#bpL_t(f)PbT^MA_IYMS{!bd$_mRjly2!RN{hQg>%@eI>+G&6rF(ajCJtT(yTfBYpL&t)F1MT3&o;+s6u2T?jq$*(U ztitmM!+*Ud-BE>me#tkf7AS64Q>kkVEh1=mxptnfC_C>8t?GVh*V}TZ&pZ)xdcFag zIsG5HnbX@zZp)mKYL9Cqye`^i1E*>)KEtjX_|wGtCRjB2j>laa{oUwny}_N_UF{Cj zT`{M3aCuZ2BH0rLSd3fmV zdP7kGw4BK%D(vAn8nb~ncxY_ea?)@XKzbWnxXN9IYz$b)#$V*Wpifp<2hV)PJ4^F9 zuYc<5da;SVg7A^RjOeR6FpbRjWLtYvr7|1xXv_WOXPYa5FN|H)hV*QHqpP_~HBSObR>Wh`SGiRYBO8;JdhFKdU~9 zO6Uxp$Xx+6H)`&{o4sfleibeEJe;bO)qf>=W~`_QOU`y`@k>6rJltcbHd#{mLx_FS zv=l#d6K?uMOD(mKuI%@imEjk=Hq~qPg#>Wqeriyf+iK~d^v$+@4|ok?8pS~U)13hI zA0DFYkHGtFs6c~$uvOp%9s6HXqxX(9X7t{}G@^bRq=xq(?#Vj{SjnZHC2RY5Eq_Qd zsP9{;;rHo%3+`+`bp;~bHnENP-TJ<)N&s5+a*BuHE%!vv*Ph}m&jhT#X#q?0=ZMEsgRp#P5d5+d6=nG#i61_$GP5t!6OcW1FC8 zYEq9Vp6x@Xa~Gm_fV+ntzMWIQo$LYO{Civ*HL*3*tiCJGb-^DBvuISD1?ml2kx86Y zboN>IGr;GQd(gE?V`IuAN+nP)t2Hl_xH70@kM435J}<15PC{73$Co=N%zt3IFA5dr z@FcZ*$vj`hSfpz~X$I!!L8b3%6t#%PNSfh4u$hwuXbdxmm+kX%tbm zgwF0}zTPgAZ=B#Yz5C|{8ZOnV!ZpS#$V{ zGV_am#WUE92BwRraI(!ykWi=fMKKJfoc!fQ5_>Y%-WV{uY!kWCR}i!2NgJKfTd(1~ zMVi?wW-CT6^QL?H41f7nE~OXz-i)&BDM~cq+d0EVAvRuvov+h2KBQ_HRknsnyWio}qHZcK^2+M$x7A&wNy zM#hU%aVy4_nSW%hTDtHili#ceJMO4!O7Mf1`<6C(XGV^T<_H}MNJ)x%!Q3wxzD0$V zp{Wj$q%~D8O5jh7?sApcql`m_BnHcuJM7OQb=sV=-%N6@B~9&U4>SxCZ;3-HBiGuH zVW-m3v`5;cErA{AyTqiNyb!%E~<#|2jgoV^dK_oc$+`+SWtBJCkwq4i0y#s<%_Xd2r1_021hLnUeHKjvt(i z=;jgw$j3wXa8egD`~BbqdIj;_z!UQ23G?u{A%EITy@4&!@ckHDHM@=52SLV1)}O<9 zQ~bG+3OB0!^Av|;%iY2kbe^S?imiH-uR-X}Cz51%6k(m@UoN+*P)gJZ!-uAGdU~^P0$N&A!53gU)1^sHb zq<;bSpU`B~Qg&l?E({8q_rz$Rvh;e#a;G<$-KDH%wbPeX1a*8%?paxk8)0LT^KiK| z8fBwT+14~@HZRwkK~e8)oJgFFSt??tTc_pJo7YL*R_)NfMmWm;a_u8{S#a)x@N8{W zj~%vntE6_u)ZZqUUfIY%do&IVCK46L;D62zIvGxEVu|-%40tX--5DGI2C^AU zi#9Lkc#M+X&g)B?qIHV$&7RL-7z@!3P#bBYS{cN=hHl-Vc4tF~6^;A@Q%Y&ucDm_gJvC?2%X39x)YEsF9UVrEb zamms(_JNUS^B_!JXg`5OG;e*Uorn8}IweILNpN=BFiM?Upi}Kms99;XA_!QbFU&c< z6ATC2n8hdoa@MAGOHE9UaGuP)pLvVi;}hp!W4QHOcMFBXLvSv*-^ZDttMO%@q5cP} zvgdE1DHzIZvlWQRsGZt!)yPeDQGd1Q?61+O)zzp8gTJ~Ww~T4q8xC#w3hZ7KViTaC zPF+b9=mtzYEAYxlhtHkZ#Ptyh3Ty&}0EbwO>s%W*K2~@vzAaCb0K}^;jZ=W-W80qe zOY~FK_t2e+sLeTrZ?JmkLoz<1H-l#03LyG9K2tywC!LN2Uq0jni`x}wlYc7Vw%l*t zmCGfHS!eKP(@c9kc&xxAb}SP-5~R9uqE{Q0uoDE*G#UC{^2{iCuiaFdF& zw3>kl6#8g@bPlK+oJ>VxjelW+)h_Nnj8CoRPYugVmyMYm&+7TKk9PQB_L~bOb%S^> za<~qOl1{zboyRu9B!!4;(2dTXdy&>phuy(W+_9bE6Hlfk&ehu_;3iSv-J01m0>>7U z>MtwWwKI-^dKKIJ+g`@1hW)wf#+Gy;1Q4!`(M2K z>3A3aa3WGXyow^h~q%;NZEXpfDq%vSPtCmyD8 zJmxVU|8l~`WRmsQBkcPO8Pr`0WNWz~>n{eOInbYAWTmm>2hU!rxi zpPEyiPEbj zRfpg~iGe0U&w}NnM`qY_devOIwYjIEv6CCg?qx{ zktyZGG@lc5y<)*Cz%hk%8+{YKz?XM;#8CqYYFjO$GoA`pEg+CfF?pm zeuRq|t75plVOy5YUO6xk3hdnvYZ%a#)Pe`)A;;v$>NBP&!wpS@-@J` z#R*lolvUeL9GN~_j3j9a+I3U3t-Nc^c3JTfo>2DwZsiH~rAyq(F@CX3ajZuE5-I;m zN}=93OY{t5E?G&IoN;7wZGtJ1j?(zO6x;J)d=^JIBpAmoi< zH01d9A}4(+!e?W?{aJ=#)s#H8(cpp-bs_&Ap_JC~>)>=+iw_}NFN`#BU-;x15y|Om za+JBOTiG*GVOMZjQbcy(YFWe8cS)bU3^mX@bC+;ueAlH_h-7~C7H-jx*?36>B4JMG zmD@Dnf*Z*CLcZZLNv?>$*LTH7((H(J&W4m+m^T6Cp=QJ98RiCxlq_PC&P#C7+=|2= zq%!Pn;w=L%C>9dV{9@#Vgn=;4@ZOf{+KoH=cx4(1nLzPq%Cq$W(r5fDCY9}boZ?d_ zWA)DndDvES7G4ZIW95G;Bx>D&GXJ5)czq3*zMwHG^~gq$4@Tr6X#`y~Sy=!Tv6`D! zrH-H3J5q=jcAftZHW92Ncbimpoeu|L!CF&?vvP%7ILU zq`J;|(zC>BIMqkIeg-Z)@?jf>kXrNyVl__iJdL_ar5@^Xc8re0`Ctm2jI8?$oAVA( zJZ6(ug8H-P0TJs5I{ZX3y_|4a6Hi!yJKLpxQr_zht>kL?@^;LDhPngHGI+%7PlJ;; zPwq%kyXwm&kZqdbzAf;~<4jjd*>sac(Ez!GoDFTMy3LdEU4Cv)KJ;kKBZHq~)a*?b zLT;raCDj7QVH`}LDu$YU#wYt|NU00F^kmA$%4Gxtx3a2f`b6ST?q?!yH)pRm9L>WT zr*XTBR96S*D-R3qD$<$2OMU0BIu+pBclu>HsON5mwZ2fRz!}yl*ejXMlZH5Avz*qZ z)93n}yu^Qp_b!u($uP%t;oXdni6!-3vLAou%;>eVOt_e$>#X@x%Nb}I2e+-6v; zmQkKP#PtwX6F?|MW2|ifQP!?JKc3QjxVZ2fJy1?(f&iE5iHYUs9q=KZjIQBxHg;** z3IkULxtPzfL>!IuQD@J~-@CrR#@=~sGC;B4O!=)XF~A>rqljxLEh>EF)*6RT-U^Z9Zcu$NxKS7PakT!2nFWZ7Rp>#eD0RsDmuDaApC0r@pO5^_}WhhdQTd z?gu{{z^fV7IIW}5>}rGYN7e}>RIq(8@59NLb|n_rL9a3ll=pJ*SBpt9EC@Z&k&)g{ zt;`OjeWdx2C^~c`(gJsj@Guq1e+a)AWf~x=-B3h|7qdh*MMDUF{{vq%0GZ^3%Wo~g zhdZ;xU_9S=s@~E<- zx)}_UZump5icj{uU{eh=RXA?0T6%H)aqbqT;XuI}jn)~uS%IE@US7mW86H%&2=uXl zAAH4-fGmOqb3G2kM83Q9Q5Jum>;hP5GI(`)!vNh;%gC@L8XcacqpCkJTR|d|m%M)^ zP5KEkU8Z9f6png5lo77R`^SK8$gm_63b zgutvD+~j!>TLtHJD^Tj%v(?22+z?glzzR>3pQSLYTYPdz{9Ra!^e5(VNJpDwNxWA+Ba;QLG)c#m)g4YlV_Hp=j>6WklU(kyVUOy z-)aac1`oCw=?G@aWlVUta0*!%(5$q~Oz@M^UOaRz!Ua2Ol8E*Zv0y%^eNNWO5iuju zv6Vg1kw9C^mPaNJy~0D5$LBCybagJmC+;e>6g$Ue_8y}6EU*pn-Zq}o;8rr;3)6x- z7@V+M;@-2ZM4NOSAB@Qm!A3DAzL{yEc>+2io}ffgtJ?UM^9bhj=9LQz{dYxF#KYM= z$^8Db_m^M=k%4`(_bpY_v`ij(<#BtCSl*o>tIb!%8`5YVyX|?A2HiDGbac;oVGth^ z4{O>?9^P!c^8G+xp4FrD_%ojjBv~e-o^gNr)Mc$-AmRP|P2M_!T;Ge@v?xT8Z!4um z3`DIEf10Mzwf0Am57GGs<`=B4pXjrSh~$t(DR4P~n^4oOdvog5UIkj?+TLOm7P7`; z9S)ehsdnEhCa&V`3qYmF{qHzZp1G-OQ6@jvYHyt*{A2})GhT4y}$idi!D46pS#;qf{S z0+Z7Ep%jnW1c%G86VH^G_s^kHg)b0F9`vw3Y)OnZcn-ID)6dmy`v}p{MLlpy&5s4+ zdABLQWlAd9772UQp%6!!!gxqgxbFq^P}=7Zd~#GdAF@@ zt+Wdn%UcERWd%H6j^B4&F>e&}@c770FMx-=v-Wh(+H8MBm!Fxo4|C3%v3#O=cY3iD zp;AWdu1~Z(w#u}cd~{d(F}aOC8Lz%#K*|+-BkU7&v%J@Ls(Ly?NnM3zrX3h!I#+zT zMzG|3L2GTb&;LaJ+I9)yJ9F1FjOH2SmUhoG4UeNSx(fR@`l+TH@Fv(zt zv`?kw)sO{qKRvVMmG+A%V@uqxy`trhiVp<2-QBQ?d*AjEcf;pa!|FH%pMCXpV!xgl zm&BTEdhf9l)6S~sg>=jbgBNTY_l9jtw4@z}T#Bf)xkMwx7a}y-o-x8bBtqSMY#I=y z&!rwRF1lUj?hv?{YR{nOcisD7PX-EZk>`G^o*e4*!bCoMrX2rJ3$N4KOIXmawjQHa zRDf-C1!Y2L<>d6r@kv@7`O?ZOIO8~GbM@GwV1y`H6bXCakFJj>4XNV09Cy)PvXo(4 zjt$#^Ug=Yb@>zHS1-fcC1a@bcL1ooo!-NBCsi)r{m80EC=iTOgDD2;HrO5{S+hqw9 zZ)_Il>tn}$kZER8cE;AmV;UcG+T2)@QD6}HdRMes&2*|TwWQxspKBsPta4x-$6`Xd zF{WYul(Sa^D=(agAykGYnNK9%_FGnKYsCOvKqsy#ML5&`-HedRwDdJke6sr;@OA4_-d>GqaM}pv{El2oQt7;Z;KeWQBMF-hOz?%g| z%mzNnO6$f7`MFUKP!WsbuIa23;k~c@JBg>IC@l{}d<4I%!KWLoU%a!*qVCJ3=Bu=} zwb;`WWHIfgQ+K!da*KVm5Bz@g$AfmRsTrG=#6jl-FWTeWSf=9Hu)3AjO+{(0fDh=! zjD&vRcXj-5x3FAw{doJM;S>~@s9uPiY;o*kU)H+?M~-&WIOiPoVY33vw}kt?Yn(sx zzJb*eV{sVEj}F+HCbYYC!t(T=cOvD=AT{2#>FBi(i({AQklJE}T7Yv=K6M>(oNCQH z=WI`Gzc6w<47nO~4Du?NdxMZ|H1VbJ9Ju=0X1DXBs=J5)eFTg4GiL=MQzDkS6NUsn zak;*UqIcQ02G)h%Q@I3&l!a$~MVA#El`k)WCZ@T`eS?bEGI~@ghjV*R%BlM4#QDi|sq~Kr za;LhC>Q4x?o;nkDk1Tjj6Qb1=M8&k~ms>rsq|>KQm3}|FNm2*_=k|?!aOf5^e1kf* z_dv@z)tfxBTn|&rdPg4njM{Nwt!+hbYqREibHW&o=rj3rVz3FE;8$EsMSXk?!8E1E zu(k5DhdUakK3`ehrZd@hj4hqlQZ|b1sUWX+NQUW3L7}2|lp_AEsae(j;r!#|@mOz! zILeTec^1*XDoqZng14fDq`G_vyAu<9%W*ui+KsKNHe-pc(^Lx-zHP1q>nhqc1{Y0B zo?w(oU|8cKpi*_>zx$#XQF0PzpC-r)qcVw|{jv`0;;LO>)Y$>Ee4Pt8hZLN4f0(v_u}CqJS6KQcC}HCYWw=OnA|rs+yFF$p@cVV zJw(XPqm#?3E78_d(7NIX!@M8`~Sk)A6TvQ9~&VS9$jEd%K_(nSA26fqK z%B_oyM0FD^b*#UFG6e%~Lqd1_f`Y|S1{LX<0H%of@aVAq$KFXAM{gGLiY2B&`-B@A zea6^Cqt2u-&nD%5xhPs&o;S;(WP03Y^Y^#o3H;O2dy?wYRm; zrgdqt$Ktu%f#7H4q?1=6 zoB>q|{?QLXBo_rvoJFOyrtehvn`_NcS_s|a3wOv!{dIKb>$<;wuQEfu3$I%==`_;j zAkxNa{n*#CGSN8GJL(r{!}}FO5k_+&O4h344nnAnB1>xUNh&LN_GG6|6I*FEJEGPu zAu!aK(yY!dv)Mo!#Rw4`*+TnqCXu6ZSMV|RN2%sDs@~PR(WO^4{vIIz9v)lD zy?PB^#cmNOie8ch{t1rdVJCCRaNMEls{1Z`INVAdskS`S42729CiUdI4^LFYk8^BG z#gp&-jS8 zfMMRju()6Adb8|n+K0#Uzzv-Yb=*)=H=2w_mjwj{KT(>baD&sKtIG1_)+ur8s9*JD zGySF>ZR@B@sZ4mMZ@s9m+beu+IaXq{oSWv^`MsL!yb$zb+-WC?a5out1oLuO*L zCbGC3Ii(3oP-jN96EyNm7H^YzyT^6&X>bl7>F3F|h``V9HaKG|X2_GNyi zy5^#WA4!O4V}c;C(nE57n8^)zK2zQx$Q(L}xUPbrB~P+&CWZ1?auLzxry9+W6H%y1j zS|7v@@|DGaljZ7OU0YT7f0q&g?z5~)UNWh-!uN7zv358>4YxAtrbqB#B{q4j`AwuD z_;6sWAn@!{%JBBrfCby2Tusr8#=A2fpI&ZU8BT;Z`z*tZT&`ApOdh(l!`qk#<1~kg znFKMWThFa!OytH@6N#qVB;L%4N}pylOh5AJq!G@`1?$^z8CR0wFu|9%Tko&K<_zhu z=hXAfiGgwy>zrhwpF!X6xfIbFqs8M%=8{t`yxMp;uS6r-xkyfR`NGIy5qU7%k;ruW zfc*{%lHsk7RhdkWRsQHEoc)kivF1Yx&T^!hzhv2uiQ>5&m9n9`VHH_B;8xw;u^486s-SL%+J|_BRn@ z>V|K~=6-fsrCwErvnxsjm0sWUz1Z_#_B3L>-IYc$sOtLOW7+bg4$sH)GBt$t&cE9s z(f=$}mI`jc{9N~4lE2;CZ{#V}c%=&Fd9{9(TE-VClt zj1)F1{Ns3=yrcTmx$s{- zi0YcrMy+gO8UtY@#W2+(@8?QLu4AQJyl5$;Nx76Mfggj1W|QbtHkgts-BEwEFCn9z zf3Xkl(cMEGj|fXC*ozOBYFFiY=`PuuKw~U^KBb_45wwL1Av)>cBz4{beVDFgX5_nA z7T4tLM7{ftCC}{#K4%OGLrF)5NNBn|g%{6_WWMNUfkPvKTT|x83YVFD2*UMTXXgBH z-iF+^^CFS_iia+aaQHe)g;@AP{8e&0s`M-Hje=q4Imc8Mti_s5DkDr&d0iv5eW#ny zkGjVRxQ``LjuBfM6a+@_zQN1Lkcj5M=bYJw53oI7Zh)X3@$JV+JB2e%o7v^;d@GPJ zo?1Mfk1w9Ey-KV;DSI?@*oiDzfYNkIPxN+H#eH1Y~y0yKlmAS zp~SQj83Y=qgseY9CWah$Kv5tgZYW8(UhE@>QC2T-9j{<-wdrP>vNj<$?kJu@dTEQ% zyi3JvAqSkq(OtrU3$axXoMx*712%V^or5wdhZ}iWu3tENwk4A|Z5a!}O2MQ*Vm@<$ z;CZ0%U>84z2qBpsDAtq~W~ffXTZZAX<7WHy9p>?f z`Yt_hzvvG8Xf?>fXG17GQJ%xn!$T@OQK-OAiNdw=uug)s9b4HC{cq7N6aq&jk{h{$ z25YNc!(J%PBxM*Ix~IbOzTR`JudaZ41=H}x2il-J68BfZ9HVpMB0k;^c4BG$`W4bNy=_!UE9O|RR+pb($C$6^pn z319uu0RbP)DZPPi@=p2N>;nW}_IMRUF^Q;TdACspp3VrZ`1H%$=Iy?rEh|RqS4^qA z+OT^TSpv0<1&?%UMSJl{X!60Cx;+Rg^I@1whJTB;maR z1A3kgLph{_B8-v6)WiE_&@f79BZU>fd{#lAPY3YSPNdQpIf4xR%+g&6xPnpKjy7*| z=l$XpgeILf(JXY}F zlMS3bzJh}~shXMwYN+j?D;yiVd>rCnoR3Kob(c(-s*RyU1kOOJ+HJadV#cK$IR55r z^I#TiuEWcsJ}&DM_l-fi1HK}2<-CCJNGJGsQ0pjzb>K|`3RAo{lLQDgK zAxhhGE37V_eQxhv8kbEq&_Gt7jLRT%EH3R-ZA2tB4q0&vjq*w%w)7XJd9)`c36to;}R__aFI(=XJd zQ(PKvTt$fSswC+AvD%99#Ir=hNB5-@ZS{|Slo*Xdu)8gKT3?z5@su=|c?xU#xiWe6 z%+U2O1Pvx^_MCU}@svFs^W)qmrkCxyniED3w%iT-lKoWG9e@ELz0 zNrmiqTgqI5HQbL2N&7pC92qd>$-AS>0J8%WO4%4Rn`B-G=S;>#nj|JHnebrKFsMf5byWm6Iwz{IFR`z3Pc4Ds~YsR?jSaMdXjae~g?N zTkKRL?wtNmZBl_Rq?@H;tPJDtNIv_@PL8caCF{Hx60o-M! zgug?}dil5yjSKhp6y94Kqd;tUU#-i9rv`P0ZqwyCKE?uDilsYY4cpsZcBj>c9! zX1^FuJ@JoGUA=-;-@KhA#*)&E$Tz8URQnJIQD^vI%z~Jd);iYWeUrY_dVo>kX(UFk z029S?EJC)!^BSk(wT5iDazyXLn(?q;J_5pBJsNRZ0}QD;*K-sE__RWS z8S?oRi#p_QK24gEa>|pq93%6ZmG8&`yQXqWj=E~>E9|H z9DMrL1GRtb?aRBMM`ctAms;sm(WfiMGLw93=yITwmSQaqN&aGZ;K8jX4GTU+Jr++z zu1IDMrpdERF#0{oPnAXwY9kXyIptXn^iJc`s!h(dM83@7B6Kj0s)gWG?0yUtcjMr- zTaalDM;^#K8R2Nu{*2z0C?Rb5s&4ri&X>WmBXgL%iuCjz@Th3^+wC1u(nlbjOP}6m zO~Hz5kSsClZ`xum<}{H;Kvi>htlERZVpSfa$x$T4{E&z z50ua&5n~aD!l{tT+S$lHPq3mb^@OHT&lKJkE?Ur`^AQ_A#`9YBn{VebNQJk3eiC<) zsn~0&WMxJsPZ0K@r*$iyYAh;2_94H=MUcDx?VCPl;<@6a0Sy^8KVbiJd}{XdM|q3=^BB~< z@AvXM4+u0paeoZLg9LO!F`J#^P>^|lJ_Oyb3x$>R!TrGsIEwu{VFIu2pPBAqWogUl zU}xdZ^`GL6bXuim=L%s!B&i4>zCU6iQe9BCD69pPW~>F&^+NLda<|Q87NaBJK|IDN zAf`WKv%8>FQA~ko0CWHb0JE7X5Gy6(XC5ADu?8}bV1Nn(V*Vqq`8_N)FVoME`ZeKa zypHvtVebkUsy{*?Qr%FZC|*`<01f~bfCs<_5C8}P_wN$}NC2b&G5|S%0ze6%0#E~J z0JH!)06l;KzzARhFauZstN=CuJAebg3E%>719$+u06qXeKmZ^J5CRATL;#`yF@QKg z0w4*H0!Ra70I~o%fIL6}pa@U`JODfdC<9agssJ^BIzR)U3D5#O0%!wt0J;D@fIi?c zzyRAPx`@NB|@P zk^sqo6hJB<4e$c+5&!|D12O=afGj{ZAP0~O$OF6ry%CMAQr>vr3e(qdN-K=rPZVwVRYpu$YUR{N)asa6M4BFq$mlAfstfiQ<|pGL>!B?t zbqeI!roshXKnql^!kh=eCUG3LFW+YsVUsuZT|6A(Zt(Fnavr@5ck&+WVLDmAyzUT7 z16S!ozh>)92}|ZZ!lWd#jKZ^b+B1u;6X`5`V`uDGOIC71aXBz>9ZG1l=YO}ls(Z#1 zgK^UGqP(b#cyuRJQHAv+hhGw39#yoE&h~{Gh4Q9JzdceLBGLO0aF_zs#;g;C+?;;A zK;Ynf+{)mfn~_nt#@6-z`zhWgeC1DlBH<(Rwj0L^dvWM?<_;upt+e>4x||OTGHXu# z7aWl{>?dErPsDUsZ;;m~e*^t@S2;jKdEyQ#93FZG8=CURxg#^5f0ATzxtgZ)23GvK?;`|KTaz}Q;2hKe_fqnUZ z_Kh&TP$F9!1y>>_8iCvNJ&FFAt>fG4bb_GhW zbhnhnpi|IIg+sTe<>A%Rs8YN19Y+uHnN%xy`D>$u{#}>0bDi7tsab!SO5*Hdr9Pdj z{>B?Mo=>tOeGI`?5 zPLjRs>bB3S?b-fM6V$~9v*XcxB<)ePc&}IrS<&+8QIQdUc%C2PBE07~z3aw0qwS<% zy-C3ZXQN2f>&MQ=oJqR`WU{56!~0Gh60p4)?3Dj9pIwrvC(SI~tDh!Gl{MBb4O7S> zJNcojlXp%*BKLuWW45K1MegKYi56V|d=yqAo?moP&_>#O^F_& zIZ8E!B2)o93Hf$%`*pb7(U+J=&B*>$_jXQ}Fq+kCuAK+T)pG8hb}Tip#S-!aOvvQV zMC3;~3JQ|*`93}yq<>0ip&5Q7QNKDTOdLv7(uXybgJ@3q@=LN1v55l~WqXm_SjMXk z@nA?rHWVG)=DMO}%)9yQOt!0!i6*v~ah>>$+N?C{go2Te{V)!4C2Jysm1xi19cSL_SSj7_Tb5;^k(AC#+QC1&^;Vt3Y8Y_`OehXRb#iH zTBF5*x$rQvotP@o&zA^^HQR7Ex3!7f1OeS9*0#bC{54WFID#A1vd7SNZl>?$r*;gv6l>S!+Pb|c#+RFt8riM4T=t@8)uggvT-WFnKR#g|7Mlid89kufb4-}w`GLZ zLQp-aWY}uGlCHVB4zy4%)DF{BWgO={$p4l zo!i@C9&rDXq{!f6W*-X1K6b(*^wXD%x;0V9HBD*k{V*yC#P1qyPeEE_dnt+4=6Nfj z=QzHs!c2woaLLGW-ExA5N&4N}_;0{v-*%V;wI5cMl~men+j6#ijM%6%s!D6O^%!wt zt zl71VEPFYCYCf}swNL}PJS}x500=qIFvm@cj{N#t}aRGs`7D zgBjP5E+H6C@$sBxPlkEq`?A5!(fk5icqsCPqs65oZYMQUCQVpIJQv}{2Uv#0_zUJa zq?P1M?t=uQZUO#Rd<$_(OIAGeH!npZ4d&v&hx=AlF1Q|iBMDu$>W|Ak#av+8xa()! zCzkpog%?=HjpdXVdd8$0UeoMW=-UdO;4KcJsimbm910JrneUcseL9n$zCOx^!9^W> z$yefd0E;^5tZhbqW{Rpuef!dB3uVAs5i`}EDNWfG8jL;j44Kb*0nGxlA<|HgRiVHc z%;(i!E*_{>nFZtRZQ^a5!jBBKaY9noab(|9d+F!lxEbMHWo;#qU>seryXwC`iQs#v z)GLT&@(St2WlBNYGi_%|VzXzWh0^90q?QVL1 zn8!=1k-T~U|D)kSAn-fg$!nyuVnP$ooIy;VZf{)qx5o3&zfi1`NRuvxUiG;GJK+sh z)|qX47dLm88M4+&dvlNlE}ZL$G3;0B?=N>Ix^z8J@sjClF3$b@%(O{JJz?AlJ|UMC z>AI@m8F0Wx%e$s1uBwdjq3@(iW5<2RhoPETC8cW;70Ps*K$bA!o0oP$JsL7Rri507 z?%0ONMDyg6RuDFzO!kw-5JcH!s)L8Ckh;=8Tnk<@6hDiC;beO!?WfA5QS8$D-c8)t zY?wAhiCFIXlCa@V(mRId|>r55xpb|YUC-Dqf;R8meDHM8ks6ACwO+p`T8A?r~@v;;7r%5?yB zd3&f416O(X^ce3HDT*x00oHsGW!r#r*0*VM%=42suZNJF8+{VH2`bxHBv>(gukeqL z5PZxk9mwtGznG=(W>CWRQ^7}CyAu!dv6${MUB2pz(Rxc%WddXGkHtS`o!^SvB)l?1 z7|z|x@wT_5S@9a_T48MhLIDrHTW7MOWbn+pSbaPyp3U`ELkSm;F#YQFEGC;vM`}4s zqpQ=KoUwhyxx&lrXSG@Aq9wA#*V@QnR%TqTZ1i=T=2FWdTG@lKvSh}godE2+`C;~P z`iQips?B~{e|y0O4r5Jxrkx*38y-mu_?o~@t31n#C|8n@R9LdmUPZ2!bgj1_7t!|4xS?sOWzz3Bu6`iM|5P-R)6D&b2n3Q90)aUHE#tnPSZ)8mGE5=xK=&BJ(ga0+@^>R){9DF-J#?r3 zSH^uy0k!x)i>LT*dm#s;+#~)s^?vcd4cMEYOf>i9t6jY*n*bUC2B;v}e}cYFAwtZ* zAmKswnxF(fW0eWAg81P;pw~b*_`Xa1&r}owv-ZE(#T7(!h*UEa{pW_F-V7y!bKUsg zd*J*30n3Es054!a^Fd%*pahVKW+*Q8eNAjoU)0zETM@GKua5KY(ly=gLpgrt^0WZC zUqXN7PKF>MKx|u}=sycs2+H!P#{+?K?SEYx_+3EctNVC-h-n@YGNcxW|5;S`eNiqA zzur1&ffD_ki`ojzogMo%w|W!_iT&^G4~@6Adw>E2;tTk@$;zjHrG>Ua34Ruqd0$xR z>94t?tx&w56&2J-g?lFk0zIIH0RiRwI~9e{UH=NjZTnws*xP^t*!UsIZ9r3*wEeGu zQFO`nB%rAzMgMN98L|6Nj-QJGEisKPQUjZF)8FlpuX-Pa4>?vwMuyz9{jWVx+JR-g zas2g`XgiePXDR%aXDE@thdzz+_uA@s{|XHQLVvDp)_nnZ{=epqwg0cO{wLcjQ^LSX z?)<&gn3ett#p{3)|6G>I2rlY;FtD{s!~VRw^H0kZg*dJK3awp1M&|hY1Nr-+(C@S$ zbCh4(aA^k=|K~RR`_jv49V)5VeKW6%{2l8Yhyr=;k3x?1S5+XI zoj?-^CqNuKp)5b!Bm>QZ+nv3L@^dfq=SbjxcY}YA z1^$zL7&7#Ka)F^l3uk9{h-w#20fq|Me#ZYjDEK?R6CV3_ zJTOE^4Z-M!qW?_#dzkQd5*qs-Nx+~X-Ou=c-oO9%ewgLgbl)0&8PMMgiSC9{{EYv5 zeC3~vqL8{je=G6#=)mu!qd${?u>y{t3;BDP z;CH-f_8$uY1`U3F5`T{u{1e|00-P)UUTI+T;Quz#eYfD(bl*s~MZe?kkNkqax3*sg z&cC|}|0EZMh!y`g^}f@9_j7gr?l1h#p)dJQ4h;kq`oAXoy9e<*XTI@w&V7G^8e#;6 TB4hFZN98BL_CMbQ1!DgnrRzZ; delta 95991 zcmV)ZK&!vVii-yFgo_3ZP)h>@6aWAK0075&caaSs7{_{dT>6q!`+y1n03ja$01yBG z0CR0@aI<~^mI@fhdUsqFa3;%stp))3q8I=y0001UZESF}F$`1z2;Z_;#m`{8rjwy3 z7=xq@holVwholVxholVyholVzholV!x17FYAuXnhpH!_<>pGsin`$Uc!TkReK10T5luXr{tHJfCRV$yiYb%4#0Fxc`Y$~9Pg#0~eG>hRgu@_}-(T#u6 zfo)9(3bip`^nJV5`gL?SyH+caDaLfAi6U#U=JrYusbyGw{S>FW61y)3>545<0mg=l zH#DWPZI(h9(~V2L7^axL%SN{P$T>bM(5ZKrzT8ILG zmx!0-jzHxqKxO|!m6hO|lT||Iruu)=iqJ|GYa#=^h#Ao82o$7themxcg6N~>yq&Tk zBagZg^PnHd>7xEhmyCw?q^$Gu-I;Gg!}iL<45Rx|jpp#^}UH z#6Z%Y?8WH z-OxlJ3`_BJG|!<-ZyO>|%FXO;GwLJpn9zp0+=L}p?}=8o6bii;+Qolo(`RjCVKe<6 zE2Zzkbro`*s_KvHEwW85_~`W(=GV0#cCnRQO_RB_Wmc>kS(0iMV6q4HqP`xepxvg` z`ZXwbW=#q3i%v1YlY0-$H?bAgKyLs;VW!)BE%`NE?23kqUUG|6fehv)3m3syTxw$~ zXd_p&D6O{rBzc%=$NGPN=%&>|RR9Y$DYDpTQiNI;vQUwnZH^OdhRIKNiEFHMRXax^ zmxhwJ5Of`2ak(88|RD5*`$)xFhUqquf znxKqqd&-}Lw1ZtKF)0|f%(T`&6xC@HwQABS(c+K1>y}(qa-4ssgtZX!cC=R2k$5{Y zQV>^EjhV`Rev8v;FKSM;n5$Iw+da9C((lRD$O9BRBYm7I`zFK8A<${_M*3*^p~zw2 zMB1dfVXjv#rB>P1dC<(xADP%gDJU7fzFJ*#5q^hop#htz>TE_+VjI~XfLYvyt*Uup z4NkBD9j4DH*^++=n@7jqGiV~?!+%Em;YIA0sJ4!b8?^3rd`Dt3Yc?ouX{oUsOECl= zeaxXLsXL6v(ZhHZg%Tr9LyR~SsnUuVRaz6HN^4?NS&;R2&BmwcRc`97Z6Y$}-=L6;tT7=xyGXV75-z52N6aXGKI#cbT1q%YA7zXIDk0s;W^-Mk`WA zwuX}>t8ss7mpGLDPo$CppB9rcV4WHFuV}#*xbd4ul!|07RuUIwN2dZ9pw^7ol3X0^ z$8BoteN(%MNWs@|oHtv2w`jAwrCf)RUmB{4)qdjAP-UmoRme_hzMC49)rs~ddQ(&0 zb_FH3v&|p}HHlFX%oqvLT8A?m`QdI@Xys7k*$#g^0cu=iDmIwN+FK~}X4VXuMCbTm ze74gKuTBx4&TsRE^dl5ChEcnUE-@>$TMtb@2TZIE6?I-zTehFrc~LD}tUFeZ7mfK3 zI2UND$9(VA!j_XV6tnabb@;B{LXSjRK3ss>SV%qdTkP#2m1sjVN$(G#7IxqX9gOto zY@L6Vv=?o@@0a+pFz2#X^w(pD~X@ZqE% z23=r=2O9iY;I?Yw_lN^@{^vF||G}F46$sJqO8kz+19bkztM&OC%O9ih>y-M8 z9H{eGyVUrjOyc*@upN~GwG7nxV+?=g9c*})4DrioJjH+R0R8Vr82l5B`O5=!zW>0X zy7)DQ{7*K-4_&Z}&fhgp`Ts!UEA4Mvy^GGDwc`NY@5>DFR~h2Zy?U6=|CKv;{;Lh~ z6K@XF`DfJV^EcwpT0{Kr_8G48H^y`4e-AkH%R%&g%LT)A{;`JN4>9i6Myc7 z!*#x~yvF!1(()_ukEQVxe`C2mewiV?j}7scJ~dqDcNi*c%>Oe({Ewa;uJgBy%pHG= zA^yg(N53rm(TyDq%-xN|zmtFBFQPXqhGx?rG`a^FBLj?RAr#_^x*c`j4aKpIw1_|8 z>^-o*zafG#K5E{bv~{r4c3(~6k2CzPSe-IuidXWdgFup8*2~4Eh@oo&0~5jnJkkO|xv{ zLJi-B_H6&df^lDcdfDigW+&ggxd#1iC=jkc4{VwQ)Y@X34&^Po#`&w-(1ojaIkaT}w`}a{g|?k)MvVSV;N-z&8wL)sEgfApqGrX( z%SH?yaj0$Tu)!;Bb<+lp+jVSNVprz_0U*GEq052qPS{?x9b$h=>^Ri+o1r_{2G-an zdx!Kq{A*iH)zH7RxDjMdfbqc0w*mUIjZcrL5?AZ?DhAZ z7LOXSbVLozoug+SwrbhnJq~KvdzXPNQ}>_c=sd^PAlIe>56(Hs_G{a?f&1I)PI|s( z$e^0NA+I3=YnKj0b4}Zj!BDe0p{!x&>&qH!BjnKFK_`DN9cHT;*We$~29@gYO`l%# zV8yaMmf5%=Ya904fEbtL2v`=_29~vi4y!wUbd^k@tQC~oxF9iQ0`g}3?mw>yJzF!9Z8v1-W_*a6v8r(JD)`GhZ+zmARCh%|8 zq`!_n->QH4b{l6fPs1Jr{~>S>gL@R*9>DV z{|oT{58T(_zMh3Efi-nUZbZV-RC!QghJOvX+F5h{?eyEC|91_p3f z`aT@|-N20iN0+0vQPkg^`g@u&j{u|L`(89`Z_RJU(C2+M-}Zy&-_ZB{HQ&d<^MRW8 zgXr_2;2#cd0=S8qa65f=fbRt70_O%-1+H52yCXEuli>MCa7Tlyq2KGkZ=lZ}>Q4rL zilKk5UU+W=Hv?QV4VwvmD>xs0Z=?P!@MnXYL*M6vzkoh3)TDb1Jo~{3^jjGGcKX}_ zzDS=JgC8-JCkF3`kxhV04pf)oY4Cdtzg+_F$ALS6e*c{&9MOnn^nE$_C)4LssDG;F zw=3ZJ5A^+X@c%@gSAu^QeO?9r+2GEl@8^Gme*w4)!CeIIVoiFN!t)yXeg*ZfqW)jN zzXse|aMx?%+yKwF(D!xV-wJL$xI4hz3GOa%e+73pxO>3eYiNu6H1GGr^MfWhy#c;G z4DJzdk7>f6gy*N|`_th64cs%}o(12JUrmZ)(!|Cp^CeZXS=zWn&|7za< z2hUr<4TQ;m5V#${?F4QpxLv>v12-JpZs11Ih0pHP|26n~g4+w+82W9jVS#r5ypIES zAh?6U9R{u(90$$`t`b}oxM~{j2=IS@OP`OV{!!r9fU5&n53T{+WN=f!dFgMaQNI!V z8T7dc{F&f<;M%~=0XLV1Euj8F>K_9>PoIO}3-mb*e!Heli{ZHwTojx{zsumK>2r_f zx4rOu9Ju4boe1uC;Ff_q1>6d7e*kwn{rw-o{}Z^C^nDfhXM;Nj+_~V+)1-fU0X$zs z-!BILQu=%u_-nvj3GOf8t^s$gp+DBb`}N>%1a}j-o59@zZXLMWz}*gRJ&kh*_fd3S@r|J7M;6D%U@8EVn;OxVjF8uuH?j<*Pm)$UH@6P0c zN$z`{$KUZti}=cgyB%@C)wh4nJn8J4%MYA>>(tXvy6T;O^NYW~;G)6bUH<*Ni*I^+ z?7-UzG&R2NA5T_^x)f zd|!UBE53Bv`(J-CYu)gd7bZu#9~nDo%1GE1~O$ZoF!Id)c>Z4twRB8*fN2 zm^A9b%f9fRctm4(__2SL|C-mbaot@{-go6|hc7?x$|GOzdEtm%qBnlEYUkVD_P^ND z^pJ-z3z!#+?wqYD9Nz)!xVN_xh?g&K>%ZJ2*Ac@&14Q_at_Ujb1cpTw2)s z?HxayJnQ>Q_F5nE-@c@6-J;$0zxMgiE5APFug%B0dye_~%%_go^#05ZM~&KdUgLXL zjemXeuCH(1u<3nQUFwuimn@ic@#rpL(QWqGzDqCr;;b?M-RX`^b;EC3_tmOnp1FGb z^7o&Q?zv^n@UMRleeS+ZUv6H#VC3i%y50}J(Yf#3XI)gj`k|F?Yd3gI&6Gvi?fv#WQUH#(hJC8l&K>3Eh-MT6K z##ipZ+D9HcXX?p!k2v?}rYCOP?c_Nf$Deb|!;9{ye&rOm_qrvC$XoHvk$ImCxwn14 zJC53~`ka4Fb-^`bh5bHx`J@%+v|n01_JKFf*ig6QwZFO1GwJ2#ulbwO2SskYbMi~; zJJ;=4J7m;eFJ5u(jKhyUtZU-n!*)IEgWq*u`Sw2^J8$f&;PiK{U9~=N!@k_Y&7Yoq z;{FFW&R^bp^rWZOExg|U!k1J2aoC=lNty z=lo4|_x^3?U7ovq?bdkkgxlArmb?FRS>0VX9+P@*V)r3iBVX=%t21%Qg81(5?{@MP zo85o6{A$YgiQ29iot6JyNz2|-U>7MWYdD+W* zKixI!sngd#`iEz2Z+DzA|E{%{&OGstH}2hZv3=ud7ma%BpGWThkn^n@J=gAV^$80e zzbo+R@*}?dVCr8Fz3K?}@L_G6hQ0W~#gBjQ5V9>g?!xjL&pKmH#}i-2ZvU|Qw0i>| zty|Lh(Gz=Zkfz- z?q84Xymd!$=d1YeSq=N0_Kq-b%Ee>PKL76*A9D5P^8f7krx!oqI*+>jfm#<{eYtd$W4X=mX|__~&JBO}x+l#Du^JLr*^M zKVMI|!*^-x{+GOeZPy>~7D(5H7#oqt8$-1T!JC%*i< z&NDY{z3P*7OS+QwlWu=!)Z^l*)eV1*!xz+yKluFfE_m&$Ny#NOPs|?n>hGUi&2K*Q z&D+EeH{3IH?eMGkcTaiw^IO-?8*|r7r-qjZf0{7;`$@jt7qmU+@}1~C>zI9p^BbJ4 ze|l@$cb%WjzGTYqcb6?ZY}M}u-FEL0d!4=JrHiI+UFw>$$A)(vOO2c`_|bnCe>-5m zrJuA-yZ8CuKla3@(~c?s>iHq#cCvl<)^&5gd-nHTyk##o*_@{nsu}#b4@OIOwqBuR8YZNuLiIKIx3Z zo}Du6)HiQEvF3?MXAPgWX#IbRjl-&bKWf=kHLV|yx{AML#FWXeEYct> z^uo8Vof`H(+4lIqF23^xdCrO6>DLclGU3sc6Hbp_vEjZ0ubIKXKlZ$6mL3_{Kfo_(y!?l6@|HZ%NIA$IpL1Z_P`wIpL2t zpK!s9HFM86D>~%S7cU-l&$QRReeQwp?ml+TL5DovIOElwW*$Co+Pxvi41Vx&;XOAu zJ~X%C#72+(;W3vTsV0!w8<^^q;7ooUnkx5a(M7@jW1vE zmG7Cyg)ifQ4=$Q~9vZjn-v77rU(etA-~sO)=Gic2;rq)PAO7snix!-9_J?O& zd*hdfUH{PkHBW!&?KSOZjM@EPM_%|y!-q}l7r0-zSh)P9=C!xnH*a9$a^J=y{&{M6 zeeVgwmJeAv$FcC$M~|~F*(?6!$_XD;HsAMj*A*KY2kpM%?Pq`1cH1M*%vk&||I%a4 z!)9EuV8QA?kFTG1R`~F576otW9lJ3;a*tb^rv3SYZ#S&@ZfL{K--XggzxKQESwr2m zhu;0U=cvzKfBDXfZ~yJN&ke5HXZ^h2?>}kd*jx8pGj7F@@7~(?g~*r39yICM_g23c zeY)x8mlr&I&trdkE&l%Abv36P=$i1ahsI93@Q+vh+qubo?Xbo>kGlGxF~i=w_5P-> zm%KXLan=rH3*R3j-u>5e{`*9E?XA0gvGB`PTh2M~yDbfG-S4>j-BrgA{9sM{NsD%o z{&&&4k6iyqdh?XGhCFq!Z~cN%H~(kA3vXU_{}cZhRepc`r3drxzSg|oIG_4Y{i8>PsHK=dn$vKDhAq;e)y^+fuvswR5)~dhCZiWB>8+ z$fp<8e|*zxU316&ZQeDD#yxBe2@%a=U*o%t1~Ka#rr(^>1>3oB1w*?n#6 zvyMl`CP&_PU&F_1+K*iQ?jx^0x$D6PfA-csTOv=NT=c*3o7Ywzu=SYsX}{m~jknKw zZqp8vkGuEsgZWEetX%)MvAzr5`{(z`Z~wTkzWwuxL8skv(#7&kw^TQ6Sy3L~*K-p_ zkDQJ8p}Du=kOm3?e}KVH5Nr$>Fkrxd0RsjM7%=$7!fw=G`8n|i(Dwrj??)Qmn+@-a z4ey)(t$*L~GyVGr!~4GUeV9_PmVwyYmA-~9*aiE0(f2JE49ET$`hG0+_on{b3rYA` z`aXpEUvK?Bj{HOHq~<#)+0~68<)rWD|C;za)Ay@r{dS@6f1{~Ci25HrOY$2?-&ebc z|10{w6ZQ9`{?w5Jb>BM{573o!?q3>rY zKiJm5^X<`ijF)78Qs#LZ?`@mGhlHfiE=sbHY@1OZiN}OCe;^`|F!K;T*yHEJV$2^A zm%!_MpX}hof4Ia4{F1;Z%Y?B&InD5q_{8Pi+%*`8LPx4FsI(mN|A2IC=tZ|hA6 z@LCs-N0jI3@mRa^+#m+!%y%dNdFESs`3AusiNu4DX(-Z?j)LFh3dG}*T$$*VJL0kG zU_6!O6u?@g%F>RlTgQq0N;WUPfKIE$Vd&GeM?UT8x zfIq~i{YlXulX-tIC`b~@#3xq^JwYKMi}6^4Lrq2KO8Fz`2aOd{G?L(ICCSs;=(F>h zr1_BY5)#w2$#K>FvcP9jCAo0UWGW_$Q3yyTpsY8f;QB6XSH=FJMBS;!s9c(>BS zNkNy@f2>%y7z^>Of|QC1O6j7hNT$@Ofb!@Mg>toWbtn#~C8v@xUXJr5SSd4ZlFE=E z`^88#gi3N!jJ4Off;~MPhnh*|>}n@@n|!=098X3a9#5de=ujdW9aptw8_PV-K zLbBJ>1SnXI$_{uQj(aeMT6Lj=#`PoIaJYn?gb90v=vF0&NHBX)f1V$k zV3zjD$y89zG434>gya!OKP6}ftlEyPjZHp|*AERu4`SU>rJ`smq;nve0r}PdkSLfi zLP9VppaQq(KF83sAHXDVKCXQ=?@RDef2Y7xaxDAXUA&K%MNit};?XOnjTS$}BdjCS z>@W@SDN%WY=?A^Z@kqQ|ND>Z3e=SN<2>2rym0;q4*$)b3QPx%;PuUwt7#JI%(ApM< zt`uR&;fb5hJk`-oUW!Mcojn2E)08Nf&;qqToOXqjPU9$|<- zne_MaF`=7}2(fm#BdxRyEl6R+P>cMtfp1dzx5-dAAD*FVV<8{n0_ng)e*v>%lGq*- zLU@Q#CJ%k5Lk^nStt%@U#c5IryAnW(O<0pmJ1(KbH7Z02;#nfNE18Adh}gCZT;8@+ zvD=907{M&_R2YLQ7_cf=gV6}+B5FUY&a;>i9T?+dsU$Gfejs28hycS%tpK6#o}M7@ zjfI3B4`$=wLo&(x1etV?e|7S9NscFY5lW5=LYmryeydX!8dXp+m~sRu;SUNPR3SU> z^e4nhI#)th`Ct&cvu7@aC5s?-o;FH$X7pE8&^*eIvNOq_cBMa&0GeM&T!)h&9Iig# zVp8yjdQoqyq}Y_ZN2tLgG788TPpZ6&ga{L0uQKx6mFZzVCS(eFf5+Ql+BRahf@2xJ zpLYOXl?hD+2nND9acKuohHVPMP)eiGfkt^tuNUDhP%T84EcsXtV|O$jN&y?&%5g2d zGw^eR5N?^APG>h*A*0Np8D$9Pop#Id$cOy0-&5q!a7uCnhGilpQv`;tLp)=(7^9Pq z8PAtDx(hGN&?3RXf0q#59ZV&YnY_&hkt@?#6d`HNwd^lmrBo#RViK5ngut|~+?_&i zcRU&L_^SC_{gPqpfrmhIm8WI0${;nSjch)IHtJX#lTI(L{6#LXfhoeZ>;Nhk5xh!K zJ-HA-5llczI%eHu38PCvLksFTN*{{In6s1I6sq1BjFK+Wf3n>b+0-w0=cF^;18->{ zpe+Rrn!a?T+Jc!y1mZPQJ=uw7a4`+T0@W@@^SFa?Km=Wnv=-7nX6?&yAt8)-B1EBb zjVZSnf>9^R8VpQpx$`B(6Os&WfCiOcM)W%#MD3G}tvN*(+98fwk><())IR)F z^s_*(e=JBabrK;6?hY5QW0IUx9*M)8sTfSzc%q8>6W&s~1FEH!Qvy9kh4lLBfEvqH zm}ddLoV@$J3b}5CcV7cfyEAW@Y2y7>W(QwPIUTxkRtEtiU~wJ^{j}OQU%7uIZH!8W z@mfaPIi+O{i#$l)#e14GqpOhovB3L)v8HU&Q0R-Dwdff)8-YaXb|n3E0V#sR07Cjlh94W zTFC}52{wrFGm;c*F}_cBRBL{Z#o|a%nUSvtEaF0O70EhRoe+hf&@BSuLS-ege|6E) z>jTnG)dZ}T8i)loaNmeo&n#o={gKFIx*#-7BqE!vR5TlAZOIhMy=`_A%$H~=`2%8_ zLo#Yfq-$J5qvzSP@+cb`96uuk$buWgJH4qImWZ(w9;^<2dK{R1I<-0IHgEA!^kqf@ z_W}x>XEWAhLpV>2>Zrd5mLrHEe@H8JuSAx?<2_zEe>EU6vm*Pl?M(*X)cEXuCoZ4z4 zAsNgH5q}WaG@@Dnq$-K%b=s4=`pV9Ee!)4WPKsshPi%Nvh!(C5@oNR zZ&%P55jO>+#ZV6vHr^=;p7~hD>jsuREualhu{}nIV8HH~M)N}ZLNFCo@Hkie79nw6_sfJ2-sc3v^NBsVU5+0Xl~

KF3k3{qF7@lW$4ne$hfv`m^72-s}evJxZ!Y>rv$IWn#1 zP+HHCX+2moA(;=|o6>YK78z={p~+WH{c2t$7hjL%wV4(wwnQXjfA==cg41S^tNEcY zL<}a%B34aI@@Pj82WIwk@+86KPXcND_1jRk;NDxRcyBX5**hJ0@-Uh`(5|8!2QL9U zyd0)eP=6K!&0GCnEvJwqOd3x`zlcX$qVmx6YegE?8Xoj3Wz7L=3C@Yo2zLP02BDE$ z2sIq4(xmboqOEw z`Hs^uX#-R#&F4OmeV;5z-fa4gN=lwm$`fg0@)4E5ywm2Ru0wK%nRJ(s zAb@lJ%vuk0NNXd)I;5L|NVT^XDOK=1Rv*DjYs^>innaL57Da<@Q-9s1{MuvjYxKoY zoA97;&on)PK)?GY^BBWwan-OIx)@8xI|aIB1!#tme^5pw9+v$f5v`T{5${|FCrXND zqN)p*_985!_~>oL>VuhuR|MKLhDChd=}|k*tuiS@5v!U>!Q0Rlj8q_X$za4C5`riP zuWt@UPzBqG*X#q9Mg7|Bn}u)}e){9<7<@iVYiFQ5&NbWAiY zQAr9)xJ<&K_S+7dT%`!Lyps#9dwo{jReYR^x6P*VK0Elj`xrP+BF63LuiyG+n^VMwR>1)y?a@#PzmfH+dej$794i3v;Gv^fp zf2m-nAWIpsoVH0?(keFCEwDkMK$QLzh7ER2OW0AQYO9v0wr+`P>z1f4WbA0Q4zN6u zx2j$439#Pb#bTFfm^Z+(0y2j{@n9;&Ev-2n16-U3ExGJ`pEKtAkTIEsoOBk#+A%Xd zn&}H4;*kimBFcCAe^J%()229IpCnR$SI-4y&TdfEOA^0Q8y_8+ z(xs4LEc|Hd>_S)-Pf#w{oQcn{<%p&1Kjy=7Vph<)-oQE38#sr0#O|wFKc+}P(kue| zD*6Zc3d)!Jf%4VWH}wP6x43)_tAbF!uL^!y3?H7ts8T~JCU&KW{(>hOZNVv!f2P3S zP}NXOk0<)eV-8DgCweZ@AB)GtAT=bBIijdJ%1ICU5$twR>_$hUFs?0)~veF zZRkQ5Phy#&`p=t{%h^oThGueSFg=_|$&xMHofKt3vwf37+I$x5#1bt#Q6`t#)N~xjM{(y=6xunQrgJzHsnV)*Dy=)G(zqal2&GCP+$J0!A)``?Glu|HH-NVMrHEp1PEgVk>F|yHR3kM@{f2>QGH#TX} zKmk9k48@~40BQNy2ag}>lS>Rg3Dg5OfjrhrWjT$Q+S^j zOGaCKvv{K96!pja?MTf^f7>Kb(78O;ES*7ud89b+r+S`k-j;rAmdgHY7G5vsS6XGV zp;aauT4gd%<;DGnY&n~@wv?vTWMx;f%2D1v?uV%IbT_ZCZ&0(kJT>#=u36qfsb%7F zi1;gc%EXt@V?T?5Ye~wG=AW-kvs)JurBZh5wl7NMs7`IaiFkGLe{k)dQ@&bNtF@}* z$-8N-rr=sl;WJB~86^wVo2~eaW@|>H*_zX6E<~tqVDDIaZ{zkZMu5*cU>kHmy#p2n zc-yy)_i;`>3bdz0!xju9(UOH7)f2b>)YN9lZPj?DDUHGW$AypngPY?>)KuHe8Q>t{$jaO8TON#PFG6V}}&EHnWcKf73@}t3^VdYB+eJy=@|i zMT&=t6^$hDMUqQEi%2|M)bm^`Qt?EBzFd-=s$7zselHr=T3>+-vHbW8mUGcQCD)?e zD)l^NMg_yFwNA5QV@2E_7>W9|NpSicO+T+5vTJ)+KM_%?kIboTlNP{P;1)pTE2a_1 zA*N8E-SiMsf8MN`8Vk)V81Zc9acJJ|`KUxzGEq@eVU;tj)g45ISFQ4L;k#etOrK#| zi6SdobY6(+W`(G3PC&o&Xzd>ro-Nlu*%rIm7sS>lIWvuaH}BDzlFQf$MW&C5ozQ~v z>&nirg5sSrSF3jO$-1!ET5uKANODw+iBW$<7%{DM#)D&RxbEkjs? zN2C4m3vB@I7uNuNQTa5}>M6Xfelmm+sry9r%wnN}9eji@DPVIF$o6l@(}Ij=2cspX zhZZW)l#H%_iQ;sAVc6=z7ndjrH6hz-c)K8De}P)TQG%z1ErPm!n6+2ewpe@V%&Oqr z^IK(gYjKZHi$b><0(!L8#Z$^w-nx`Kn2-SxS~E*rOmfZ1(c@Amn`ZsO|5j%p<(q4nP>*d4_5|N+c&d?QVI@fD%015wBk2fFJiJ4&_rW*$@m*Lxfw>v3LrO`86rzE zw06>#4ri=XkvNVDWHQc=j6w!Gp=`(^CXs7;L>mdRWrA!W8e9Fb)Ef=NBN87D*`Q)X zVL1DkO@W=O?Jh+%o|IqF)-Lh*O|2Cde^6`91=Lz|0kwTpjQWMPNHMBQztAQk%0*Q~ z2c-q8piD;ECk^Odbza>f)`+SiPic*aPd9h+2Gtq#;6WxsWw}=$5=2H`44>jckKvP; z_D77rN{ZS8&FIq-r}_a=cbLqLKZ(T*KLH+EYx&9_R4!_9k=tjqW@+fvE0WNKf35id zFJ^TtL-SK7I}vwY3J$nlFDhAB~BWm-Y`n*qF#rSnX)yZYKr;9t8Z%*@lxzHMwk*9imPgH8ISV#=dC73++} zyJH?|I=89-)47=43pV@eb#`-$wn$5h<4}v^&=$wc7Oi$fX?p;*92KakYDV;GB(&DA z2vG`6tSV<&@hu@Tr$<>1(MAoOjN&r~wb5p7 zm+}^-Z@9-e9$V?+k*T+g;0l?gggF>$M6IQ(m!--6c2AWO2hFRLQz&JvLn)$qO3%}R zEmp#qX+t_kLx0O{%SR%aRAXHWJIIkVVyYUJEef2qA1Rd1bNJv~~{ z?N7$cCu!xAqEyf#gnq~ zOzx>gmoSqiC+TT^$ajjd5c&X9JKEjX#fwVq1c?l7Onx3t`1zz9)4b6HGGPhrTTIZs z`E|csLf-;us$zgDe<+$%5OE->fF`c`7D%IK61Ng~XjspgBmf8vqbAcx@Cu_qd4p7v zSTGK8mE7RRx)1b)3^~C}MQ$50puPohQR3mrDN|y)jv`VI zwUCk$F2*QMkRrSAZQtG&HxI#pQfPgbsR2E-jMQgjo=g$Ee~b_*&B>%%aL1U|?fG%wT1F5ie z>6~a!j$&jri6zD*({fPa;zLGUym1LR*)I!yX&Z~dm09q%XIN%+K;tmPRtwRD+>2~l zAr?Cnf1*!`WIP!4OP!h+sF$G}$Z$VhbQ=Qk6~LGr;k$sf^a+2UXhIkVN&POsG^f7l za3zN^C&LWmDs+a)vfQ+t3|tkj+V%6Tuem1Imd8!)r$I6!p_SCY#(bKg3gy|LVq zR+}%=8g9d0vD>g$?4}oF`z%`fX#DG*V{&s%n?m(ky4 zCrlWf1dHuQMc=2t`;inX74s*1(IJJDq^#CtWt|*I#{`AJM5{>kEf!bScIrHCnAOpa zfbyHn0-yW}9)vFX(5@sDQF3ip9zvod!!(ikfvhP?yY3~E8hVT9-{eeXq==w-)_iZ* ze?ALZah6^dr4wkG1o^T>K}t&6g@Brf;j7^=GspAksOim}O%6eFOHnx@1{{@GVJyR0 zn}l$dA@)oNTJ7>OzHb#TK{V2>>_BB#)tMw0vg)Fkrp8$Z>36c!-JgPXva;*sY*$45 zC^LtI1_y5pA{z$i3NM|=On;vjmY(I&e<0#`j~kn8PKks-J3IZooE_f~jCe7L-h14d z|9Fo`u0}3?WetnY=By)81xMyGk;(QcV(kP;qAS^im=J8?5HjOngJ$ovg~f;vi`zn} zgaa{-LM&|)dW2vkL9%!9vmKmE>Xl?6nz_J%<@<*45XR6b*@>V`5Wf5ecj zQ;bBif05xA2#GUM#>}rfmrGZZ&!}lCJEutUqS^4-&iNYZTasnKoJa4os=NQ)XeQ+cV>wQUVfyX)h^dPN`ePp( z-|3mXkF6;~g<1x`kKEvCxGyei?>f)MwF3sbIuW2d0UT=HQI99kO0*kVAqUYA*U9PP z;!AmvU<2(6ljRUz6u8K}@wAcD2;Rta(T+!;Taya8FiXjlt`!$Kd<|h3KHhAM$6e~<`O~HXl82C^+dzB4Qu#LL(08V ziJEVbYSOHA&Ge`H(z<5X>=5Byfw%i$RY6amuJtJ>KdyxQvbYV7f7)yVjR7$08(R(dLTiJCXcjFRn6x;CMh7}=dVHJv6SUe^3@i4{o za8%amed!BHjcX$PPQTP1(=1a}Zq_tvW-l-pL8t(&hZ@Y3KZHTuDJFO+fmjst^=uVh zI;3KuOn5aC+4*Ipe~v0dMcLR6K3-`CwG{%ka5#cQ$f^ug5%_2&sJd^yED;kD$(naE z3CZGW6}+$v&{tKjE2(GMRb5E~E8m!fl!_i>iHfn=GL9nR>Xs2zSvOBex#ckZsrHJB zRBff_i*&`YQZLo%4GZxwI+ucP<1;}*tu$I6z*z;@wIc7Ve{MEeDsjf8k^$Gz6*uHO zM58_+3@xFvR~hw0vZ7bwF(!FT5~hPl9WSkFpG2Vw|>KqG(PXicxdHl!|1d1W~hwaSW(luT^bQzA0zY3)n`WXSWriEta|& zK!!EU68JcFf2q3y*{LA8jx!X9-~~BePUhd!$XtphE)m2S>buOBpg+dhkUYd6mWj5u zJ_D}63<@t;SI&6W`5EI8Miy^E>D1ZiC;eF>%{dOkSt4zSbsQ?oQOXMQ2AWurwk^_! zf!d0Irhq#5GdxPlBU$H*aS0C@ikxk2C2J9p4a&Uwe@nJPl$IEPSGb-RNh}_bl+kPCJ-HJ3Mni9E=6h1^3?&RH zK~@NpAuRndg(K6)x>1yw!j<)%>GsRE&5*O0G)y8o_#Bgr)$dIvojz44=Q^2cN0Wz! zZPA`Ze`aq|`#kZ@8Pvj}^>16}6-dr6nQ5?+ z%rsa_W*V#|GY!Stl+a*UKUjA+QUT5WLe}1cR?DiASILssY6Se4%Eyqa@8&rSDyk-R zFZizHEIrj9nNiuFTUEF31yVmR$v9UDZI?=fe{j16gmy8;Xo+bSmrs zYZX$G(44GAxYvYrzWrVtq|*bhqZe6YNH?Q9nA%LuT(fpYIV##dP|T{~R_gNv!q($q0Vmetw#Oj7h5c3~M)QV0R7GS+bW0;kpYL@DW2hXpp zaZEKww2kxQT>onfxItOHM%DqHMka))Unoib6;Q*EoUV8(>0xrMCJnSn3~jeYRGa4L z{(5{xVm&HwrtjPEor!Hc<>ujaK5p{Hf9x$F)i4>^IYZ`KkcLP_5SgyU8xGZ(c>-;V zw)*02yjCwE%b1wgBA^M`!9tY@34v6*16Hd3uz)X6RcnM10I&G`vg{9b@Dj8$GB<>p zt(wG;pr3?TP{6_j-zMoQ`VW3Kh%;k^VZH(NFB6nwUHs6IZiIu$zRd`;Wf6_}y zsu+s27;CC*z!hjcA-kQ6On@`lUMwz&>=rCOzDewOUt6Hi1k830i%Y^|)q+LDFL@ud zP!z~FV)UE8W+NrX=5IIM{Ov<2xXifCQ;fpUHRqr$09^|n^rl^1Kx3DhK*8vmy9M&2 zYr!pGhOPxP09TReXUjO6-Io%xe=}>YqEg6it)iosJcpVo0u?on_Avo+Eekeh)0_}u zNDraUMe6Sq$r);#NHaPN$W`!sPgnqUClv-3Rmxn2R)|UpX>&)PDp){q8cRx>a}sk+ zc%vgucXvulssdR~?*og5?IJZJ#?iQlaPU|GpyZU1@Kx%K1<{>NQDX&Ey;G28ZLqCd zwr$(hWuwcsZQJ_Fwr$(C-DTVAvblTx=d2yE=hcXqxR_V@W{&Y>6u_8nnpRT>2V8Zj zI2?TTlPgNY07n$Ff$Dj%J6M!_-rQt@Edx!2&q)^1S0-NtC8J8>E5~SpY{5osHh%oN z90{Jn>qBY2H&JS;If^nRRT5o-hW=375a6EIpiiE%4W2kj^8bCGm9c`ZCLFv}(FHlm z=m~~0WfMUY<^Yf+B?tC*mI9>00>sQb4^~qxSmWD(lJM&6XvmLQ$!T)5VOPN)uAGRX zy6zw)(86*4SDKL$z0N-`uo8WJ^++ULt5D&l%MKkE@=W|W#qZMLU@2^3@R=T8bb=Iq z;rD4!bb9Y~jEZN&;Kz>gkxAj!fdd{${k#kVOqUKvs6*Ks;(N& zNL|-E72b_DBqrbH^KY{dd7I4!GvBG|wF%dcy~qz{(nXx(St`nciJ1#b zF$*c_qI5Pzq2N+~viDycFC{@yW?JvWO@nVie>IwVP1We)8o%yo3iP;>eoEx_%p%bE zf-X_cKhOYnHeglYLDFoJqmYLQNQtN|tR$MqDBA_Fi;XbqFmIE)v*Z*?tiV&7vu_LG z;Q`MUg*uUG&Tfy5!_a>ot>qEl^#XQ4?l?PFOVqE!$FlLl0Q!2zbYqTWTxfU#1in=w zb@S88Je_6mKNvmW6YhFYZ>D`Qr=W?!Ce-5#9Y_G!_0px)2dV+;#U9tE?xHQl>q%a9 zlU&ZWcZt5)9N0UW$tqWSgjtBV7FWGoT6bZoi>PPw)`$rYom`FBv`*6Tbhp%|@fH7A5_8KDqc&1e=PZW-tJvnOL2L*05d=B%Z>4%K z+QXp}rpq6+xbXw~8v1?hPa>Mo%P|#4{^Z!(`nEW^m9d_|VZYP;ax#;}aoJa1_o8t7 zqQ3hL$hf*GC9REd2Vm_IXJgQBWO| zcYp(H0t`gUhSh{Pt%=%AWy;r6zl?>SdwuIwnzxjvWmqPCL?E}W*=%4 z7D-wnYLs}E?6TYOG_;qa`x6BZZ<_BKR5l-kliJgLhz5VJ&F`Q*~Q4-4kByaAO&;N^lLjh6ySlCqa_ICH)gV$;&0|Go*b1 zf)RvN=ej1dFOk0fF`aXYp;gg~@>=gx^q|M7z^`r6a9gh7K}?rKyj=mHfBI1a$4LKX z)}n?P$N^(P8x8?=K;}OWvcUa;;{Dk3#K?ca^k$8K%pc1?EM(8Hv1R z^Lynn^Cd_KBP%;nqcG!V*6x(aswEvP;uIqB=WV0rWNj9aA2@rLDmm$F6-h?`IvjN5 zN^22x`CN#c_SOs5wz~PN$iklC5mEcYqnjeeUWSh=_|!IgnOY}0Sy?e-bfae_GDCiW z)Hf1D1fQM=oQfUJ4<}>sRkHVw%W~SPWWu$eXyXH3n&*3N3T#{ePxc@D7yyjVto-?} znZ7ve?Ii3Amh98CX^!O(*J~yKjGKtU_HAilWu=}3om$1y>yCodTzN%I`ksnR8qGO1 zo2F=KlSru`X+~!az3jQn@n1fwK7GH{4QrzH86wrZ>&oVujXn0fSIu(db2J&@myKH` zH(W6TED+^BE}7-E>zL7%EZSo&ZmS9y?!w*R2+!BUMt<4~VVRV|&SGeQ;$|nARZV#y zd5NW}Boq*Bw}gv=LMej;vu=0@(C|m~QK^`5Cmd;rJeu@#rz155WuYT+ zCK|W~M?3dmk|831g2ozPl6ngC;;0&OcU+7V6__wku^+zTUkG9#DT;}r$j44gC8>nS zlxZUVK%YI~#aqc*24bQVN5X|Mnw}UL1$LZFHZ7_uivqmZp{a>_pr|X3ZQyRZ5}3sB zdxlN}mbxQWIXK5n4mnh<;Q|jwHTw&?Ip1&}Pf1Q}4oNtkp~M7mMqZGNRvtHoeAiIS ziAA<8R~00L2mnt8)dOczdgp!20)~a zN20JOSw{fW~pYvXIg-FMM6be}W$pp2jZz@2$lH8cu^g zY;u$1`9cnG&~^gA*d2{3^W5w7ZCRQS4lBpoImcUa1ZVdVdiO5mJ*0$86uzWJo98M-<(Hb0 ze0(3%Zz`%7F&9)eP3aL=5aDK#alfE3ysa)ipLoT#BEfAc+d1Yf zoQAIL9rt8FB60E)yYvb?+U}#-Va+J|%3i0Fj(Y0#dn4Fg)K63(k=&A4qC)|hKiMPg zU!~iWM9>x`i}vOEYFr)j*vsjKoK&={EmyR>p9Hi*%<)-Py)7lBN>dg3*6K}VI%`rN z^>>B_Cvvd~DoDlHb5Yt*tkM(@osPgzgz9t}=?z0bef4i)K@8BafCu%$2d#38i-BO6 z!FaX}Lt+Q6N(kY86UAW}X<0*vtzkOVYFYFu=-mb&1g(DI-xlDk&XvE!Z^9l0F+POv1mbMj}cQab726nt{Aw9Gz{bIMNmVpv75rL0d>)aW8h)nrp_}DGeu4) zD)B~6U{Sx1fwWq}YdKNHD(Jx@tflu;%qRK+UiPJ8V{n33>khmunpTF0Y9VjvZ|)ni z;M;H;A6uWQ3a%1*C6>L4-2g7jYCArrOSMTKj}`f#vsj6C2{#ckRf%Z8#1>ZhUP4(u zCDd>?jE2q}8KXF-zK6JZORfU7mwFv0OK;AuenFQrjy;;4t4=MNrgE#j!J<3zha>_3 z+H(;}#u74D0<>+X-QXtE)lv=98kO@(g85|5cCiEvVK77}RiqWLZnyx@-=CHLx|Q6x zjm-!mVcSP0R~4PIOKzYk;FOY^-C;Uc!1-&WwbLR-QaXwSVcGGAiYXrIY&k~DIfZu_ z;Z!3e`AZE@oIaIK%Loce@(fBQqVZ~gm!oE^@nVfM_LuChE9sOpSA|6Ys#gh z#FWGldb;M5+WT*GBE%2o`42bj`bAY4dNJ4ZCvLFmmi4Yyl%izQ!7rJQrd^Fb*VrR` zcx|mku)663eyK}CMULRvynij)7ot10SvS@P6fa-qeuN%+_CIV9Hzrj9sq7u9TO~3W z_(an7+T++&tb$;OfZ*y42)sWwRkcClY9;x~BVY3PLq}9@rU#J|{y_(1s3+HBD z3A)^5qk9$zN`7=>ai$r}jT9e*&RHAsZ3ff4FN-eRAAx_v7!)uer5h}QWF+(mb18mYXaxTe{ z#AHddBPlYuV5vo&g_(MaQZxoc|9{FZafP-Ko%!xBW|Y zW9@x%yRhe(g3Pv-PF>S0>TNe)Zqe}c6 zoFtv90sFGG$+FHG@AJDZF57nNkI*JhWksx&UBf> zj^TN5!DoYjK{r~GMmdN9j78+M#Ni>uyY?zSSkwbFuKYfF6fSHo`$kUYMM)8bNgQQ&l+jCqQv;mll^sX&AlSns`S^M%S#+Oi`&O*F&A0P!SCCmHL)*Upl= ztWR4%&wI5Yl()9g&a|~l)F{-c84fv9xthCNj^0GTgIkS=`ledZjU7k#KsJZPZI|=s zHC@vOrp?eo36C#C7yXzWt%BYOXU|C`K)0h{3t2sMH`CL5)WP*sXB+#+^(V8+vg2*m zap-ue!v{RFuAf01el<4RT6{E*T3GgI!wtJua2B0P;o7R$Jq~LVq{B7H$cl4g?(jg^rpAIR-l8y_taf z4z!+az50lFoo^ys6yST^A0Q<6%VyHbWof^|$e`ym~vunPd zx%M8g@EzEbJ84hSJTb{VWZ|J}iyd=rtquV6s^p-}cMvm-?J8cjZP42x);z45G2?Z* zkpdgD7H+chCe;2)dxRJrzhCUu{%vbfjaP;NGQws&f-dOlsu5|yHfvx~I1OVMKzQSjX<> zLM%@y{DZ?iB!mlvjY;Rx{#=M)k385k7g+anZRO~k_pXwa1u`Y1bt{UnzTE8o%}oBN5Z^>hgv zXmASQ`CP)V9qmUMqZAjwY^}|G=R1hTjzu-+kJ)ZFCT2Md4zmKz$>3}w$mpYEHCf^Q zVvYvy^L>Z~Ymze@kiS$^jzeDiknMQoYPq|bHHqMg`{(Pl%4>_$MLH6)39IIyP9gx} z)#MeH6}iTa#v?-1tP@i>=~cerfSGyW^kU2I5b~@qbn*ROBzo(9wj1=h&rTBb72`h> z3wu8NLL>tU*L?xQ`frZ{eXj?wPB0{=1AgNkXgd%2bVXTX%}f1>fDX^ zhM$i6Y@Aia*VEnfxYhB-;Gb&dtT&f$O$7q@@B6u3-i^7W(xCR&y)7SRXKohME{spB zu>aZ!0$)Tq*n&5XKRf|l0}IC!uaLQ69|ABwSga6Z&OwI?0RncP&{I8*yw1Oz8~d3M z_FSmn*u((?LPMUXlba8ri+RrPh7RK1JIZ}aZDiY9zTWOD^QrER4nF&Q(nCki`QL1@ z`uDpg_DvbRKEI_YGz>2Aa=WK`SRH=Pj(geJl;24?&^Q~nR(2>=hW%z zb^C&1={UTl4 zh^w)d&7oz)mi6r@?OmoWvYZ2Pt(Sg$0?pVLhD1`lRTT8RT#&Z!^}1Y6$yeOXtGeNM z4ZstKH-%t#YvC~PGtRXi55Cv=%5rVHNY!U<&&69!D)DX&pXwFl^Hq2oq^PnVx^=Q{ z_wfWM@W;nHuw--2#VP0)MT*DI1l9iNXoh=c&YVT)k>rl;wU=CZ?|t{`^It6K z_kA3PZN>yHhjF;$e7JljnupO<_St9;@af&}x`x85~+pBmepPIBD1^6~;Q zZmzU_p~n1eG%VmdW=GyCIePL(>%yjX8>50t?2W$eUg z9BtIHcl7BB1uW2UO-5oRdt#dV0wxHFS)_T8ejZ0Vz$VPC`huYSvWgBODf8Om>xg9Bq!a-MYQrFX z#6<3Qsr&uXl1>$3Zeq8{vS}`=VOub>J;LZggk)|7rR57Jp_{du`rUOy(u%HnU%>OG z4o)QXBXT1pVDnOgLax~~ezkVGPfkTEKX9Ct-(034WX+sZzXyaCU2->4>A!LnWY-t! z`W1C_#m0KbhE8zvChDVH4axy&*&N$#+t}v4mQ;s2)qlEr(9?xL4i8uwE*A=u9glE9 zu*V8tBp*y#JyG&;?~fOfLZ9VvIte*kaN&h(g~mN#uh=`ysa5lds$`!#&##XIK)^L^ z7Y<6hQa2m6RI{3!=Qq>x=oR9ZDTp(#wfzv5La(c2bqMg#QLbRtD^mc=6g(F1NN$|X z3u2972#I{1GL0L9;C^RKdoLOyX< zP|E-XJ*;u}7mO`A?%>}V6O)>W+ffO0yNl`$g5SkOReK^xm@P$%7yE?}FUyZDi5OO^ z*FL)jJAGCCT^6jEDJ~<1$1hqnpNe@Yr(Iv*kQlj-j~LltJECN+Z?JYsN98$uqB?l! z-0cK^-;=9He?b0shVUPSJI!bTf(rEC>>({?0m2R8e})j6FqW3xus}dAG-=3-5VQbH z0R<~^^3{&8DwfPx1~u+$&Q?k{ufIpUcP&90cRA)TPsxjE2AGMEw9xjS3@~vPcZI(bKHX^&91DXm`IAO(-cPM_bI= zfF3KyF`Yc!2KUk;!J~{bq*2E}=^IDpVh<7ZDW!-iu`(j1$uaIpvl^>NFm3By6C$r9 zsK`}P#-JUM!C>WE3=baOE?=FV>)wt(dH;2>X9ClqtlZeTKEJLHmTf;z-hBYqdux^^ zXlb5k8V)=kt^|FZy@29U%BgeeGo;(2GJ8c2n)thU( zo$ZlRO#9Qi>)PG#9Yjw@R}KJ=x2rp_&6bYd?l#C-PNi^u+ryiuK;hKox~DlQWUv)D zVUXipORRDRmjBOI{~qJyYY<>u1gC84_AJ^r_H`&Ia+bu4umOrLQavMLcF+n0s)-(4 z2A9oa-&-$a{&pF%Tb(r02?s$Q~?gH(-*;=#+~4SJW5|Eu4n z1?T~nS-Iau6C0!9hQ}lvjTE$hdA#e1C*I#Ns9AAlrwhK}kN$PhMBzW9uN_4HCsB9b z{_RvbJyY(3jkQUkF)KjdI4OFUqbesPsRy>sV?&1*5#Leq|To5ARi#+G8 z0fMEMDGRy?$?grzC2}84om&%QN5f^M&SI z3{?B~1Sjyykv!Re7t~hiJ#5WiXe4c*C|EOowJ32kyqm-BicFX|tS&@63?^+XMMjMs z?054dOm5^O{{;YxzCE-Gn`R`2?miXz#_WTxBWdv`lLgn`NTH8ODiLHWT_=kdbGy?3 z@4E$yJmTxombrGyZ`{&jgRwAZ;T!`-Jad*&!M}5R11vL0f$EKfLrp8DcrDZ%eGQ>9 z=bj^#B*|}JYfm__o(4&Jt6|4LV0(nkD+P*Rht$dd8oNE<;$O!ZJU?&^b`F3S{z5OBdAGKhK zZ8z3Pnw4`W+UH}(5(j44-aSS*nETEZ#NIHhN%g1kR#E<~#kFD&sfs4(ryx5Pf6Knu zzy?rnA|mxZxw+{Ig|S=YSk=pCufb^rCuvOO!i%T`MmW3&cR@2J_#_zhT|Y!DL>75^ z8xv8$A{1%rA8-Ym|0Bv40!HP%PV_hDN2MVEEJN(KK3^{icP!NA06he2ViHAHY?lD5 zq7i(>I>8j=JlO2n{4J--euQuEl>CprPXu5`q*owcLs;wwQv_A;0qw~*YEKKcg>cr2 zKi1e-pgfQTgXT%`nxbc*0&<_x{djEP2!YLio4E8%EwZg6$Nz#i)^p=u#1)%E1(z-o ze?(UaJlBn}7heGVZw{KGT~Pdtau|}b^mtoX02LuZI{@t$A?Mgj6fmcN7;`Sk5I4Y! zy8`d`th|!0D>zLCU&vjjO_Hml`K7=TR>d5qM_dCR`DbmeR(wNWR8b?-Hwe!MObTAw zW?zh0-Uc6=p`D{>{>?rq#XhSZY;;-011`70QYJJfSS=!Yis3&;V%flc+0X|E(Pi+n zeg85-^=zjbY!C*!!x;5^I$$b@Hk%F^)ZhVbug=uM5ENH2EAc zW6x>^Wi1=K0Z~|J8YeENdqW+I)l&ca6655Fwx%piaqmd#TiiF)r~^!B729%bH>1YG zCy>Vj(C(W)oRHppiV)A#iT(Ga`*+sjU314v)^s+AorM{Y>$X$mz$Rn6L7fcFOHkdBAyuE}U923E?AMh%7DD&p;z)4+&uL3{g$r>vr4S zV@xI;BFfMjU=RNE>NXElsIV9e%eorgp|(3K$4kG1u|C{Viv+tI6+mm1eW@que4~W# zQ|tS(KJx3LZ=*Egw=6NO-%X{AfBjn1}*iBA0$tN38YU=I`md$vU-g zv@E&dCs%U^K1KrN@oNPfqootgx8Q-%g6Y~So<`vF2s80`jc|afeaQMtWbbk&>xN<_ zm@E9g#&IyMUi5>ad|ud{-0-C$WRsblmpgDLa%cE4Bs!kvnfzq)r}!F9CkCg$y^e%* zFEMt$x32M!8W8J07G6y=PpVoq(*8o~txe^QJk-#*1s215s|A8y3&uyktwD$T_;_yN z(+r*hs8)LVEXM$O$|yY_Kmi)r{6e_WH1_jg9GI@zHi)LX9Q5*}ESJCBfUQ7Vg$Ve; za2eZz+_(#kMuGbfx3#yGDVyqhNEM>tgRoMN7(!8E8pTG_Aa5ZUyrFLD*jMglpmXiE zUA184KTxx94g**ZqLA*#jP1WQWNz6N;HZrEW*_s#1O@?qV1Cc6Uy#A7$={F-%gUdX zrkrp@QjH)qMc31oLEZ-Q!pBEQ;y=vu;p7>)C#IiMf1j3l$2dzs(C013Nprx1?iP&6 zo4BbGInOQ@Bl>Xpz)st5Er+bQKZuy_oLY@YDne1FL}%GJha(2jqi={ ze+&QrJV$K^dl0su|5N$5`5i(yEBvP(Ss=~$|5Lh7kVQ!UDbOIu82tZKQaVL|& z8PW*rzrzCiC@@QBvu>v3}h|De|H{x%wPHNfq(+N(gqeGNdcQqo1CdP zZ!k8WL=9o3JbXNia-L>A5iBVe!;=AdrxpnAy1bppI{Ef0y^ejA*LV2zv7-DrrwbY+^^wf935=4G}ip=;MVmzKGJE zh*E=pc1>r=EjiCKOqxC)hyofkR&EV4)SJ4|Z+D$a^Hi%woPy#iODxr7`cjnuA2+SF zkGrlP2P54Tn1vw_p30`M>ue*<=oeqYC(FfOmMI)g>CKCs30saG8G};{ z)o8nxuy}}JQSv?ihz+zm0h=#xdY8~?aIh)sS`EIdD zJ-sGrzZ0Xj3mBfNR%;4fzK$B}^ud5vs;&<&2s$;6z*DLGs0#*k3_yOHjXL>sLjn}I zUX-Jv?9?*cPw_cr1t31|ojeF6KT0)Y3J$`n$qHxh&WUtm86~R)WXE*`MZ?>t3naCM zCn3$$2+7~x-BsD;1y&Hz{P$EcOzyrg&?c-N8d7@QTaJxmcH9n`f{BxXd4(B==A;>j zv*rqstC5zvuJX`E^-Y(zg_63GA;O?r7tLhG+4Td$2H$w8DBvpidUH^r-lW5V9AW;6 z+pRet1#JCseE0*VUsCY||A1viJ7aLX5rvSEUoYG%f|ed*rjU46NTL3oDOOX4FrNjZ z79r41y{(1}ojJ|3EilzzpSld+zia!J(@YhbWCn4RNe~ecS*gmfg`i&4(S0{Uo7!x| zhan*sY}k)w3Xl}B*jS^ixR)K3Y6d$LZP|;gB$!mIIU&M~6KiDwb>mdP$!yPV1iz*o zOq*{%h0)*vAJWJM5{*u3%=v(VQ}wD%Q)Z7ygY?n^HTtQ}TH7QR8P zCbJE_Tnq&@i=rOruaaC;2btiaSA$?WxeU-Ln|Z9Fv!DlcjAl`DHLNH*ma{7HrJX4c z2@sWPMjJkKW&$kMd5@On5|JtVfk+~dycRP9EVbaLfphjrb(EjhgxPN#7*jJv-fzSn z38ek0H~=9(EK(hKL!iYa%KPBf218vbSxX97wSi_pl%$tZDvf9@|G%|XODlppX!~^P zEPI9hocTHiXR(mR(Guz|ja$W$H$e2k3(Sq4o5GM->Qk;WYc5YH^LvJWnM7vRWl)Az zzbbG>z;N8*Tp$J$h33Zd)I@`lvnNVSa>E=2832;t?<@txl}?Kbbl2sQi9PA5{*~>) zJt57u-k-w#<`kz!zkuZ~DUz7y0J*o$pL?s(9CB;FZ>|lBQ5_L%=B)3YueOiwANyu0 z2nBlhK7ed{{guL--!@BZ?-5MxX|pG?86I+;wd0&yx^R4CbH%k`G*tO zO7Ud2%KT$MfZ>g3uL`SXqUbI3cUEaPbO7ELjiMF9j+)gcv?>(k9^WKl2Z+GDftHkz zvUA( z)Yo~}S2<|HM1=T-({Ueg7 zY{>!YZ9Y|3!gi}o>Qt4Q5uF2NGXe08=IRku^#1}?_Ft|&8QG3>ziPi%u=US4(6Cph zlX4Z1pK}^8ZUTi5?xwf=vPNqAEl4=yKGj4OYF%ILe;{#euf_%QR5N(0l~{n}Wl!2k zwbhy1CD)4yTB{Gw|42`Vv%b;MayxI6UO*m7lNrNz1{Vj3DZQx)VrDFB&kB&Vx&_@V zOfrcnXd7t_HyOK~YbMW5?BmVA+rNjK8|3c1n&?XIcfUCAx0`E&&J_n`+0+F7g1HS> zJa;ev7ESsQ{+r6tE?3IOMMtky8wATb)s;4ay+P6=@21=XG@IEGHbTxXc_`8^wk4SKb_9$D57k^+r$tLd~> z7D>?)@F@Mq?DtRxC^My;idnWuW(8K1U3srqmXrU%F!VmyOftv{gb|aBTi6}L@>y>5 zWJD(C6Eu)hG%b*L5h{~iJt9@C7kIvfB@`OR8fdaPiZ>8s%rPKT;uFBqYW4$nBKv#s z2SH`nDp1?MsqJ)}Vcf*TZNIckPjwQ!&;Z-55DKa@Wpo1Amjk*0<)$QI|J_ajtRP9E zjrjR9^gFX47jZuZrjfs*YZ+i1V<=IPjc8F@(=bf>4@4^Zw&$65yZLY`u(OwaJbgA# zyxH9h#~;JCpkK!DH=$6DHg=mVG#e#$?owW1TpY8T;{N>9az+^;rlzwl8lUXM=P^8N ze5C<@AGGf)J~+?~{A@`j2Y-|mz+G5YX$5Uda%<5znlUT<0tJZLm}EkWg)rZmoKF*H z3vsC@url5PS9JyUn3Tg_X`yayZvmehf=$?L@JIXklE}c~xNOleje6ewrZM4|VN&^E zESMP1U;5TBjSMO+iTaJZ`oQ$qfBT`QM_g0*e6M8qJapaiZ~cj>io+5|4ZEbimMhe6 z@7kh!a#cX&Hw|Fz5X9#Sq^P@!s}`bJMo-y%HRA{^^#Ev=|9LOOzhAj!a^(UuY}n0J z$v(DXKLeHQ2xS7X z2CxcAK~Df8d{X$ZhjJsb4kHH(s1^C{zs)NpCO^rAffe;fV00H^VonNGkuSEFm()y0 zA%%jDJ0dj!5a+ds8w9`d_90rpkj1=`_i&=DgC=4o4MtblSbSXDMl4wHC9R8#4f}Kh z3?Qyd@`I{O>x&rWpL(+c!umL1m_D`Bik830_$#~&iPIhR_i4*_5x*v;K8VSzJ`4F6*|+! zce>D?(;}E4$kk%-l_1|~(TkqKr?WjJT0KLF#ks`3l4sPSOnDHPC4ie?K*uS65Pn6p zLY)9Q<(VFm$&lk08|}zB_pG25$$wg1*2CypT`0(=R%8s-fIjcel=wBOl8GM8l*^%s zyS|6C(NVE+@)Zsu#J0_fFIk8Qh4z~_yZDnxGlgOmoJ8RQtl4w5um08XMZVEgYNC*x zA9`|qjeRm5QD&j4404%zH#DeU!Go04Y(xNj*w*9a;8xG~->tXzk-xTh<4>YiyFvQ_ zQJo_+gbW;$sv+YFZhiKbmULJ@(EB%Cs|>kgz9w;wVeJr3yTvwHGG-5l$8_GXX7632 zc6#qY571di-8;(ql55C($3*X$+J{Wl!nZ}wn2=QE9@}hLE5}yA{37ayN28KV5Ptys zT)!Ru7=R6le_;z)qb)mo5Qqx;Mqa?j4i99wO$6f{?87x*6yj47rw%DiSTP_MmETj|0I~wZOCeFAxpVzXN16l%Se_ zv>iBF$^JR4@o)DXG{;v9E~K7=D@*pUqX&Ex32+zkywlCNAR|sWJ65b1MCkzA5nDKq z6715FsT}JUk2AVCjwphA?3~Vvzw7?KzEeQcy%a8+>Mg-WekxZ;n&Bvjm;Y!u`{G%^?#wN#{t@ILjT{-q&)&`yyDf>vRRaB+&5jaA6S4@VA(KeK#Z@awMCy3tlo(917d zULPV}b4k(Hx^;KKY^;*KyX5qE0|YvZtSPI`ZVlNMK~EY`yf`GzROdD;U)9n^aQha7 zV%i-{V3F&{m?#O2h74!A$0sxVbo#y@r zmMdu>$aO%Zk_`|im-L%I)mk=z914B9ZNp7qOglcI?yj2U<5jrIxza95+r+0~oArTV zFg8z25MNPPu?Yu!oHNL6&3J^r+7EfGF!{KG(x^DAwNGu@PTs17R&mY}Mm?JpYd9uT zVR+-q)?b~}^RbgL{vy9)jphagQaKr3X;jr7V=Ck(Ka_@MZF=I7(^baRT=Qvqi>fH-4v;q;_%yQ&OB>Dkyn_VNKP-R&^@w zC1=IfsYZuFXX5G$fxLz&i$uy^R&jRww5(io&o+SXs7ANXDn)BD`AM$c&^^2c4`Vf{7In%i#qhOtKAphu%HwWV71!k$ zPmlv@EgZy9q%?_#R*P$5M+7RF+R#4IeHK@Qf7NMFOG1>wZgB$pzNvU@c$VQ6;KH(K z2a#&GDnZi`R=*1rLces_((Y#w&YcjyDZhJP-36rdT^^i_QZBbpn!fTQ50Dg5VUkTy zL>e1XL=ei2%!nMK>gnIs4Np_+!45G<3?>2c#80Lnq2h3f@%Ni=3vvl4P?oKfwF?Ir z`G)-LbKT$euyq9b-{zfqh<6axEd0ORL6<=U2N1U_FIk5&o9U4u{TCWmRe%m8zc_>UVoWr(?-I@h4#$jp@yPL z^FHJ#Pm4h?I>Lh2bX4pi(06u)VSkuE5F`8+?oBhkzZ8GxF$mpvR@ z^V2eFSEbAA!$@`S4(H|T-~4q9?#==IK|WUwz$0G}-TzYVVWjzba}(4?mNUzk60G>h zf@P>zez(*o9*p9^{?5E;;Hj^TdbO`~Xiw@dq<9BkP$tb)pp4vCrq!<1oK0(U2l{6me2C-t=^2T?$;*6p*1 zY$&q;2N2^F)vt{kG8YEGr-Pf9<2NnIQSRwFCi!zFbOA>hrn$bKsLPdPA=ED+XgPtF zr#6_n_JS%xFHOzNV0Agn;85LQ-L&4ZKire042uIrEIpe}x3ke+%aO;WU1L4EBOP#Y zjYudOBblwK12we~T}dn3D#rjJzOh8>kk;iLo5C;l2EXe}s%jlV)a;XW$AFaPZ-N8y zTBcvFyo}a#=r=uyaZ>>tK&pMvaYqZcADqpEd<;;!8f^-!TENRiGBoWPY~|C1gR1t( zAn2g{ej z_`N4*jrmh*BrGD;COa6>X_2UZG&tQEO(Lm{=zLBxMoMEO@$u_GR5E;Elg-gNapJ*l z9*e!^_%68wV4VyB+WGnnqVGIaSDTw z&-s`0)7j%A%KHl+E7RNdQ=MDsH$4YXJ(nZ@R@|OL>qgJJ#V0Bu_d!z2+GxpI7qq3) zX$274aQbn@cX1M9=Z(P^+z^;T>NHT+524DUurnC3mT~;#QOyD#j-h>VTmBY)YBl@8 zRIKYN{mr$xc)F_V4R~?+@ZZK~p~4OIagySp@05ahAs;hdfHZo&dZ}OpL8#!lQOFYCi^&|>p&jE{D_!8aXjUs@zO%#j z?ww~;?&pDGdymq7L_b_kYwx7U$}(xi($kOV@7a2Hj}Pb_*!tLES=E*P%A(iciJ1#k z&56`5S!yiRO%;XIf_EEGpLn9JT9z5$Xf6Gl{8xI7ent#1!PU?iMZC?D8;B$BJvxUy zkJYymiWlm^?(=45qwnmEb70Z`QFa zPpHb|Me_u}0V|SjOhm!yw~6=dn2R|nn@~(4BGJ2V^nJcHtMmMlN_@$+^k_LA`r~u- zae2|Oxb|0n?`xCCh`Pf@JJuf#t>~%nsu0!7gJgpyvtrANxx1RB{rZYt|7pG)Y#_!H zVhR;(xZnNJj`2`Z1p30#-sc%PF_T4aD{-W^^-4B?&TwwGKi9Jx>b}x~4PR)sfp?ZV zAnKdX2ADZCJ|0kvO~X-3CeCB-`M^wz?QmMGOHTD);FRH>rv zw$L5{K!|YVcbsaP=EM3VI$~7%TKtJ4` zYRBg0=Jo-piJI*(4e3T*UL|cA<%f!gzA0%>A~{>Avtd-P05y4r-*^zB}I^DHZM#DS2jw9KLF!8cBk(R;vPPa4&l-K zw;}d+g7<#`XRCnT|7kCpKaK88`yu*i%%6&9pg>{+Y_(>Q3CAkvnCxTGx^~TIn-P70 zm*DoQ2=>$Sq~fCr{=mJTL0d`NubJ>7|9@`bz}A@lo_V`qt)rQnX!QOmxr}&1r@v`A zHc2Fbzz1UD*9%217TWE>Z+Cq6y9;Od-Wv~=e+N!{M;-7$)z*yH6X|&8-aGU<*rD%OSRi4t_cD-9PF%-SV zTlhBZYsa#5FVrgdb|>dDO#SmpLw|OO6IW1k?YTSOKp^}$G|3Rt{kHaJ#j7|Hw$C6@ z5)FZV#{O(+z4r#)cLJ|`_tGS-wz{2zT3uaPFFu{KgZBRXuj1|Lm($+y4_C2)(dIKt;9B+sFj|j)F3Iwv)i|!+%{047%rg z5#a3d%fRB}okV-By|#Af-`{EFoiXF<9&Q=Yg@^tl|J1~l>jlv|Z-)J%eU|vc>3^Pm zIC~pgW%@-px=l0}7%8vnMUj`flpBA=41Yeyd1EfLqr?)`ZqrYr;ZCoG6j8p4n4AZG zdSCxOTqF7S*;cSGUjayZuYV2z=*wXlp_;F%*O5|2<5Qhx*E)1iYa^^k3-R!ldAJ*F=*>r&aL*UWOOy5nQBSh{v;jVNQ=2OQ#Unq$s);9hD$DX%z(^p4( zXd+5<{4V-O=Tn}i?Uexa^$y0!2a*0$nA*VqQ;@SyP94l z+5ZAiO9KQH0000800YN*cek|h2YCh&$9i{MzxM`I$~yo6J|F=A9hVV<2NjpByuu5A z?R{%=8^@LCcm0a7SX%&M5R$XGsk&5XUqzNSifu{PqO;y*Sg0Y;AV(U^AeaG3#Owcl z&!ZowA2R?c`gJ3eEdn$BJbn7~dG|r~_RW{yAAViTRMn`%mvdFM#iCH_qwMJ_pIxfM z-%lPtILJ;eizZtXrOM#%Dqpu*y~vt>Wx1L>cu*`?H9Wvy4<0Pm^)j2__i3xv%O=xr z-@=moK>so zU*Eod^}L!vN9&Ou(_)>^+M=#(r5E}Ezn-gEvCK>J@=N~7@7nFEs4mQ_?<%Z+q<_3? z-1k*p&GROM|5kJJtlBJBTYOXjY!0#p+5s5W>uk9x+v2c3f2Uv&SMX@n7R%zFIgV(Y zT^H?T*0lL~Q5Nkso58T#bzW30G+)i(|82Hu)EZj6fXbCupsrx93KptbZQ3!uYS(p{ z4c}Mw`G~65n-*SWSNXcg&r8*RWDRuNz^czD4`yZFGz9Y3JR)`UAcN0bEwbsfsET$v z9X6_5jQBZz@aJR-Gnkh3#f4gD$Jwj8Qdv--F(~j71`A~@N(@Iq$)&{*U6h(T5SSBH zA&l`$TfB)1HxBE}SlpASbh`v}Yt$3~JX6zBEtjyK$JvX#Y?Li(zd*l#IjrZof_9ra zTjc9(SSk3IRyYUz#C0vz2v=rNTx`}kqa|!8Tp~iCriN9_8_{S`IA1Ho$8~+B=CHI% zt+Mt~WzbIvn>X96=6S1{(HO8GYp-D#^?I4N&5>vkQ8k+vvo>pM7;d&!h_TQdyg;<( z?M~>{Wb;L$)U3IpLK7cAM%aLSH_A z0snt}@)TNnJbCFXCS-abEpa)ZZ%WFN1gGi9(M@Zzm$6?A%-+cT0*~_=^ z?FgtMl>g%&Qe$C%C_JSaZ%@93p8n&%j)0URcvTLxYyil2^7_ly*(aZTf<&-@)m1>% z@la5>t+F!z=k^hmrOop6&C6`Dsq|R`o905bv^SAJx9VaGbhq57F%pML0iB0>@CptF zI3#Ml%4C?eTSF;~!#K8I6;=LKx?O_{@{EmxO+`55Z9@GtyBZ{9Z((dx;i+PVgyxIJv_+2T%;8)|p0niUjM;sqUoi@Gf9Ye|Jf zKRC|pC_Kt$&|X2r>(W)0K}svKUdi#c_QKh}A5fhDmm02CbXo4hQyT(GRq z;Y4>G0_fX+{74+LYjuG~yDLF(ltaQ1T|+fow4C^$Vg?Xj!}Op#EC-9U1~93zxG2sM z%t&Qt^`-)D=uq9{%avHW16m@Ow3!w#Afhus6qp;3z6~SOB_1p=aXzlM0FVWs44m=Q zdtWzjY~oa@)LNNd@(Svw@d?zMz_{E5}5O&pG0@hsC<-EyGcu+YZi)h+@V3>6DLCZLhR#mE@0~8y|%Zqvq z`)ZjSp2l_2A9M$Z1)SKFuqSIavr8Vp5Lltj5?Q}_u~xLBaaNxnkJ1?Y{ANO!cauH( zd-imHJ7ElipWE#4b4DiqI;3UaVln6!zI+b!Y$RHeOII)TRA+L1b%ASW9Z@2xLMmgqQM3$ zuaHNuJoR~w_=byyK(jq8k&Xa>mibL~j)>i4gY47p)=%}!0W3eB4JOP=p~x|eATb*K zIQC%*ausZuX}wr9s+ExSU!>vd@eCHq&~p!rpHaI4y}1X5x_S1&^Yum+&H7+)UQcI# z>PF62KS@uNxt^XG9>^&I8Ob!C1RLkG$FROiaM6E$jk^I(>X>3h6F-nY_L@0&+%aW9 zt8mWvsC&}D$=!C=PrvM*Z|=@ZppXTURun!6uin@l`Wu4PgH!Ijs4X(rB z$e$y=YC6_EWPtaZ-dnXSW;M`aAm%GvexPJ!(a^>g1IdP#`5bAHEnlrbD5S%k&UDS3 z!DdL(fC003AZ`)gx`fj)yC}(CI21x6MkbQfr>;MVO!nQC2k0JANJ83w>WjJ{ zo( z&ECsN>la1zKki(|!JV8*ArZv+!i_!*@;lRU_4@F^~m*{npeXB}!h2kdg`48c3 zvDiKwXAen33|~6?duWJ%=0h4P{%p3@LzGqp_mDz=Z-M$LGEu9k(FJ+D5r}8@c@KA# zQd#ZEBE2;#sG${-iAg}#5(P(No=93HNRI4^e%lTOuXe z!@c;lBoO3=a|&l6OQZRs$V(RgVakOyT{O6f(b)W!>p> zEC!M+%A-PjJ@&@g3sen&+fIEq^gp!W2fPc}gfvAhHWptLHGHqSv~>4_2YP=-n~D@0&I&|d*u@2P zQHnhfP#aHL`NE2_K$CzuX-Q#dc8^uAKrzyqXbryO*c?KWe#mq?g+?_#i+W?%!;g`yG_G;D_ql0)0TD0`F*ZTI52M3wLE53mR> zy-vRgfbmR!mxg~4prrY}54bLi*(D-A^4is$6(i*6Qs%6(YVs|L#b~er0R8`WmFu^t`!9W>cxeVxt9x>4-cL;iKpxEmax4I2g~ zXg-tb9o;bP)XkDdW|d}Sf34YAYf2cqtVfEVy_uSC)@*-o%*%EXp^t&vZy%2&@@NaA zu@}}XGP(l`+01`z3N-zbAPt~E?#mUhaMCRaowgpWhSB#&tNG+hn8b@U!Dbu$b|#b~ zRR5m>pLeF8lF`9E6#$(tkSwoI*26;@T7;R;K&(Ffe!PY1`AyLrf99AeYL15`T)2cJ z{X!cV5D$M8A`{}6Jinz$f?9(P$rh{NxX;#r$Y_v6l?~a~VSdu*nxrQ(n~$v?qU}=USIT+|2z|aBves1*#;@`VoE_Oh$Tq^L zfVFuW)eF%)%};1hs$iBlx57A~=9Q)y@6qo7w-tZ-9emIBd-5-7{MKow%J#A$X)<74 zp}HNnGxChsRZ!zApf>RrObLhh3eJ1czBXu(lgTzaeLVi`j1ZcA4Y5<(yVew<8gXna zAW(-?DMfXNn~5zNBe$}+0dU30YBK~Z5IBk-dIyW`yA0xoi6rrO5}5&y%gwTCCT)#) zKMa4`lWm?t(K8h6>TCMy74&{U>r=$gW5zCvzGE2F*tRzc>JUNERozx0;El7P-L_-5 z9Y#1Cboe53-teEr$`xzmo*o6$qBp?E6$%|uJ$Rt8Z#yjW<@r3PZFUp@Jl^cKMX}*e z?LZJ@KM?gi%E-TCD0-j3^Wg}!k4TS`1@eCzD3H*`AzF`r$WEEJ!XQ`G*$_R*N9;o0n69?u%zzbn>f~EfZfQ;+7_7;960!><;be!0*d>c| zjwxb7rL;8~@IZG&b_DCkfKJiA`*XIUk=P^kyr@V~hqxvrR`#4aq9_IaZdDhXlI37p zu_OFvoUJYk+Lc@ZN!#zmY0KUjc1wRIoHghjHp&g)H)x=IY`mg2_M35$(?8jUfx69N zhVAw!Atyz(sD}eFVO;8UJ9c<9b&~UR^LSc5Myi#e72eaOZyt}v__usK8abTS;$itX z`xA^e@fgtz7V7k{TH#nQ*aH8aWntM(ciEU0pg-?W7aQRbU0=fvp-trqZJ>V;7eS$N zwTeqHTDSlJtL_KH9P5E0`lza*!xB*PLbZ)%7}VT0#2N?mF!qVd9?OBQIen`!-Aqfn zBJQ8AwV}To{Zp=#CWsrPgbbCK4h~Nc>qZFN4WPmAe+P?l2&?F~qDp9|#C8t*8Y;P= zk{T+pp-t3KqvEV#;C<5&%9ABiNiZwV$ThRm6IGTLV!<@0wSkg|W{jNUZ6w$k(@ zv6*hMqsDs5{`g=cPqk(FIIZbLA88Qc_&5PDl{FttQ)5}5qJu8$Q&C_5kO4{$BkUIk zRr+um6B+UkiLSv;^7(vdjz(%PZL&Q0uiWGlE@LgI5&tuXCSZ&Z8LW>dY+^c#67=)E$!nXHeK#dGWQ3H6O zo$NtD_wFd&O&N8AAA43Np>Tn=0EPs`r1AS0iul>?+Y$;Xoh^0JXgZueH9S-HdA5lm zbPl`meau$aH4g2rpJ$K#Mi?4gu4^X1SYwz0Z_T!5{nn6cj-`LD?U;A0?d~`^2sizB z)$eyW6sRc2gL^8pLiVDU!Y1{g%IRj@O=#uFM|_}^hUCV4=E_PG(N<65gBH2WB)9r6 zFjO0}?#3pBjuVk3<@R#e5Z$U#u0iGE8tTqiJ7OQ_H_ljRK?Xwczm^<=VP(HvIV8d- z*rnvsu;DLe3wnQY=I(>rEF>I<(@LQZaXuY=##Z z7u|H##$hct$5?jI(qMF9Si=!#t{~G>YU-?C)MJ6V%rE&DH#iFTQw? z*d*++r#F9z$t)3{y`;R<=d$rtwJKW!7gEd~WpDAo5&W*z_ju|fI4g+7(66N4NE|Gv zNqKAuZqD`e&qxU!E*_@y5UZ4~$6&_n1F=u_2A_Wx;BQhCpcX`5r#{9+W=^k%{(7E0 z6+gilyo>ThJN?#&KAo3Rlt`RxNG2&@WOz!!?IM5D{VZ@a#LpXO>+}pzgQP$aOwQg% zyk3Wguy?Ntdd&G=E>*T_)Mj2EI^`2`jN5ncVzXY=a3HT1K;SP?)Sqi#LFoznh6$bN zo@~NGb;XKv){5((c++bkd$sL--3OEwPEImxVrCMjFEzkSDox3rW_9c$#{v3+=j)3N z*;;=$h}%HeRI5`Dm^V&!i=y&*kN%dXZp{&09hEyf+D!3K2SoPgp~HH`c(7QB#l#>? z@NdLPLg_zY(~c5gP%edy;$FS0M%W&bz4~BGCNteIe}Ev`sTZCqy>HAHUSsnR+eqk$ z^%8k9G}YT%)!?Q>JdJnwb3oAr>;mlxquqZX^4V+yZEO=7313{dTfg49^@oN;jI*yh z)H$hSJ$INVFPj|}wKI;`Hlu0_U~c!-Z%2t34%t^F-iv-P>;lB1SJ z5*G!Bf9_H}?YEgCahCsXywmm0!ffctz@`IXhyYy1&eTpxQ^WT+K%d+wkHtxs;Lm^f z^l|2C^UGJPJs@Tg=+TgRu>;~ zBB)@zVFxpH!)l1%%%K(F4kl4T2Q`jDeTN=_?0c(GiAb29Bsuc=6F%yL2Azi_Y>ajm zlY3`(>8^DRcZT7ck#)wTkC=ao!T>U|wz^nY6XScXV8J_2?%l_=KUO%cCH#NxdSd@P z$8q;zyuumK(Wub7cz9du*ErpMkCkg59|;ZFUEHrkEG_yb%zZ>Ya2c*LLoq4%W~r9v zfFAnEIY(g|7Z^?$IY*HcZSdJ*wN6O2DJYJ0i^N~JIe4~^J=qIv?0&fm{PZvpo>uB5 zCq82mViF~Vl58D&5_)PWotJ-0zY}Z0Z(*1E??&->54yZB66_FV5}it=Htuj*p6q{k zeWFU#HdaB$d_Gz^Y-`eN&YRXds1ofb?AU}mE}Tk-Y94#2_=$Bd+BJ(S%($SjNH-nC z7h&_wFf}-dj7n%q$3m`828xuY?PhiQ?;Jb;Us2%Dm?iYIcU7ial&AiVvp1)Q zpPii!EZGLa&e@daLZ=-cy?Zn>NO-IMq*e{sbAyB;Tdx$25EZOAWND*&Ek)i`Di7nt zZBtqnnBaL0M?LbM3RSH-|0_C!i}gVBj*AN6h!EfsSdui%!GV8I4MjRDadtO)yE+0X zpK>4Zb2DyqTeqDG4?&Jkf)UMX%=Lm9kF-!*0{w4H;OJAWPi~AwXaj%N=LM|CdMh;H zeBe_7r)6%&6xsrKIA}O#`CKc;M81-*y z)%(o{RvgIq0jEO!+zS%etH!ZcB~-atrBtSM&DxsH0XKhFburgr?!Z`ne(Y8L&N869y@T_&u;4>*q!YVycKBnx1Z3r>(b?krn4*7)_z4Ck`YxVrjHPz{gVbhoe>zI*ld>B%4_A2vAYPsf4By^l;DH}a)Q=*U91lH2LX zHj!p53}3=e94mpJ1NNh#Z1R{&l7(0lHO-9%V>%q*^t+~nkgdMIqQ#W@HUlMZd&QY0 zb$(iL5GA_R1jD*TYH68o=P;=VKqk?bD@2+!y*_`}crlbhrTyUT$HZRS zzMhqvxe}4TL0*htP1lAg@scfDduC%>Lhhu^-@}U8RR27jOb;j1Ixoj&j5wuAqFXYo zR)c?*?^7_Vi4dCT>(6o!G>5bZ$w}KdY;FKU*jeo+fa9lw#0S|f%x5r2=V%ub(Rp{)LTTdI4ga3A ziB1y|)ERoZn^5wJX)%M8j0HuDs>Bd~#kD`2jUaqa-FCE#=^#v+lAwkmv;DJAw!447 zO~r!Pf`MWL(uR_rVT5oTX9t3n?d%pmKcrVrt2C*(q2K!GC}E=bF}-vef4HI<`MeWn zYzf`3?Xt6R18cwiLZ^_kuM&EA`AKb*)3{wsPm)}!TkR(j)J&X^2@}#%TOz@T`B>0M zy4VTHD8U+z)8Bld?WpXAauPmi6?lIpg}4uDlG;knx(lqE`vEJQHTMYC&D~+JS*5}H zK}~jnb+g*%43P%YK*CIiJKbuew=z`XXMb}eDM@p6OCBk)jNPbvW&h&ex-PER>*PR;JqtL9>kxg!+GFo;BO$ zd0iH>O!HP5A9cW2FY_&h2hl}b`uamFeWB-)0wzM`YhV_=1g`RP?`{hRY)O(_3kwyE zr-d~0YY2Ow%1niI=xl@ea6aXNIpLYA6ly6~wZ6)4qmrDdPS|Ff95kj;?)PGouYsy5 zo0u~9(3#>4+k#Wqr|&Veoq2yC?4TGIbk*Nfr>I}i6%#a`Vaf;M7No&whW$Z&o)qg# zqh&MYVq8z^1Ab1=p0rcXhxzlw6S`N|7&!PkePVvi?k@-SNJ*V3(SFB=B7a@Sk~sSv zir_m_OdMp-IogY~M5yt;R;+z8G9ziAj2a2(?He{q)163HYQ4tmTj_t^67nIDh0$|X z<#z_=(V<0y;V=IU5$+gEeCQ#x+-Hm7yU3K8Yn`>ty2 zMITJa?0qSzlb*K_ti39(j4~t85jTS#C!>ARgB)b!(#>iVw#k@tSKI52sVAzM2nE>7 z%f@B&oYiZ*JJdI<@&bQiI&_wd-u!U**_l=|2{(OnVv|cIJXmZ>R-NFQi{8!wF)#Dw z`aRuWf+#j?P*X)&jakz|x0~=DQ%0Ypv;nNF>lLCVr9{^E>u$0?Nx%wi$1Rw8k;nIz zwpEeP_m*mPXQ=>d4$a;y^DXknvVzK;n}_U3_SO| zj-<2E1KXjvJ2!DXYuFPTv2UNFOJu~;Zd zk`27wVY4FjqRwIKXM&`6Mn7w`g+N`9C!(958zg~SePbrIqPqzOg>(Ruy71K*q%)S8 zu>R4QOAlq8+|K#;=ZlISO#4{eb+j3I8-*{SFR4@)Bsx+kT&nMl^4?eTk0T0ml9n z=J%h(`64-JT0+lojkVlj_9j(Ql}I#9+G_iV^9?%+6Z(H~91GY|mjae3&kic|lH2JI zVhenbcOZ=B?+F`thkLQm*9yINlx#WP0Wo)+^sYi@h$Ft=#iA1m2ZSv^Jb=@jZ$T%Ikg_Aa^9=-DDWxgdkDpSSa$*NSGs%^cNu{+5g@eXvafjoT ztFIb7zBPX$?{NsqS}fx9UMUc-l3Pz0wIwb6%%gS4MU4~QMVHVunPoWjIyf`K>^=k1 zUJT*7vE0;U=ed&d7DC7OaA1#y*p)#f#^~^C=#lsse%&MTpdPtQ>ae}t1T>`#6sy6Z z;rF*3I``1WMd@9XCwuJFUZKy6zK;Ce&4W>bgxY_*!{qger{Q&pxX=Tn>6k2J?v5HZ zwSz1>lxPrNiS+4U@~(k)#@XO{FhX@-pnb)+6QtBbOMm|-JvQLn@FtcyaGS@2zqioj ztZa^})%34#U%z@@&7k-?&ZjjmP#{F;To!0VU#c9bzxGk#D`c3@4(}LoHOu`ZVTC>GCzHN>Ud(+Z;HUKy|Kfm!*XCx-)m@D{eQdNR=1R`qRve zFa8`AvoJRpL~4kQz58aZmYn+E2taoJTI3G^m>p88HwW72Y8b1GSs|AM8VDeNO|CB_ z*=P~x#>}pfgLJ~mPFIQjA`KkBp`W&VF%5C>Asy@F`u7kzcF7r$h~Y~Dw-GDs_Uok6 zMjxt`oQ(tiVF!tZFHgt z@45DG6*h8@=X|#x&)LOpMlRNW?diL-eto!`La+O*E+&TLc{*c1dxc>O7}&4I#n(<8 zM{eJuz%yW{Tw5Sz>a$Sa>3Sm8hi33nLKLJy^WKYzHV

cY2~hN4BMuRPL;7z2 zDdCW&sV9e*(Y3gw))kFf9*rr#)QF76tWBp2zO_0#IH^4EEU5<7mvb$DyhD@jcv)i3 zHqx-aqA0GhJuhTqVOum6bW0P|yztC}*rHml+AYa_@PjUlAG=%SST8E?wt9^>ty~w4 z3OxE+y3VIXPfjvdDgR8W4QciUQ;CMjIZpk#x^aSc|^etoV6sHi%%{kqAY>2qT4$Q6LU(WZ)yRa+)RI@Eo ztC+P8Lt10zJ1tn$r}s!j3Q0bkSbh)=REOZ|MKf-R<|k^Hh++kQCXfDB?9%Vq+Cd*N zB|#wQZj`J#5KG53+~E|^)tIK5+co{X&jgU@*yKqJ4+DdSOfPCP7otiV?K?B{Kdu+% zwm)B|rMK;I_Jf(uk-10fvA!=WJNoQLd8-};uSLty1@OmmXnnrHy}Ju#2_&_&N$r)p z?Ibz^TJnIkUK@Y{hKHA-90)Ie)bVdC7vs!F)Qxpat#K<}`ff8sPbo}}OSbiB;(;Ar zaUK=nE{>EAmK%-ALwgSN5QXysudOu>Fdj+oF)w5Q540_xD>{16iza{-un6|3A-5D= z?$A4sV)XJiQ4%SikPWqTC>Zk@9LxDc8e8pqj^})mGQ3I;&0Ry=wa;RIfQW6^J@VUa z-_*dbApwraoNcYsoLMQw}m) z?jHI_s7EH*1fl?dz)|qsm0(Q78o5K~wfXJ}xTzIITp%P|~7G z_+qp1!&BGR*$DB*{WcT!fr8E?)QoBIv>}=~ z#A}b`Mat&AxkMSkq+b!5IJcCukv~@QPq~*qoTp3J2c15D_tJ+mp@n_;FgJIiYY2|b zG{^n?vBkMxzbOXx*<>=ISrE8qDXVfRuziA-;J z1~*ram?hu?NozwXjNChV$*XW~G;r$;GAS1s8d05py{P-V(pR93;}ND@Y`UCrJUq5$ zUu`iVXZGr--N4h+?&+ zfvb3Sdj2c>x~%}?K8pEiBy>Z+4D=CP7T*Mft(m&8&K?$947us5&vW@jW@1J$g$2ct zJp8wR$sZRFvwzF<1%5=~bs1TUC)Hnz%Ov{JtOouW24nLEBxDQQV}D&YEK!%vDudM? z$lj~%w9L;HYET9X42brFS3A6a-z1Fbwudkwfgn#uBt1ooHu!smf8P3OqA)`Yzj$qe zwRvCg3zs}bLUrr|Y7_|GyqVKSs0WLye6}fn+U=d8q3iz!pc3v^4-9Aq8GMT3E<>*C zj^&;3&`dhoK7OT;5VD~bfRb_ykDg3W=BW4G#fNQuov-Jtj@AC%!JU3#o2M4$XTAUe z7#kgI-S%Cg1g@N>5vP(IDoa$^`Rs=u$64ZZ?;xxx2et%x6=!RA609KUgLXI-Ch*9A z-p{OED)i=B2~D+5(eL@RaRBHJ$Wx>hy3ls<$bOSr0kEst#BeR@yEx34@1m%!B?jfd zoY?thjcla)$0je0gO6+}(UCA|xZ@}rrgG#M2Fwmar{!~rGH4Seba4k9Iz;(?T+2F6 z6_e;KF#gS@>%bey(8UVygfic=2n2_JJ}hn`B2B{HB53HAi7Djt^g?H1{Onqz*vs>W zq$nOq*GK*1idCW1iqq%mc%UOgqR9PHXc3OYtU4N#dWEBi=M4s=!=SpFlDddJn!>m! zVOzux>15QFBr?RFh_v+TPoQSTxmSAAcXVn*cCIsgrPWO=Bs_`pi^#|^?@!tL-#nONw&6Zca2to-6G>(z9=S(U{MeXO+=*T_J5?n)%ra7r&cD5fsmeV&0$p}d5`HN?=6kwEz?K-5@{pvFq@Kd`Uxf@w?FnE==Fd4r`+t-~M?dvo^z$Uj@w&Wq04q1^wegcA z&53g6L|Jlt^!l)rIs2O@;~TAh-6?b26hAO`j=lt@Kix{?B%Lzw>Rf?MJS0vjib6s( zAncn4Bx0ccf6eCOPrzIh6)d`62sO&+0-jgqk|GxWQa0;nGNM|q*L(-{% zo4g|D2)qTHWJT$jEfY$fZ=qQ>WpEw7IebbZp9YG7p|Bt70u}mHMt;3W@b_pgNGA|9 z8L0M!H*!#au-oLNQN2BVe@1?m$#wlQwA5aczn)$r%)G!UGBEo?-P)vd);j?}Xd~)0 z`FbLuU2)<$7x1cWN+KARLQy$p8Ut#xyjXG+G53M_?8hdPnHWvs?mLnGI>_kt0fNItI*r@##>VhA#Ldem*uY%au zXp4!^*X#-hv8Mdis=8p`6|;j$if2OKvgXuudw7^!yTY!W=)uJ*5bluo=a%-Lu}H@iG*yK4npg}lbk*7ZSY-GO^b9dXx1-f#atB7Mo9T0|O z>g!8?5o?@F>>QY|D;>Fip3k#K*+pHOu*{kc8{t{VAxX%Fz6%mgXs<53*YSX_?h{l< znr!I49u6aLSJR*eEvDACeiy2BF(9~mb_RHN+BW&_o$x-g1pU^l28Y2{i6BI)6STC< z(+wVJ0F1MQzW7MrcIN3skFu|R%o68dbhDd(=7uda$3OhY|9MxpY2PsDWI7#8-qnS7 zrpd?1AP?0@I7yy`7rV(&8MkCKzMU)4lXpw<2C*O*gui()ZXJoXY~Wlv$(H7OWk#2{ z+rA7h)8O7%!&vMS`sPElKTyoWL&jNhW00pi2B1f7dp41g&JjN(KOmEPX_Q(>ub(!5 z_6XkNOpedt=oo%yok);@N_v`5k+0HfIm#JxOvCR6k~zA&y}0jbLe2%SUs=1ue9_u9 zxU*xt`FNHJBs$J?n>0+wx|UMwFEX`t1Y=-+7TS^!3-_Eit1*H3Lxc-I_+!kG>NP7K*SPArBF zs+byJ5@L%!L;x2*OZiF~-&Q<{^<5wA8KEyX*;YfMhyu3ftpPP-9Ts$DAI(OnvUJTMVd z3bM?9qBL|J_JQHA^~Pioil>1qCWdGSMR739)opG1w)Y@m%j^u?$5%G54qbg`3L}@i zG6^`oBH7K{h{)0}pE^rNJQ{o9_aw0*CsngPtrji_bTKF|01lR7{bVsAUI@S5Z#LB3-yJ$PJ zEPqN$_R?E9DT^DyJ%~Zp_GZ<@?K%GEz-62wIyuRnHgxF)$~gYWj`eI0KO^R`(jY5P z@e+HkhTYh#kYZJd|Kdo0NsCBDD?Hy#{umn^7^juOk7bb({10v@6D0jEmTM5=Za4g7 zJIjocL~|Tc7o%1fM$O$K@g!cu{dhI}+ylR6#{PZ>6N(@71@51s9KO~P`O{6->dwfQ zXs?04DHb`6&e>s#t-z1Ckh|li(XRWcGkbAxRE=hWU(yEjw1WSC87?1RRHNts1YAT~ z=C;xgKW1Jcs2_fuOeQj92&B|^BfiPKq129P8NWknCjQPUILCg+vRAJ|;mQgX z-QL{Zo-Aqi(hhu#THW(P`D6-sRw@B2i@Vg%m06@NOtPK-tT)=553ZrT`W+EcB%v8_ zQaRRO;HuoP0)yy(a7)*lFv&iaIH_eKa(7nU7FH`8~nYn(g$c30H9gZV{Sfq;D_OVc3>Cp<92 zsY4b&x(RnarrPQz^Btgj#f;@o1HO(vzYY?N3Kk7|*3yxG=Eq0`i^kE3(qe#q9DvU} z7+UHdoL$oh_)G6+sI4#`r>1lmub%<3ACRLqokzOdW_8%shbV`4L-J=)<=`h$=Ro(Y zu-vWAF&|&)Wz7Y@OBf6E!tFedC>N>TiQW8>_T3X^ZSa*@`ihexJ;L2blc0Fl~G_r^jX zW=+1Xd1%f-h!M#GV=92*JcP^>?cEai_h|2+z{y8{K8-zne4Gy5XKv$^)Z>TLf6G?N z>nE%bJ0x}7#kIp-$R`t6%1>XI9fkb1Jux7OEJ4+K zVh%g{a%^G;clf~!pH?C-vw0QjW+-V$@2Gp(*i;Bk zf<;JwYSd5T$9K%Z&~bDcJ{>p_QG7T zU7w?Z(rDf(O`^Lr#sl$PLW=IkI>&im6L(;LB^T~z*qXN_v7soN_dx?vvxy7 z84k8Xxy>pygFTb4wb!7YWd(zxV+bdK-7jI7=Sp94j916zvl%AoQDHf{E0!I}h2N02 zDxevK{T)STE{@LJE{nmo^~GjmDi0lh)2kwecVxLr0~#u4LT0_J3fANMqGlF}R!~3P zbz`e$7C5G@o3FgHZ~HC=cdR#w39I#-!4D|(J*rJlQxwiBzr5n~R2TP{%f2g_kkg?` zJQ8G%#83ZM2O}Hvt~%wHo~zG7N0-6U*@JL5pwYKbHY@Le;PtzC<73H+`QGt=)UiHH zF#3JE%G=9hJ{E$!PJ;FSFS?nSTFqDj|Czm3bE(Ayz$d7BgS`}s$_Hq>$NVUm_LvXy0v%|BOhy^Ae74$8a1Y=;B^9$`$Ox11-r|-@St_-io(F{BN3!2omDRhZo%Ld*t6} zdw1+@5oeIZsDzxUeMZfq&b9`HyKvH%>h{7D*Z|%p(IS)yjNi3?yj&xHtPqNO-HVlc z>1RW73Cg5uiBq4smY<-<(HhVD$z+mbDg{rgM;JojryCtpBWCPGxmyexKY}DW29-Qc zfLUboeoh{ybF&R4_ewT>!)K0(X`;Ka=K0FFgj#l565l}9(RT)rt3OCf=PYFtxczK5(M0q% zOGFZB7r*a;Tk8_-Mu|hzZ#;aII@zrL)uPJG=xv9%X03N&QA3V5$mSqqwqLk0xaUN6 zM6zi!$sX0HW{SFn*q+d-h#q#3`1FyM(0F}-wrfv4S69NL05+w6;t1Y$Jv1&)7Q&+m zIV}x6n&*`Wg0F?5Jy=BCKuh%FtFv|2c6A_e6f*j^G?H zNso|af6BK{*h%aKVB=6^M0R&PiX^X+L5-DycevSomu|0r^LC+sF<|#yX>|u^2ii-c zXku1s1w3Spxq?WKdDs!XFmz#gP{EZNv7afqavCk4_5s&+v@cEw_i8!97r=-aHo zd3QD2J!sZ{_SUC4b(*h4b9%FX9c|f1UD^HX%06n8K1id~wLL#-m+ncs1Qh+FUg=k* zSNf<^`c3PUheqTeKz}NeX8;Fk;+XM}SBfDrH1YIYol|#ZU7&5_?AW$#Rcza~Q?b2cJE^2% zR&3j;*tTsuxplvLUe5go^JTR**BoQ?URvzRkjZRgE8-(b!Th|OfA~>FyKyDHO?m!p zi=8pGO*?okJ@onw^1o5fKiC1q(Ekm9cXa7>apKxY46LZ z-$MsZ>l)O!EfQ5wFd6J5!|(>TJj^q8+1-nzF#@^nzxjQh9vbz8nlcb5E3MDHbSIh0 z%RO&)`U*xe!q#5B#?Ln|K!bmdYT61+k9THIOCm67nBL+ub!JcDE#B_eTd&sPj96X# z(mUTBD^RRRY6wh+T(~Ls1-=b@9H*5`1G5GBUvD4emZULxK93crd13jvKW~SVUlpen zdw@Q^PcN&g)Gtkpe}M2Sc-`Q6zbd=-Y}3)4>%E6DSBp27yu75DY_@Epn}r{$n<+3* z`9I@&GnvYr)At8JVE6pyEwF^hhi$3-5eolk@cnewR``rd3(;7!BU=wPyj3xo(USSV z4V!N{0y)U6%H$3LDes&Yzrl?nHE%webOiq0LTHGN_Wn^ge0Lgx5a`W%VF8uG8AbXN z{?FQGm}+9=Jfxe;P&5MjJG!ozmS!fW;W5w*;`(fl1A4@8pu%Mp!{_;H(XqY!%xx*o zqzjcGcKDi#o|EXy9X!E0K`@-nL*->GQ_WW7^T_4z@@oNYgav2kluoyq_*Oz=?O$N8 zL{xJI%TGG@KJDgJbncWeUg34;aGSqX%hP487i!ewFOKm`@edpxng}ejrM3Q#hG;F!wGpf9x ztZZxQH8?i))YnZ$$I5@5q;!ERfUb-8u_vf0Q$rQ%Zz1p!qJxUIS3R?k00x50;f0-_ z_LFn9x)`X$9OcKknDZ|X0lG*l*%_#5zmYIe9z?p!@SiKdm3#lvCwuXJw6noq zdXjAjyJBa2rOWc?0i%3~T3kv6pSUoy!lErh(ENZ5t{QgBJ^7W=mOLOG&Y1v0#`9Jo zpIh$lxGy*jVx`!%?5m|EV>~vZ3N&UTrCDhw+L<4W9aARCE=p(PX1{7yW@fW9Y1S-r zw(t=#25EkW=bF?SOfNa2iG~in&BrWUdVULK>cgx_By;5&RhC~1wsINpx2qd49>XaG z+)g@{}|fe+Xm%1jFdfY!8Hxo`E<&g7eo+xFQGug z!`Sj#58>o-Qq9!B#gkowis5TCsSDI7Y5_%jjX6KaYIPFj*9L)RrZR}sCu#a-t$aZ@ zbDzmubeIyG*Cr+U5E_}$HeIU)TT_)q68D}jp0ECG^IrH z-iH4<8pYkfb<8vi1wLGZzoKhd@>tEPwWce}PqVj2=tBv(aSMFDQkj|o1h{b6alHk; z+!xh0{CPSh)44!FAN#Y7fIcvH=6^ge3=qXl_~BBJs*PQeamBRtM+* z9c-cwkZui5gg^AnhmNA!Ni6{@b#@k;?H;{9S39-MAjh!T-(<;@dg{n(9@UlmtU^Hj zJ_@$Lwg$=4XjK7ppsHyassKKK8xGB~Z11jHT81is5DMBTehGgoty>iUL=Vo`wQ7bm zIyjMXzww4!V&jAQgPEqR2H*ku?2#TvhBe%)l|S^_zk;P{tV3Z{uZBb>>iMLgFb`*o zJGYT*7F1g+{?;nG1dZ#eL&<@`oaF*ZK5;>Skvu~kw2huVl&BE+)mcG+Ndhk`b=mOI zXl&LfBwfao{CoxiLYE;NGyg7QL`&9JrowIUe+tI8pyN3%@KO(hkw~$Oj z$ZM}AhT%nb6@gh`gO&`(mU_oPYd7DtubC_h!Rf+t?2C$357)@RzQV2$+VZ+|#Q&i5 zfWQ=K72spV_LO?LGRp-F+8k6&3B5U=b!bI_671%-U|RW2!263xXA$TF(NYfDy1E`v z25YV+{t}5WOdB7Ivg3^r(AXT|u`^}cHG9=*mkut^p^btitfe27r4$AcB_W~}dKYQI zQPW_avkoa}dV4Rb!7lwumXLy*$-Hj8EWmW<&Xoo?mQ@~$Jc#yBiB6(N{}H1t=cybD z4r#pb9-9D4nJ`C>%n1Gd>&3rkPN<0>s;RY^*|ZTYqi+xH-l+FfZ&3RNthc6+lAf;h zz)d4>pF!axI%-`P#+mPBAu%64wQ!Gphf9kRGJ+xM7bpwuimc4dEOr;KHYh~fMqRX> zffPb^rR7_O&$9V_WHCT3A-A<}rP zjccv7iO>qMe+M$!Mp&Dr_S{nVGr04(^cFUrYwoe*`9jcG+~;{K5_BeyP<3!wYQWoe zG-5_G^Wv4aZa`?vE2cS-EaZ&C%#g(c;zTa@Q$ytbEc<_um*jl(l!rp4yYg=UW^WN2{jYEXLrdHzdLN^x8yN;%i*@r#9i5NbR-2s{X7v z!-X(*)iSh^ZLb3RLcS%A5!C0wBrF+}B&s5Fvj)3Bsy85t@NS^ZQ(c8d$zj8Q2%KHk zpXLm>agfI;)WTpxZ}#h0bQItr$(-oUk zj!fIW5|Qa(AhEG<3R@8l_9W?mm7rD0s>pP@QtZ;|mB>j>trYm+vfM7hgc5&7IU@fu zI3B6#Oo~wg;L-xTf;Z4xd9k01!<@I|muL?SWPEXk z+l+;tczsnoPl=nq>y+>2L&xE<(GngEwiscE)1j}0@B?(CwQQ$ zTj)BJQ*J>*{Z0asKi5X@A3F#S^dft*z+1OlC6D|TkOlG-n1;Bconq_~8No#+pFoNF zr1H@fl|0Ebnp2-9dKytn5gR5d{g?y}Xuz)L15b2>=Vj0TU<5&NEthHUCg#=wvwWZL zR>#?%=0{gondfpK-aGOM!_n^tR%goK;Cz-AE*&Wm{OYLKvvxOqmQ0n&+JqI%s_2;L z7&nqd4fN^l^x&Q`?y^FO<(FE=D|Ek`or!PLNi81#IS@aMM*RonoAMo|B0Lq zH&SyyQx##C$ZDx@Bjt@qld%_g<>1q*PvGwitT(c$>j8(ePk6hU3f4EfF2_+BKM{RE*`ur~ zg1#N~)Gp~m63KrS&aMVtxd1CXUB=b%eOSMTI`?6wJy`*YFNu*IDUxe7czMj3!e;Yl{3moe%Nh0&k+Epg#8W3Ls0-3^zvb^cqqb)Ia;@(4kyy zGkj;*vY0O@adiPM=<%AX1ywe$7m#kHHuM?iR?MUgP1zwzA6LND`G#RjCWT!8XFA5? zxODUIGRWVX+;tOQbp|!3ICx`@*Tx5>=G1*W(a@x0DekZxD&vTiDsK`8BO>9{H$oU8 z*1lTAWmojOnI=M`W!osABiJ!C6d5x=<;`$pBCMDH=%u9?~Yq;;(Zy1(BLt? zYJEblbB4)L#_Y;%sx^Xs&D6;>lac)b`eirpa%xzOyLi#;L9!cXwbJS9>a&8W4(5Jl zOS9yy=yLXTYwiU%865o&QPf`8{fB5+w{kcy{GTb{!)=G2f0L?Tc2hK+4xT)TG*oAu zsf#pT|4n0f>!$0#pYSaFn^grf6z#|rghd#OJ0U|TdslzFP90`BEt7Cq%w!n-So z?tPX@k65~ZWM=M+_y>9#Jf@kMaf1y7otUvJ;vjRyHEjw<-4o_72k)6f#Igj zo)5khb={k1f*D5{Vz{hip2JyGfiW_)sOuxxq`x&yf$~q8b?;L&bNlh5D?XrjUavJt zfShW?7(R;VosU@!w2oNPpeb&I0NR4t->xyzEuQKaMZFBaTI=t=t0ocn8V%M<2cpC5( zGu9V4Pbjl2DRkNJJJu|J_3Lh+TMShbBnL` z1%_)CQGvBipD;}AhFcN*O0H3(!Y}frNCBCAW|WMGn24s*@##V%uaf-? z{fq!af>69&&l{okEYGECJnM8T`62v}@Q^b6pb4#=F)HB_0`d1qlr>Z8@Vu0mOHamq z@DPLAqUYZlre=9DH&b^&5r{EHvlL}B|J+FZ!gY9K=aFLddrD~C#dU|SG>UIz+=#fn z{gQg88N6-SE(CT&O@E=r2dHh)Nl`Y-6@}|Im8-_lg+Cr`yi_-mL?badkj(|b`*0o; zb2re&pqw#DH%G<=371uq%mrFkpldn-JiDuHr%S=7&u~EUOs-_$&qEQ9-E<4Q3UJto zAMr!r3Z!*El9_jo)Z4u}+odiLyu6T*d*d*6ucpH16{|{E`gxy3U+0t0ng~h*TOA%x zHw)l5+oqoybz{K_(Z>C7jq38Yd?0o8klvF;C*e3hvq0SiC#^k|@$VQ?#;WsluS+Ms zkaZQ8qn>@C**@Wczu?*ZFSDU|yj(JUVf(LQD|$vr_)5$Men4TgFoK~XE*F?q^b2UT zyNN;JtsL(mI`C?xePqFG{@7ms>MQsZfLvfORayZFT>vi`RV=(+OT$s@3E(!T(k_3O zR7jMHodu?_;tx|7gsMwieFZ)TqPR^vJ$a6i(Tga4|JuI*>d)U*bNgPSS`u*>ORyfw zO5*Pc^q=4)TirOAZuU3TYwckxh!MFH6)lX&?XwUxv{JFM*v<_94zEPd>9mrV9AR~T zyWc=A)IFaB?H^}=U)`0T`UQChTuLe-))||_R?<8b zrce}6?1pAR8Mv^^d0JUNh3KM8_^z^8UNJrDtt`)>LQs!_mBkNyqWC1eS8igMqV;Ex zt}nv3+lUgg0i(-;>JlfGB@QKW2P*y#c)Y9f$qIvwoDu5`-*wW=#aU(YL%HFU zAvvVUm&Dk_zLKj~$e(l8{!u=@3jnh}VnS`dazwACl`k^OySWPM{C4ecKX}Nz1$&xX z96M)AKk>Tc_iNsEi*XG!XRg$j{@r~Gd1XC;DEZc7h!^+1!;#=M2d@}B)+UNQCg1LG z9uA$KHNkiK($t3&h{a*NMYMO09-h^t?`N4UR)S*JrZPkxj6{@J$?agL=6_m+QHoJP zU}?G~{b4|ea+&~U*oq^}v*h-B6=8=o`&QCkgcpOndBy_U@-S#vx5+$r2WT*ai)jGh z!Nmr1s=CQwZ3~aV13zzfv8Vt|qAT@2Z0V+a%Kf9T3Vscu^whOz(!N^)d)kf*InOrT z?CqUwhf4Ny!kMR=?31ftzN;qIuE=gZnOzU`%{oNQ{dkD zMxNv%R*X5{d9kX9$&`juecIH1R0AmKSRbz;GU+bo8@()T0xe5#TdC}l7>xqZ;)CI#Acj>cr{3Kq}6}5h?y72{~JG{%`H8>-{ zVNKf7jzl1n*}K0JhpcI0o6BgAffXx>NJiNK6W1+!!M0bDR!yQko?bUe(gU4IgWxA3 z>&ID9fV67t@5klUHsQ^P8plh60t67HbZ;p<`jrVFa2k<~jgWR1RY&uoM)Ax0POv)j zMrR@DSf&`R_D=`z;F_NbGt02(-E-(<=b2wfkk0rb#fwUX5&n4LCx_on7y*A@R|hsL zfhRMZH$?Me*D@V9wy1G=70xjz)cHxgVv8pU=>Ja=^zG>F{DiG02!zi-a~7%TGmy?F z&2VOgPYfM0=;U*W+O5C-^>%n0T~cDH!NRgZf)}AXN>Ml2KeZ5iUcNMXlSq6;^PNXz z`h$(;Rp@7Aj!8YL#TKcLv4>6{GhAu(#MDF%C+9l`gzpnIwKA=MjIm!>|De83^sMBk zOp~Zk^W$%+6J!<+PoTbL(`W_G>s%apFH`9nR!Fb5q|Kziaivdv93kw?JCqHNpcPL| z5iI%6ZwRR}_yh9vlL>g>R!9;}OzVT2Lm5T3ZPM+ZhlVKxuii+d#(Wc|;rRkE_eOW# zehEKc)8GQ}Z~2wA?0LO`--LYj_))DDDn3`VwK6*;e0axyRbOb)S3loS zac{nI@Y%vJ5rONv{^47{=!-?;beOt^uq)#!5RmZ0)YjCi?>w)^WZ%aMV$}^`@!%m% zCdJPrNs@Un_k!bX{4gnlyAxj-H*G7~vPr~`YS0I`wn$x%9k-&;mE7ejUNW+?siiz) z&2*5}rygHQr@nW>JN%%&v+PpmuZ6ka^C@3@?ytbl+Nj=NQxO%wf7amIj81|7xLDUjoGFh9Q5GNv5VPMCMY zfhK^)&)nC3$EQitXtK{;H#%4veag(#ChE{A*HNm88a2pA-)^sn6iukHFc6eBQd$nw zUBpO9qo_uU@2Y{r2K*I$jyMC2bJ6;<<|`%)yG0(0_V?Wp+a$OX%cU~RxC67Y3w=HX zkSRCe@`-QiE&r?5F4p>hCehyet$i#|X7is(QI6+c)9}aXl}x-MR!8mr&;1*`$RTYH zE)C>f7ngQZP*8npiMKnnqYISrwbU9QSCu&p)K-z2V;D2m9&YF#6y0OX@Z^Aw!5BTd zV>0t%UNa}T3CTCL#W`y794(|^G0Wft^9Q|oA$zsOKbgR%$1Sx0oE!*P(QJfG#)zT#v`stnK_S`IIWe$1iSZYsfD_wUjtepg@kQIO3p)CBMF z|L|jkF;xRAZ0En=-XJ=3x>YM|HbOlOt7jC8wrBOe{y(*HuN@!<=06EC2oEWF5I6|P zC`=m4JQQ{sL3ck)TC4{E065o#zX-Nfb5G0m0MG%QM!2%L;@QpyK0X$Ql_ykFpd_hq zWX4_Myus$9{y@g%#C-$aYN=ZTv)~}!YJNB8IWzw${VUu%E;AeA0~e%Mf%?}zYata5 zylU}mu-vRH1psiU9^|l*f@C{_ENWUh(5lSi{2iFYUJnbeV}yEBAHBhYG@vbCf`?=lt*4Bbbzk^Zv@IX5S3q9(5{ZP(fFPMc0+dlfx%_V*Bs4 zC6*ad4?i4HBh~Tz(6%6$^j?dDLL9*Ydt8gwXRsTUqo= z%pOtBn%M-O4sVs#no)rD2Bj~=cc;Q$a`Y)g_ku0MrE+zm=PoLc z2Az??pny{nH=X$%Z<>coL#DEbmK)}-Zt`{rJ&rF59OWa*M7R8BjFGl(>u$5s!!~Iz z2!$>zDXnGLfqxM-_nPG}ngd5+DE_45Fx;UL7?+G82fME~W*^PJ(x$$c+ zF90S`c|nhZ4a&PQ=D4C1x{Hb9Or0qxr<~sU4)3W{$?R*f(#!##gpb>?)#+pVZ4iBO z^Vm^t#ZyK%Nz##AI&f4KnYb6>2xdLPgESnVeU|m$rwRY+82?`rHXD&Y5(DjJm>;GMLA@%{t(EnFTHc!7`mE`@e-_v6eoFLE76c01pW7zJY$p%uBcC

_%^>RUK`0Mo)-_T}lv9)Tr zg!!p2n?F>qfu0hoh3jSq!MDc+8fQQJSkNrj?@60e8)L>PmCOSK%{x~mkC;(?-SDUkDddji40*n`rY`sgk%Yz%>V-Wu_|$MFrPWM)BNJ;@P9JNngbte3{V4hz|iw|Ke(;LjVOZbAisEeQ1mE?zNp+ zWblz^>?>7*pj|YZI(Of7kGEKM99ezyAIoo#wM5tnZ54X$+}!wOl6bIw3&ecZp6Z4r z8eZHub(eKZ8MvZ0WynLzx=m@50 z-Zds<R#7L0?Lxa4U~Bc(}tjO~T4P$H!buf2VT<-dnI%m7h@CzNZLgfKF2LNmg7$ z$^9F{lg5rx-Y1(K9=TPSm_-;DOB{5^0j=K9j+1G!FVBXRcl@NxWS(h8z4+_ZIJ94v zecX_S1I?{Kxruvq& zbl5g^X9F_n2o2Nsi#Qk8xxlY0R6zuM6=oA#RugJd6N=}G#@;QiIZM+w| zTgAZO)%Qz306YpsL@mDBdTc=4?H@Fd@b63Pc<(k19QGC*So|5gF?I7gL-yhcbeJ?={G zpNj`*ffsKKXEkhEkZA%#DM)Z-wO}gX*~ixng>%UjJ#8{-aeGT|)r3{o@Qb84QYTS{ zq^X=S0onsaxj>qu-iH75^#}xVuE|De_78!GFu%jPxr^ubS7Q!);UPz@Fh{wj8W}H;q#Jx@`A6ZxyRl^nQlSFI0#R8uUMIMV zmlt=t?}FTvcKu@XJG|-_TXRj!orPCnT_zU;Xeub`*9g6>p3M=P4wBI}&sa`B-$%}8 zt#h&s1KZ6$I?%U4AQadmIeG2D7rT>&3w%LQ!0NHw?&4;$w8M-fdVA-8j|6`4zsmww zMYtZefi;KV&tJ|4%XA>N!ZX}aGd@GZSKEP-x7)cmg}I;<>z-iHKW*RkH0>q6Kdb)$ z4SIXN*X9IYZq-GTrFSnZdBZbXvd{Xkdi_TqB~M39P( z3ZPJ2n#Xw{E@{8$Of=lCb{MG7N)*|Slh z7j9IW=O(z(uw)-c@WU^N^V6;w9%wv-A-MUr5WCr8M3E z9Xt{NoxC_nN&2&K(AAMYMc1}w>fX^AulVOV&@t!!Li6e-e2mWo?-d7^fz`nQ zU7JW5)bWQB&d$lyK)yV7R4{4FFp5d#= zmlg$Yb8jQUEja^(zd4fC+ia@{5iZ5VQ_%pxRq4VbBoEt~Z!jI+S~DNJ{E{~Ih1Emf zd&8{vZ`rBMb@CZ}7ZyXjRCP-R;I8hYnAXo}ry3wT+cu^=*7cXH*~Q3{2wa@5NF%$E z;`PqYwS}8!UZOu#tdsUsq z1I;kZoZnmJbEeDKPHHI#wJ_9UgQx-8&1t^KPb5SJXkyqeq_4aLJHmcZ4VCwoWY+Ln z7T$fV_)@zcGTy-HKKQ>8Z$wEuSam&H?g1c@2sc8p3Auc7@i+=`c^#Z~Pmg2|zpQpv zI&(VrICIIQX#mQmLXfU=)ErB(E@c&v59H(lW!T^65`!%i9P|J48Q@SLAOaTQ(enU33}jcxn0zfU$LEd7cNt9)x$` zr|b4myJ{RFCRr<}G_L`2Yu)uThP#kGJ+A%!@+J0Sc-}g z#2i9VNpN7vRWC~OM?hKNvbmmTYNFHbaQMeTzBg)o^yDu;Z}l~w5svdAqLvQ%&rO6n zm_rcb(!+pCd*|_cjTS%Nw)Qr^<6@3qUErAr^*avZ<*Ev$z)e-}eQT&vBF;%bo=e6h zf52_gN5pS%+t61|_Bsw_{la6Pu}$ZE;shiz5}_MuD`j-P*%=?H)GhdSXz>jD+A=%N zDmZr*+L_O&ji0NQP%GE#4m}uv>7Pp-iqSaL?Cdi*svLNKZnnVnjm*R`x3`jTK`uB4 zSE3K{Db7a(fk$x%8!&T>9l10k(6h}LATG-!x;q)xH#=KNgyU~_1HZvRU8|W^dhWcu ze$q+Zo?kk$y^(hk6;jSipOJq?6(sog)CrV@+tla^@Q%F=UjKN~CEAEa!t+wUV85Fd zusSH41-gub0!w3pp1Go^!*^!wSUL|=wBS`$QB9r?m{Uix{fJ+u+$pw`*ORU9vj+&*`C+x#{)@=?~DAG!<0U5wypOe)>qA}V{pvcz*#7-NC zR?Uib7XO|X=Aqozj2H6v^&T*uT0o}ru8K;a4^@= z{!9ZQ^A4Qscb$&0tTPSEUmtR5DEBSW0Yq?}%RL`5|9BL?IovSQio5u~zSMNBNL zEcfCg6Ydqhr)J@EFPQncR#`w-Ajc&HNK!Fs2QdES1ZjUEx+`uq$XdFuRGg%lu`trv z>qF5|fe#$&XKtbJTY5KKnKZo3E#U_lBzViCj0RE&i3oBekRn8-W0YoME0pf|x`Z%y z_W^TehN)hrQT#qGN{{1frqK%9ojz>Y?FUxu8TiHK)3osG`J3wfJ@9k&lrc3x{^%z3 zC!!N@IelgdF%^)iu+>c%S?s%6Fkqq|g{>6zYpk3D^_`3mAPHCW(FU2nRE%z>UAssn z&47rXLI-!R)CqzsEBc^thp?BCj+-$V1sKoxdH!wG|B@;2k9q%!8rqe2A;@pZUDFr^ z4m*aVeq+0iM&gO&I(DA9{MyfY8|XEia>-wc>{%D9b> zt?+HI^&x|BN4PvI7FdK~(zV961y*EtXCT$dfjWnFVp?-y)GCh`lRC-F+hl;mFbm=^ zHKU?Jy^rLo@B-aqKp*zQ_Y&QFyTN1)|JtkluR}R8qNw`T{gs+r6CDl9)1zVdb8>%# z?+bWgGKaP8Mr@`lr$${gbTV9pUy$lzB_<}70YzJ2K&M}YynfC0cMbhkf{K|IUBEm~ ztnW#dXfM6mhsb_dbDTU(VXHN_vFjgrI|aYv_Ax*NLli|rR>6)n-1ZO*7Dk^D9omD2 z^wM#mzec4lK>U4bERj-FVO<85qzvuTxzt)vy8^7dn@4Vb)yp6<=q)>K=M`PKEsan(FReX7R`aI2?g0|Ej^}J1Rr*2RqpCF6pO38!2w475m+&E$nX4+`?&+;d+0pB zJOZNabyTTydGe`0vq=e}o3m=6X{hPnpBFjs1vxO+{?fY!IwBBMW){FSE&xBuE3uj{ zyLlJ`&4xXJ@4cLLjFcxZ0mfWzp;*kdJH^_$@afT{HMp>OVq->1to7^kXqg?Y4dqfc zftM%Uy~sJBl5SxAizLo+$liWJI4DQl)W;>ulo4wt)}?-i>8o{{f>G*rc#w&UcWksA zloHUo)`cyvn_>BAC>QjgkU&S)Q7o;c5Nqz*o0A89HQzT22iRx-vVuc7x#TyTfFsPJ z&=YpGMnFrNxU!pxmEjqglVstnkeT6RihIu66k+( z6J5hXZcS7gIo}dL=hc&QSxBjfG$FHE!hG>rzcy_gXGfn(Ag)=BQrNVXyP!q=J&w{Q zrg9e6ZDWqt*5s2ykn7nYudWCeiH@g#bnoE@fj#2Zd1y1?)e`RzkxL0rRR_7UC2`_z zAb%RG%pLoH#v}a11O&vxAr{)cKiNg8bu*)+-n5=(9&mg;Qr*3@Gh`9*gF=MD5B?_i z1#t?%F$w?<%^Fg)g0N(9rHqmce6Sp>`hsJ6zMND9GZB8CcI%A^<-WgukxD+Ic7uvm zq!*WMTTT>3i}N>~FBi)@-L9H22Z1Wx1;A{-R<2l4n4aR*0D%oH1E8#CCNV+5b|p}2 zQ<)SO6jiXHc#8L&M7jt|e1{u2DQW=G4X_9zF2<=BH&t>C-DJ%w6Q3b-6AexHNCawv zc!BEG`z#OEera4Z!K<;ob4bF<1_z^X9tI6XQj2)TypxQ`9|jCUA9P3h*&lx@{k5FA z?oE~iM!Q*xfI;zea#SeKx_6_QKILiP4GM8XSerb#9$7-{Th>gcrbu7X+q3o|T8Lp$ z8D#y=%n|Jc!tW7;1lqt?a*HiE_~p)fx#r-9=(~C!2esKvUyq-pVsRu^Oe1>`|CgJ_ z$5eYNm}eg5>{cFZ>9fjjF|IJeZl8e)>_;%M!b1E6!nzOBN2+CMgz@2#hsJD56IogY zEHRRu%q21SuWOGCDrVVshfiKvz6DY)^d7VAb+XTeI7+_!jLHzCXMw{?09emTNd#;z z^@D7!NM#VxiR6J?>rPt3wpS>t(2zIQ^Uurm9_L7t?om*|9-3TJj>8Wq3XHPC8HAxV-Y!1a-$*hd-lyr1mnJ5C0rOdn%NFa=j2xsly~W)k zdm=Pyyim`4#w-)T{n*-Yy=ZsPoX{K2ubEL+BRN+0rEp#>%6YVsTrIEC3YJ;mQoLGF&g zv5vAgMOUgLL%w6}%x0}imU065Ix*>PM<__-TO3wa(Z2^~DW6rfqia!~7?+T*T_@D$ zBd)ZkVJFpx^;=w8p00&}V{T?;@l`oRxcYOW0ws)*i?VBJG_sm^m;J2TS*j*h!;gD^2->a^Lys?G66vtyO4?X$m5#|)0?s3p@ z#1;e2#-?>>Th)TZ!Iu?3>kp$e{p98WGoMAcDd(i{+DQazxd4SX*rELJUW-%`9==h; zx%sPr)KV_$(vI5YzL{%FGzab(bFQH=h#`@&+4qk5>IZ90??$>7j}p@AiNbtfzq!!u z;ihm0At&`FG*!0#mGs(Qu$0J(ROg|at&MP)$%=D?gUhU8=lQw9+soFw?r8q8+Zr-b zf3Y*Mp69}YpL6Nj)|Po@!?SH9`Lkl)HL2xyX7%{JRWx+btAZC9hqC?TMk&6kl4XBg z-z0lDZ$#YtuB8up`}nQueP<}s@whCioIZQ2cS72cQv-$s26ym+kx_>B+KMD^*ppn7 zB!+)f0%THIjm;4q1wy7Uq)Ij1B2G(_vVU++$iM>kSq4(o?Uo&vud6~J;nxygDK|RZ z1zVlh{E#*`ii2pTyu|B$;&%VA zJ&%LrRAe5HXQiMpApPXX_ucpps@ z&3nV0qGK$zhzR}1n+OB2Hfpu*>(@0=^nh~Q7l~~0XhuZMPAfG=GS-&9z~gxDvg^*_k9tBz+^mgz++!K1vvOQbwZQK5 zG>3Np0gxq5(c^1bSb}ST3wVFpTZZ+8>5PWla`90jY^1rVS`h&WusF}vT|U!0d?s4c z>(3Hj!Sovl36rF5`Ef}#OuOs84NA6?cw!+l3Jh7WJ0I>%sgt*hyF6>357?L#F6^ZM zr$V03bnZUATFxSi@GjHUJ?_io-sqi0ym6GQM57c|LC6l1N+;8-K-4UV_Z_r09nqtl>Abro#%Igm}%-=+|} z0kvs(@gVjg8G&rP`7ilI))bO)pjbSh$h5pEevK?Th~BV97|=rRm0(?kAax+nX|R7; z68OHQobj9V_&dr0KY8x7<|`kB8Wy|E?~k-@zPK2Etf#}c4G_1?{p_p#Tt>Z12BpEt(DQ-U2Qu)rwAqRFLbtZyn2_eI3Aw+Y6R#QB;dOPmGrHPm zZhYEz5r}mJ&~T#2Y6{4|5K2L<5Yh4xeC~a+U`FAM1Rrx?TJc_HHF>;}G1GwQ#-;aE zQe9Qc_0P@1r7tKu`Z>T=41)pbLMK@`o%K3l{2(K4B^O3nfs1Ky{ro1| z?#OHvX%pfq|3F2NZ5B-L0?JsEZM5YvJ$#`5co0-oIVo-E5`oDY9Nd=gh*qd7??|$c z%0v7@us^}Fam3?%MZ82$Wp!+H_hS)-I0^7LnwEZ(MC!9QW6b}{d}!ap@|cfoWTsNo zoB-Rki#p_mJlfJoyRpH_z$fP%(^moi;~6#Aj9pc5Y1%7nc03xUVAJV0J{@@U6k2h; z{aRz*Nj`^Iv?f}5bBo{6Ol$U)EMOMJ7*

?Ihpt>IxY8Iu9OJ*sw4Hy(0p3=1g>c%iz(1i5E=A(%)3eLQ zsEhFphrfM2G~CyQc~a;|<|h^^R4AHsp01Ctt}kBgk3KaW4BZ%e-tQC&ja^lt&QwG_W(PSvuEVj)H5Qb|KYThP`f*5MUpQ zq**9nA2ve}*`3`Nfe>(4IEhTyrE^l;SFc$#q8Aky(nR=+?6oyxXvMYj6NAk zV#z61#Mfgr1E$={v*J3J-gaUv_}^zqq9x1a@VPW{CKaW=8~x-5BfQP?wOugY(WV$v z{zxpW^J@=bt4>IoY(ofWin~;7v;1!NBPMft1u=EhSZ#UFz~W1-5bW;uj8liu=XLb? zyP$w%_a_(QoZOc#yQ&J2DtG!huzw99&A*Rxmk(Q%^FDLD2O&>=`(*wG5Ys*R!}qnX zTpPZGoPvADLp6Q((=Tlzub&nNF3DxY93Tn@S=fPQo^ivyUIl2u2@rFU#*X!8gFkG$ z&#LS4Bt9&Rfl;UgZEO6W0jq!Ycp#dMwcO?|bMM9XFp8JWb+ zsA-BN0JdVc-@cYE8tWN)`8U!CgEH0Yh<`y|tiV}8fPzL!q`w__$^AbI5hD67Iq~^% z3z@l8`cR>S%QTJPJqgt7!FVn56~kDP980TkAlIPi);lEU4*&ga9W@#E&rqu@Ety~t zIy9l$md_>`h+`qiPtJ1b^gYjt3liy7`2PJr06##$zdK=_fjc~}=UiIN+>2_xpI1Cx zsIa<)8kKdw!x4;siKN=VPRNfHS<0zF*-+Ie-x1yHWgGQtFzyZl7@@`GcE?aH#sV`vJp|Ez%lnni0r|73OX{DSqE&2e zpPQi=kljJbKQR&4o0FzfK_45rQAsr{`%Z~PMEzpAG#YPQ5@$PQ<0wxuHGnx*L{;-C ziCgvr-GK7sv*p}JjVe!p(P+Fi+zGn&yhatZuTGJCEsw6tH65PNZD!q=a9s)MErfu0 z{Y|TNj+Co^8_)E%E1{_`pV^;&N6wQgDQzKAvwAzkr+C)=-mqk9PANR zovVVv%}f!xPcBi)#_)7C>- za26K!!mrXQlgJCx$Iuj0=1fnx59HDIP#T)i+_S$s0{H$rQxlk6>(7fou6TQCU&=LC zaB4<>qa6y=L`7_;^q^W#p|R`2dhTQ!3TaFGnN%H>2muhL zjcMHv)Ycq>ljUhK@mA`P1h83&VLtU;gjElJ0mv^Jma-gl?VqVk%&tp+8>RuAM7CRn zp&N8hH#Rqnlz`Wkd}c1C5{{BALrOR@$DUrN2svP_@KkGsGn>{TDWu58MbRD>lB)^F zrYw8iOxE3Oinq$sd@0dQikDRKt^{hA<$MmKL=SW)qRVF7$q*a%D4J~S-Rk%}%ibe@ z{4i=T#ZiNjrLQJn+zW*|lZ;hSIjM`NnVMTekn^G0n+VCt*xDwRPHrc{WDB_ztM$(0 zEa;>}J8@E@mI8bxE1mrTFW8`uqy2**E5le0(Ifie?k*Hyw1s&c9hWDZQ~a|d-YZ9M ze|qs^{PXMQFJ8WS{o*;Rk9VOI1xLhD=63{f=;t}YcPv=@zUy|66)DMAVISkfabSNj z=HwCVK#g7SzfJ5t<*V`$$AX_>^LtCg)z#;PaGiENdQvK;8~t&{0rh_VJ5pYUp#AJOl}hcJY2btz$J&i|Fg z{QBU_gM){UPB0+wATjgiz&R^d7%{njU}Tg{>Pvav=Y)54_%^v?oT`HaIZRb(H{ zCvxP26C2#;_PAu=216Zx%Ls(`1O{**{2NBK1wa#%f@+>@pxwdwiN0ZpuxCs7o7<2R zAuMXA@)m;mbH68iI?SJx3t3fuu@E)R^e9yC#t)&7<-dDJtb2OHNXL36a6DdBC!D7s ze18YAYRXNhV;?ay7tW@X$7nTpOm>y|HS3G`BY6`L|HWq>3TQ@up-?o54rMV>Sl%2H zoKj#k=`#4@dA^!b7A8K#JE9_Y!SNdN$59?Uy9n_b{wjl?V|EuTjPw#teGt$4oH+`F z?jpbH%WDb#$u0%OeMZr++M!6n<*Gi-PfNUU(>!|c;KAs@=K}V4esaIU1lUJIPT91>tui^uthcS^lR%c?AvO@bi!i}j) zh8WOTW_PLeVfLF3<$cyC0QLt3?*0EY@J|Q|TK2m$4E$zS$id0s@V!xA{C6icTSN7N z3KRkc5A&*z)`v=!ko?_DpWdPSOg}Jue9WJG?zlBwf zk^!Tw$0kV%n@1Fr`XM0r<1h*b7XYkQ&9u2Axd3kq2j$@84R zRAs@*SLgMabqsQqwi3KbP@`o_ipqLvZm^Cug-^P(zy@&b0s~MWfMwIeFHxd~P`@M{ zf_hf7&N@qfLZBII1REfXnWx{o*UPkxs2H$s6%LD3Za`&lGPu_p^z?{>QBY=^zo&(d zznToGgx$KARC-@Fqri;R6o>IqMiWzU!+?66yW}Hw|7nRCMq2SQ^G#KXwAL7&o&*k~ zDQ=MBM@fN1=zaRZUiRavL>oz!A=XpWeb~7_6X@lC+!(-UiL0{-ny)bdcef?1(YMTE z5R2_$Lp;KQGDe=!l1pEi-01obRIKu%oRSAHN)isCN!ZWqJ;qaWMa^yiO^YD}nD~)S zfP%ZJ_+Z+I7;Cc6*;4C;3}IAgFz2%m@G!DvS(ZNTqX`Z>we;it5+kh8)Q@%FJClzY1cVF z10FPy?B2z1SyhD;cPW@GX;M1K^5#D0lQAr3HBZ|3K9 z{Z4MS>AvZ*fFsC7AxV}C8y0_j94JP|{oeACsXFpt^aWM4cyRFOi-Us)Uwnz>Qv1Ds zm)}4AcUZ-fp0OuB7=87n`{nBT9}d8m0^o~3KR7t})4{{RiJec}Ftq9gIvObM2k-$- zEa)Y=UOeP4?>#jvd-z_#_+?EpJWW#^E2pXrEF>JyxT3zqpO5#%L=P z-*mrjE(CHqi(OEqB5{Er;qNvW(rrAC0?rj!&5UcCbW9h_m}`*3i}6|?pv=mDv$f>4 zszFa(l>cCD#k<(2V5{(Y)6-58H=RhPShM}O~X*^&c9UX@$ zA%IAS9dsWNsXi3f1iV|K?OoSnfP{6iS5}ZcRWb^j4V`ZS&MQ&Q=$nOq1hw^@z8q@d zq)f&)OKymqw0nBxR%%d{JQzn{MKUAcg@lJRUi{{@o2>T)-95;`@$Xh$JoftVCqkRz znk{Cca9L=s#!G&m!;ruU6E593&?!gkd};GB!(}>X^B)=h(uU!^pfv@2^-(U0y|4>; zKza}kHeO@+msl4cd*D%j7dN6ojG#BTZNg(RubZNuxQjQ{L_FRxX(A}blJ*QvlAz*k zn{T~rA?Ojo$s{O6x7zcq$9=uzGh$HCf}z-NutAgciIOu57+VwLtQ9X!GWnRB;37N2 z_|sy=&XKN0uyUQH9K7RtY0I5)5^zbw*%hCWh+sRemz>DcylQBFC8HC>+O=|KS;! zE_8WFSz2csaB{fdWmDz^wL`&!+Sw!5x7L$T4(b!HOo&_v#I=#=T;C!XB6rmFEjXIm%$robfj--|MTOK86`X*EtBYX4Jo zq(&4Z{K?O5q%7A~3nC@RYCt-;N#OUbFLqUn2QG_9&eJX@C*_y!aA_$xdXzoot<1RP zxRQ2MgC9Yl3eU*b2xy7;A zgLWXtPUuE@UiS`!tl558vOUBXV7TrTypVh$5~dr7A?_7`r?%S~EU; zm`DS@7r$07L85PdlPk<9e2zXELaDcqU^l|&$xi6NJl%#qFsnDE&y(HgvkO8=e74gk zd9xhnxVC8sPsi;~v4|3*wQD!hO^N^wJ18WDMm2*~dAu(pyk7YSjgeEe>RI``u0Bs!3J zuyIN|$57!I@~eLIM%Ngg$QY_<48jC5h9@@~Lv_-9j#9663tH2{o*kTCfR)oYu}!mB z&OsYEP28F0Dq5p#Y&>0VuX)okgnQY)6{@y>cXpSyi@Op)!J8LQ{hvX-I^`zRM6e`4cRlQ zW_T|i1Z-)2%6X!i>9w;PxQ59pb#C^tkB-;YbFhs4u{z)~ubdV$IyJz@eAZH$5>mZV zT3bq!5}4+-#JSC-LOa%Wq2Oq{9JmmF2-i;_|5imRtsefFtiv4{ID%9a=Fd12Qvy!k z8>!ogO(TMF{>zgMgXc_#`Mk3r0TOj+;SJK0vcnes@rJ?}Z5>cJGq$_}OPeHb#hZ&> z3V03^4=o)S8pU=>4`cYB@Vmc|{PyR~hUtk4h&VDb8E~W=KGST_R|K);cyn)mCaqwH z{X?Zl>CBaYbuZKGL_J$1@XigNY#ur*2^$t}AAk%k)Ik=~YO!IowEq>2FnAczB@BZw zVy@T=yj`3wV$-GgBH-rB-#Y>7Vtgu*r-gnVHWbG|6 z%|g=tH(ntGlo${B!iN8Ok}V%){rlOMkN))aR}cRDD1*6wc<|L%55FD^;N_#t%sw`H zlzlN63{uk!a11k0!$)6!`NbEH9%X;}>fpiGU;hbTdos*?Tqvt{_Tb4NW*ox<{~55~ zMV0CjLWYfRCW!PE?uM;@KVeHrpe9Mlw(5Aq&GtMX2LZX+^f!0?eo|?oVEoNBiusN( zb{ln}olfT2RJOcll4H=cY<}^|5u`gouVk0GmOBym<l7^GrG5eot8#^C2n=}MoIsFlg<;yKn&!zDo# zlduOYUnKWxa^Z=7M>g`CE2kRIq{Bb#(ZDPpTtgq1-T2gE>_nV=AZDe(K*4YVq61W~k^hU22 zTXNaZ#j^u@QTBVLz8(y+|K|OEkR3hQg3=E21ZUXY=ILP5Jk`igo$uGH&U5n4-9H#? za3#3OAO`V{;wB2FeFv`i7-($cCc+1;DH{!OeKi9^e;hYuQtWHbgRm3NI(5wMXl!0t zZn3Z6-wtYj*M_b0kJ;_q^51pG!@oVN)`)$3TGKT+RsLy;#QwE6sTT|Afm$P4ZQpfc zwkQw*_=CM|lgAOT1S}U=zbg>gKozXPhU^{TmE%votsmV9&Us5C8cOg^*uwO{9&QsL6>wA7c|?8>jSJ>?&kjHyVaj$?=4 zWZ@`(_MlkvTw?&SAqZ?w9mEU5jy5TOHVo(ax5X?OJWP{sdlKUVjX)-g6ovYG zh}+EwcJ2)ZqH+)>C%WZ9&}0IiQyT82O?((4WOWg%fe;~f@|f9ev<~Qod5sKbK#`Jc z^?tiMTB2`vk7HS663rPt3#QrUPGqL_qTj)?vGo{$vSC?X+-QwXzi#iGyIvQAwEs4L zr?tiC>|ZPbv5?%14hCO9r!<2`z3FZ}=tjT{VMheafT)c2w*F1ylweroSD=J1+jC0- z<3Y6J90X9zI69#ur$QD&z(`MyH2?YbYO_8)27(uS5UEb0S3EA-t(S;(HonF*30~{& zE#}LuugaUr8GUk;oFni3jggV@iR`U^3z4+ze4w38v{#byb26e5HS|8b@!R`D7ne7b zye~SbUowqL@9)*$Th^GEPsCZ9{f#r_WoUV43n0ri_f2sM1Zx5LJhcIqq5$Vuq-^b& zH_0i9nw4$_irFXz#aCG=$p#k#SV#EjE0P>)F`o=84TuAFynl6Tes|=sMEEmSKsM!C!8s4BlL;!p(%d zK{BmPVA?}%u9kB;8TrUyO$P1Q_L3Y@>gl~5bd+BPCmXc6U(#tke?cU}8QRYd(x-&f zEN$qMk5QInq(db}m+Z>RE9onLgx%&L*udF@wZZ5%^}XMA_x&k#b(BGvL^Yt{AYoH7 zf0!L}8=8n>r=4XmTvBBusR|=vw9Om2MMi7Y_LM0SIn?SXTWlZx3%g zx^Ax6B)Q~bnbOYE_JxjLQWS`kn*#6K@k0NioZ7g=^#Yy219+YN-d+X zBo$o0h}wrn{nWcI39S1y4CJ`61z+W`ly}p)Vlh#nsy|e6_K{@f`RyvOnYEqwB?l7i z2>9Pkd&iCoGI19Y--4udIz?E$Hd08%X1+aL|EaL~l`e`EUn0zZRa|+W>Km>SdCyZO zmx$(F1Dtj2yxuHMu1=6nTgk~32O3XLk z2Pw1S##}%Ijkobmq6UD+u7$pkV&$AJvVAxw9*7;x%7m5girs}kcqy zkN*=Zo~vG;Q9`4TVj(4lG6BsFA)$ERCcsS{9-CJrPF-T^!Cuz-$W;)Lzw1tE~It-&B9sKX;$nrUD z0S@V~^iPUB?(^r)Yup17>qQ$}JZ^&{KL3Br=B0^$(UCuo33ea0K#*=gSd;g(kPN7w ze{PbEMvgpi?3PFviZ>u-pYAW?3)OGIvYtzR+Z|r3~&E$N%?H= z^UnjbjYppjn~e>sW=?)M^jPfPLC5L(HmngYdpGweJrbwI4&F#M)QxsflN4-^<@o=1 z>W=<@quRRh?UDI}^UNFRo)Vd}yT7>xR@l4;pm*_xCG0;EQi+!b@?xyIIi$QTQVVU` zUl=uYmt;T_ZE&hIZr&c$kz-aM_lCNUba6yXl`D;0ieBAExC?ft3YI$^mA5y1H^|)U zza+UPlNbjrO@_%(Is7d_#6zUqdz+f>1C-WF9;uV(Z5GW#FUM3|j(dWr+^ z8;;v9IRWQOiT8tVPk(%cC%m_pwe{)d{4z5Qi-Ol>b?Ib_hw$pbDEltIrsVD~1j1qV z5_+kM^$K0siVB!r0@=z53g=XZG z04L9jvR*gbpUHhgHBvw<+>tCz770|Qnxb(V9QFwUay=m-VEfYpB9h>}7DYIs9Y>NY z0(8oH&Di!#!a-JD>@~&wDjL5fl<1*<9mp)E zyU9V7!&FxF9FzD=cthQ@q*g~kkF=5|h?s4XFIj@&@5oCwQ#+3gneon^szt-37b;0P zCw@lR59AOEq!;I_kr)DAzb4N6RJH)W83BG(kx9ySq;e~B%c?>fiXn0mSXI44tjBH@ z9l>0!7yX`$$4p!UTElTq)&W|7%Xkirm*GFrRAJzGqcsKZ^|PSzeVRtzcfXU6R%4-t zvWOn_DhzRfR=uu-&OwJqT@x`9=p)*Sa6>1nCF*^)k1hB~y#1#^kAd%ool&qFy$fKn zt}2j~u*6=!b}86ccJq#4ZaQJzO$tJeSEY2&Z*`(?;<_na_46z-6Ad1Jn@lc7wq6#P zL7Vle(=u0;oe3Wl(Msn5^~p$v5RI@al_44kGA%HDKL1R0&XxKTzB#0>$AX!BL)_Gq zx#r=r=E#Sj=0HS;_&TP;5aM+dfD6BD zhMa^!bRNW2L*1KqIyu^ZPbo*abts$i#95Nz32NZ7^|+jz^EE0sSWr1%3H-WJpeZwb2wB4A8dT?hJ2FowMvep^%MrS7 zXq?{r`gK&4FSZ*wl;tP5 z&VW+Y65d$l*QC6MvbZ4q-FRquy$N(yoVf-BX7Zq~SYQ!dy!sa50l)N^_bHn^G+-*O z5@1g2iTO7>kcUZsn359X5_$7EtwFt~dkZ_8$`mm34>NLJp`ln9_5j3`!KdJBT*Q^y}Q4y-1_g z()+{Ay^;*$++p%lim_L(F)`cR%CC@U{%$ij+J0sx=lkwSS>S%dQ?tMeUM{9p`F1#4 z%H~ynMHPZzd#tlrzS=a~sLHSzp{eJES~-t*2$4lGwuPSkwC^e?4EA=Xi!`_ zI%}4?jJ#(fk%~fchjdxa=X8Nbc3!?6|8}r{VM*|S{54VW&ywtZa~k^{C1bXy#g;D* z11pBXiIKnr*@Dl#1EZ3##~QQIU~qGJ8eyZP=+4-L(ST1j;zlzUL(8jc78N7mtRuV# z#~T$^@%f{5(&I+WO&v-n;;;xy{GD3jG<-l4rmB~Z%ZiOC69Szdi+T)Z?~2&AUN?YtE#!>?otg!%pGo6am!mg{yEMw&&mqg z<*ILM7=pLa6I*vFEx}G9n~Qp%oMhFK8+t5%5_Z{fzq%*etV?icFSv}^f)4=Zh3MFd z$0g&2)y3(MV#UVhZCy|>Y8D;?>7+qLI+?Dpm_=_J;tsuyv*+pENz>g5Jrt_?Nwy(~mlIV)MtB=GC;g^1O0J^r+u|jY=wo z++|_)Pfss^^M|C9TQ8?{uNqK0h84LSbYpU2T(dR-jNpax=hzV<2{BBz+GX&M7I@kk zu$2>%8f>SBLZ{A)t@{5cmyw@GY=RmjkKSaC!8b&pTL^gVD7?;!5w^rnLFIMlcG4ug zs07X4SW+=E#>;jJuxz?NtDQAeA14&A||0>CN^`$m1pE^BvVWxF*#5|KeV zy#IrZ;PvaFx&TaF(V5GBa{=OftE?jhNAA)&)m_n%nftDr&AUz)E&~O7gK?7PSq5Zo zna$LjskZzA8(#N)-X?3M4K+&|WAv$W@rs9AtrJj`ZGGCA^ z#EZ$Uhz{$y$5De$#xorSZU;)RM2x3IDnI!6w)3R97dz`Po$p|I-|+U$4eu!nJW|rP zz;{kZFetR1Z(#LZjO{Uhop(0A-`4t07!3RpNCwRV9q_Y{j;wf6||Gub%ABVX&M91!%_~fbnNF%Rl#T3<(!Dc z2YrDMq}su>UgTv(WhQt`m{8Cjjq%ZtJ^qSTj=j2+pEI%PKz6QwuD)*aGuqr^4{1|5 z5*R9xK3KtD_Xa} z52BY;W2L@nump{N@yKbLDZ>BpRs+@fzZP7#gjr0KPR|hSJP&yMjro~RUTY3*av-%S zOpHo^VQn6_tuhPL1wzS$!{ zy(~93#F0G;E3i0i3`YU@V{=1C0TCwQ4$yc;srt)VjBnz8>Z83In=3l+EU0;WQ!w9p zV`2V$k-`%evTfP|lkE29R})Mu?m4_C69E}7qBPct{`Yt4{^Q?{;9oeWK`$LCAl+HO zBstJ`2pd^{gLlf<;Fh4)$u2fFhmXyQ!8AOc$05cFd0KKd(X?53u0iMO>n6l;rwVJw zImFG3>`5FBQnNcaABob~1({TW1&r>0DwqZq?X;lc0;E6SIe(6UaDK=Qa>b04t5rcY z5y|1UWM^T&Mb4*#drFFQhBRJ_y0`cWccW!_#)KcU7 zWC*8!2)v6H>%eye&udk$8~2+s#deB@5F=shFyAU{H( z*jJT3(c+_H!2G~Z>`nTxdn{rG2)cxs;kAFv=A#Ehq@#n&l8o_lf3XTyqBVSNllU7jExMzu#pb`Gk7fbUS;Ee5YBo;%2S1Z z<6%ivz%h}R3WF^heVb-)-w>*lqtsU$_Ev=M(>NPxb#~Wm(*eaOEF^4|m5rOVP$|M$ z#wb7)2#))9Rgg<5C{@*TjSeEn^d*|78E zU9VPOdH5ter8gRNp{6G`(V~`@qorVfND3f5p5+sezeMJt7eKMo4U$FfLvA81xk#~O zN(=vlF-64W-i@u|IgX-KH@Fn116^+Oj`{?515Rt`h~rKBOBX@<&nE%g#fv zZ#qB4u2^4mQM_Rw@9P(D-oAMLAw!qGKf%a1r4iheV9{PrSpr~}>aN4spA)oyioF;E zYVbwboxSY!4{u+XFy*qU=k?h&=Qn6CYj_Fd(zI+Q>jswwrbi(>C^H5JSG~rvEc5ao zZ0N()@ur1;^VL+IQO*ay6&z(S(*m4aFqXkYXk~!W+yEv*I)bH_42! zgqNKGooDOKv!~y^cs_pn;-~N5jNknH^5v_4$IA$lyvE&Q{>yTO3C3ENC6Jp)+7-%t z6Lx5QVj^guGS$@uo*vn`#bq|(%r}xNXQ;ALoc_XM5RR+Lvs8)4dR3l(m82<i-sbAx-uXf%rzmH4obBU+;WKN$mcjM8FvVW^DiwimT zSDduNEH@MGsJwgmElT6IZbHh1NVrQv@#)2JimA@1A7MX!nxkQ7BBS&`r7jK)Q;(DG zMD4BJURHy|S(sI1XIZj;%mg&^+5$sPiYal4Y?SMH+t`b~b+s?A8&!-ELjAaIWyb}= zt!x+reZlyErf}f!z$Pq0;6$~wIhx4BM(+Exr<1G><-``GQZmU_;pshEi}8Beut?of zqTbfk*8X6(qw7cD;|A5`en+Rc>t&n!O$_1qa38J9T`*5}g4yYRRcCX$S>a(+DO97b z#%?Y^r<$)9Rd!`mXM3n}&G3%;e%8m5Z&zp*hy4aV46@I&{+0b2MT(=Oa62)jXfs!g zrq7{sf)g5e?ac)!MV2!v@}~4lpEXs;^9#*1x*en<5XY5J+Ow<~|4obdSuw2qa#B-VSvNFyt zlbUYsS`Z;E+m2^6I#eO>6qnzHgyOB_`o2 zP<{Q$8%EPJv?LkY(GOI?8ni+hnt}Rfnk1Q(*MO?Y4$D|sH-|{z|Q%;9qC{&j6fX&aNsW*#k9P*HT+@v zm;7oYQk^($D%6rB6-p25ACW-Y0=N1v2}A*Zod!2TbRW9-Ps#0u{6k50%gR_&U|kDr zC-|_vjk#9XDz>nbAQ+7eMtU`?R}Z;%yzPU7N$4eYB(h9?6cLG2whE4Y=(BgUU-bsA+ym8IkUZY`(8Kt(neu}?W$)$X}F-8&{Nr?Pm3i8--WiEjIU z)_jjzE=_q6X=%%isFr(M@`45+tI8!Rm8Lp(X$*LhPtFVLQR^p%@L}W4z^bK8!eh`( z&>WD;Iw}_8Ev>rqp*jS~9dlLB@rn$1xn;#+3Tt!f@>8HKbUGDhVqDwrs#@I+lA2+C z(G_e|XtZZM!ky<*uRx4Dm}mrny9+pfN`)c`5c2szzcf<7N%GfAlQWENHm{b&sw@^# zdS;lJ+&Gi-x`dM^ntAfs3&<-b3`CWkraj|IIwoTWt<@a&kjIp0Fiuo}C?&RP-doOoJ4%KL8 z9O{8HY%zeAD@JBWHh%2WI9$OmvT@|N9W=kl_O3c=@@qVvA7tNv(vJUpN%XT8Wga|^ z{ffgAe&-Jl;lpS&GGD)B!=8kH44b$8)geS|OlSB4cWLb+R@(=h z<-@Es{%H`Bp6*Wy_P`kvZIJqk09%B|`1y;M<7Yp7|Km?D-n<$A^z(Nw-h_PW+P!)8 z{g2&;ISjCcM;j5{WGrHso!FJv@2f@s5j@_YSk)dim+;s&^Rvg0;(vSq^(~ z@6cUp#pJ}`U6fDM3N^OVx}L}OlHPeSUqUZ$&g=DjnsLc=Ke;JaFq1-}Sm{19D6xW7 z%uCo@k(Gf^pl{@yZ-&c%u{6MIEm^aB*}b@T3=?9JT#L7?(ygh?yeTapSkTq8xmnQi zi*6%e)8`b_DpAL`>7I{;irH`PPkJMgtf1X&BIw|~PrjZc2z4zz<8d-9kOyG}vC zk*a{XvkK264ELIUbVn8L`6b_^TA;XHO{K0ew1}YL<=T0^qU^jYw5t21U2n^sKJ!G( z>G=j|=JbE)W=?M>xh-=_sy(ie@VaQ54VkUN(&~hf5sIZ6QXv_xQ;GwZ;%Sppo0O@UP;VO3-vN2#G8-J1if<9Sc9X#_D z?<~#dysE2z>%}Je3c^PMGor8Rz%(-7lWpydscnr8=`ZmXq-(l^`sJ>WHnX%qwX zPj>>;e|U(pKLYQ!!IWCwgRKHD=-B_78ohUO$%5m z@rdTv^QUM^Oo=<=tA-JQ z*gHa3D2#q->WZ>i;180(B0!VdSjdw3V#nNnZE2K;A$~VZ-qr!sq}dpB!8geZZZ(4m zAKL^)Q%X?PL!K=ilSnsEMtaX7ycht_%K9m_?)FEKqOI zicI3HqO;Gsp8-Cf+=H%F8XHp{Q7VCYS*>}Y#FarMdvuqh@OfdabP~cMKEB*JVFuHG zeNm_|CwDv0pBJD^xl|Kgj$N{}H+6vThT5eXL+Q#xRGJoBuYoH z;w1Uuj0`f#a$ynxxVb{Y7YY0UM`bA~XQ6X*rt}D=rV9SGX8nu^_{{#yUaMR;3n>%kx@s z8Cnz9gbs6<0&Y=E07BU;B(Sk1DHNfZul!=O_r>GRbkz`9357nsHK473J9}e;a->Y2 zLsI9YbF~iWTAA&VHEcW!cMW1JEUP+?h?jhA`6P>3sc z2;#~5M)Di=>=w~FcnBCKH&KqS)xK-et+(~xY=7lhO)a}FXwsQeD-uH*x-lj8YKKaS zhB#6*8yPQ7#jO}yW|FmkYU#q8On$Q>?6{+@DZvk3?pxaEof$bUnj>^5ASEg41#`b( z_!bpbhNe11lGaqYD1kpQy318&k1`G!k{B#s?yx_L)M<0dely9rmNd1aJnyd@5) zj9hC&hMh`B(;jJ)wgh&d?-G-8@I<%AcJWj zrr0NW2h6m)jEO*hC)_q{^nM~n*e`;nuy4;BSDSI9!q7T#=hG@;`EkS0<(!MsAg)w~ zc)L?SmOhQjDl7L~{ObtSj!i`!arU1$YFiHh?@Y$cJ2>2}s@_fk=fO1#0Z5>~WlGW` zIeu_5qMJ($ARiCi!%1Dt?DvBc=oQ3w15e18C(Og+hG;W?^#-;?!}nus)$BHI9|Rd6 zS$_`aP4VYOD%_~@&r=+ZEq4oF(0P_lE>;fkZnkzppySH07uZUvzmJffmzAicjo(Ze z5mj;ynmW8ez3dIhTi&MtT75=n_3EpNMrKmbaK-`EReRL?d)vc;g3VF29RK$>KfHcH z7xb&$k_Om+e?pT{OWBRpxiBbb-V>vN%F^o{%bnh2c9*i6)lOel5!CT5xo2fDZiJ0V z&co%>Xq1gUWn0sr*}PnD21UKIaUyXxW~qpoZk?7>Z(b*LTeU;`8sRAW%e9Z-Wx=@% z!n3tiJ$BgQt&-XqQ-7OadSxR6?a??em`GF{gF8Ea=wvvti6!25G3c?NVsz}%_QP}V8mUYx(g z$b_y#q)L{p6k4Fe8M<|c+MNv{Ry6VtOev*p+v%p0_0*h6FV7W)QBU7xddPx#Mo9yj zdZ8zO#3f7D*at?Q&4Vy?q5T9B(Y*DYb{_5@>Xa01B*EEf!zgubfljqMp=PDkiXdQ# zzA)$bPB0v7V-}+X$XT1#Ej2MY!g(_He&#K5k58O`jp5dB-7ORj55c+MejjIouEv*r zhWa0@%AUW4reG+u%~l{LqjqY`RU0vuvBu5)eN_*mhw__jPz0uZmZG)@7Q zk8OL>FVRm`-$QpQqBiFgzQO9D56Sq5-VB;~D}d5hC z)D7aj$l*F9N;>s!cOKgalN2JZK{q;k?nPQZ9d-vhamRLsPdu5HI9G3zfSW{tcWY+P z2pn5Xs=usg*UmTw>Q!v>Z+jW58usU^8(Y$a6gZm=z`~3XOzl*-)1wlTU@`N5Q@0fL zUXCPp==>mf1p(#BB^|_^dy86Y8}82+PFk7Hy{a}~%xEp!5GXH<^ z?tk&_r{i7x!-+`o@Gjbtg1q~G9u++PeGC6;4%}_Ic>6@xQuzSAEp;xzKkA&4#y1_u zf-~cM%akZ!AuaN@Uh!M{=roAX<{l-JJ!WqKI6=`pnofsl+*VcZGmGPwp*=RbGF!>p zop_kW@tDVa{L2X!lS$ee_|U1FKZy0Rpj@PD-JyIEo>sT8l~qHQ_46@*(s{WXT#C%2 ze2Lc4eris6Lh03V+Ao=$iCR9JC+B!~&?iB@6ORZdyRv)C96ND;@Y=AOWJJ7X8&$>Sm|xwA8KCL*?R<>UEYa%eYI!UuoOGRGFzCSK zG#_phNx$D<24jw+y-)RjrRoFvk{*%0wE)K<-$0q~JQey%z^Bk@{cJK zNFVG2#@~3lzqvR!#Q2DEv~y*+k#;VQUDky==zCmEXMK+gKME`V6Yg{|HOS4bbOUjE zvwPk21+Nd%7FYR15deH5Fkb25rO<`|fT#XN@))!!{($I(KFO4SI<`u9w3=(Rot%p) zLqgf4{dVYjH|KJ~dE4PUE8Y2*)&1p-JeGHMRd#p0H*r`Vgx>!Y$#!k($TwU^9tmZL zYwp(Mkd~N^j$3+kU1~P^uj!=px#yhAclZl;hwq|O{j$Ht?%fp+AlIW!;wH>;qmYye z2b>l~)qR6%uLr$uVbE5&M~g!bUPOQC}=^bO;GTqAlTN)Hqp{DpZbtbh1zs zOOzG%do%iFNAXGTVj(eJWd}$D=-8J0%e81FbaIR8FX0t`Mifwt7l0TxQZ!gtp>On zU>Qi&=fcBbrRn~XuR3BqNRJ-Lq+<_$LmYaFVW~It?sO!b@9N=ms<2NKxYpZw?hf}1 z&cgI&eLfQRD4djP5ll==<0MWZZTPHR1~r%K=h!$e4+=3cNKIsG7L0q!JDPf*PA3^c z{mx$*)!>jz?7`$A?1lvSTC6%}<||6s)LUOo5{*Ld(mJXYg>8Hmai_N% za1_0kA})2@(|HG7tns4HQ4dLfXBp$1^_2wba?d{Yu%9|&qO-xmwLe}-!wwpgYtG*P z!e_R~qpm?dNaO5W-T?G3BZOkWN2b>&v#89W=EH_CRn4TX#C~dD)Eq~iVQ&VnCdN;! z?m8~iHucp%9n%c3{9kfEFRmVAIO%6I8&ew1;I~R;jyH2)u=fA{X500vY1*}?N7{PK zT6vNKC%PYc2D4nVlyHnxVFGl{0WM}3uge@LZJ}|Hs1cd|#)t6R+6se9_?Lzn2X>35 z{tl^`^i)RG)x$vzTs(57uL28vv`@_Vn!lSw3UO3FVR3h3(x@J#R*XYJAJxttJx1ia zKl@Sm{%e+_+Yz;z6Z%NVtyuQrcBRV$2FlTFr9UxhI+j0D@YaS09TZbSTMKr=W&BOu z>MXhfH`+fq2U?7!&s0nd>0I9pNVFogpiOhFV7G7Gnoj47^hJ!sXbpXDFZ{AqPi*mxv&sS;@T~2G54abK_DaQHDkfe6qeC-qFG==b zu3%IcU*Ve~%HmAxT1E8k@@4B#lxDQCV&gf%Ru|Ke_qXOZ5(#9rr&}dQ-`0m$=^@|iUu42$c zC^0(8zGY|FmpLmO!?sHM++Vv;v@oRR)Q+=7a!<*ISc;ip0D_IB=cb{QQS{Rx z(R7W}mDr+b-B`xiAz@I)cwa|oO&xgnRcYddtrCNylYFGE%ulH1<^ipFSLbH1$ZEHB zhUIg)=C@ffyNMgZ6|ZcBi|oQRN30=s_J(E%YL958sRbc@64Z-z$@U~AvVPAZd^59Q z5Z6~F+aA33)F*FsDsuwm_}Z*w+@($D`|1RJ!pUdW+tSps5jZ38lRV;S>Dbxog@ci^ zwD6hcN~6n%{KcRgT-aT0T`vywg?pB>8P4b0#6N4qYt29i-wu*WqW0abAmZUVzJ8Z% zc;xWdJW{WoRCMgbjI(5?&p8?cP3}j4SG*N>ove$)3LOt#;XwnzxeM{B^{_LZpC+OW zZLD>}B?fL#7-j}m+u}8e4l>hK7Q^@5Glt8H6jSgsPnBWlRG8*qo+5G!XLNd~wz*c& z^^dvf>U#}1v%51{{RptDj1RA6_N!pZk`sloIK_KynKnt~ky#lY%J$qUzGA*Iqk{qQI)?|CR`<6q)w@7CrZAJM<1)DwAJ5hbS+l?Msn2o z#GSG>Gc#68s|3YoC|NGHIGH_JHrPj9QsXgGdm?z$KGb3p%cxL@rkmAqy@5z$;~>>d z{P=!!*BgIFJoF`18a(wndz*pwL=EGM?6}<#>D&ccudJCgqs4SD1Bq@*k5b%+*j3aL zB@R8W3aU9J2ZX7aEx$-IxSMQhg6xObM37W=akChG8D8AtzyI_MGXkQ#b0@yqlddW-RS=)IGMDJbZs4%&w=9DMZbIXNE_?|_Pfa@C zvKHt_cNAh4UEySLoHGN?{AQZF=L9X87vfIPRC&kwnzLfb!~*hL8huo5@s*at7TjK2 zThUM3IN`AK?QztOZZo(ahj<{0fHOTqjem#eyGU=Qk;VD9A7Gt&&$4 zeK^;i;E2IU`)vEu*!jg-{vu^AC0%Of0Z}+I`SdMO;*`kDx+^9}uPJ=^WcR>q`;|=; zbdxceGY5eHrP^36GXmd;%^%P5=|$D8T80&II!hLugl)pk@A(R3_T9p#4uW5c`0oA) zoQPFWaZI+VqHI{IQzI5eeL?VLB9ulm8&$)(t)44+-9xJJU&bqZ_AtP{aWB7Bl zxs@j@W-Kqdf3{QZc(EWB*5A8~t!%}-1a&CW9NNymrY|0b?lQG%g%(9c@(sLqBH_$4 z9WgK{6xXNw4kWm>dBmUWY{dH~m@~c-l?f`Een_o~JXOQwKa-achI#kEL zb0Q`!<@I}O{FMR+l8b_iH~b44^viqGLlP6syz%2+O((I^)6nK3X~ryxXe5~$b!c=> zaf_j4!{m>k5X#pyGDk@l2Isd&AtBMfhBG*>Tcid>0G z&Odz5zYf1&BjW9ao(7RI%Q9e@;r{e|`J!24{d+Meo(&=iNDsZu?? zGZpWS!-)7!r}z2h>$7JlcQ#FB(A}OeAhQb*J=#t zRX--1L)z0?(oAmzjl}SwzA<~l+?u?%+G5=+aIfKWyRTvRkayKxxF~d-3ed=RJ~JCZ zoD~=Zil>?dWI{HM&E}-s!TV+VvtJ9)BrzS!A$1&1^pI;u0<-V&bVw7zmg+y~Nj^Uq zf@|K#*k{E!6_wH8qOs_u=JsP~>RHl!ibY&xq|@jC07n42`<{;XH`R4s9Dc5Ee>UB_e_AUlI*7`479 ziTbEuVumBT&Xxlni|^(4YQD$wz4y?E%|{Yz*u?WD=3iH?S#HdMac7$tqLEdA5zn(?)Jgp1y#o;m%AvQshRatEt1o_7dE?b9G3eA%@a zS;Qlp&lImQ9?3nF)47sNhvrnp&>GyLRwu%vX!~G}eM-&{qGF;%5Sxw7&S3JGahH7dJ+~?8qE(G-s?A%JnbQVVX6R|Bhk4w%9zDGG;+VW@2jptqzHlq8g1SU>-H?doy4Zl+&ZXC-XZc5M$S;QyDh3|!uhm!Z zJ`f#astR2RevnC5n9P7CuA5U{I<&OTVYv}RKAeQ@DLcg)&i>L8T0x(u{LFaRE3+rA z3{y=HbzLZ4h(ki%1AOiB1uLtLo8f~%>I%{f#3}{NY90cw;I~O6tRYKk=bMZ4^vdI% zRv*;9V9o5eYlS@UD)Q63ztZmFt2;f)ov$8QX~~?xP;@G(xprb4_JJS#Bza+MV#L4D zaOz>>+NA%ZS{5NIU#OzU3Fq0U`(DdB*&C%vrly7U2-ESNc?xFkC z?0JQbr>>RrrN4!H-DcfRuCPttK;`JM)7>;;>!~{LOJRjm+2SwIntTXp(gDi`66dKB z?57z$8S9>O`>5LeOB<=nOY2VhkdPmbwUX?tKH_0EIpM#+@^X>w>X!F9BpwxL>|Saz z$4yCUAU%k{5I68HR)04X;iPoo5YRtFtH3cSP0M}}0WY0$B>0YPU?HWhJ2;<#{NjVo zre{*>vB8ir-A|uR=#feS4vo-TQlH8lc^ga(A>DG9L3Bk!m5x=M2ppSM#TB(r;=+&J zKH#?S==ra2g;VbMs&-zD=>)$%q~CNNAd*6wI`&A7)$REFVANjQC}Xa^?P*_|V(n-} zS>>*1swc_MSEq3rgynJS)IO$D37i%Rrbx8JNQ3rzq>rd$1fXu>dN{)4FG*0oJiYhz z;`X8kI5z!Dbv;x+pX3X=b2~;?&JNtqg1B14zm=VLW=7?l>bsbbG+#g?NHzXMnJ!~g-_phF= zjR$q$y06%Ae3Vu~9Ly#0aHAEweQ9f$-yDvu*j1lqM=85BK3h-*kGmc{cS6?OWR9IW z+xnbg+nnY*_?fEzH1JIkj^Q+~Xwr}q7e9@Q)tKD1i}tN z=iLkr?U+E?r=dSMjP8$dw|{N4rP0Kml{~D7oP^nz=Zzr?MYmWizj*EvB_@)q-_nEs z&YvDJQkC=#($k5LDZ{s!Narr2_JkXXaNtIAI9+J>xnK8`%5{PFf#vrFCCA1==kCj% zJ+ z{W-7LTzjjK%3j(Bqbj!&u$!pZjFTp9kno3_FMZNR(Ppu*s5$MjcNMg=Xtw)J?;GP`8V!B#vgyAguxcgd)jKet66 z-TAn*W0)@gEGpHw)**#JZY8PUi{=wElb^ef?bU}EmEz`IIlSg)XQTYQ72r@~3@96Z zQHBCf1l#&kbek;&#dJ;KYj&|ElT@AI{<8Qtaom}%9)23rr{4-XsJ84nBrUbCC^8P|&!*RdU#`Amj{-Y-xfoawi?gJTgTgrU_S{7lcoNWqc=gKvm8;p%FK2E6` zUvyZEVzS3UX+uBL@l(CaWr9A$=?htfmF`{sT<;>vuCHAW(=oq771h?Q z`6|DA4?~v5$Q`fiNnkN&Lotd>nH$Fz1(#qz-+cP)?Z7f}k33Q{(a4(;hRn9!l+mnl z?Pm;`B2^-hbrd#_83I$-GLU_+g?YscmA21D47VYDR+B1+$6@+pw+4jzvtbt|iX&#NZsY2d?R(6)lU+@=FdhevUXK}a$AsnK^((T1f$k!eCvK#KKS7|>^g3- zZcDkPtJ;&FHZtyYEZKa*dU7R!*S&c?nu&zNWK5@jCK*aEy?n?glgF$=raMRq(aHCr z!3t49On8MU=TeA&6-Pi9gl-@l{}@5;UVw`)1)=vg7T z(u<~}C~nJaknDbaF&$WfQx?6f#bUD9es%A^uYrv*K}RRV9`N6k3&WklTU}+RIfx-!Fbz+1 z8AO}guryC}ODZ}e49B6*Pmv_YjJIprOrE3YI7{}PX%Bg;*F8Xj!6?1ZqY(=cV6VK; zsiCWk7blmHeZi^Jx04->?`R%>Xj~xF!(8*inr~s1bdteNpR4B=WTB}oTV&cUW3@*m zrAvxhLl4?>v41+qQwi@PARZ|kbA>eRY;P=e#iO>qWXa9;bf}XhIu*vO+g{(B>uJNELw56!3NaSTzWOmm5rn8r6gBA*Wpr+ggvx@jQCg+>^lIh?!vcJjg~?QTs>`i+8|6uFAv4>iiQfZa zzeGR4ByI~f%y~^Y7Pa499WLp6LNm%yd-uh~ToAb|erD1`&G$(mG#_0Sx-by%3%a@K zvq>1DoW^P1&plQ(kvTMfFK#(&W_hhv0PPSJ_B%BqO;L(Ys>P$Choa-%3oa`@Z`+I& zxxIYllI&Of=&-^Ju3)w(nP-pv$Z0UajLxC^amX9fxlpH(`y&RvXp}yLl80ZM@fNs; z^TVR}?DJ^ldmpkmM!h({i=~BP)Kxgk;xx<=g#ErzThl%vl@R{i6o(7r3vvP4#aAfz zeH6_B!s!?kUc3a;PaaHuw#J&G?!mib;3p5lF$6e5+cejL(mrZ`^OUr!%YzxN!KL7OPEgxt4skWfkna|tb4 zQGV%tWBM!Zabq*aFf_`+-u^wm9`G8&o(!?5h?BSP?e%f`W$fM!E`~aOMS53^vvPEQ zoc?3VRVT~L5o2#9sq>r9Fr(P79JY6Gk_{K0-QhY#b7@$rrWZc&-&ew7C9BPj5k`52 z$1Q`dgn@frz+Z&#ElSD|);O&#xO+;aZBQdRY8_FQQ2S=RzF@9PqCNger><23DtMP@ zD-$)TUr40dDqOD+Yd*7h$P8_LNE*-N=yT8=ZvkEV42L_usZn}+%$F-J+-Y|zxnSMY zC_FrDCzG92VbKvU+52m4TM(`r+jI?aCR!yLY58kCod_T9*VUK9So@TxzG%QSx(q4g z_^m&VM}M8P{J9XBwawGHoM)63ItI?odp^^WVs`JSC>*`Zcn#4gV>=5+evs)I4)_8`K`K>q$7TrHDV zBz8}e_}7XwpLys+BQRKS^6tG9|KiE%7ixQF21h}?%VfkaPWG&y9=nWRJNF0FHf(Dv zNv%x7#Cfv(m}XfiT((6UEnfa{kE;EG{{sYs&(_%ds11#}?1&QC9Le!+);TaDSCv%N z4ROW_7~FY>2t$-^a?iCnY0q!*>U;f{CzPTtmtPOL)45+Fa@ST{bmkCkMVbW=eL*#C z^U}dP!4m4hgNQoXXvN?vvvlu6Ge{f+s935<^~M5F_^!e}!6Q^PhV6%pk-uMTg_v7LGy54kd`=a_<1J)70_K~Tnq?t79)}vpkXIQ3#h)U%+3_N|+3^7afDj-8 zZUe*s2|x;v0ptJ$KndIdr~qn!2A~D#0D6D{U<8-|X5cQs0k5l{k@0TniIbZ=;0#<-EU<23!c7Q$L z05}3pfHU9%xB_l~JKzC$0$zYO-~;#qetp3Iff}F|r~~SO2A~mW0-AvqpcQBX+JO$B6X*iE8@buxnhrO+ zIoPxRH{AyR{W=n1x#@6Sb^H@P@nr3X0MvqpAURm06EHMA=*~NrHWcJ=SXR11( zoN7@k!MR5md{kePv|OOeTc2H;Szz=nb5wNH-H|gr$DT$%6z&z`B9|tgn%*x)L=>1l zmE(g?PvC86dJZtHb$m3Pk6#hVh*3R6zT-`P|9EMso+u830XeasmJK1t*Y^#!`lWWrw+Ee zCq?Uto=*1ogn@^>uI-bZKJKe?7LQ*$l40-Nf9j=iW_Mv2NYT@@iEYh-k^g2+sL$S9 z;y))-{XSZ%12@pfX#n}3#vX^miD7=3@PM1`0VjRW$qJG<0LDKdls#}-7=IR=?G^E8?o?g=Eq><%H1qNg+4T{TX$<+W{oyuEG;R9NA0e~>eKV7qb(*uj;32hFX*l7z{B*wIJU-JCw}Ds6Sbojs z(Y-6FORad1@`G}p?LYXWd-e0~HHPqcrATMPM%j|r5r_DayC^C5)}&dz!1k2K@cw-4mQXNa2>Uf_BmVTy0!VUz3HzVO{<#QP;BpA^0Ii44f{rL1Vbw4+#Rth^d zb*%AI`sUuVMaVP03YfY0K5xGGnT%8!KcwMoH>H`grZBI^-kLBy**o^z5wIu5}ocf1bUWPjh<6B>uw|tr<-p25l^p{z>xvS*p+ z^!47WYrGHA(4UBoX>L~B&bwk9jrck)h}( z&_I#yI5Zlc01q1nJ_LsXz3bgICfsk?zK)RMMG)4?0`5~lWti1gpyY_tCf_RvAvBNg*EHxVRrKw zOB)917%_B0Yrcen2Z0!VHx|Ju&p;E`XR`hAb{e;^(5`MO7t_}tRmzKe2}F!=n-#6E(;TVC*<|u844Z#mtA-MUGSFV^AF!P zOB`6K=9Bk`NGA3=7-V0#CFK+rrLR*7DH+So435e~!C6pz1{~ggX?%8jU*!1|>XrD; zH)m)&^yrzg1pXyuB7!VY@OnAUyHr2z6^p3cSB=#T1_Yn4sgH+!S63d7AiZa)6A?ar z-5mc>SZyP?B%05X-EGG`Fo(dT=mCttXK7~pXxS_-!;+<%iaS&0p*0(^^ba3{ z4sPfG*LFsA0LL^2vy@=L&L;H;I$OOXDyJmL!&KirAHP}D=$ESt{Dor_X_ux3*a|w= zC=TTDmQK4{hRzSb>p?S?a!O5&YFTmj&V0@BBLx*gXPpKhYC-;)GcJQr3fL|d5i6(1Rll-H~OI$AK;ZM$f z>i--%^-)q>5e-Jk#*6CWUVkTc@%$sn=f_9i!dnGH%x$Pl{2wG&*@mdS+H9qu9Aopa zwK;S@WH@$rDD_xSL6A0CayvM**%UW0nB}`(hStjF6yvBdgg`$QbX-bGSbx8DnMY*k z@p2%}yVHwcJ*>6r_O1w=H%O=sc|USZ37RtLE2NoO(5-+tP%2SaQx=X8rGVAjE#bW$ zdJ(>_y;Ro1OPI(2-P_RiG1ep__dsyFy^8kgm9~o_p8A^UQ^q2@0R&FAQAuAV<`j*lWouzpl3BhZbgoOE50$H)&&axH2G5&JPHxTd zv|T13jBVFlDiSMs4rJ`8qWa{`+FR7k*-V^x+EAe_d%LLEV^H_b&k0n?sCxR+th39Q zT##sO+(p}=kKdcwYP99+1E;hNr7_(O&Ft-PH8s)Df*O_J{> zai_zPDv9t&53aB0ec$pvzwfV&Kb`KW+YGioYHKA~4z6aF&NrD4H7U08MZ*Kn%CdCB z8{%a#8jAM+u+n>XE#`40q`n=M+_H9DAN`7Jah*7ftDB>LqPS7^9QlIoV9NrEDrMBh zd3?9@s_M|ZI&@0w9shjPt$!h$O` z&XNL&eWbXwl<3IM z$b_p(Y}`J56DxWz_-j$Uw#kP9QbQnkApXx(82qOT zc+bel#?i*z$cWR~M@R}3CxSE{HP<)$F87|uRsCT7+aMn<56``_gl zZ&!VFMgoC|2|^$&|H}@66b66zogJ3l1m~vyyDIGe#r~=aAGX&7r@Uz&>}IfpqLV1F zmy;-DY=2j$hhbyWh6#bVgWJeoH-LYp!r*$CQ!||Prid(11Y+4D%()p(0vm6J-=_Ih zJ9pEikDwYDQZ_??(EiB>{uTz4ul@=phEcS@$!fv!%7@89jC zF$q&|gWrbTX@e8pgcggeILm?pC&AYHZ-ImRze4{hu=5;6YghyV5vM`8=|cT!jly7= zvwxPhq76<$`>TQT#>YOpfP42B#Q$^v{tZ>&L4zHNpb^0?+I}}1PCMAjN5VI=X<$n2 zaIBjm`ajuDP0~Ohz72mD(fQ~mmJk-({(JfFWT*~Wql0A^_|kB(aF&}AA@}JR zLT*7ILm=|MJB+Od8qCEDjgsTe@g)Ywl+Ehj4a4YjlZ6Oo)$x04`gef!a4q^h z`%TC1rcfD2Ump#EK(HkdZ~8+2Hbv0l@Ab%DMnn7m4o|#JP+YhX`mfd^guO9AM}sMJ z{%)K9eIx=-T>o8@MAzRLo^`@+-|S9#pd3U8KbTq<*y6bXpqP13%uUkPuOxxkf0Dom z*FQVMwX{|75!ef;-2dq*|GU5r9lt`^4F5iW!CB__^y>i5m_`53DA<2b5`T|_{mHo- z4F8?)*Ql5v)2}-I9t-<7_KQE}KmB_!GDgkw_g4FRQ0(7)m56x%^F7%JvjDn z>?O^uMu#e_}z8n&T$+@6PnUvHTgo%XRC4v);u1-L?J`TNmvA zC-zrI8@AX3$GXYzC*arLU)R!qreBTGp8vaWaOI diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index d2fee19a..9ae420f8 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -5,7 +5,7 @@ DOCKERFILE := Dockerfile.$(IMAGE) GIT_BRANCH := $(shell git branch --show-current) SMLP_COMMIT := $(shell git ls-remote https://github.com/SMLP-Systems/smlp.git refs/heads/$(GIT_BRANCH) | cut -f1) -WHEEL_SRC := /app/smlp/dist/smlp-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl +WHEEL_SRC := /app/smlp/dist/smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl WHEEL_DST := ../dist define copy_out diff --git a/scripts/docker/repair_wheel.py b/scripts/docker/repair_wheel.py index eee53262..de77f505 100644 --- a/scripts/docker/repair_wheel.py +++ b/scripts/docker/repair_wheel.py @@ -51,7 +51,7 @@ def main(): print("Install with: python3 -m pip install auditwheel patchelf") sys.exit(1) - wheels = sorted(dist_dir.glob("smlp-*linux_x86_64.whl"), key=lambda p: p.stat().st_mtime) + wheels = sorted(dist_dir.glob("smlptech-*linux_x86_64.whl"), key=lambda p: p.stat().st_mtime) if not wheels: print(f"ERROR: No linux_x86_64 wheel found in {dist_dir}/") sys.exit(1) diff --git a/scripts/venv/run_dora b/scripts/venv/run_dora index 19d91ffe..94cd42b1 100755 --- a/scripts/venv/run_dora +++ b/scripts/venv/run_dora @@ -20,7 +20,7 @@ mathsat_src_dir=scripts/docker/external $(dirname $mathsat_src_dir)/run_mathsat_build \rm -rf $mathsat_dest_dir > /dev/null \mv $mathsat_src_dir $mathsat_dest_dir -pip install ../../../dist/smlp-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl +pip install ../../../dist/smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl sed -i.bak "s@../../src/run_smlp.py@smlp@" regr_smlp/code/smlp_regr.py cd regr_smlp/code smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots t -seed 10 -log_time f From d13ed8cd30dc34e503c192f6ddfb0e4d40c48c86 Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Wed, 25 Mar 2026 14:35:39 +0200 Subject: [PATCH 03/11] Updated expected results after merge with master --- .../run_smlp_regression_venv_expected.log | 961 +++++++++--------- ...p_regression_venv_expected_diff_report.log | 100 +- 2 files changed, 576 insertions(+), 485 deletions(-) diff --git a/tests/smlp_regression/run_smlp_regression_venv_expected.log b/tests/smlp_regression/run_smlp_regression_venv_expected.log index 5f1483d3..119215dd 100644 --- a/tests/smlp_regression/run_smlp_regression_venv_expected.log +++ b/tests/smlp_regression/run_smlp_regression_venv_expected.log @@ -6,792 +6,792 @@ Initiating 3 worker... Initiating 4 worker... Initiating 5 worker... Initiating 6 worker... -Running test 3 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric response in training/test data only -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test3 -mode predict -resp y1 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_unlabeled.csv" +Running test 6 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test6 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 10 test type: prediction, description: basic et_sklearn prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test10 -mode predict -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 15 -et_sklearn_bootstrap f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test10 -mode predict -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 15 -et_sklearn_bootstrap f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 16 test type: prediction, description: basic nn_keras prediction test from saved model on new data with numeric labels and two responses -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/Test8_smlp_toy_num_resp_mult" -out_dir ./ -pref Test16 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 18 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test18 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test19_model -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 24 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test24_model" -out_dir ./ -pref Test24 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 26 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +smlp -model_name "../models/test26_model" -out_dir ./ -pref Test26 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 30 test type: subgroups, description: basic test for subgroup discovery for numric responses -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test30 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -feat x,p1,p2 -plots t -seed 10 -log_time f - -Running test 37 test type: doe, description: doe test with four levels with box_behnken -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_three_levels_real_nan.csv" -out_dir ./ -pref Test37 -mode doe -doe_algo box_behnken -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test30 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -feat x,p1,p2 -plots t -seed 10 -log_time f -Running test 45 test type: doe, description: doe test with four levels with fractional_factorial -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels_real.csv" -out_dir ./ -pref Test45 -mode doe -doe_algo fractional_factorial -doe_resolution 5 -log_time f +Running test 40 test type: doe, description: doe test with four levels with latin_hypercube_space_filling +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test40 -mode doe -doe_algo latin_hypercube_sf -doe_samples 20 -log_time f -Running test 53 test type: discretization, description: tests discretization options -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test53 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 48 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test48 -mode discretize -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_num_resp_mult_y1_verify.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 64 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test63_model" -out_dir ./ -pref Test64 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 58 test type: optimize, description: basic dt_sklearn optimization test with numeric labels and integer grid as domain and without scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test58 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 77 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test76_model" -out_dir ./ -pref Test77 -config ../models/test76_model_rerun_model_config.json +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 70 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +smlp -model_name "../models/test69_model" -out_dir ./ -pref Test70 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs Running test 85 test type: optimize, description: tests alpha and eta constraints specified in command line in optimization mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test85 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test85 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_verify_vacuous.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 92 test type: verify, description: test to detect contradictory constraints in verification mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test92 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_verify_vacuous.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 93 test type: optsyn, description: basic test for mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test93 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_witness.spec specs_path ../specs -Running test 99 test type: optimize, description: testing that the response and feature names can be taken from spec file in model exploration modes when the responses and/or features are not specified in the command line -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test99 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 101 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test101 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test101_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_certify_witness.spec +spec_fn smlp_toy_num_resp_mult_beta_verify.spec specs_path ../specs -Running test 103 test type: certify, description: -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test103 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test103_model -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_certify_witness.spec -quer_names valid_candidate,grid_conflict,range_conflict -quer_exprs "True;True;True" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 107 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test107 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_beta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_beta_verify.spec -specs_path ../specsRunning test 5 test type: prediction, description: basic dt_caret prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test5 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model t -use_model f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 111 test type: unknown, description: smlp toy basic test to rerun saved model using the model rerun config file saved during model training +smlp -model_name "../models/test110_model" -out_dir ./ -pref Test111 -config ../models/test110_model_rerun_model_config.json -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" -Running test 19 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test19_model" -out_dir ./ -pref Test19 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_system_stable_constant_certify.spec +specs_path ../specs +Running test 117 test type: certify, description: certification test with knobs only where assertion is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test117 -mode certify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 25 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test25 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test26_model -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 123 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test123 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 3 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric response in training/test data only +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test3 -mode predict -resp y1 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_unlabeled.csv" -Running test 36 test type: doe, description: doe test with four levels with sukharev_grid -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test36 -mode doe -doe_algo sukharev_grid -doe_samples 125 -log_time f +Running test 9 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test9 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -model_name test20_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -save_config t -save_model_config t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 44 test type: doe, description: doe test with four levels with uniform_random_matrix -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test44 -mode doe -doe_algo uniform_random_matrix -doe_samples 20 -log_time f +Running test 16 test type: prediction, description: basic nn_keras prediction test from saved model on new data with numeric labels and two responses +smlp -model_name "../models/Test8_smlp_toy_num_resp_mult" -out_dir ./ -pref Test16 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 52 test type: discretization, description: tests discretization options -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test52 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 27 test type: prediction, description: checks nn_keras prediction with nn_keras_seq_api t +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test27 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_y1_verify.spec -specs_path ../specs -Running test 63 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test63 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test63_model -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 34 test type: doe, description: doe test with four levels with full_factorial method +smlp -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test34 -mode doe -doe_algo full_factorial -log_time f -spec_fn smlp_toy_num_resp_noknobs_verify.spec +Running test 42 test type: doe, description: doe test with four levels with maximin_reconstruction +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test42 -mode doe -doe_algo maximin_reconstruction -doe_samples 20 -log_time f + +Running test 50 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test50 -mode discretize -resp "PF,PF1" -discr_algo kmeans -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 72 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test71_model" -out_dir ./ -pref Test72 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 60 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test60 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat spec_fn smlp_toy_num_resp_mult_free_inps.spec specs_path ../specs -Running test 83 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with numeric labels and integer grid as domain and with scaling objectives -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test83 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -beta "y1>7 and y2>6" -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1/2-y2;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 82 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test82 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_optsyn_vacuous.spec specs_path ../specs -Running test 93 test type: optsyn, description: basic test for mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test93 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 90 test type: optsyn, description: test to detect contradictory constraints in optsyn mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test90 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 99 test type: optimize, description: testing that the response and feature names can be taken from spec file in model exploration modes when the responses and/or features are not specified in the command line +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test99 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_witness.spec specs_path ../specs -Running test 101 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test101 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test101_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 102 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation +smlp -model_name "../models/test101_model" -out_dir ./ -pref Test102 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_unsat_eta_verify.spec +spec_fn smlp_toy_num_resp_mult_stable_verify.spec specs_path ../specs -Running test 106 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test106 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_unsat_eta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 105 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test105 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_stable_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_synthesize.spec +Running test 112 test type: prediction, description: smlp toy basic test from SMLP manual +smlp -model_name "../models/test110_model" -out_dir ./ -pref Test112 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -use_model t -save_model f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" + +spec_fn smlp_toy_system_stable_constant_verify.spec specs_path ../specs -Running test 108 test type: synthesize, description: basic test for dt_sklearn in model exploration mode synthesize where synthesis succeeds -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test108 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 6 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test6 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 118 test type: verify, description: verification test with knobs only where assertion is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test118 -mode verify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 9 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test9 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -model_name test20_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -save_config t -save_model_config t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_system_stable_constant_synth_fail.spec +specs_path ../specs +Running test 124 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is not feasible because while beta constraint is feasible the assertion is not feasible therefore optimization is not performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test124 -mode optsyn -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 2 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test2 -mode predict -resp y1 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 17 test type: prediction, description: basic poly_sklearn prediction test from saved model on new data with numeric labels and two responses -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/Test11_smlp_toy_num_resp_mult" -out_dir ./ -pref Test17 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 12 test type: train, description: EV-SI real life dt_sklearn predict test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test12 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 23 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test23 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test24_model -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 20 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +smlp -model_name "../models/test20_model" -out_dir ./ -pref Test20 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 29 test type: subgroups, description: basic test for subgroup discovery for pass-fail responses -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_cls_metasymbol_colnames_mult.csv" -out_dir ./ -pref Test29 -mode subgroups -psg_dim 3 -psg_top 10 -resp "PF 1,PF#" -plots t -seed 10 -log_time f +Running test 28 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and functional API +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test28 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 39 test type: doe, description: doe test with four levels with latin_hypercube -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test39 -mode doe -doe_algo latin_hypercube -doe_prob_distr Exponential -doe_samples 30 -log_time f +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test39 -mode doe -doe_algo latin_hypercube -doe_prob_distr Exponential -doe_samples 30 -log_time f -Running test 47 test type: prediction, description: tests options -pos_val and -neg_val when re-using saved model -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test47_model" -out_dir ./ -pref Test47 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" +Running test 45 test type: doe, description: doe test with four levels with fractional_factorial +smlp -doe_spec "../grids/doe_two_levels_real.csv" -out_dir ./ -pref Test45 -mode doe -doe_algo fractional_factorial -doe_resolution 5 -log_time f -Running test 54 test type: discretization, description: tests discretization options -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test54 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 53 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test53 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult_y1_verify.spec +specs_path ../specs +Running test 63 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test63 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test63_model -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 66 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test65_model" -out_dir ./ -pref Test66 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 72 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +smlp -model_name "../models/test71_model" -out_dir ./ -pref Test72 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_mult_free_inps.spec specs_path ../specs -Running test 79 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test79 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 83 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with numeric labels and integer grid as domain and with scaling objectives +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test83 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -beta "y1>7 and y2>6" -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1/2-y2;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_query_vacuous.spec +spec_fn smlp_toy_num_resp_mult_verify_vacuous.spec specs_path ../specs -Running test 91 test type: query, description: test to detect contradictory constraints in optimization mode due to contradictory alpha global and alpha bounds constraints on FMAX_xyx -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test91 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 92 test type: verify, description: test to detect contradictory constraints in verification mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test92 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_verify_vacuous.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs Running test 100 test type: optimize, description: basic test for sat_threshold option enabing usage of objectve values in SAT assignments that prove optimization thresholds -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test100 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test100 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_stable_verify.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 105 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test105 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_stable_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 104 test type: verify, description: assertion verfication test with wrong spec that does not assign a single value using a singleton grid or range with equal max and min +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test104 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_cannot_synthesize.specRunning test 8 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and two responses -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test8 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -nn_keras_epochs 20 -nn_keras_seq_api f -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_cannot_synthesize.spec +specs_path ../specs +Running test 109 test type: synthesize, description: basic test for mode synthesize where synthesis fails +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test109 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_cannot_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 14 test type: train, description: EV-SI real life poly_sklearn prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test14 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_system_stable_constant_synth_fail.spec +specs_path ../specs +Running test 122 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test122 -mode optimize -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 21 test type: prediction, description: test for illegal symbols in column names -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_metasymbol_mult_reg.csv" -out_dir ./ -pref Test21 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model t -use_model f -model_name test22_model -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" +spec_fn smlp_toy_system_stable_verify.spec +specs_path ../specs +Running test 140 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test140 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -trace_prec 1 -trace_anonym t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 5 test type: prediction, description: basic dt_caret prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test5 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model t -use_model f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 31 test type: subgroups, description: testing resp2b in subgroup discovery mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test31 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -resp2b "y1<6;y2>6" -feat x,p1,p2 -plots t -seed 10 -log_time f -save_config t +Running test 15 test type: prediction, description: basic dt_caret prediction test from saved model on new data with numeric labels +smlp -model_name "../models/Test5_smlp_toy_num_resp_mult" -out_dir ./ -pref Test15 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 23 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test23 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test24_model -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 32 test type: unknown, description: test reusing saved model by using configuration file +smlp -model_name "../models/test20_model" -out_dir ./ -pref Test32 -config ../models/test20_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 38 test type: doe, description: doe test with four levels with box_wilson -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test38 -mode doe -doe_algo box_wilson -doe_cc_face ccc -doe_cc_alpha r -doe_cc_center 2,3 -log_time f +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test38 -mode doe -doe_algo box_wilson -doe_cc_face ccc -doe_cc_alpha r -doe_cc_center 2,3 -log_time f Running test 46 test type: prediction, description: tests options -pos_val and -neg_val -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_pf_mult.csv" -out_dir ./ -pref Test46 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model t -save_model_config f -use_model f -model_name test47_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" +smlp -data "../data/smlp_toy_pf_mult.csv" -out_dir ./ -pref Test46 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model t -save_model_config f -use_model f -model_name test47_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" -Running test 56 test type: discretization, description: tests discretization options -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test56 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels f -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 55 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test55 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs Running test 68 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test67_model" -out_dir ./ -pref Test68 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -model_name "../models/test67_model" -out_dir ./ -pref Test68 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs Running test 80 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test80 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test80 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_alpha_asrt_verify.spec specs_path ../specs -Running test 88 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with beta and objectives specified in spec file -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test88 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 87 test type: verify, description: tests global alpha constraints and assertions specified in spec file +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test87 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_alpha_asrt_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 94 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test94 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 96 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test96 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_witness.spec +spec_fn smlp_toy_basic.spec specs_path ../specs -Running test 102 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test101_model" -out_dir ./ -pref Test102 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 114 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual without specifying resp and feat in command line +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test114 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model f -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs -Running test 104 test type: verify, description: assertion verfication test with wrong spec that does not assign a single value using a singleton grid or range with equal max and min -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test104 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 120 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test120 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 110 test type: prediction, description: smlp toy basic example for predict mode from SMLP user manual -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test110 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model t -model_name test110_model -save_model_config t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" -Running test 7 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test7 -mode predict -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_witness_certify.spec +specs_path ../specs +Running test 128 test type: certify, description: Basic regression test in certify mode covering all four possible outcomes when certifying a witness for a query: the witness is stable +smlp -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test128 -mode certify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_witness_certify.spec -quer_names query_stable_witness,query_grid_conflict,query_unstable_witness,query_infeasible_witness,query_poly_intercept_sensitive -quer_exprs "y2<=90;y1>=9;y1>=(-13);y1>9;y1>=(-10)" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 11 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test11 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 149 test type: prediction, description: tests the mae loss function MeanAbsoluteError and sample weoghts +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test149 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mae -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 15 test type: prediction, description: basic dt_caret prediction test from saved model on new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/Test5_smlp_toy_num_resp_mult" -out_dir ./ -pref Test15 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_verify.spec +specs_path ../specs +Running test 156 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training; adapts test 154 by consdering multiple responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test156 -mode verify -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse +Running test 4 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and one response +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test4 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nn_keras_weights_precision 2 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 13 test type: train, description: EV-SI real life nn_keras prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test13 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f Running test 22 test type: prediction, description: test for illegal symbols in column names -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test22_model" -out_dir ./ -pref Test22 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model f -use_model t -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" +smlp -model_name "../models/test22_model" -out_dir ./ -pref Test22 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model f -use_model t -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" Running test 33 test type: unknown, description: testing -config option with subgroups mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test33 -config ../models/Test31_smlp_toy_num_resp_mult_args_config.json +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test33 -config ../models/Test31_smlp_toy_num_resp_mult_args_config.json Running test 41 test type: doe, description: doe test with four levels with random_k_means -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test41 -mode doe -doe_algo random_k_means -doe_samples 20 -log_time f +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test41 -mode doe -doe_algo random_k_means -doe_samples 20 -log_time f Running test 49 test type: discretization, description: tests discretization options -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test49 -mode discretize -resp "PF,PF1" -discr_algo quantile -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test49 -mode discretize -resp "PF,PF1" -discr_algo quantile -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_num_resp_mult.spec -specs_path ../specs -Running test 58 test type: optimize, description: basic dt_sklearn optimization test with numeric labels and integer grid as domain and without scaling objectives -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test58 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 56 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test56 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels f -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 70 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test69_model" -out_dir ./ -pref Test70 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 69 test type: verify, description: nn_keras verification test with model_per_response training +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test69 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test69_model -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 86 test type: optimize, description: tests alpha -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test86 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 88 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with beta and objectives specified in spec file +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test88 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 95 test type: optsyn, description: basic test for dt_caret in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test95 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 112 test type: prediction, description: smlp toy basic test from SMLP manual -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test112 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -use_model t -save_model f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" - -spec_fn smlp_toy_system_stable_constant_verify.spec -specs_path ../specs -Running test 118 test type: verify, description: verification test with knobs only where assertion is valid without stability and fails with stability -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test118 -mode verify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec -specs_path ../specs -Running test 123 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test123 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 2 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test2 -mode predict -resp y1 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 12 test type: train, description: EV-SI real life dt_sklearn predict test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test12 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 18 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test18 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test19_model -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 27 test type: prediction, description: checks nn_keras prediction with nn_keras_seq_api t -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test27 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 34 test type: doe, description: doe test with four levels with full_factorial method -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test34 -mode doe -doe_algo full_factorial -log_time f - -Running test 43 test type: doe, description: doe test with four levels with halton_sequence -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test43 -mode doe -doe_algo halton_sequence -doe_samples 20 -log_time f - -Running test 51 test type: discretization, description: tests discretization options -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test51 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 94 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test94 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_mult_certify_witness.spec specs_path ../specs -Running test 59 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test59 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 103 test type: certify, description: +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test103 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test103_model -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_certify_witness.spec -quer_names valid_candidate,grid_conflict,range_conflict -quer_exprs "True;True;True" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps.spec -specs_path ../specs -Running test 82 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test82 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 110 test type: prediction, description: smlp toy basic example for predict mode from SMLP user manual +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test110 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model t -model_name test110_model -save_model_config t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" -spec_fn smlp_toy_num_resp_mult_query.spec +spec_fn smlp_toy_system.spec specs_path ../specs -Running test 89 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test89 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 116 test type: certify, description: basic test in certify mode when system is specified and is used as the model; p2 rel-rad needs to be 0 or very close to it the witness to first query to be stable +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test116 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 98 test type: optsyn, description: basic test for et_caret in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test98 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 125 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test125 -mode optsyn -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_basic.spec +spec_fn smlp_toy_system.spec specs_path ../specs -Running test 113 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test113 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model t -model_name test113_model -save_model_config t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec +Running test 146 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -out_dir ./ -pref Test146 -mode optimize -pareto t -opt_strategy lazy -model poly_sklearn -resp y1,y2 -feat p1,p2,x1,x2 -save_model t -use_model f -mrmr_pred 0 -model_per_response t -split 1 -spec ../specs/smlp_toy_system.spec -doe_spec ../grids/explore_doe_two_levels.csv -doe_algo latin_hypercube -epsilon 0.99999999 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_query.spec -specs_path ../specs -Running test 119 test type: query, description: query test with knobs only where query is satisfiable without stability and fails with stability -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test119 -mode query -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 152 test type: prediction, description: tests the huber loss function Huber and sample weights +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test152 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss huber -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_system_stable_verify.spec -specs_path ../specs -Running test 126 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test126 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 159 test type: prediction, description: tests the msle loss function and sample weights with model_per_response t +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test159 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -model_per_response t -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae,cosine -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 1 test type: train, description: basic dt_caret training and test on labeled data with single numeric response -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -Running test 20 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test20 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 19 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +smlp -model_name "../models/test19_model" -out_dir ./ -pref Test19 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 28 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and functional API -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test28 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 24 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +smlp -model_name "../models/test24_model" -out_dir ./ -pref Test24 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 31 test type: subgroups, description: testing resp2b in subgroup discovery mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test31 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -resp2b "y1<6;y2>6" -feat x,p1,p2 -plots t -seed 10 -log_time f -save_config t Running test 35 test type: doe, description: doe test with four levels with plackett_burman -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test35 -mode doe -doe_algo plackett_burman -log_time f +smlp -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test35 -mode doe -doe_algo plackett_burman -log_time f -Running test 42 test type: doe, description: doe test with four levels with maximin_reconstruction -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test42 -mode doe -doe_algo maximin_reconstruction -doe_samples 20 -log_time f +Running test 43 test type: doe, description: doe test with four levels with halton_sequence +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test43 -mode doe -doe_algo halton_sequence -doe_samples 20 -log_time f -Running test 50 test type: discretization, description: tests discretization options -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test50 -mode discretize -resp "PF,PF1" -discr_algo kmeans -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 51 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test51 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 60 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test60 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 59 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test59 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat spec_fn smlp_toy_num_resp_mult_free_inps.spec specs_path ../specs Running test 81 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test81 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test81 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_optsyn_vacuous.spec +spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs -Running test 90 test type: optsyn, description: test to detect contradictory constraints in optsyn mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test90 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 89 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test89 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs Running test 97 test type: query, description: basic test for rf_sklearn in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test97 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 111 test type: unknown, description: smlp toy basic test to rerun saved model using the model rerun config file saved during model training -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test111 -config ../models/test110_model_rerun_model_config.json -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test97 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_certify.spec +spec_fn smlp_toy_num_resp_mult_synthesize.spec specs_path ../specs -Running test 117 test type: certify, description: certification test with knobs only where assertion is valid without stability and fails with stability -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test117 -mode certify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 108 test type: synthesize, description: basic test for dt_sklearn in model exploration mode synthesize where synthesis succeeds +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test108 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 124 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is not feasible because while beta constraint is feasible the assertion is not feasible therefore optimization is not performed -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test124 -mode optsyn -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 121 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is feasible +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test121 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_configuration_verify.spec specs_path ../specs -Running test 141 test type: optimize, description: basic test for compress_rules option for dt_sklearn in optimization mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test141 -mode optimize -opt_strategy lazy -pareto f -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 4 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and one response -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test4 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nn_keras_weights_precision 2 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 129 test type: verify, description: verification example with demonstrating all basic result scenarious for assertions +smlp -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test129 -mode verify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_configuration_verify.spec -asrt_names assert_stable_config,assert_grid_conflict,assert_unstable_config,assert_infeasible -asrt_exprs "y2<=90;y1>=9;y1>=(-10);y1>20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 13 test type: train, description: EV-SI real life nn_keras prediction test on labeled and new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test13 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 150 test type: prediction, description: tests the mape loss function MeanAbsolutePercentageError and sample weights +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test150 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 26 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test26_model" -out_dir ./ -pref Test26 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 158 test type: prediction, description: tests the mape loss function and sample weights with model_per_response t +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test158 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -model_per_response t -sw_coef 8 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 32 test type: unknown, description: test reusing saved model by using configuration file -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test32 -config ../models/test20_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 166 test type: optsyn, description: basic flat tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test166 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 8 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and two responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test8 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -nn_keras_epochs 20 -nn_keras_seq_api f -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 40 test type: doe, description: doe test with four levels with latin_hypercube_space_filling -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test40 -mode doe -doe_algo latin_hypercube_sf -doe_samples 20 -log_time f +Running test 14 test type: train, description: EV-SI real life poly_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test14 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 48 test type: discretization, description: tests discretization options -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test48 -mode discretize -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 21 test type: prediction, description: test for illegal symbols in column names +smlp -data "../data/smlp_toy_num_metasymbol_mult_reg.csv" -out_dir ./ -pref Test21 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model t -use_model f -model_name test22_model -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" -Running test 55 test type: discretization, description: tests discretization options -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test55 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 29 test type: subgroups, description: basic test for subgroup discovery for pass-fail responses +smlp -data "../data/smlp_toy_cls_metasymbol_colnames_mult.csv" -out_dir ./ -pref Test29 -mode subgroups -psg_dim 3 -psg_top 10 -resp "PF 1,PF#" -plots t -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec -specs_path ../specs -Running test 69 test type: verify, description: nn_keras verification test with model_per_response training -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test69 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test69_model -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 36 test type: doe, description: doe test with four levels with sukharev_grid +smlp -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test36 -mode doe -doe_algo sukharev_grid -doe_samples 125 -log_time f -spec_fn smlp_toy_num_resp_mult_alpha_asrt_verify.spec -specs_path ../specs -Running test 87 test type: verify, description: tests global alpha constraints and assertions specified in spec file -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test87 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_alpha_asrt_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 44 test type: doe, description: doe test with four levels with uniform_random_matrix +smlp -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test44 -mode doe -doe_algo uniform_random_matrix -doe_samples 20 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +Running test 52 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test52 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult_y1_verify.spec specs_path ../specs -Running test 96 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test96 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 64 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +smlp -model_name "../models/test63_model" -out_dir ./ -pref Test64 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system.spec +Running test 77 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file +smlp -model_name "../models/test76_model" -out_dir ./ -pref Test77 -config ../models/test76_model_rerun_model_config.json + +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 116 test type: certify, description: basic test in certify mode when system is specified and is used as the model; p2 rel-rad needs to be 0 or very close to it the witness to first query to be stable -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test116 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 86 test type: optimize, description: tests alpha +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test86 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 125 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is feasible and optimization is performed -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test125 -mode optsyn -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 95 test type: optsyn, description: basic test for dt_caret in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test95 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system.spec +spec_fn smlp_toy_num_resp_mult_unsat_eta_verify.spec specs_path ../specs -Running test 146 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -out_dir ./ -pref Test146 -mode optimize -pareto t -opt_strategy lazy -model poly_sklearn -resp y1,y2 -feat p1,p2,x1,x2 -save_model t -use_model f -mrmr_pred 0 -model_per_response t -split 1 -spec ../specs/smlp_toy_system.spec -doe_spec ../grids/explore_doe_two_levels.csv -doe_algo latin_hypercube -epsilon 0.99999999 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 106 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test106 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_unsat_eta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 152 test type: prediction, description: tests the huber loss function Huber and sample weights -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test152 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss huber -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_basic.spec +specs_path ../specs +Running test 113 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test113 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model t -model_name test113_model -save_model_config t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec -spec_fn smlp_toy_system.spec +spec_fn smlp_toy_system_stable_constant_query.spec specs_path ../specs -Running test 115 test type: certify, description: basic test in certify mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test115 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 119 test type: query, description: query test with knobs only where query is satisfiable without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test119 -mode query -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system_stable_certify.spec specs_path ../specs Running test 127 test type: certify, description: certification example with knobs only and fictitious inputs with values fixed through their ranges -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test127 -mode certify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test127 -mode certify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs Running test 143 test type: query, description: basic test for compress_rules for et_sklearn in mode query -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test143 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 159 test type: prediction, description: tests the msle loss function and sample weights with model_per_response t -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test159 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -model_per_response t -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae,cosine -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test143 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec -specs_path ../specs -Running test 164 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test164 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 160 test type: prediction, description: tests nn keras tuner bayesian +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test160 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -nn_keras_metrics msle -nn_keras_tuner bayesian -nn_keras_layers_grid "2,3" -nn_keras_losses_grid "mse,mae,huber" -model_per_response f -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs -Running test 168 test type: optimize, description: basic test for rf_caret with flat tree_encoding and modelper_response in model exploration mode optimize -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test168 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding flat -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_y2_verify.spec -specs_path ../specs -Running test 173 test type: verify, description: basic test for nn_keras flat encoding for sequential api -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test173 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -solver_path /nfs/iil/proj/dt/eva/smlp/external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" - spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 178 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features are not scaled adapts test 177 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test178 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 170 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response f in model exploration mode optimize +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test170 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 180 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features and responses are not scaled adapts test 175 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test180 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_feat f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 174 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test174 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 182 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test182 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path /nfs/iil/proj/dt/eva/smlp/external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +Running test 183 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test183 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 187 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization adapts test 164 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test187 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 191 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response t in model exploration mode optimize adapts test 169 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test191 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap t -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 120 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test120 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 195 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 192 by setting n_estimators 3 and then discrepancy between z3 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test195 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_witness_certify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 128 test type: certify, description: Basic regression test in certify mode covering all four possible outcomes when certifying a witness for a query: the witness is stable -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test128 -mode certify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_witness_certify.spec -quer_names query_stable_witness,query_grid_conflict,query_unstable_witness,query_infeasible_witness,query_poly_intercept_sensitive -quer_exprs "y2<=90;y1>=9;y1>=(-13);y1>9;y1>=(-10)" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 148 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test148 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 199 test type: optimize, description: test to demonstrate that in pareto optimization and optsyn modes with multiple objectives when beta constraints are not present SMLP results are not consistent when different solvers are used; this is due to fact that when a subset of objectoves are exemined in pareto algo +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test199 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_verify.spec -specs_path ../specs -Running test 157 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training; adapts test 155 by consdering multiple responses -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test157 -mode verify -resp y1,y2 -feat x,p1,p2 --model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse,logcosh +Running test 218 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test218 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs -Running test 166 test type: optsyn, description: basic flat tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test166 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 223 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test223 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs -Running test 169 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response t in model exploration mode optimize -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test169 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 177 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test177 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test177 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 191 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response t in model exploration mode optimize adapts test 169 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test191 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap t -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 189 test type: optsyn, description: basic branched tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn adapts test 166 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test189 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 196 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 and test 181 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test196 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 193 test type: optimize, description: basic test for et_caret with branched tree_encoding in model exploration mode optimize adapts test 171 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test193 -mode optimize -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_no_input_beta.spec specs_path ../specs -Running test 200 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test200 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 201 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and there are beta constraints +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test201 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input_beta.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_no_input.spec specs_path ../specs -Running test 109 test type: synthesize, description: basic test for mode synthesize where synthesis fails -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test109 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_cannot_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 202 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and no beta constraints +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test202 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 121 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is feasible -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test121 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 206 test type: optsyn, description: optimized synthesis test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 125 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test206 -mode optsyn -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_configuration_verify.spec -specs_path ../specs -Running test 129 test type: verify, description: verification example with demonstrating all basic result scenarious for assertions -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test129 -mode verify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_configuration_verify.spec -asrt_names assert_stable_config,assert_grid_conflict,assert_unstable_config,assert_infeasible -asrt_exprs "y2<=90;y1>=9;y1>=(-10);y1>20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 219 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test219 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type integer -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 149 test type: prediction, description: tests the mae loss function MeanAbsoluteError and sample weoghts -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test149 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mae -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 224 test type: correlate, description: basic test for correlate mode and tests the Shannon mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test224 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_verify.spec -specs_path ../specs -Running test 156 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training; adapts test 154 by consdering multiple responses -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test156 -mode verify -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 167 test type: optsyn, description: basic flat tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test167 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test167 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 171 test type: optimize, description: basic test for et_caret with flat tree_encoding in model exploration mode optimize -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test171 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test171 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 176 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api f for nn_keras in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test176 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 175 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test175 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 190 test type: optimize, description: basic test for rf_caret with branched tree_encoding and modelper_response in model exploration mode optimize adapts test 168 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test190 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding branched -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 178 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features are not scaled adapts test 177 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test178 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 198 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 and test 183 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test198 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +Running test 182 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test182 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 206 test type: optsyn, description: optimized synthesis test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 125 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test206 -mode optsyn -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 192 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test192 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs -Running test 142 test type: optsyn, description: basic test for compress_rules option for rf_sklearn in optsin mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test142 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 203 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed adapts test 122 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test203 -mode optimize -pareto f -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 153 test type: prediction, description: tests the logcosh loss function LogCosh and sample weights -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test153 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss logcosh -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 215 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test215 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 161 test type: prediction, description: tests nn keras tuner bayesian -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test161 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -nn_keras_metrics mape,logcosh -nn_keras_tuner random -nn_keras_lrates_grid "0.01,0.001" -nn_keras_batches_grid "32,64" -model_per_response f -sw_coef 4 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 220 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test220 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_num_resp_mult_optsyn.spec +Running test 226 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test226 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f + + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 170 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response f in model exploration mode optimize -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test170 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 145 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +smlp -out_dir ./ -pref Test145 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +Running test 151 test type: prediction, description: tests msle loss function MeanSquaredLogarithmicError and and sample weoghts +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test151 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss msle -sw_coef 3 -sw_exp 10 -sw_int 0 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_verify.spec specs_path ../specs -Running test 175 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test175 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 157 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training; adapts test 155 by consdering multiple responses +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test157 -mode verify -resp y1,y2 -feat x,p1,p2 --model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse,logcosh spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 183 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test183 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 164 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test164 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 189 test type: optsyn, description: basic branched tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn adapts test 166 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test189 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 168 test type: optimize, description: basic test for rf_caret with flat tree_encoding and modelper_response in model exploration mode optimize +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test168 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding flat -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 173 test type: verify, description: basic test for nn_keras flat encoding for sequential api +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test173 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 187 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization adapts test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test187 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 193 test type: optimize, description: basic test for et_caret with branched tree_encoding in model exploration mode optimize adapts test 171 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test193 -mode optimize -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 194 test type: optsyn, description: basic branched tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test194 -mode optsyn -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_no_input.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 202 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and no beta constraints -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test202 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 198 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 and test 183 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test198 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs -Running test 203 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed adapts test 122 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test203 -mode optimize -pareto f -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 204 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 123 +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test204 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 215 test type: correlate, description: basic test for correlate mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test215 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 216 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test216 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -Running test 150 test type: prediction, description: tests the mape loss function MeanAbsolutePercentageError and sample weights -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test150 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 221 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test221 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 225 test type: correlate, description: basic test for correlate mode and tests the adjusted mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test225 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -Running test 158 test type: prediction, description: tests the mape loss function and sample weights with model_per_response t -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test158 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -model_per_response t -sw_coef 8 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs Running test 165 test type: optsyn, description: basic flat tree encoding test for dt_caretin model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test165 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding flat -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test165 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding flat -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs Running test 172 test type: verify, description: basic test for nn_keras flat encoding for functional api -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test172 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -solver_path /nfs/iil/proj/dt/eva/smlp/external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test172 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 174 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test174 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 180 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features and responses are not scaled adapts test 175 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test180 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_feat f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 195 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 192 by setting n_estimators 3 and then discrepancy between z3 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test195 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 190 test type: optimize, description: basic test for rf_caret with branched tree_encoding and modelper_response in model exploration mode optimize adapts test 168 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test190 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding branched -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 199 test type: optimize, description: test to demonstrate that in pareto optimization and optsyn modes with multiple objectives when beta constraints are not present SMLP results are not consistent when different solvers are used; this is due to fact that when a subset of objectoves are exemined in pareto algo -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test199 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 197 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 and test 182 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test197 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" -Running test 219 test type: correlate, description: basic test for correlate mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test219 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type integer -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 205 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 145 +smlp -out_dir ./ -pref Test205 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 223 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test223 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 217 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test217 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 222 test type: correlate, description: basic test for correlate mode +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test222 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 218 test type: correlate, description: basic test for correlate mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test218 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 227 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test227 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -Running test 224 test type: correlate, description: basic test for correlate mode and tests the Shannon mutual information -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test224 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 147 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test147 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 160 test type: prediction, description: tests nn keras tuner bayesian -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test160 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -nn_keras_metrics msle -nn_keras_tuner bayesian -nn_keras_layers_grid "2,3" -nn_keras_losses_grid "mse,mae,huber" -model_per_response f -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 153 test type: prediction, description: tests the logcosh loss function LogCosh and sample weights +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test153 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss logcosh -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs -Running test 179 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn when resposes are not scaled adapts test 174 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test179 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 161 test type: prediction, description: tests nn keras tuner bayesian +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test161 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -nn_keras_metrics mape,logcosh -nn_keras_tuner random -nn_keras_lrates_grid "0.01,0.001" -nn_keras_batches_grid "32,64" -model_per_response f -sw_coef 4 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 181 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test181 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 169 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response t in model exploration mode optimize +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test169 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 188 test type: optsyn, description: basic branched tree encoding test for dt_caretin model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test188 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding branched -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 176 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api f for nn_keras in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test176 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 194 test type: optsyn, description: basic branched tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test194 -mode optsyn -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 179 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn when resposes are not scaled adapts test 174 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test179 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 197 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 and test 182 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test197 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path /nfs/iil/proj/dt/eva/smlp/external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +Running test 181 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test181 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_no_input_beta.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 201 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and there are beta constraints -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test201 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input_beta.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 188 test type: optsyn, description: basic branched tree encoding test for dt_caretin model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test188 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding branched -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs -Running test 204 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 123 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test204 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 217 test type: correlate, description: basic test for correlate mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test217 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -Running test 221 test type: correlate, description: basic test for correlate mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test221 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -Running test 225 test type: correlate, description: basic test for correlate mode and tests the adjusted mutual information -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test225 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f - +Running test 196 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 and test 181 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test196 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 192 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test192 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 200 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test200 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec + +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 205 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 145 -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -out_dir ./ -pref Test205 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 141 test type: optimize, description: basic test for compress_rules option for dt_sklearn in optimization mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test141 -mode optimize -opt_strategy lazy -pareto f -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 216 test type: correlate, description: basic test for correlate mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test216 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 148 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test148 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 220 test type: correlate, description: basic test for correlate mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test220 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 155 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test155 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -Running test 226 test type: correlate, description: basic test for correlate mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test226 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 7 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test7 -mode predict -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 11 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric labels +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test11 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 222 test type: correlate, description: basic test for correlate mode -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test222 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 17 test type: prediction, description: basic poly_sklearn prediction test from saved model on new data with numeric labels and two responses +smlp -model_name "../models/Test11_smlp_toy_num_resp_mult" -out_dir ./ -pref Test17 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 227 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test227 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 25 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test25 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test26_model -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 37 test type: doe, description: doe test with four levels with box_behnken +smlp -doe_spec "../grids/doe_three_levels_real_nan.csv" -out_dir ./ -pref Test37 -mode doe -doe_algo box_behnken -log_time f +Running test 47 test type: prediction, description: tests options -pos_val and -neg_val when re-using saved model +smlp -model_name "../models/test47_model" -out_dir ./ -pref Test47 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" -Running test 107 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test107 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_beta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 54 test type: discretization, description: tests discretization options +smlp -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test54 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_basic.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs -Running test 114 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual without specifying resp and feat in command line -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test114 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model f -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec +Running test 66 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +smlp -model_name "../models/test65_model" -out_dir ./ -pref Test66 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_fail.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs -Running test 122 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test122 -mode optimize -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 79 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test79 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_verify.spec +spec_fn smlp_toy_num_resp_mult_query_vacuous.spec specs_path ../specs -Running test 140 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test140 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -trace_prec 1 -trace_anonym t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 147 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test147 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 91 test type: query, description: test to detect contradictory constraints in optimization mode due to contradictory alpha global and alpha bounds constraints on FMAX_xyx +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test91 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs -Running test 154 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test154 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 +Running test 98 test type: optsyn, description: basic test for et_caret in model exploration mode optsyn +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test98 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_system.spec +specs_path ../specs +Running test 115 test type: certify, description: basic test in certify mode +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test115 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_system_stable_verify.spec specs_path ../specs -Running test 145 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -out_dir ./ -pref Test145 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 126 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +smlp -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test126 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 151 test type: prediction, description: tests msle loss function MeanSquaredLogarithmicError and and sample weoghts -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test151 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss msle -sw_coef 3 -sw_exp 10 -sw_int 0 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 142 test type: optsyn, description: basic test for compress_rules option for rf_sklearn in optsin mode +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test142 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs -Running test 155 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training -/home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/lib/python3.11/site-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test155 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae +Running test 154 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training +smlp -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test154 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 Initiating 7 worker... comparing Test1_smlp_toy_num_resp_mult_y1_dt_caret_tree_rules.txt to master @@ -1178,7 +1178,6 @@ Passed! comparing Test14_smlp_toy_basic_training_predictions_summary.csv to master Passed! comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1186,7 +1185,6 @@ Passed! comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master Passed! comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1194,7 +1192,6 @@ Passed! comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master Passed! comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1232,7 +1229,6 @@ Passed! comparing test19_model_rerun_model_config.json to master Passed! comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1240,7 +1236,6 @@ Passed! comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master Passed! comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1287,7 +1282,6 @@ Passed! comparing test22_model_rerun_model_config.json to master Passed! comparing Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt to master -Passed! File master Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_new-col-PF .png does not exist File master Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_new-col-|PF |.png does not exist comparing Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_missing_values_dict.json to master @@ -1328,7 +1322,6 @@ Passed! comparing test24_model_rerun_model_config.json to master Passed! comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1365,7 +1358,6 @@ Passed! comparing test26_model_rerun_model_config.json to master Passed! comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1454,7 +1446,6 @@ Passed! comparing Test31_smlp_toy_num_resp_mult_ranking_resp_feat.csv to master Passed! comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled_args_config.json to master Passed! comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master @@ -1543,7 +1534,6 @@ Passed! comparing test47_model_poly_sklearn_formula.txt to master Passed! comparing Test47_test47_model_smlp_toy_pf_mult.txt to master -Passed! comparing Test47_test47_model_smlp_toy_pf_mult_missing_values_dict.json to master Passed! comparing Test47_test47_model_smlp_toy_pf_mult_new_prediction_precisions.csv to master @@ -1715,7 +1705,6 @@ Passed! File master test63_model_y1_smlp_full_model_term.json does not exist File master test63_model_y1_smlp_model_term.json does not exist comparing Test64_test63_model.txt to master -Passed! File master Test64_test63_model_trace.csv does not exist comparing Test64_test63_model_verify_results.json to master Test 65 Failed: @@ -1760,7 +1749,6 @@ Passed! File master test69_model_y2_smlp_full_model_term.json does not exist File master test69_model_y2_smlp_model_term.json does not exist comparing Test70_test69_model.txt to master -Passed! File master Test70_test69_model_trace.csv does not exist comparing Test70_test69_model_verify_results.json to master Test 71 Failed: @@ -2490,7 +2478,6 @@ File master test101_model_y1_smlp_model_term.json does not exist File master test101_model_y2_smlp_full_model_term.json does not exist File master test101_model_y2_smlp_model_term.json does not exist comparing Test102_test101_model.txt to master -Passed! comparing Test102_test101_model_certify_results.json to master Passed! File master Test102_test101_model_trace.csv does not exist @@ -2716,11 +2703,9 @@ comparing test110_model_poly_sklearn_formula.txt to master comparing test110_model_rerun_model_config.json to master Passed! comparing Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt to master -Passed! comparing Test111_test110_model_smlp_toy_basic_pred_unlabeled_new_predictions_summary.csv to master Passed! comparing Test112_test110_model_smlp_toy_basic_pred_unlabeled.txt to master -Passed! comparing Test112_test110_model_smlp_toy_basic_pred_unlabeled_new_predictions_summary.csv to master Passed! comparing test113_model_dt_sklearn_tree_rules.txt to master @@ -3626,6 +3611,9 @@ File master Test172_smlp_toy_num_resp_mult_trace.csv does not exist File master Test172_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist File master Test172_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist File master Test172_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test172_smlp_toy_num_resp_mult_verify_results.json does not exist +File master Test172_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test172_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist File master Test173_smlp_toy_num_resp_mult.txt does not exist File master Test173_smlp_toy_num_resp_mult_data_bounds.json does not exist File master Test173_smlp_toy_num_resp_mult_features_scaler.pkl does not exist @@ -3642,6 +3630,9 @@ File master Test173_smlp_toy_num_resp_mult_trace.csv does not exist File master Test173_smlp_toy_num_resp_mult_train-reg_y2_mae.png does not exist File master Test173_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist File master Test173_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test173_smlp_toy_num_resp_mult_verify_results.json does not exist +File master Test173_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test173_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist File master Test174_smlp_toy_num_resp_mult.txt does not exist File master Test174_smlp_toy_num_resp_mult_data_bounds.json does not exist File master Test174_smlp_toy_num_resp_mult_features_scaler.pkl does not exist @@ -3835,7 +3826,12 @@ File master Test182_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does File master Test182_smlp_toy_num_resp_mult_missing_values_dict.json does not exist File master Test182_smlp_toy_num_resp_mult_model_features_dict.json does not exist File master Test182_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test182_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test182_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test182_smlp_toy_num_resp_mult_optimization_results.csv does not exist File master Test182_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test182_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test182_smlp_toy_num_resp_mult_smlp_model_term.json does not exist File master Test182_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist File master Test182_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist File master Test182_smlp_toy_num_resp_mult_trace.csv does not exist @@ -4112,7 +4108,12 @@ File master Test197_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does File master Test197_smlp_toy_num_resp_mult_missing_values_dict.json does not exist File master Test197_smlp_toy_num_resp_mult_model_features_dict.json does not exist File master Test197_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test197_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test197_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test197_smlp_toy_num_resp_mult_optimization_results.csv does not exist File master Test197_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test197_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test197_smlp_toy_num_resp_mult_smlp_model_term.json does not exist File master Test197_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist File master Test197_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist File master Test197_smlp_toy_num_resp_mult_trace.csv does not exist @@ -4278,5 +4279,5 @@ File master Test227_smlp_toy_basic_features_summary.csv does not exist master log file does not exist! Do you wish to copy the new log file to master? (yes/no|y/n): No new tests crashed (not in the masters) -Time: 36.19023111661275 minutes +Time: 33.25827042659124 minutes End of regression diff --git a/tests/smlp_regression/run_smlp_regression_venv_expected_diff_report.log b/tests/smlp_regression/run_smlp_regression_venv_expected_diff_report.log index 0f81984d..71387358 100644 --- a/tests/smlp_regression/run_smlp_regression_venv_expected_diff_report.log +++ b/tests/smlp_regression/run_smlp_regression_venv_expected_diff_report.log @@ -314,12 +314,60 @@ 737a737 > if (p2 > 0.30543398172847647) and (p1 > 0.7651434488432218) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples =================== End of Test10_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt diff report ================================ +=================== Diff report for: Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/Test5_smlp_toy_num_resp_mult_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/Test5_smlp_toy_num_resp_mult_rerun_model_config.json +=================== End of Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/Test8_smlp_toy_num_resp_mult_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/Test8_smlp_toy_num_resp_mult_rerun_model_config.json +=================== End of Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/Test11_smlp_toy_num_resp_mult_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/Test11_smlp_toy_num_resp_mult_rerun_model_config.json +=================== End of Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test19_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test19_model_rerun_model_config.json +=================== End of Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +75c75 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test20_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test20_model_rerun_model_config.json +=================== End of Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt ================================== +75c75 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test22_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test22_model_rerun_model_config.json +=================== End of Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt diff report ================================ +=================== Diff report for: Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test24_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test24_model_rerun_model_config.json +=================== End of Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ =================== Diff report for: test26_model_dt_sklearn_tree_rules.txt ================================== 6d5 < if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples 7a7 > if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples =================== End of test26_model_dt_sklearn_tree_rules.txt diff report ================================ +=================== Diff report for: Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test26_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test26_model_rerun_model_config.json +=================== End of Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ =================== Diff report for: Test29_smlp_toy_cls_metasymbol_colnames_mult.txt ================================== 95,96d94 < smlp_logger - WARNING - Range plots are not supported in this version of SMLP @@ -335,11 +383,23 @@ < smlp_logger - WARNING - Range plots are not supported in this version of SMLP < =================== End of Test31_smlp_toy_num_resp_mult.txt diff report ================================ +=================== Diff report for: Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +75c75 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test20_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test20_model_rerun_model_config.json +=================== End of Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ =================== Diff report for: Test33_smlp_toy_num_resp_mult.txt ================================== 95,96d94 < smlp_logger - WARNING - Range plots are not supported in this version of SMLP < =================== End of Test33_smlp_toy_num_resp_mult.txt diff report ================================ +=================== Diff report for: Test47_test47_model_smlp_toy_pf_mult.txt ================================== +83c83 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test47_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test47_model_rerun_model_config.json +=================== End of Test47_test47_model_smlp_toy_pf_mult.txt diff report ================================ =================== Diff report for: Test58_smlp_toy_num_resp_mult_optimization_progress.json ================================== 17,19d16 < "p1": { @@ -385,6 +445,12 @@ > "p2": 7.0, > "y1": 9.0 =================== End of Test63_smlp_toy_num_resp_mult_verify_results.json diff report ================================ +=================== Diff report for: Test64_test63_model.txt ================================== +27c27 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test63_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test63_model_rerun_model_config.json +=================== End of Test64_test63_model.txt diff report ================================ =================== Diff report for: Test64_test63_model_verify_results.json ================================== 12d11 < "p1": 1.0, @@ -497,7 +563,7 @@ > smlp_logger - INFO - Executing run_smlp.py script: End =================== End of Test66_test65_model.txt diff report ================================ =================== Diff report for: Test66_test65_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/Test66_test65_model_verify_results.json: No such file or directory +diff: /home/mdmitry/github/smlptech/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/Test66_test65_model_verify_results.json: No such file or directory =================== End of Test66_test65_model_verify_results.json diff report ================================ =================== Diff report for: Test68_test67_model.txt ================================== 0a1,97 @@ -600,7 +666,7 @@ diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_ > smlp_logger - INFO - Executing run_smlp.py script: End =================== End of Test68_test67_model.txt diff report ================================ =================== Diff report for: Test68_test67_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/Test68_test67_model_verify_results.json: No such file or directory +diff: /home/mdmitry/github/smlptech/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/Test68_test67_model_verify_results.json: No such file or directory =================== End of Test68_test67_model_verify_results.json diff report ================================ =================== Diff report for: Test69_smlp_toy_num_resp_mult_verify_results.json ================================== 6d5 @@ -613,6 +679,12 @@ diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_ > "p2": 7.0, > "y2": 5.078784562647343 =================== End of Test69_smlp_toy_num_resp_mult_verify_results.json diff report ================================ +=================== Diff report for: Test70_test69_model.txt ================================== +25c25 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test69_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test69_model_rerun_model_config.json +=================== End of Test70_test69_model.txt diff report ================================ =================== Diff report for: Test70_test69_model_verify_results.json ================================== 6d5 < "p1": 1.0, @@ -712,7 +784,7 @@ diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_ > smlp_logger - INFO - Executing run_smlp.py script: End =================== End of Test72_test71_model.txt diff report ================================ =================== Diff report for: Test72_test71_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/Test72_test71_model_verify_results.json: No such file or directory +diff: /home/mdmitry/github/smlptech/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/Test72_test71_model_verify_results.json: No such file or directory =================== End of Test72_test71_model_verify_results.json diff report ================================ =================== Diff report for: Test77_test76_model.txt ================================== 0a1,110 @@ -828,7 +900,7 @@ diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_ > smlp_logger - INFO - Executing run_smlp.py script: End =================== End of Test77_test76_model.txt diff report ================================ =================== Diff report for: Test77_test76_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/Test77_test76_model_verify_results.json: No such file or directory +diff: /home/mdmitry/github/smlptech/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/Test77_test76_model_verify_results.json: No such file or directory =================== End of Test77_test76_model_verify_results.json diff report ================================ =================== Diff report for: Test79_smlp_toy_num_resp_mult_query_results.json ================================== 14a15 @@ -2028,6 +2100,12 @@ diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_ > "y1": { > "value_in_config": 9.0 =================== End of Test100_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test102_test101_model.txt ================================== +38c38 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test101_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test101_model_rerun_model_config.json +=================== End of Test102_test101_model.txt diff report ================================ =================== Diff report for: Test105_smlp_toy_num_resp_mult_verify_results.json ================================== 6d5 < "p1": 4.0, @@ -2039,8 +2117,20 @@ diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_ > "y1": 9.0, =================== End of Test105_smlp_toy_num_resp_mult_verify_results.json diff report ================================ =================== Diff report for: test110_model_poly_sklearn_formula.txt ================================== -diff: /home/mdmitry/github/smlp_python311_refactoring/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/test110_model_poly_sklearn_formula.txt: No such file or directory +diff: /home/mdmitry/github/smlptech/scripts/venv/smlp_package_venv/smlp/regr_smlp/code/test110_model_poly_sklearn_formula.txt: No such file or directory =================== End of test110_model_poly_sklearn_formula.txt diff report ================================ +=================== Diff report for: Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt ================================== +79c79 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test110_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test110_model_rerun_model_config.json +=================== End of Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt diff report ================================ +=================== Diff report for: Test112_test110_model_smlp_toy_basic_pred_unlabeled.txt ================================== +79c79 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test110_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test110_model_rerun_model_config.json +=================== End of Test112_test110_model_smlp_toy_basic_pred_unlabeled.txt diff report ================================ =================== Diff report for: Test113_smlp_toy_basic_optimization_progress.json ================================== 17,18c17,21 < "x2": { From 3f374d68cd3e64114bef74ac2ef1ef02888fa566 Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Wed, 25 Mar 2026 15:54:08 +0200 Subject: [PATCH 04/11] Updated after merge with master --- .../run_smlp_regression_expected.log | 1277 +++-- ...n_smlp_regression_expected_diff_report.log | 801 +-- .../run_smlp_regression_whl_expected.log | 4283 +++++++++++++++++ ...lp_regression_whl_expected_diff_report.log | 3293 +++++++++++++ 4 files changed, 8541 insertions(+), 1113 deletions(-) create mode 100644 tests/smlp_regression/run_smlp_regression_whl_expected.log create mode 100644 tests/smlp_regression/run_smlp_regression_whl_expected_diff_report.log diff --git a/tests/smlp_regression/run_smlp_regression_expected.log b/tests/smlp_regression/run_smlp_regression_expected.log index 72d8f684..1b37e71c 100644 --- a/tests/smlp_regression/run_smlp_regression_expected.log +++ b/tests/smlp_regression/run_smlp_regression_expected.log @@ -1,990 +1,799 @@ Calling 8 workers for multiprocessing... Initiating 0 worker... +Running test 1 test type: train, description: basic dt_caret training and test on labeled data with single numeric response +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f + Initiating 1 worker... +Running test 2 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test2 -mode predict -resp y1 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + Initiating 2 worker... +Running test 3 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric response in training/test data only +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test3 -mode predict -resp y1 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_unlabeled.csv" + Initiating 3 worker... Initiating 4 worker... +Running test 4 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and one response +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test4 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nn_keras_weights_precision 2 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + Initiating 5 worker... +Running test 5 test type: prediction, description: basic dt_caret prediction test on labeled and new data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test5 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model t -use_model f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 6 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test6 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + Initiating 6 worker... +Initiating 7 worker... +Running test 7 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test7 -mode predict -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 3 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric response in training/test data only -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test3 -mode predict -resp y1 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_unlabeled.csv" +Running test 8 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and two responses +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test8 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -nn_keras_epochs 20 -nn_keras_seq_api f -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 9 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test9 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -model_name test20_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -save_config t -save_model_config t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 10 test type: prediction, description: basic et_sklearn prediction test on labeled and new data with numeric labels ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test10 -mode predict -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 15 -et_sklearn_bootstrap f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 11 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test11 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 17 test type: prediction, description: basic poly_sklearn prediction test from saved model on new data with numeric labels and two responses -../../src/run_smlp.py -model_name "../models/Test11_smlp_toy_num_resp_mult" -out_dir ./ -pref Test17 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - +Running test 12 test type: train, description: EV-SI real life dt_sklearn predict test on labeled and new data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test12 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 24 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -../../src/run_smlp.py -model_name "../models/test24_model" -out_dir ./ -pref Test24 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 13 test type: train, description: EV-SI real life nn_keras prediction test on labeled and new data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test13 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 14 test type: train, description: EV-SI real life poly_sklearn prediction test on labeled and new data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test14 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 30 test type: subgroups, description: basic test for subgroup discovery for numric responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test30 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -feat x,p1,p2 -plots t -seed 10 -log_time f +Running test 15 test type: prediction, description: basic dt_caret prediction test from saved model on new data with numeric labels +../../src/run_smlp.py -model_name "../models/Test5_smlp_toy_num_resp_mult" -out_dir ./ -pref Test15 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 16 test type: prediction, description: basic nn_keras prediction test from saved model on new data with numeric labels and two responses +../../src/run_smlp.py -model_name "../models/Test8_smlp_toy_num_resp_mult" -out_dir ./ -pref Test16 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 37 test type: doe, description: doe test with four levels with box_behnken -../../src/run_smlp.py -doe_spec "../grids/doe_three_levels_real_nan.csv" -out_dir ./ -pref Test37 -mode doe -doe_algo box_behnken -log_time f +Running test 17 test type: prediction, description: basic poly_sklearn prediction test from saved model on new data with numeric labels and two responses +../../src/run_smlp.py -model_name "../models/Test11_smlp_toy_num_resp_mult" -out_dir ./ -pref Test17 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 18 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test18 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test19_model -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 47 test type: prediction, description: tests options -pos_val and -neg_val when re-using saved model -../../src/run_smlp.py -model_name "../models/test47_model" -out_dir ./ -pref Test47 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" +Running test 19 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +../../src/run_smlp.py -model_name "../models/test19_model" -out_dir ./ -pref Test19 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 20 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +../../src/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test20 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 54 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test54 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 21 test type: prediction, description: test for illegal symbols in column names +../../src/run_smlp.py -data "../data/smlp_toy_num_metasymbol_mult_reg.csv" -out_dir ./ -pref Test21 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model t -use_model f -model_name test22_model -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" -spec_fn smlp_toy_num_resp_noknobs_verify.spec -specs_path ../specs +Running test 22 test type: prediction, description: test for illegal symbols in column names +../../src/run_smlp.py -model_name "../models/test22_model" -out_dir ./ -pref Test22 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model f -use_model t -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" -Running test 68 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -../../src/run_smlp.py -model_name "../models/test67_model" -out_dir ./ -pref Test68 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 23 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test23 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test24_model -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult.spec -specs_path ../specs +Running test 24 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +../../src/run_smlp.py -model_name "../models/test24_model" -out_dir ./ -pref Test24 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 80 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test80 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 25 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test25 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test26_model -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec -specs_path ../specs +Running test 26 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +../../src/run_smlp.py -model_name "../models/test26_model" -out_dir ./ -pref Test26 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 88 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with beta and objectives specified in spec file -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test88 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 27 test type: prediction, description: checks nn_keras prediction with nn_keras_seq_api t +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test27 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs +Running test 28 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and functional API +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test28 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 94 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test94 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 29 test type: subgroups, description: basic test for subgroup discovery for pass-fail responses +../../src/run_smlp.py -data "../data/smlp_toy_cls_metasymbol_colnames_mult.csv" -out_dir ./ -pref Test29 -mode subgroups -psg_dim 3 -psg_top 10 -resp "PF 1,PF#" -plots t -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_witness.spec -specs_path ../specs +Running test 30 test type: subgroups, description: basic test for subgroup discovery for numric responses +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test30 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -feat x,p1,p2 -plots t -seed 10 -log_time f -Running test 102 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -model_name "../models/test101_model" -out_dir ./ -pref Test102 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 31 test type: subgroups, description: testing resp2b in subgroup discovery mode +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test31 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -resp2b "y1<6;y2>6" -feat x,p1,p2 -plots t -seed 10 -log_time f -save_config t -spec_fn smlp_toy_num_resp_mult.spec -specs_path ../specs +Running test 32 test type: unknown, description: test reusing saved model by using configuration file +../../src/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test32 -config ../models/test20_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 104 test type: verify, description: assertion verfication test with wrong spec that does not assign a single value using a singleton grid or range with equal max and min -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test104 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 33 test type: unknown, description: testing -config option with subgroups mode +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test33 -config ../models/Test31_smlp_toy_num_resp_mult_args_config.json -spec_fn smlp_toy_num_resp_mult_beta_verify.spec -specs_path ../specs +Running test 34 test type: doe, description: doe test with four levels with full_factorial method +../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test34 -mode doe -doe_algo full_factorial -log_time f -Running test 107 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test107 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_beta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 35 test type: doe, description: doe test with four levels with plackett_burman +../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test35 -mode doe -doe_algo plackett_burman -log_time f -spec_fn smlp_toy_basic.spec -specs_path ../specs +Running test 36 test type: doe, description: doe test with four levels with sukharev_grid +../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test36 -mode doe -doe_algo sukharev_grid -doe_samples 125 -log_time f -Running test 114 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual without specifying resp and feat in command line -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test114 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model f -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec +Running test 37 test type: doe, description: doe test with four levels with box_behnken +../../src/run_smlp.py -doe_spec "../grids/doe_three_levels_real_nan.csv" -out_dir ./ -pref Test37 -mode doe -doe_algo box_behnken -log_time f -Running test 7 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test7 -mode predict -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 38 test type: doe, description: doe test with four levels with box_wilson +../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test38 -mode doe -doe_algo box_wilson -doe_cc_face ccc -doe_cc_alpha r -doe_cc_center 2,3 -log_time f +Running test 39 test type: doe, description: doe test with four levels with latin_hypercube +../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test39 -mode doe -doe_algo latin_hypercube -doe_prob_distr Exponential -doe_samples 30 -log_time f -Running test 9 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test9 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -model_name test20_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -save_config t -save_model_config t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 40 test type: doe, description: doe test with four levels with latin_hypercube_space_filling +../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test40 -mode doe -doe_algo latin_hypercube_sf -doe_samples 20 -log_time f +Running test 41 test type: doe, description: doe test with four levels with random_k_means +../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test41 -mode doe -doe_algo random_k_means -doe_samples 20 -log_time f -Running test 16 test type: prediction, description: basic nn_keras prediction test from saved model on new data with numeric labels and two responses -../../src/run_smlp.py -model_name "../models/Test8_smlp_toy_num_resp_mult" -out_dir ./ -pref Test16 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 42 test type: doe, description: doe test with four levels with maximin_reconstruction +../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test42 -mode doe -doe_algo maximin_reconstruction -doe_samples 20 -log_time f +Running test 43 test type: doe, description: doe test with four levels with halton_sequence +../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test43 -mode doe -doe_algo halton_sequence -doe_samples 20 -log_time f -Running test 22 test type: prediction, description: test for illegal symbols in column names -../../src/run_smlp.py -model_name "../models/test22_model" -out_dir ./ -pref Test22 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model f -use_model t -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" +Running test 44 test type: doe, description: doe test with four levels with uniform_random_matrix +../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test44 -mode doe -doe_algo uniform_random_matrix -doe_samples 20 -log_time f +Running test 45 test type: doe, description: doe test with four levels with fractional_factorial +../../src/run_smlp.py -doe_spec "../grids/doe_two_levels_real.csv" -out_dir ./ -pref Test45 -mode doe -doe_algo fractional_factorial -doe_resolution 5 -log_time f -Running test 29 test type: subgroups, description: basic test for subgroup discovery for pass-fail responses -../../src/run_smlp.py -data "../data/smlp_toy_cls_metasymbol_colnames_mult.csv" -out_dir ./ -pref Test29 -mode subgroups -psg_dim 3 -psg_top 10 -resp "PF 1,PF#" -plots t -seed 10 -log_time f +Running test 46 test type: prediction, description: tests options -pos_val and -neg_val +../../src/run_smlp.py -data "../data/smlp_toy_pf_mult.csv" -out_dir ./ -pref Test46 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model t -save_model_config f -use_model f -model_name test47_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" +Running test 47 test type: prediction, description: tests options -pos_val and -neg_val when re-using saved model +../../src/run_smlp.py -model_name "../models/test47_model" -out_dir ./ -pref Test47 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" -Running test 35 test type: doe, description: doe test with four levels with plackett_burman -../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test35 -mode doe -doe_algo plackett_burman -log_time f +Running test 48 test type: discretization, description: tests discretization options +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test48 -mode discretize -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 49 test type: discretization, description: tests discretization options +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test49 -mode discretize -resp "PF,PF1" -discr_algo quantile -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 43 test type: doe, description: doe test with four levels with halton_sequence -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test43 -mode doe -doe_algo halton_sequence -doe_samples 20 -log_time f +Running test 50 test type: discretization, description: tests discretization options +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test50 -mode discretize -resp "PF,PF1" -discr_algo kmeans -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 51 test type: discretization, description: tests discretization options +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test51 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass Running test 52 test type: discretization, description: tests discretization options ../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test52 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -spec_fn smlp_toy_num_resp_mult_y1_verify.spec -specs_path ../specs +Running test 53 test type: discretization, description: tests discretization options +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test53 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass -Running test 63 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test63 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test63_model -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 54 test type: discretization, description: tests discretization options +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test54 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 55 test type: discretization, description: tests discretization options +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test55 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 56 test type: discretization, description: tests discretization options +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test56 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels f -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs +Running test 58 test type: optimize, description: basic dt_sklearn optimization test with numeric labels and integer grid as domain and without scaling objectives +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test58 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 79 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test79 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_query_vacuous.spec +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs +Running test 59 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test59 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -Running test 91 test type: query, description: test to detect contradictory constraints in optimization mode due to contradictory alpha global and alpha bounds constraints on FMAX_xyx -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test91 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 60 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test60 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_y1_verify.spec specs_path ../specs +Running test 63 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test63 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test63_model -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 99 test type: optimize, description: testing that the response and feature names can be taken from spec file in model exploration modes when the responses and/or features are not specified in the command line -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test99 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_y1_verify.spec +specs_path ../specs +Running test 64 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +../../src/run_smlp.py -model_name "../models/test63_model" -out_dir ./ -pref Test64 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_certify_witness.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs +Running test 66 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +../../src/run_smlp.py -model_name "../models/test65_model" -out_dir ./ -pref Test66 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 103 test type: certify, description: -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test103 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test103_model -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_certify_witness.spec -quer_names valid_candidate,grid_conflict,range_conflict -quer_exprs "True;True;True" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 68 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +../../src/run_smlp.py -model_name "../models/test67_model" -out_dir ./ -pref Test68 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 69 test type: verify, description: nn_keras verification test with model_per_response training +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test69 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test69_model -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -Running test 111 test type: unknown, description: smlp toy basic test to rerun saved model using the model rerun config file saved during model training -../../src/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test111 -config ../models/test110_model_rerun_model_config.json -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 70 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +../../src/run_smlp.py -model_name "../models/test69_model" -out_dir ./ -pref Test70 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -spec_fn smlp_toy_system.spec +spec_fn smlp_toy_num_resp_noknobs_verify.spec specs_path ../specs +Running test 72 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +../../src/run_smlp.py -model_name "../models/test71_model" -out_dir ./ -pref Test72 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -Running test 115 test type: certify, description: basic test in certify mode -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test115 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 77 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file +../../src/run_smlp.py -model_name "../models/test76_model" -out_dir ./ -pref Test77 -config ../models/test76_model_rerun_model_config.json -spec_fn smlp_toy_system_stable_certify.spec +spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs +Running test 79 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test79 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 127 test type: certify, description: certification example with knobs only and fictitious inputs with values fixed through their ranges -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test127 -mode certify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 80 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test80 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_free_inps.spec specs_path ../specs +Running test 81 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test81 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -Running test 142 test type: optsyn, description: basic test for compress_rules option for rf_sklearn in optsin mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test142 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -Running test 2 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test2 -mode predict -resp y1 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 12 test type: train, description: EV-SI real life dt_sklearn predict test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test12 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 18 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test18 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test19_model -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 25 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test25 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test26_model -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 36 test type: doe, description: doe test with four levels with sukharev_grid -../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test36 -mode doe -doe_algo sukharev_grid -doe_samples 125 -log_time f - - -Running test 42 test type: doe, description: doe test with four levels with maximin_reconstruction -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test42 -mode doe -doe_algo maximin_reconstruction -doe_samples 20 -log_time f - - -Running test 51 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test51 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -spec_fn smlp_toy_num_resp_mult_y2_verify.spec -specs_path ../specs - -Running test 60 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test60 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat +spec_fn smlp_toy_num_resp_mult_free_inps.spec +specs_path ../specs +Running test 82 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test82 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat spec_fn smlp_toy_num_resp_mult_free_inps.spec specs_path ../specs - Running test 83 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with numeric labels and integer grid as domain and with scaling objectives ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test83 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -beta "y1>7 and y2>6" -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1/2-y2;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_verify_vacuous.spec -specs_path ../specs - -Running test 92 test type: verify, description: test to detect contradictory constraints in verification mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test92 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_verify_vacuous.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec -specs_path ../specs - -Running test 100 test type: optimize, description: basic test for sat_threshold option enabing usage of objectve values in SAT assignments that prove optimization thresholds -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test100 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_stable_verify.spec -specs_path ../specs - -Running test 105 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test105 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_stable_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_cannot_synthesize.spec -specs_path ../specs - -Running test 109 test type: synthesize, description: basic test for mode synthesize where synthesis fails -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test109 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_cannot_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_constant_synth_fail.spec -specs_path ../specs - -Running test 122 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test122 -mode optimize -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_verify.spec -specs_path ../specs - -Running test 140 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test140 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -trace_prec 1 -trace_anonym t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 147 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API -Running test 1 test type: train, description: basic dt_caret training and test on labeled data with single numeric response -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f - - -Running test 20 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test20 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 27 test type: prediction, description: checks nn_keras prediction with nn_keras_seq_api t -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test27 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 34 test type: doe, description: doe test with four levels with full_factorial method -../../src/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test34 -mode doe -doe_algo full_factorial -log_time f - - -Running test 44 test type: doe, description: doe test with four levels with uniform_random_matrix -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test44 -mode doe -doe_algo uniform_random_matrix -doe_samples 20 -log_time f - - -Running test 50 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test50 -mode discretize -resp "PF,PF1" -discr_algo kmeans -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -spec_fn smlp_toy_num_resp_mult_y2_verify.spec -specs_path ../specs - -Running test 59 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test59 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat - -spec_fn smlp_toy_num_resp_mult_free_inps.spec -specs_path ../specs - -Running test 82 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test82 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat - -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs - -Running test 93 test type: optsyn, description: basic test for mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test93 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_witness.spec -specs_path ../specs - -Running test 101 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test101 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test101_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_unsat_eta_verify.spec -specs_path ../specs - -Running test 106 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test106 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_unsat_eta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 110 test type: prediction, description: smlp toy basic example for predict mode from SMLP user manual -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test110 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model t -model_name test110_model -save_model_config t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" - -spec_fn smlp_toy_system_stable_constant_certify.spec -specs_path ../specs - -Running test 117 test type: certify, description: certification test with knobs only where assertion is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test117 -mode certify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_constant_synth_fail.spec -specs_path ../specs - -Running test 124 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is not feasible because while beta constraint is feasible the assertion is not feasible therefore optimization is not performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test124 -mode optsyn -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs - -Running test 141 test type: optimize, description: basic test for compress_rules option for dt_sklearn in optimization mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test141 -mode optimize -opt_strategy lazy -pareto f -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 148 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test148 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 6 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test6 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 11 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test11 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 15 test type: prediction, description: basic dt_caret prediction test from saved model on new data with numeric labels -../../src/run_smlp.py -model_name "../models/Test5_smlp_toy_num_resp_mult" -out_dir ./ -pref Test15 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 23 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test23 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test24_model -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 31 test type: subgroups, description: testing resp2b in subgroup discovery mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test31 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -resp2b "y1<6;y2>6" -feat x,p1,p2 -plots t -seed 10 -log_time f -save_config t - - -Running test 38 test type: doe, description: doe test with four levels with box_wilson -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test38 -mode doe -doe_algo box_wilson -doe_cc_face ccc -doe_cc_alpha r -doe_cc_center 2,3 -log_time f - - -Running test 46 test type: prediction, description: tests options -pos_val and -neg_val -../../src/run_smlp.py -data "../data/smlp_toy_pf_mult.csv" -out_dir ./ -pref Test46 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model t -save_model_config f -use_model f -model_name test47_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" - - -Running test 55 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test55 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -spec_fn smlp_toy_num_resp_noknobs_verify.spec -specs_path ../specs - -Running test 66 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -../../src/run_smlp.py -model_name "../models/test65_model" -out_dir ./ -pref Test66 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 77 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file -../../src/run_smlp.py -model_name "../models/test76_model" -out_dir ./ -pref Test77 -config ../models/test76_model_rerun_model_config.json +Running test 85 test type: optimize, description: tests alpha and eta constraints specified in command line in optimization mode +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test85 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult.spec specs_path ../specs - Running test 86 test type: optimize, description: tests alpha ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test86 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs - -Running test 95 test type: optsyn, description: basic test for dt_caret in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test95 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_synthesize.spec -specs_path ../specs - -Running test 108 test type: synthesize, description: basic test for dt_sklearn in model exploration mode synthesize where synthesis succeeds -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test108 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_constant_synth_fail.spec -specs_path ../specs - -Running test 120 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test120 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_witness_certify.spec -specs_path ../specs - -Running test 128 test type: certify, description: Basic regression test in certify mode covering all four possible outcomes when certifying a witness for a query: the witness is stable -../../src/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test128 -mode certify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_witness_certify.spec -quer_names query_stable_witness,query_grid_conflict,query_unstable_witness,query_infeasible_witness,query_poly_intercept_sensitive -quer_exprs "y2<=90;y1>=9;y1>=(-13);y1>9;y1>=(-10)" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 149 test type: prediction, description: tests the mae loss function MeanAbsoluteError and sample weoghts -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test149 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mae -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -spec_fn smlp_toy_num_resp_mult_verify.spec -specs_path -Running test 4 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and one response -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test4 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nn_keras_weights_precision 2 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 13 test type: train, description: EV-SI real life nn_keras prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test13 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 28 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and functional API -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test28 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 41 test type: doe, description: doe test with four levels with random_k_means -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test41 -mode doe -doe_algo random_k_means -doe_samples 20 -log_time f - - -Running test 49 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test49 -mode discretize -resp "PF,PF1" -discr_algo quantile -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -spec_fn smlp_toy_num_resp_mult.spec -specs_path ../specs - -Running test 58 test type: optimize, description: basic dt_sklearn optimization test with numeric labels and integer grid as domain and without scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test58 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_y2_verify.spec -specs_path ../specs - -Running test 70 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model -../../src/run_smlp.py -model_name "../models/test69_model" -out_dir ./ -pref Test70 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat - -spec_fn smlp_toy_num_resp_mult_free_inps.spec -specs_path ../specs - -Running test 81 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test81 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat - -spec_fn smlp_toy_num_resp_mult_query.spec -specs_path ../specs - -Running test 89 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test89 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_query.spec -specs_path ../specs - -Running test 97 test type: query, description: basic test for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test97 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 112 test type: prediction, description: smlp toy basic test from SMLP manual -../../src/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test112 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -use_model t -save_model f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" - -spec_fn smlp_toy_system_stable_constant_verify.spec -specs_path ../specs - -Running test 118 test type: verify, description: verification test with knobs only where assertion is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test118 -mode verify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec -specs_path ../specs - -Running test 123 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test123 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec -specs_path ../specs - -Running test 145 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -out_dir ./ -pref Test145 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 151 test type: prediction, description: tests msle loss function MeanSquaredLogarithmicError and and sample weoghts -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test151 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss msle -sw_coef 3 -sw_exp 10 -sw_int 0 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -spec_fn -Running test 5 test type: prediction, description: basic dt_caret prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test5 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model t -use_model f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 19 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -../../src/run_smlp.py -model_name "../models/test19_model" -out_dir ./ -pref Test19 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 26 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels -../../src/run_smlp.py -model_name "../models/test26_model" -out_dir ./ -pref Test26 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 32 test type: unknown, description: test reusing saved model by using configuration file -../../src/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test32 -config ../models/test20_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 39 test type: doe, description: doe test with four levels with latin_hypercube -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test39 -mode doe -doe_algo latin_hypercube -doe_prob_distr Exponential -doe_samples 30 -log_time f - - -Running test 45 test type: doe, description: doe test with four levels with fractional_factorial -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels_real.csv" -out_dir ./ -pref Test45 -mode doe -doe_algo fractional_factorial -doe_resolution 5 -log_time f - - -Running test 53 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test53 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -spec_fn smlp_toy_num_resp_mult_y1_verify.spec -specs_path ../specs - -Running test 64 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response -../../src/run_smlp.py -model_name "../models/test63_model" -out_dir ./ -pref Test64 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_noknobs_verify.spec -specs_path ../specs - -Running test 72 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model -../../src/run_smlp.py -model_name "../models/test71_model" -out_dir ./ -pref Test72 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat - spec_fn smlp_toy_num_resp_mult_alpha_asrt_verify.spec specs_path ../specs - Running test 87 test type: verify, description: tests global alpha constraints and assertions specified in spec file ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test87 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_alpha_asrt_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs - -Running test 96 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test96 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_basic.spec -specs_path ../specs - -Running test 113 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test113 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model t -model_name test113_model -save_model_config t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec - -spec_fn smlp_toy_system_stable_constant_query.spec -specs_path ../specs - -Running test 119 test type: query, description: query test with knobs only where query is satisfiable without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test119 -mode query -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec -specs_path ../specs - -Running test 125 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test125 -mode optsyn -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system.spec -specs_path ../specs - -Running test 146 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed -../../src/run_smlp.py -out_dir ./ -pref Test146 -mode optimize -pareto t -opt_strategy lazy -model poly_sklearn -resp y1,y2 -feat p1,p2,x1,x2 -save_model t -use_model f -mrmr_pred 0 -model_per_response t -split 1 -spec ../specs/smlp_toy_system.spec -doe_spec ../grids/explore_doe_two_levels.csv -doe_algo latin_hypercube -epsilon 0.99999999 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 152 test type: prediction, description: tests the huber loss function Huber and sample weights -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test152 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss huber -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 160 test type: prediction, description: tests nn keras tuner bayesian -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test160 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -nn_keras_metrics msle -nn_keras_tuner bayesian -nn_keras_layers_grid "2,3" -nn_keras_losses_grid "mse,mae,huber" -model_per_response f -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -Running test 8 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and two responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test8 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -nn_keras_epochs 20 -nn_keras_seq_api f -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 14 test type: train, description: EV-SI real life poly_sklearn prediction test on labeled and new data with numeric labels -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test14 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 21 test type: prediction, description: test for illegal symbols in column names -../../src/run_smlp.py -data "../data/smlp_toy_num_metasymbol_mult_reg.csv" -out_dir ./ -pref Test21 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model t -use_model f -model_name test22_model -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" - - -Running test 33 test type: unknown, description: testing -config option with subgroups mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test33 -config ../models/Test31_smlp_toy_num_resp_mult_args_config.json - - -Running test 40 test type: doe, description: doe test with four levels with latin_hypercube_space_filling -../../src/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test40 -mode doe -doe_algo latin_hypercube_sf -doe_samples 20 -log_time f - - -Running test 48 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test48 -mode discretize -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - - -Running test 56 test type: discretization, description: tests discretization options -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test56 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels f -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs +Running test 88 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with beta and objectives specified in spec file +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test88 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 69 test type: verify, description: nn_keras verification test with model_per_response training -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test69 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test69_model -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat - -spec_fn smlp_toy_num_resp_mult.spec +spec_fn smlp_toy_num_resp_mult_query.spec specs_path ../specs - -Running test 85 test type: optimize, description: tests alpha and eta constraints specified in command line in optimization mode -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test85 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 89 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test89 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn_vacuous.spec specs_path ../specs - Running test 90 test type: optsyn, description: test to detect contradictory constraints in optsyn mode ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test90 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs - -Running test 98 test type: optsyn, description: basic test for et_caret in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test98 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system.spec +spec_fn smlp_toy_num_resp_mult_query_vacuous.spec specs_path ../specs +Running test 91 test type: query, description: test to detect contradictory constraints in optimization mode due to contradictory alpha global and alpha bounds constraints on FMAX_xyx +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test91 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 116 test type: certify, description: basic test in certify mode when system is specified and is used as the model; p2 rel-rad needs to be 0 or very close to it the witness to first query to be stable -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test116 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_configuration_verify.spec +spec_fn smlp_toy_num_resp_mult_verify_vacuous.spec specs_path ../specs - -Running test 129 test type: verify, description: verification example with demonstrating all basic result scenarious for assertions -../../src/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test129 -mode verify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_configuration_verify.spec -asrt_names assert_stable_config,assert_grid_conflict,assert_unstable_config,assert_infeasible -asrt_exprs "y2<=90;y1>=9;y1>=(-10);y1>20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 150 test type: prediction, description: tests the mape loss function MeanAbsolutePercentageError and sample weights -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test150 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - - -Running test 158 test type: prediction, description: tests the mape loss function and sample weights with model_per_response t -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test158 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -model_per_response t -sw_coef 8 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 92 test type: verify, description: test to detect contradictory constraints in verification mode +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test92 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_verify_vacuous.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs - -Running test 167 test type: optsyn, description: basic flat tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test167 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec -specs_path ../specs - -Running test 121 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is feasible -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test121 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_system_stable_verify.spec -specs_path ../specs - -Running test 126 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test126 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_query.spec -specs_path ../specs - -Running test 143 test type: query, description: basic test for compress_rules for et_sklearn in mode query -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test143 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 159 test type: prediction, description: tests the msle loss function and sample weights with model_per_response t -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test159 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -model_per_response t -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae,cosine -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 93 test type: optsyn, description: basic test for mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test93 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs - -Running test 168 test type: optimize, description: basic test for rf_caret with flat tree_encoding and modelper_response in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test168 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding flat -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 94 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test94 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs - -Running test 174 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test174 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec -specs_path ../specs - -Running test 183 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test183 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 95 test type: optsyn, description: basic test for dt_caret in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test95 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs +Running test 96 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test96 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 189 test type: optsyn, description: basic branched tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn adapts test 166 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test189 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_query.spec +specs_path ../specs +Running test 97 test type: query, description: basic test for rf_sklearn in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test97 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs - -Running test 192 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test192 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 98 test type: optsyn, description: basic test for et_caret in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test98 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs +Running test 99 test type: optimize, description: testing that the response and feature names can be taken from spec file in model exploration modes when the responses and/or features are not specified in the command line +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test99 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 198 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 and test 183 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test198 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat - -spec_fn smlp_toy_num_resp_mult_no_input_beta.spec +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs +Running test 100 test type: optimize, description: basic test for sat_threshold option enabing usage of objectve values in SAT assignments that prove optimization thresholds +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test100 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 201 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and there are beta constraints -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test201 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input_beta.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_witness.spec +specs_path ../specs +Running test 101 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test101 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test101_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +spec_fn smlp_toy_num_resp_mult_witness.spec specs_path ../specs +Running test 102 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation +../../src/run_smlp.py -model_name "../models/test101_model" -out_dir ./ -pref Test102 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 204 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 123 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test204 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - smlp_toy_num_resp_mult_verify.spec +spec_fn smlp_toy_num_resp_mult_certify_witness.spec specs_path ../specs +Running test 103 test type: certify, description: +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test103 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test103_model -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_certify_witness.spec -quer_names valid_candidate,grid_conflict,range_conflict -quer_exprs "True;True;True" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 157 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training; adapts test 155 by consdering multiple responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test157 -mode verify -resp y1,y2 -feat x,p1,p2 --model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse,logcosh +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 104 test type: verify, description: assertion verfication test with wrong spec that does not assign a single value using a singleton grid or range with equal max and min +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test104 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_stable_verify.spec specs_path ../specs +Running test 105 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test105 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_stable_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 166 test type: optsyn, description: basic flat tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test166 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_unsat_eta_verify.spec +specs_path ../specs +Running test 106 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test106 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_unsat_eta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_beta_verify.spec specs_path ../specs +Running test 107 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test107 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_beta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 170 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response f in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test170 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_synthesize.spec +specs_path ../specs +Running test 108 test type: synthesize, description: basic test for dt_sklearn in model exploration mode synthesize where synthesis succeeds +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test108 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_num_resp_mult_cannot_synthesize.spec specs_path ../specs +Running test 109 test type: synthesize, description: basic test for mode synthesize where synthesis fails +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test109 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_cannot_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 176 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api f for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test176 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 110 test type: prediction, description: smlp toy basic example for predict mode from SMLP user manual +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test110 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model t -model_name test110_model -save_model_config t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" -spec_fn smlp_toy_num_resp_mult_optsyn.spec -specs_path ../specs +Running test 111 test type: unknown, description: smlp toy basic test to rerun saved model using the model rerun config file saved during model training +../../src/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test111 -config ../models/test110_model_rerun_model_config.json -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" -Running test 179 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn when resposes are not scaled adapts test 174 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test179 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 112 test type: prediction, description: smlp toy basic test from SMLP manual +../../src/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test112 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -use_model t -save_model f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_basic.spec specs_path ../specs +Running test 113 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test113 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model t -model_name test113_model -save_model_config t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec -Running test 182 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test182 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" - -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_basic.spec specs_path ../specs +Running test 114 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual without specifying resp and feat in command line +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test114 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model f -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec -Running test 190 test type: optimize, description: basic test for rf_caret with branched tree_encoding and modelper_response in model exploration mode optimize adapts test 168 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test190 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding branched -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_system.spec +specs_path ../specs +Running test 115 test type: certify, description: basic test in certify mode +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test115 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system.spec specs_path ../specs +Running test 116 test type: certify, description: basic test in certify mode when system is specified and is used as the model; p2 rel-rad needs to be 0 or very close to it the witness to first query to be stable +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test116 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 195 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 192 by setting n_estimators 3 and then discrepancy between z3 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test195 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_system_stable_constant_certify.spec +specs_path ../specs +Running test 117 test type: certify, description: certification test with knobs only where assertion is valid without stability and fails with stability +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test117 -mode certify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_constant_verify.spec specs_path ../specs +Running test 118 test type: verify, description: verification test with knobs only where assertion is valid without stability and fails with stability +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test118 -mode verify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 199 test type: optimize, description: test to demonstrate that in pareto optimization and optsyn modes with multiple objectives when beta constraints are not present SMLP results are not consistent when different solvers are used; this is due to fact that when a subset of objectoves are exemined in pareto algo -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test199 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_system_stable_constant_query.spec +specs_path ../specs +Running test 119 test type: query, description: query test with knobs only where query is satisfiable without stability and fails with stability +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test119 -mode query -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs - -Running test 203 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed adapts test 122 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test203 -mode optimize -pareto f -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 120 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test120 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs +Running test 121 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is feasible +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test121 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 206 test type: optsyn, description: optimized synthesis test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 125 -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test206 -mode optsyn -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs +Running test 122 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test122 -mode optimize -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 171 test type: optimize, description: basic test for et_caret with flat tree_encoding in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test171 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs +Running test 123 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test123 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 177 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test177 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs +Running test 124 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is not feasible because while beta constraint is feasible the assertion is not feasible therefore optimization is not performed +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test124 -mode optsyn -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 180 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features and responses are not scaled adapts test 175 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test180 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_feat f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 125 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is feasible and optimization is performed +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test125 -mode optsyn -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_system_stable_verify.spec specs_path ../specs +Running test 126 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test126 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 188 test type: optsyn, description: basic branched tree encoding test for dt_caretin model exploration mode optsyn -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test188 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding branched -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_system_stable_certify.spec +specs_path ../specs +Running test 127 test type: certify, description: certification example with knobs only and fictitious inputs with values fixed through their ranges +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test127 -mode certify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_optsyn.spec +spec_fn smlp_toy_witness_certify.spec specs_path ../specs +Running test 128 test type: certify, description: Basic regression test in certify mode covering all four possible outcomes when certifying a witness for a query: the witness is stable +../../src/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test128 -mode certify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_witness_certify.spec -quer_names query_stable_witness,query_grid_conflict,query_unstable_witness,query_infeasible_witness,query_poly_intercept_sensitive -quer_exprs "y2<=90;y1>=9;y1>=(-13);y1>9;y1>=(-10)" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 193 test type: optimize, description: basic test for et_caret with branched tree_encoding in model exploration mode optimize adapts test 171 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test193 -mode optimize -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_configuration_verify.spec +specs_path ../specs +Running test 129 test type: verify, description: verification example with demonstrating all basic result scenarious for assertions +../../src/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test129 -mode verify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_configuration_verify.spec -asrt_names assert_stable_config,assert_grid_conflict,assert_unstable_config,assert_infeasible -asrt_exprs "y2<=90;y1>=9;y1>=(-10);y1>20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_system_stable_verify.spec specs_path ../specs +Running test 140 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test140 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -trace_prec 1 -trace_anonym t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 197 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 and test 182 -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test197 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 141 test type: optimize, description: basic test for compress_rules option for dt_sklearn in optimization mode +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test141 -mode optimize -opt_strategy lazy -pareto f -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_no_input.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs +Running test 142 test type: optsyn, description: basic test for compress_rules option for rf_sklearn in optsin mode +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test142 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 202 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and no beta constraints -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test202 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_query.spec +specs_path ../specs +Running test 143 test type: query, description: basic test for compress_rules for et_sklearn in mode query +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test143 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_system_stable_constant_synth_feasible.spec specs_path ../specs +Running test 145 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +../../src/run_smlp.py -out_dir ./ -pref Test145 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 205 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 145 -../../src/run_smlp.py -out_dir ./ -pref Test205 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f - - -Running test 216 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test216 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f - +spec_fn smlp_toy_system.spec +specs_path ../specs +Running test 146 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +../../src/run_smlp.py -out_dir ./ -pref Test146 -mode optimize -pareto t -opt_strategy lazy -model poly_sklearn -resp y1,y2 -feat p1,p2,x1,x2 -save_model t -use_model f -mrmr_pred 0 -model_per_response t -split 1 -spec ../specs/smlp_toy_system.spec -doe_spec ../grids/explore_doe_two_levels.csv -doe_algo latin_hypercube -epsilon 0.99999999 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 218 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test218 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 147 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test147 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 148 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test148 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 222 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test222 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +Running test 149 test type: prediction, description: tests the mae loss function MeanAbsoluteError and sample weoghts +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test149 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mae -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 150 test type: prediction, description: tests the mape loss function MeanAbsolutePercentageError and sample weights +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test150 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test147 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 151 test type: prediction, description: tests msle loss function MeanSquaredLogarithmicError and and sample weoghts +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test151 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss msle -sw_coef 3 -sw_exp 10 -sw_int 0 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 152 test type: prediction, description: tests the huber loss function Huber and sample weights +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test152 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss huber -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" Running test 153 test type: prediction, description: tests the logcosh loss function LogCosh and sample weights ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test153 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss logcosh -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 154 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test154 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -Running test 161 test type: prediction, description: tests nn keras tuner bayesian -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test161 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -nn_keras_metrics mape,logcosh -nn_keras_tuner random -nn_keras_lrates_grid "0.01,0.001" -nn_keras_batches_grid "32,64" -model_per_response f -sw_coef 4 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" - -spec_fn smlp_toy_num_resp_mult_y2_verify.spec -specs_path ../specs - -Running test 172 test type: verify, description: basic test for nn_keras flat encoding for functional api -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test172 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" - - - -Running test 217 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test217 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - - -Running test 220 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test220 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - - -Running test 224 test type: correlate, description: basic test for correlate mode and tests the Shannon mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test224 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f - - -spec_fn smlp_toy_num_resp_mult_y2_verify.spec -specs_path ../specs - -Running test 154 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test154 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 - - -Running test 225 test type: correlate, description: basic test for correlate mode and tests the adjusted mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test225 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f - - - -Running test 215 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test215 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass - +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 155 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test155 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -Running test 219 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test219 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type integer -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_num_resp_mult_verify.spec +specs_path ../specs +Running test 156 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training; adapts test 154 by consdering multiple responses +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test156 -mode verify -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse +spec_fn smlp_toy_num_resp_mult_verify.spec +specs_path ../specs +Running test 157 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training; adapts test 155 by consdering multiple responses +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test157 -mode verify -resp y1,y2 -feat x,p1,p2 --model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse,logcosh -Running test 223 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test223 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 158 test type: prediction, description: tests the mape loss function and sample weights with model_per_response t +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test158 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -model_per_response t -sw_coef 8 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" +Running test 159 test type: prediction, description: tests the msle loss function and sample weights with model_per_response t +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test159 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -model_per_response t -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae,cosine -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 226 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test226 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f +Running test 160 test type: prediction, description: tests nn keras tuner bayesian +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test160 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -nn_keras_metrics msle -nn_keras_tuner bayesian -nn_keras_layers_grid "2,3" -nn_keras_losses_grid "mse,mae,huber" -model_per_response f -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -../specs +Running test 161 test type: prediction, description: tests nn keras tuner bayesian +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test161 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -nn_keras_metrics mape,logcosh -nn_keras_tuner random -nn_keras_lrates_grid "0.01,0.001" -nn_keras_batches_grid "32,64" -model_per_response f -sw_coef 4 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" -Running test 156 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training; adapts test 154 by consdering multiple responses -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test156 -mode verify -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 164 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test164 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs - Running test 165 test type: optsyn, description: basic flat tree encoding test for dt_caretin model exploration mode optsyn ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test165 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding flat -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs +Running test 166 test type: optsyn, description: basic flat tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test166 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 173 test type: verify, description: basic test for nn_keras flat encoding for sequential api -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test173 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" - +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 167 test type: optsyn, description: basic flat tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test167 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 221 test type: correlate, description: basic test for correlate mode -../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test221 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 168 test type: optimize, description: basic test for rf_caret with flat tree_encoding and modelper_response in model exploration mode optimize +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test168 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding flat -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 169 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response t in model exploration mode optimize +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test169 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 227 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information -../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test227 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 170 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response f in model exploration mode optimize +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test170 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 171 test type: optimize, description: basic test for et_caret with flat tree_encoding in model exploration mode optimize +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test171 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_y2_verify.spec specs_path ../specs +Running test 172 test type: verify, description: basic test for nn_keras flat encoding for functional api +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test172 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" -Running test 164 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test164 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 173 test type: verify, description: basic test for nn_keras flat encoding for sequential api +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test173 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs - -Running test 169 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response t in model exploration mode optimize -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test169 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +Running test 174 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test174 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs - Running test 175 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test175 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs +Running test 176 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api f for nn_keras in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test176 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 177 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test177 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs Running test 178 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features are not scaled adapts test 177 ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test178 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 179 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn when resposes are not scaled adapts test 174 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test179 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs +Running test 180 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features and responses are not scaled adapts test 175 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test180 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_feat f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs Running test 181 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test181 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec specs_path ../specs +Running test 182 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test182 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 183 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test183 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs Running test 187 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization adapts test 164 ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test187 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs +Running test 188 test type: optsyn, description: basic branched tree encoding test for dt_caretin model exploration mode optsyn +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test188 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding branched -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 189 test type: optsyn, description: basic branched tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn adapts test 166 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test189 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 190 test type: optimize, description: basic test for rf_caret with branched tree_encoding and modelper_response in model exploration mode optimize adapts test 168 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test190 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding branched -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs Running test 191 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response t in model exploration mode optimize adapts test 169 ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test191 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap t -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs +Running test 192 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test192 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 193 test type: optimize, description: basic test for et_caret with branched tree_encoding in model exploration mode optimize adapts test 171 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test193 -mode optimize -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs Running test 194 test type: optsyn, description: basic branched tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test194 -mode optsyn -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs +Running test 195 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 192 by setting n_estimators 3 and then discrepancy between z3 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test195 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs Running test 196 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 and test 181 ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test196 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 197 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 and test 182 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test197 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 198 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 and test 183 +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test198 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + spec_fn smlp_toy_num_resp_mult_optsyn.spec specs_path ../specs +Running test 199 test type: optimize, description: test to demonstrate that in pareto optimization and optsyn modes with multiple objectives when beta constraints are not present SMLP results are not consistent when different solvers are used; this is due to fact that when a subset of objectoves are exemined in pareto algo +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test199 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs Running test 200 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results ../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test200 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f +spec_fn smlp_toy_num_resp_mult_no_input_beta.spec +specs_path ../specs +Running test 201 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and there are beta constraints +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test201 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input_beta.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_no_input.spec +specs_path ../specs +Running test 202 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and no beta constraints +../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test202 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec_fn smlp_toy_num_resp_mult_y2_verify.spec +spec_fn smlp_toy_system_stable_constant_synth_fail.spec specs_path ../specs +Running test 203 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed adapts test 122 +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test203 -mode optimize -pareto f -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -Running test 155 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training -../../src/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test155 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 204 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 123 +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test204 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 205 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 145 +../../src/run_smlp.py -out_dir ./ -pref Test205 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 206 test type: optsyn, description: optimized synthesis test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 125 +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test206 -mode optsyn -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 215 test type: correlate, description: basic test for correlate mode +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test215 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 216 test type: correlate, description: basic test for correlate mode +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test216 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 217 test type: correlate, description: basic test for correlate mode +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test217 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 218 test type: correlate, description: basic test for correlate mode +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test218 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 219 test type: correlate, description: basic test for correlate mode +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test219 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type integer -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 220 test type: correlate, description: basic test for correlate mode +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test220 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 221 test type: correlate, description: basic test for correlate mode +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test221 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 222 test type: correlate, description: basic test for correlate mode +../../src/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test222 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 223 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test223 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 224 test type: correlate, description: basic test for correlate mode and tests the Shannon mutual information +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test224 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 225 test type: correlate, description: basic test for correlate mode and tests the adjusted mutual information +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test225 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 226 test type: correlate, description: basic test for correlate mode +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test226 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 227 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information +../../src/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test227 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -Initiating 7 worker... comparing Test1_smlp_toy_num_resp_mult_y1_dt_caret_tree_rules.txt to master Passed! comparing Test1_smlp_toy_num_resp_mult.txt to master @@ -1157,7 +966,6 @@ Passed! comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master Passed! comparing Test7_smlp_toy_num_resp_mult_rf_sklearn_tree_rules.txt to master -Passed! comparing Test7_smlp_toy_num_resp_mult_data_bounds.json to master Passed! comparing Test7_smlp_toy_num_resp_mult_model_features_dict.json to master @@ -1251,7 +1059,6 @@ Passed! comparing Test10_smlp_toy_num_resp_mult_data_bounds.json to master Passed! comparing Test10_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt to master -Passed! comparing Test10_smlp_toy_num_resp_mult_model_features_dict.json to master Passed! comparing Test10_smlp_toy_num_resp_mult_model_levels_dict.json to master @@ -1371,7 +1178,6 @@ Passed! comparing Test14_smlp_toy_basic_training_predictions_summary.csv to master Passed! comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1381,10 +1187,11 @@ Passed! comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! -File new Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist -File new Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist -comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master Passed! +comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1422,7 +1229,6 @@ Passed! comparing test19_model_rerun_model_config.json to master Passed! comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1430,7 +1236,6 @@ Passed! comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master Passed! comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1477,7 +1282,6 @@ Passed! comparing test22_model_rerun_model_config.json to master Passed! comparing Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt to master -Passed! File master Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_new-col-PF .png does not exist File master Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_new-col-|PF |.png does not exist comparing Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_missing_values_dict.json to master @@ -1518,7 +1322,6 @@ Passed! comparing test24_model_rerun_model_config.json to master Passed! comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1526,7 +1329,6 @@ Passed! comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master Passed! comparing test26_model_dt_sklearn_tree_rules.txt to master -Passed! comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master Passed! comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master @@ -1556,7 +1358,6 @@ Passed! comparing test26_model_rerun_model_config.json to master Passed! comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled.txt to master -Passed! comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master Passed! comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master @@ -1623,23 +1424,28 @@ comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_trai Passed! comparing Test29_smlp_toy_cls_metasymbol_colnames_mult.txt to master comparing Test29_smlp_toy_cls_metasymbol_colnames_mult_features_ranking.csv to master +Passed! comparing Test29_smlp_toy_cls_metasymbol_colnames_mult_missing_values_dict.json to master Passed! comparing Test29_smlp_toy_cls_metasymbol_colnames_mult_ranking_resp_feat.csv to master +Passed! comparing Test30_smlp_toy_num_resp_mult.txt to master comparing Test30_smlp_toy_num_resp_mult_features_ranking.csv to master +Passed! comparing Test30_smlp_toy_num_resp_mult_missing_values_dict.json to master Passed! comparing Test30_smlp_toy_num_resp_mult_ranking_resp_feat.csv to master +Passed! comparing Test31_smlp_toy_num_resp_mult.txt to master comparing Test31_smlp_toy_num_resp_mult_args_config.json to master Passed! comparing Test31_smlp_toy_num_resp_mult_features_ranking.csv to master +Passed! comparing Test31_smlp_toy_num_resp_mult_missing_values_dict.json to master Passed! comparing Test31_smlp_toy_num_resp_mult_ranking_resp_feat.csv to master -comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt to master Passed! +comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt to master comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled_args_config.json to master Passed! comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master @@ -1650,9 +1456,11 @@ comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction Passed! comparing Test33_smlp_toy_num_resp_mult.txt to master comparing Test33_smlp_toy_num_resp_mult_features_ranking.csv to master +Passed! comparing Test33_smlp_toy_num_resp_mult_missing_values_dict.json to master Passed! comparing Test33_smlp_toy_num_resp_mult_ranking_resp_feat.csv to master +Passed! comparing Test34_doe_four_levels_real.txt to master Passed! comparing Test34_doe_four_levels_real_doe.csv to master @@ -1726,7 +1534,6 @@ Passed! comparing test47_model_poly_sklearn_formula.txt to master Passed! comparing Test47_test47_model_smlp_toy_pf_mult.txt to master -Passed! comparing Test47_test47_model_smlp_toy_pf_mult_missing_values_dict.json to master Passed! comparing Test47_test47_model_smlp_toy_pf_mult_new_prediction_precisions.csv to master @@ -1901,7 +1708,6 @@ Passed! File master test63_model_y1_smlp_full_model_term.json does not exist File master test63_model_y1_smlp_model_term.json does not exist comparing Test64_test63_model.txt to master -Passed! File master Test64_test63_model_trace.csv does not exist comparing Test64_test63_model_verify_results.json to master Passed! @@ -1934,6 +1740,7 @@ Passed! comparing Test69_smlp_toy_num_resp_mult_training_predictions_summary.csv to master Passed! comparing Test69_smlp_toy_num_resp_mult_verify_results.json to master +Passed! comparing test69_model_data_bounds.json to master Passed! comparing test69_model_model_features_dict.json to master @@ -1948,7 +1755,8 @@ File master test69_model_y2_smlp_full_model_term.json does not exist File master test69_model_y2_smlp_model_term.json does not exist comparing Test70_test69_model.txt to master File master Test70_test69_model_trace.csv does not exist -File new Test70_test69_model_verify_results.json does not exist +comparing Test70_test69_model_verify_results.json to master +Passed! Test 71 Failed: Error in Build stage: Data file does not exist @@ -2537,7 +2345,6 @@ Passed! File master Test96_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist File master Test96_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist comparing Test97_smlp_toy_num_resp_mult.txt to master -Passed! comparing Test97_smlp_toy_num_resp_mult_data_bounds.json to master Passed! File master Test97_smlp_toy_num_resp_mult_et_sklearn_y1_tree_rules.txt does not exist @@ -2708,7 +2515,6 @@ File master test101_model_y1_smlp_model_term.json does not exist File master test101_model_y2_smlp_full_model_term.json does not exist File master test101_model_y2_smlp_model_term.json does not exist comparing Test102_test101_model.txt to master -Passed! comparing Test102_test101_model_certify_results.json to master Passed! File master Test102_test101_model_trace.csv does not exist @@ -2935,11 +2741,9 @@ comparing test110_model_poly_sklearn_formula.txt to master comparing test110_model_rerun_model_config.json to master Passed! comparing Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt to master -Passed! comparing Test111_test110_model_smlp_toy_basic_pred_unlabeled_new_predictions_summary.csv to master Passed! comparing Test112_test110_model_smlp_toy_basic_pred_unlabeled.txt to master -Passed! comparing Test112_test110_model_smlp_toy_basic_pred_unlabeled_new_predictions_summary.csv to master Passed! comparing test113_model_dt_sklearn_tree_rules.txt to master @@ -3619,8 +3423,19 @@ File master Test160_smlp_toy_num_resp_mult_data_bounds.json does not exist File master Test160_smlp_toy_num_resp_mult_features_scaler.pkl does not exist File master Test160_smlp_toy_num_resp_mult_model_features_dict.json does not exist File master Test160_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test160_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist File master Test160_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y1_msle.png does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_msle.png does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist File master Test161_smlp_toy_num_resp_mult_data_bounds.json does not exist File master Test161_smlp_toy_num_resp_mult_features_scaler.pkl does not exist @@ -4517,5 +4332,5 @@ File master Test227_smlp_toy_basic_features_summary.csv does not exist master log file does not exist! Do you wish to copy the new log file to master? (yes/no|y/n): No new tests crashed (not in the masters) -Time: 37.54873522520065 minutes +Time: 31.98271323442459 minutes End of regression diff --git a/tests/smlp_regression/run_smlp_regression_expected_diff_report.log b/tests/smlp_regression/run_smlp_regression_expected_diff_report.log index 31607643..7db436c3 100644 --- a/tests/smlp_regression/run_smlp_regression_expected_diff_report.log +++ b/tests/smlp_regression/run_smlp_regression_expected_diff_report.log @@ -1,331 +1,411 @@ +=================== Diff report for: Test7_smlp_toy_num_resp_mult_rf_sklearn_tree_rules.txt ================================== +94d93 +< if (p2 > 0.4000000134110451) and (p1 <= 0.75) and (p2 > 0.7000000178813934) and (x <= 0.6666666716337204) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +96c95 +< if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.4000000134110451) and (p1 <= 0.75) and (p2 > 0.7000000178813934) and (x <= 0.6666666716337204) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +97a97 +> if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +=================== End of Test7_smlp_toy_num_resp_mult_rf_sklearn_tree_rules.txt diff report ================================ +=================== Diff report for: Test10_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt ================================== +6d5 +< if (p1 > 0.7673577288013687) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +7a7 +> if (p1 > 0.7673577288013687) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +21d20 +< if (p2 > 0.565498446377692) and (p1 > 0.21566598080828134) and (p2 > 0.7262518305173236) and (x > 0.03081251758592215) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +22a22 +> if (p2 > 0.565498446377692) and (p1 > 0.21566598080828134) and (p2 > 0.7262518305173236) and (x > 0.03081251758592215) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +44d43 +< if (p2 > 0.05282566885129813) and (p1 > 0.9621611074368288) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +45a45 +> if (p2 > 0.05282566885129813) and (p1 > 0.9621611074368288) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +66d65 +< if (p2 > 0.10769168804757841) and (p2 > 0.9843916629018533) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +67a67 +> if (p2 > 0.10769168804757841) and (p2 > 0.9843916629018533) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +73d72 +< if (p1 <= 0.9643084043470717) and (p2 > 0.7106753814549537) and (x <= 0.7383051325780686) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +75c74 +< if (p1 > 0.9643084043470717) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.9643084043470717) and (p2 > 0.7106753814549537) and (x <= 0.7383051325780686) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +76a76 +> if (p1 > 0.9643084043470717) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +88d87 +< if (p2 > 0.37253817607301204) and (p1 > 0.5069297996847564) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +89a89 +> if (p2 > 0.37253817607301204) and (p1 > 0.5069297996847564) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +96d95 +< if (p1 > 0.8097833990164955) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +97a97 +> if (p1 > 0.8097833990164955) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +103d102 +< if (p2 > 0.30979522099243256) and (p2 > 0.7974478444662528) and (x <= 0.7718040145802331) and (p1 <= 0.9832575130198419) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +105c104 +< if (p2 > 0.30979522099243256) and (p2 > 0.7974478444662528) and (x > 0.7718040145802331) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +--- +> if (p2 > 0.30979522099243256) and (p2 > 0.7974478444662528) and (x <= 0.7718040145802331) and (p1 <= 0.9832575130198419) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +106a106 +> if (p2 > 0.30979522099243256) and (p2 > 0.7974478444662528) and (x > 0.7718040145802331) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +118d117 +< if (p1 > 0.6478692095949636) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +119a119 +> if (p1 > 0.6478692095949636) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +126d125 +< if (p2 > 0.5083941302766997) and (p1 > 0.9604900148513215) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +127a127 +> if (p2 > 0.5083941302766997) and (p1 > 0.9604900148513215) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +134d133 +< if (p2 > 0.38305688667253446) and (p1 > 0.2547155522064844) and (p1 > 0.7231216464299344) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +135a135 +> if (p2 > 0.38305688667253446) and (p1 > 0.2547155522064844) and (p1 > 0.7231216464299344) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +141d140 +< if (p1 <= 0.5519522472992318) and (p2 > 0.33294736609465786) and (p2 > 0.7309142946144038) and (x <= 0.6016799023753295) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +143c142 +< if (p1 > 0.5519522472992318) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.5519522472992318) and (p2 > 0.33294736609465786) and (p2 > 0.7309142946144038) and (x <= 0.6016799023753295) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +144a144 +> if (p1 > 0.5519522472992318) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +155d154 +< if (p1 <= 0.5757871147132418) and (p2 > 0.39527962049249543) and (p2 > 0.6458938131907435) and (x <= 0.6096057855068803) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +157c156 +< if (p1 > 0.5757871147132418) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.5757871147132418) and (p2 > 0.39527962049249543) and (p2 > 0.6458938131907435) and (x <= 0.6096057855068803) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +158a158 +> if (p1 > 0.5757871147132418) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +204d203 +< if (p1 <= 0.8026946730924279) and (p2 > 0.05688018773537138) and (p2 > 0.4364986121396114) and (p2 > 0.6777798791052577) and (x <= 0.6717608829535293) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +206c205 +< if (p1 > 0.8026946730924279) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.8026946730924279) and (p2 > 0.05688018773537138) and (p2 > 0.4364986121396114) and (p2 > 0.6777798791052577) and (x <= 0.6717608829535293) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +207a207 +> if (p1 > 0.8026946730924279) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +218d217 +< if (p2 > 0.5397191641571555) and (p1 <= 0.7132398065706901) and (p2 > 0.634045960603359) and (x <= 0.6156377025174744) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +220c219 +< if (p2 > 0.5397191641571555) and (p1 > 0.7132398065706901) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.5397191641571555) and (p1 <= 0.7132398065706901) and (p2 > 0.634045960603359) and (x <= 0.6156377025174744) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +221a221 +> if (p2 > 0.5397191641571555) and (p1 > 0.7132398065706901) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +233d232 +< if (p1 > 0.6806756624396312) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +234a234 +> if (p1 > 0.6806756624396312) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +241d240 +< if (p2 > 0.4943563777461445) and (p1 > 0.920448066629678) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +242a242 +> if (p2 > 0.4943563777461445) and (p1 > 0.920448066629678) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +264d263 +< if (p1 > 0.7209493553829144) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +265a265 +> if (p1 > 0.7209493553829144) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +286d285 +< if (p2 > 0.9499151226831205) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +287a287 +> if (p2 > 0.9499151226831205) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +300d299 +< if (p2 > 0.4160567286499109) and (p1 <= 0.7821608059025187) and (p2 > 0.7500101945124135) and (x <= 0.8482964849267818) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +302c301 +< if (p2 > 0.4160567286499109) and (p1 > 0.7821608059025187) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.4160567286499109) and (p1 <= 0.7821608059025187) and (p2 > 0.7500101945124135) and (x <= 0.8482964849267818) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +303a303 +> if (p2 > 0.4160567286499109) and (p1 > 0.7821608059025187) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +357d356 +< if (p1 <= 0.6838882522116826) and (p2 > 0.0626814738207876) and (p2 > 0.3422835304420513) and (p2 > 0.6023659933821865) and (x <= 0.7909361110756394) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +359c358 +< if (p1 > 0.6838882522116826) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.6838882522116826) and (p2 > 0.0626814738207876) and (p2 > 0.3422835304420513) and (p2 > 0.6023659933821865) and (x <= 0.7909361110756394) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +360a360 +> if (p1 > 0.6838882522116826) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +364d363 +< if (p2 > 0.40336603038169727) and (p1 <= 0.7296289325364069) and (p2 > 0.6471116276536257) and (x <= 0.4249370103517018) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +366c365 +< if (p2 > 0.40336603038169727) and (p1 > 0.7296289325364069) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.40336603038169727) and (p1 <= 0.7296289325364069) and (p2 > 0.6471116276536257) and (x <= 0.4249370103517018) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +367a367 +> if (p2 > 0.40336603038169727) and (p1 > 0.7296289325364069) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +379d378 +< if (p1 > 0.5139580394672034) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +380a380 +> if (p1 > 0.5139580394672034) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +387d386 +< if (p2 > 0.8305540819794657) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +388a388 +> if (p2 > 0.8305540819794657) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +395d394 +< if (p2 > 0.9449580902908733) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +396a396 +> if (p2 > 0.9449580902908733) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +410d409 +< if (p1 > 0.5120007179267708) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +411a411 +> if (p1 > 0.5120007179267708) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +417d416 +< if (p1 <= 0.610545704891228) and (p2 > 0.6501177091384854) and (x <= 0.7877861639695688) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +419c418 +< if (p1 > 0.610545704891228) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.610545704891228) and (p2 > 0.6501177091384854) and (x <= 0.7877861639695688) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +420a420 +> if (p1 > 0.610545704891228) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +424d423 +< if (p2 > 0.45482464342137086) and (p1 <= 0.6847267941034989) and (p2 > 0.7332769920461923) and (x <= 0.7843162575827582) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +426c425 +< if (p2 > 0.45482464342137086) and (p1 > 0.6847267941034989) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.45482464342137086) and (p1 <= 0.6847267941034989) and (p2 > 0.7332769920461923) and (x <= 0.7843162575827582) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +427a427 +> if (p2 > 0.45482464342137086) and (p1 > 0.6847267941034989) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +446d445 +< if (p2 > 0.054852516881587224) and (p1 > 0.8053342741007611) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +447a447 +> if (p2 > 0.054852516881587224) and (p1 > 0.8053342741007611) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +461d460 +< if (p2 > 0.35838276676758324) and (p1 > 0.6469177723149386) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +462a462 +> if (p2 > 0.35838276676758324) and (p1 > 0.6469177723149386) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +476d475 +< if (p1 > 0.6586678422329332) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +477a477 +> if (p1 > 0.6586678422329332) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +483d482 +< if (p2 > 0.6300749409152544) and (p1 <= 0.9968296801656623) and (x <= 0.37232690124052253) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +485c484 +< if (p2 > 0.6300749409152544) and (p1 > 0.9968296801656623) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.6300749409152544) and (p1 <= 0.9968296801656623) and (x <= 0.37232690124052253) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +486a486 +> if (p2 > 0.6300749409152544) and (p1 > 0.9968296801656623) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +497d496 +< if (p2 > 0.37996283470651265) and (p1 <= 0.5025284804881217) and (p2 > 0.7656687748432474) and (x <= 0.5092783321373655) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +499c498 +< if (p2 > 0.37996283470651265) and (p1 > 0.5025284804881217) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.37996283470651265) and (p1 <= 0.5025284804881217) and (p2 > 0.7656687748432474) and (x <= 0.5092783321373655) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +500a500 +> if (p2 > 0.37996283470651265) and (p1 > 0.5025284804881217) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +505d504 +< if (p2 > 0.22414377714700243) and (p1 > 0.904138504017209) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +506a506 +> if (p2 > 0.22414377714700243) and (p1 > 0.904138504017209) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +513d512 +< if (p1 > 0.8937446875002909) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +514a514 +> if (p1 > 0.8937446875002909) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +535d534 +< if (p2 > 0.07422745696931493) and (p2 > 0.545370601481947) and (p2 > 0.9300510326789317) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +536a536 +> if (p2 > 0.07422745696931493) and (p2 > 0.545370601481947) and (p2 > 0.9300510326789317) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +542d541 +< if (p2 > 0.7889165751584417) and (x <= 0.9135558618761394) and (p1 <= 0.8191675724550931) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +544c543 +< if (p2 > 0.7889165751584417) and (x > 0.9135558618761394) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +--- +> if (p2 > 0.7889165751584417) and (x <= 0.9135558618761394) and (p1 <= 0.8191675724550931) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +545a545 +> if (p2 > 0.7889165751584417) and (x > 0.9135558618761394) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +549d548 +< if (p2 > 0.09689574087825406) and (p2 > 0.33636798963203285) and (p1 <= 0.8197608151565123) and (p2 > 0.6876566959859812) and (x <= 0.6647538383146583) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +551c550 +< if (p2 > 0.09689574087825406) and (p2 > 0.33636798963203285) and (p1 > 0.8197608151565123) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.09689574087825406) and (p2 > 0.33636798963203285) and (p1 <= 0.8197608151565123) and (p2 > 0.6876566959859812) and (x <= 0.6647538383146583) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +552a552 +> if (p2 > 0.09689574087825406) and (p2 > 0.33636798963203285) and (p1 > 0.8197608151565123) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +556d555 +< if (p2 > 0.16337258050375272) and (p2 > 0.20940975581022525) and (x <= 0.792321881648303) and (p1 <= 0.574456778622445) and (p2 > 0.7807007138854876) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +558c557 +< if (p2 > 0.16337258050375272) and (p2 > 0.20940975581022525) and (x > 0.792321881648303) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +--- +> if (p2 > 0.16337258050375272) and (p2 > 0.20940975581022525) and (x <= 0.792321881648303) and (p1 <= 0.574456778622445) and (p2 > 0.7807007138854876) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +559a559 +> if (p2 > 0.16337258050375272) and (p2 > 0.20940975581022525) and (x > 0.792321881648303) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +563d562 +< if (p2 > 0.7007179663985585) and (x <= 0.47267748344348626) and (p1 <= 0.5654538877147501) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +565c564 +< if (p2 > 0.7007179663985585) and (x > 0.47267748344348626) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +--- +> if (p2 > 0.7007179663985585) and (x <= 0.47267748344348626) and (p1 <= 0.5654538877147501) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +566a566 +> if (p2 > 0.7007179663985585) and (x > 0.47267748344348626) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +570d569 +< if (p2 > 0.30978463977099613) and (p1 <= 0.6320834037065894) and (p2 > 0.6127691589194908) and (x <= 0.6760619000417248) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +572c571 +< if (p2 > 0.30978463977099613) and (p1 > 0.6320834037065894) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.30978463977099613) and (p1 <= 0.6320834037065894) and (p2 > 0.6127691589194908) and (x <= 0.6760619000417248) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +573a573 +> if (p2 > 0.30978463977099613) and (p1 > 0.6320834037065894) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +578d577 +< if (p1 > 0.6819941814439344) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +579a579 +> if (p1 > 0.6819941814439344) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +600d599 +< if (p1 > 0.5931505284240239) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +601a601 +> if (p1 > 0.5931505284240239) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +631d630 +< if (p2 > 0.3774126001528523) and (p1 <= 0.6553700527434098) and (p2 > 0.7353104082940054) and (x <= 0.4155191624469929) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +633c632 +< if (p2 > 0.3774126001528523) and (p1 > 0.6553700527434098) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.3774126001528523) and (p1 <= 0.6553700527434098) and (p2 > 0.7353104082940054) and (x <= 0.4155191624469929) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +634a634 +> if (p2 > 0.3774126001528523) and (p1 > 0.6553700527434098) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +638d637 +< if (p2 > 0.23018040425618197) and (p1 <= 0.7102524464532046) and (p2 > 0.7248063887234734) and (x <= 0.7892120132227867) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +640c639 +< if (p2 > 0.23018040425618197) and (p1 > 0.7102524464532046) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.23018040425618197) and (p1 <= 0.7102524464532046) and (p2 > 0.7248063887234734) and (x <= 0.7892120132227867) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +641a641 +> if (p2 > 0.23018040425618197) and (p1 > 0.7102524464532046) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +653d652 +< if (p2 > 0.21158053456413584) and (p1 > 0.5586915479780601) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +654a654 +> if (p2 > 0.21158053456413584) and (p1 > 0.5586915479780601) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +661d660 +< if (p1 > 0.6795984351446844) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +662a662 +> if (p1 > 0.6795984351446844) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +677d676 +< if (p1 > 0.8126054242312002) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +678a678 +> if (p1 > 0.8126054242312002) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +684d683 +< if (p2 > 0.26886312862339573) and (p1 <= 0.8950548846717248) and (p2 > 0.6824239181038759) and (x <= 0.7910962169102163) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +686c685 +< if (p2 > 0.26886312862339573) and (p1 > 0.8950548846717248) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.26886312862339573) and (p1 <= 0.8950548846717248) and (p2 > 0.6824239181038759) and (x <= 0.7910962169102163) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +687a687 +> if (p2 > 0.26886312862339573) and (p1 > 0.8950548846717248) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +698d697 +< if (p2 > 0.25803846924474394) and (p1 <= 0.85781003667871) and (p2 > 0.6174925668996046) and (x <= 0.4784058184220548) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +700c699 +< if (p2 > 0.25803846924474394) and (p1 > 0.85781003667871) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.25803846924474394) and (p1 <= 0.85781003667871) and (p2 > 0.6174925668996046) and (x <= 0.4784058184220548) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +701a701 +> if (p2 > 0.25803846924474394) and (p1 > 0.85781003667871) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +705d704 +< if (p2 > 0.16944414757631912) and (p1 <= 0.7608029128801092) and (p2 > 0.31309513934344707) and (p2 > 0.7650359471516853) and (x <= 0.6112039253981838) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +707c706 +< if (p2 > 0.16944414757631912) and (p1 > 0.7608029128801092) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.16944414757631912) and (p1 <= 0.7608029128801092) and (p2 > 0.31309513934344707) and (p2 > 0.7650359471516853) and (x <= 0.6112039253981838) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +708a708 +> if (p2 > 0.16944414757631912) and (p1 > 0.7608029128801092) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +728d727 +< if (p2 > 0.29657478970316) and (p1 > 0.8279177280272906) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +729a729 +> if (p2 > 0.29657478970316) and (p1 > 0.8279177280272906) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +736d735 +< if (p2 > 0.30543398172847647) and (p1 > 0.7651434488432218) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +737a737 +> if (p2 > 0.30543398172847647) and (p1 > 0.7651434488432218) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +=================== End of Test10_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt diff report ================================ +=================== Diff report for: Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/Test5_smlp_toy_num_resp_mult_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/Test5_smlp_toy_num_resp_mult_rerun_model_config.json +=================== End of Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ =================== Diff report for: Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt ================================== -87a88,111 -> -> smlp_logger - INFO - PREDICT ON NEW DATA -> -> smlp_logger - INFO - Model prediction: start -> -> smlp_logger - INFO - Model prediction: end -> -> smlp_logger - INFO - Reporting prediction results: start -> -> smlp_logger - INFO - Saving predictions summary into file: -> ./Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv -> -> smlp_logger - INFO - Saving prediction precisions into file: -> ./Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv -> -> smlp_logger - INFO - Prediction on new data -- msqe: 8.026 -> -> smlp_logger - INFO - Prediction on new data -- r2_score: -1.032 -> -> smlp_logger - INFO - Reporting prediction results: end -> -> smlp_logger - INFO - Running SMLP in mode "predict": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/Test8_smlp_toy_num_resp_mult_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/Test8_smlp_toy_num_resp_mult_rerun_model_config.json =================== End of Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ -=================== Diff report for: Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv ================================== -diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv: No such file or directory -=================== End of Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv diff report ================================ -=================== Diff report for: Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv ================================== -diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv: No such file or directory -=================== End of Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv diff report ================================ +=================== Diff report for: Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/Test11_smlp_toy_num_resp_mult_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/Test11_smlp_toy_num_resp_mult_rerun_model_config.json +=================== End of Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test19_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test19_model_rerun_model_config.json +=================== End of Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +75c75 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test20_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test20_model_rerun_model_config.json +=================== End of Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt ================================== +75c75 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test22_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test22_model_rerun_model_config.json +=================== End of Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt diff report ================================ +=================== Diff report for: Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test24_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test24_model_rerun_model_config.json +=================== End of Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: test26_model_dt_sklearn_tree_rules.txt ================================== +6d5 +< if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +7a7 +> if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +=================== End of test26_model_dt_sklearn_tree_rules.txt diff report ================================ +=================== Diff report for: Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test26_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test26_model_rerun_model_config.json +=================== End of Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ =================== Diff report for: Test29_smlp_toy_cls_metasymbol_colnames_mult.txt ================================== 95,96d94 < smlp_logger - WARNING - Range plots are not supported in this version of SMLP < =================== End of Test29_smlp_toy_cls_metasymbol_colnames_mult.txt diff report ================================ -=================== Diff report for: Test29_smlp_toy_cls_metasymbol_colnames_mult_features_ranking.csv ================================== -2,3c2,8 -< PF 1,0,0,1,0.36363636363636365,0.48104569292083466,FMAX(xyz)_9.0_9.0_Bin_0__FMAX.xyz._4.0_4.0_Bin_0,FMAX(xyz),9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,FMAX.xyz.,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,5,none,0~~4~~9,1~~2~~3~~6~~7~~8~~10 -< PF 1,0,0,1,0.36363636363636365,0.48104569292083466,FMAX(xyz)_9.0_9.0_Bin_0__categ__c10,FMAX(xyz),9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,5,none,0~~4~~9,1~~2~~3~~6~~7~~8~~10 ---- -> PF 1,0,0,1,0.36363636363636365,0.48104569292083466,p-3_8.0_8.0_Bin_0,p-3,8.0:8.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,4,none,0~~5~~9,1~~2~~3~~6~~7~~8~~10 -> PF 1,0,0,1,0.36363636363636365,0.48104569292083466,p-3_3.0_3.0_Bin_0,p-3,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,0,none,4~~5~~9,1~~2~~3~~6~~7~~8~~10 -> PF 1,0,0,1,0.36363636363636365,0.48104569292083466,categ__c5,categ,c5,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,4,none,0~~5~~9,1~~2~~3~~6~~7~~8~~10 -> PF 1,0,0,1,0.36363636363636365,0.48104569292083466,categ__c14,categ,c14,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,0,none,4~~5~~9,1~~2~~3~~6~~7~~8~~10 -> PF 1,0,0,1,0.36363636363636365,0.48104569292083466,categ__c11,categ,c11,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,9,none,0~~4~~5,1~~2~~3~~6~~7~~8~~10 -> PF 1,0,0,1,0.36363636363636365,0.48104569292083466,categ__c10,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,5,none,0~~4~~9,1~~2~~3~~6~~7~~8~~10 -> PF 1,0,0,1,0.36363636363636365,0.48104569292083466,FMAX.xyz._4.0_4.0_Bin_0__categ__c10,FMAX.xyz.,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,5,none,0~~4~~9,1~~2~~3~~6~~7~~8~~10 -5,6c10,17 -< PF#,0,0,1,0.45454545454545453,0.49792959773196915,FMAX.xyz._3.0_3.0_Bin_0__categ__c19,FMAX.xyz.,3.0:3.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,categ,c19,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,10,none,0~~1~~5~~7,2~~3~~4~~6~~8~~9 -< PF#,0,0,1,0.45454545454545453,0.49792959773196915,FMAX.xyz._3.0_3.0_Bin_0__categ__c4,FMAX.xyz.,3.0:3.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,categ,c4,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,7,none,0~~1~~5~~10,2~~3~~4~~6~~8~~9 ---- -> PF 1,0,0,1,0.36363636363636365,0.48104569292083466,FMAX(xyz)_9.0_9.0_Bin_0__categ__c10,FMAX(xyz),9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,5,none,0~~4~~9,1~~2~~3~~6~~7~~8~~10 -> PF 1,0,0,1,0.36363636363636365,0.48104569292083466,FMAX(xyz)_9.0_9.0_Bin_0__FMAX.xyz._4.0_4.0_Bin_0,FMAX(xyz),9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,FMAX.xyz.,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,5,none,0~~4~~9,1~~2~~3~~6~~7~~8~~10 -> PF#,0,0,1,0.45454545454545453,0.49792959773196915,p-3_3.0_3.0_Bin_0,p-3,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~5~~7~~10,2~~3~~4~~6~~8~~9 -> PF#,0,0,1,0.45454545454545453,0.49792959773196915,categ__c4,categ,c4,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,7,none,0~~1~~5~~10,2~~3~~4~~6~~8~~9 -> PF#,0,0,1,0.45454545454545453,0.49792959773196915,categ__c19,categ,c19,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,10,none,0~~1~~5~~7,2~~3~~4~~6~~8~~9 -> PF#,0,0,1,0.45454545454545453,0.49792959773196915,categ__c14,categ,c14,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~5~~7~~10,2~~3~~4~~6~~8~~9 -> PF#,0,0,1,0.45454545454545453,0.49792959773196915,categ__c10,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~7~~10,2~~3~~4~~6~~8~~9 -> PF#,0,0,1,0.45454545454545453,0.49792959773196915,FMAX.xyz._4.0_4.0_Bin_0__categ__c10,FMAX.xyz.,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~7~~10,2~~3~~4~~6~~8~~9 -7a19,20 -> PF#,0,0,1,0.45454545454545453,0.49792959773196915,FMAX.xyz._3.0_3.0_Bin_0__categ__c4,FMAX.xyz.,3.0:3.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,categ,c4,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,7,none,0~~1~~5~~10,2~~3~~4~~6~~8~~9 -> PF#,0,0,1,0.45454545454545453,0.49792959773196915,FMAX.xyz._3.0_3.0_Bin_0__categ__c19,FMAX.xyz.,3.0:3.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,categ,c19,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,10,none,0~~1~~5~~7,2~~3~~4~~6~~8~~9 -9,21d21 -< PF 1,0,0,1,0.36363636363636365,0.48104569292083466,FMAX.xyz._4.0_4.0_Bin_0__categ__c10,FMAX.xyz.,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,5,none,0~~4~~9,1~~2~~3~~6~~7~~8~~10 -< PF#,0,0,1,0.45454545454545453,0.49792959773196915,FMAX.xyz._4.0_4.0_Bin_0__categ__c10,FMAX.xyz.,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~7~~10,2~~3~~4~~6~~8~~9 -< PF 1,0,0,1,0.36363636363636365,0.48104569292083466,categ__c10,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,5,none,0~~4~~9,1~~2~~3~~6~~7~~8~~10 -< PF#,0,0,1,0.45454545454545453,0.49792959773196915,categ__c10,categ,c10,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~7~~10,2~~3~~4~~6~~8~~9 -< PF 1,0,0,1,0.36363636363636365,0.48104569292083466,categ__c11,categ,c11,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,9,none,0~~4~~5,1~~2~~3~~6~~7~~8~~10 -< PF 1,0,0,1,0.36363636363636365,0.48104569292083466,categ__c14,categ,c14,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,0,none,4~~5~~9,1~~2~~3~~6~~7~~8~~10 -< PF#,0,0,1,0.45454545454545453,0.49792959773196915,categ__c14,categ,c14,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~5~~7~~10,2~~3~~4~~6~~8~~9 -< PF#,0,0,1,0.45454545454545453,0.49792959773196915,categ__c19,categ,c19,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,10,none,0~~1~~5~~7,2~~3~~4~~6~~8~~9 -< PF#,0,0,1,0.45454545454545453,0.49792959773196915,categ__c4,categ,c4,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,7,none,0~~1~~5~~10,2~~3~~4~~6~~8~~9 -< PF 1,0,0,1,0.36363636363636365,0.48104569292083466,categ__c5,categ,c5,NA:NA,NA,,,,,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,4,none,0~~5~~9,1~~2~~3~~6~~7~~8~~10 -< PF 1,0,0,1,0.36363636363636365,0.48104569292083466,p-3_3.0_3.0_Bin_0,p-3,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,0,none,4~~5~~9,1~~2~~3~~6~~7~~8~~10 -< PF#,0,0,1,0.45454545454545453,0.49792959773196915,p-3_3.0_3.0_Bin_0,p-3,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~5~~7~~10,2~~3~~4~~6~~8~~9 -< PF 1,0,0,1,0.36363636363636365,0.48104569292083466,p-3_8.0_8.0_Bin_0,p-3,8.0:8.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.75,PSG,3.0,7.0,1.0,0.0,,,0.625,2.75,1.0,0.5289,0.625,0.4,0.34,0.7273,0.6066,4,none,0~~5~~9,1~~2~~3~~6~~7~~8~~10 -=================== End of Test29_smlp_toy_cls_metasymbol_colnames_mult_features_ranking.csv diff report ================================ -=================== Diff report for: Test29_smlp_toy_cls_metasymbol_colnames_mult_ranking_resp_feat.csv ================================== -1,12c1,12 -< FMAX(xyz),FMAX.xyz.,categ,p-3,PF 1,PF# -< 10.0,2.0,c14,3,1,1 -< 12.0,2.0,c15,4,0,1 -< 10.0,3.0,c1,4,0,0 -< 11.0,2.0,c9,6,0,0 -< 10.0,2.0,c5,8,1,0 -< 9.0,4.0,c10,7,1,1 -< 9.0,3.0,c13,6,0,0 -< 10.0,3.0,c4,4,0,1 -< 11.0,4.0,c15,4,0,0 -< 12.0,2.0,c11,7,1,0 -< 10.0,3.0,c19,7,0,1 ---- -> p-3,categ,FMAX.xyz.,FMAX(xyz),PF 1,PF# -> 3,c14,2.0,10.0,1,1 -> 4,c15,2.0,12.0,0,1 -> 4,c1,3.0,10.0,0,0 -> 6,c9,2.0,11.0,0,0 -> 8,c5,2.0,10.0,1,0 -> 7,c10,4.0,9.0,1,1 -> 6,c13,3.0,9.0,0,0 -> 4,c4,3.0,10.0,0,1 -> 4,c15,4.0,11.0,0,0 -> 7,c11,2.0,12.0,1,0 -> 7,c19,3.0,10.0,0,1 -=================== End of Test29_smlp_toy_cls_metasymbol_colnames_mult_ranking_resp_feat.csv diff report ================================ =================== Diff report for: Test30_smlp_toy_num_resp_mult.txt ================================== 95,96d94 < smlp_logger - WARNING - Range plots are not supported in this version of SMLP < =================== End of Test30_smlp_toy_num_resp_mult.txt diff report ================================ -=================== Diff report for: Test30_smlp_toy_num_resp_mult_features_ranking.csv ================================== -2,13d1 -< y2,0,5,9,6.818181818181818,1.9917183909278766,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y1,0,5,9,6.818181818181818,1.9917183909278766,p1_2.0_2.0_Bin_0__p2_4.0_4.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,4.0:4.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y2,0,5,9,6.818181818181818,1.9917183909278766,p1_2.0_2.0_Bin_0__p2_4.0_4.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,4.0:4.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y1,0,5,9,6.818181818181818,1.9917183909278766,p1_2.0_2.0_Bin_0__p2_8.0_8.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,8.0:8.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y1,0,5,9,6.818181818181818,1.9917183909278766,p1_4.0_4.0_Bin_0,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.32,PSG,3.0,6.0,2.0,0.0,,,0.7,2.2,1.0,0.5496,0.7,0.5714,0.4762,0.7273,0.6749 -< y2,0,5,9,6.818181818181818,1.9917183909278766,p1_4.0_4.0_Bin_0,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.32,PSG,3.0,6.0,2.0,0.0,,,0.7,2.2,1.0,0.5496,0.7,0.5714,0.4762,0.7273,0.6749 -< y2,0,5,9,6.818181818181818,1.9917183909278766,p2_3.0_3.0_Bin_0,p2,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y1,0,5,9,6.818181818181818,1.9917183909278766,p2_4.0_4.0_Bin_0__p1_4.0_4.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y1,0,5,9,6.818181818181818,1.9917183909278766,p2_4.0_4.0_Bin_0__x_11.0_11.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y1,0,5,9,6.818181818181818,1.9917183909278766,p2_4.0_4.0_Bin_0__x_12.0_12.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,12.0:12.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y2,0,5,9,6.818181818181818,1.9917183909278766,p2_7.0_7.0_Bin_0__p1_4.0_4.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y2,0,5,9,6.818181818181818,1.9917183909278766,p2_7.0_7.0_Bin_0__x_9.0_9.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -15,16d2 -< y2,0,5,9,6.818181818181818,1.9917183909278766,x_10.0_10.0_Bin_0__p2_3.0_3.0_Bin_0,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -< y1,0,5,9,6.818181818181818,1.9917183909278766,x_10.0_10.0_Bin_0__p2_7.0_7.0_Bin_0,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -17a4,7 -> y1,0,5,9,6.818181818181818,1.9917183909278766,x_10.0_10.0_Bin_0__p2_7.0_7.0_Bin_0,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y1,0,5,9,6.818181818181818,1.9917183909278766,p2_4.0_4.0_Bin_0__x_12.0_12.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,12.0:12.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y1,0,5,9,6.818181818181818,1.9917183909278766,p2_4.0_4.0_Bin_0__x_11.0_11.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y1,0,5,9,6.818181818181818,1.9917183909278766,p1_4.0_4.0_Bin_0,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.32,PSG,3.0,6.0,2.0,0.0,,,0.7,2.2,1.0,0.5496,0.7,0.5714,0.4762,0.7273,0.6749 -19c9,12 -< y2,0,5,9,6.818181818181818,1.9917183909278766,x_11.0_11.0_Bin_0__p1_4.0_4.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 ---- -> y1,0,5,9,6.818181818181818,1.9917183909278766,p2_4.0_4.0_Bin_0__p1_4.0_4.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y1,0,5,9,6.818181818181818,1.9917183909278766,p1_2.0_2.0_Bin_0__p2_8.0_8.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,8.0:8.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y1,0,5,9,6.818181818181818,1.9917183909278766,p1_2.0_2.0_Bin_0__p2_4.0_4.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,4.0:4.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y2,0,5,9,6.818181818181818,1.9917183909278766,p2_7.0_7.0_Bin_0__x_9.0_9.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -20a14,16 -> y2,0,5,9,6.818181818181818,1.9917183909278766,p2_3.0_3.0_Bin_0,p2,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y2,0,5,9,6.818181818181818,1.9917183909278766,x_10.0_10.0_Bin_0__p2_3.0_3.0_Bin_0,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y2,0,5,9,6.818181818181818,1.9917183909278766,p1_4.0_4.0_Bin_0,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.32,PSG,3.0,6.0,2.0,0.0,,,0.7,2.2,1.0,0.5496,0.7,0.5714,0.4762,0.7273,0.6749 -21a18,21 -> y2,0,5,9,6.818181818181818,1.9917183909278766,x_11.0_11.0_Bin_0__p1_4.0_4.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y2,0,5,9,6.818181818181818,1.9917183909278766,p2_7.0_7.0_Bin_0__p1_4.0_4.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y2,0,5,9,6.818181818181818,1.9917183909278766,p1_2.0_2.0_Bin_0__p2_4.0_4.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,4.0:4.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -> y2,0,5,9,6.818181818181818,1.9917183909278766,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.32,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659 -=================== End of Test30_smlp_toy_num_resp_mult_features_ranking.csv diff report ================================ -=================== Diff report for: Test30_smlp_toy_num_resp_mult_ranking_resp_feat.csv ================================== -1,12c1,12 -< p1,p2,x,y1,y2 -< 2.0,3,10.0,5,9 -< 2.0,4,12.0,9,9 -< 3.0,4,10.0,5,9 -< 2.0,6,11.0,5,5 -< 2.0,8,10.0,9,5 -< 4.0,7,9.0,9,9 -< 3.0,6,9.0,5,5 -< 3.0,4,10.0,5,5 -< 4.0,4,11.0,9,9 -< 2.0,7,12.0,5,5 -< 3.0,7,10.0,9,5 ---- -> p2,x,p1,y1,y2 -> 3,10.0,2.0,5,9 -> 4,12.0,2.0,9,9 -> 4,10.0,3.0,5,9 -> 6,11.0,2.0,5,5 -> 8,10.0,2.0,9,5 -> 7,9.0,4.0,9,9 -> 6,9.0,3.0,5,5 -> 4,10.0,3.0,5,5 -> 4,11.0,4.0,9,9 -> 7,12.0,2.0,5,5 -> 7,10.0,3.0,9,5 -=================== End of Test30_smlp_toy_num_resp_mult_ranking_resp_feat.csv diff report ================================ =================== Diff report for: Test31_smlp_toy_num_resp_mult.txt ================================== 95,96d94 < smlp_logger - WARNING - Range plots are not supported in this version of SMLP < =================== End of Test31_smlp_toy_num_resp_mult.txt diff report ================================ -=================== Diff report for: Test31_smlp_toy_num_resp_mult_features_ranking.csv ================================== -2,7c2,5 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,0,none,2~~3~~6~~7~~9,1~~4~~5~~8~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_4.0_4.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,4.0:4.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,1,none,0~~2~~5~~8,3~~4~~6~~7~~9~~10 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_6.0_6.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,6.0:6.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,3,none,0~~2~~6~~7~~9,1~~4~~5~~8~~10 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_7.0_7.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,9,none,0~~2~~3~~6~~7,1~~4~~5~~8~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_4.0_4.0_Bin_0,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,3.0,6.0,2.0,0.0,,,0.7,2.2,1.0,0.5496,0.7,0.5714,0.4762,0.7273,0.6749,5~~8,none,0~~1~~2,3~~4~~6~~7~~9~~10 ---- -> y1,0,0,1,0.5454545454545454,0.49792959773196915,x_12.0_12.0_Bin_0__p2_7.0_7.0_Bin_0,x,12.0:12.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,9,none,0~~2~~3~~6~~7,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p2_6.0_6.0_Bin_0,p2,6.0:6.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.8333333333333335,PSG,4.0,5.0,2.0,0.0,,,0.6666,1.8333,1.0,0.5413,0.6667,0.5,0.3889,0.6364,0.6286,3~~6,none,0~~2~~7~~9,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,x_11.0_11.0_Bin_0__p2_6.0_6.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,6.0:6.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,3,none,0~~2~~6~~7~~9,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p2_4.0_4.0_Bin_0__x_10.0_10.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,4.0,5.0,2.0,0.0,,,0.6666,1.8333,1.0,0.5413,0.6667,0.5,0.3889,0.6364,0.6286,2~~7,none,0~~3~~6~~9,1~~4~~5~~8~~10 -9d6 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_3.0_3.0_Bin_0,p2,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -11,15d7 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p2_4.0_4.0_Bin_0__x_10.0_10.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,4.0,5.0,2.0,0.0,,,0.6666,1.8333,1.0,0.5413,0.6667,0.5,0.3889,0.6364,0.6286,2~~7,none,0~~3~~6~~9,1~~4~~5~~8~~10 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p2_6.0_6.0_Bin_0,p2,6.0:6.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.8333333333333335,PSG,4.0,5.0,2.0,0.0,,,0.6666,1.8333,1.0,0.5413,0.6667,0.5,0.3889,0.6364,0.6286,3~~6,none,0~~2~~7~~9,1~~4~~5~~8~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_7.0_7.0_Bin_0__p1_4.0_4.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~2~~8,3~~4~~6~~7~~9~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_7.0_7.0_Bin_0__x_9.0_9.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~2~~8,3~~4~~6~~7~~9~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,x_10.0_10.0_Bin_0__p2_3.0_3.0_Bin_0,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -17c9,12 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,x_11.0_11.0_Bin_0__p1_4.0_4.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,8,none,0~~1~~2~~5,3~~4~~6~~7~~9~~10 ---- -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_7.0_7.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,9,none,0~~2~~3~~6~~7,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_6.0_6.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,6.0:6.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,3,none,0~~2~~6~~7~~9,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,0,none,2~~3~~6~~7~~9,1~~4~~5~~8~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_7.0_7.0_Bin_0__x_9.0_9.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~2~~8,3~~4~~6~~7~~9~~10 -19,20c14,16 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,x_11.0_11.0_Bin_0__p2_6.0_6.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,6.0:6.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,3,none,0~~2~~6~~7~~9,1~~4~~5~~8~~10 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,x_12.0_12.0_Bin_0__p2_7.0_7.0_Bin_0,x,12.0:12.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,9,none,0~~2~~3~~6~~7,1~~4~~5~~8~~10 ---- -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_3.0_3.0_Bin_0,p2,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,x_10.0_10.0_Bin_0__p2_3.0_3.0_Bin_0,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_4.0_4.0_Bin_0,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,3.0,6.0,2.0,0.0,,,0.7,2.2,1.0,0.5496,0.7,0.5714,0.4762,0.7273,0.6749,5~~8,none,0~~1~~2,3~~4~~6~~7~~9~~10 -21a18,21 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,x_11.0_11.0_Bin_0__p1_4.0_4.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,8,none,0~~1~~2~~5,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_7.0_7.0_Bin_0__p1_4.0_4.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~2~~8,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_4.0_4.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,4.0:4.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,1,none,0~~2~~5~~8,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -=================== End of Test31_smlp_toy_num_resp_mult_features_ranking.csv diff report ================================ -=================== Diff report for: Test31_smlp_toy_num_resp_mult_ranking_resp_feat.csv ================================== -1,12c1,12 -< p1,p2,x,y1,y2 -< 2.0,3,10.0,1,1 -< 2.0,4,12.0,0,1 -< 3.0,4,10.0,1,1 -< 2.0,6,11.0,1,0 -< 2.0,8,10.0,0,0 -< 4.0,7,9.0,0,1 -< 3.0,6,9.0,1,0 -< 3.0,4,10.0,1,0 -< 4.0,4,11.0,0,1 -< 2.0,7,12.0,1,0 -< 3.0,7,10.0,0,0 ---- -> x,p2,p1,y1,y2 -> 10.0,3,2.0,1,1 -> 12.0,4,2.0,0,1 -> 10.0,4,3.0,1,1 -> 11.0,6,2.0,1,0 -> 10.0,8,2.0,0,0 -> 9.0,7,4.0,0,1 -> 9.0,6,3.0,1,0 -> 10.0,4,3.0,1,0 -> 11.0,4,4.0,0,1 -> 12.0,7,2.0,1,0 -> 10.0,7,3.0,0,0 -=================== End of Test31_smlp_toy_num_resp_mult_ranking_resp_feat.csv diff report ================================ +=================== Diff report for: Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +75c75 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test20_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test20_model_rerun_model_config.json +=================== End of Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ =================== Diff report for: Test33_smlp_toy_num_resp_mult.txt ================================== 95,96d94 < smlp_logger - WARNING - Range plots are not supported in this version of SMLP < =================== End of Test33_smlp_toy_num_resp_mult.txt diff report ================================ -=================== Diff report for: Test33_smlp_toy_num_resp_mult_features_ranking.csv ================================== -2,7c2,5 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,0,none,2~~3~~6~~7~~9,1~~4~~5~~8~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_4.0_4.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,4.0:4.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,1,none,0~~2~~5~~8,3~~4~~6~~7~~9~~10 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_6.0_6.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,6.0:6.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,3,none,0~~2~~6~~7~~9,1~~4~~5~~8~~10 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_7.0_7.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,9,none,0~~2~~3~~6~~7,1~~4~~5~~8~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_4.0_4.0_Bin_0,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,3.0,6.0,2.0,0.0,,,0.7,2.2,1.0,0.5496,0.7,0.5714,0.4762,0.7273,0.6749,5~~8,none,0~~1~~2,3~~4~~6~~7~~9~~10 ---- -> y1,0,0,1,0.5454545454545454,0.49792959773196915,x_12.0_12.0_Bin_0__p2_7.0_7.0_Bin_0,x,12.0:12.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,9,none,0~~2~~3~~6~~7,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p2_6.0_6.0_Bin_0,p2,6.0:6.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.8333333333333335,PSG,4.0,5.0,2.0,0.0,,,0.6666,1.8333,1.0,0.5413,0.6667,0.5,0.3889,0.6364,0.6286,3~~6,none,0~~2~~7~~9,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,x_11.0_11.0_Bin_0__p2_6.0_6.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,6.0:6.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,3,none,0~~2~~6~~7~~9,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p2_4.0_4.0_Bin_0__x_10.0_10.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,4.0,5.0,2.0,0.0,,,0.6666,1.8333,1.0,0.5413,0.6667,0.5,0.3889,0.6364,0.6286,2~~7,none,0~~3~~6~~9,1~~4~~5~~8~~10 -9d6 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_3.0_3.0_Bin_0,p2,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -11,15d7 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p2_4.0_4.0_Bin_0__x_10.0_10.0_Bin_0,p2,4.0:4.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,4.0,5.0,2.0,0.0,,,0.6666,1.8333,1.0,0.5413,0.6667,0.5,0.3889,0.6364,0.6286,2~~7,none,0~~3~~6~~9,1~~4~~5~~8~~10 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,p2_6.0_6.0_Bin_0,p2,6.0:6.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,1.8333333333333335,PSG,4.0,5.0,2.0,0.0,,,0.6666,1.8333,1.0,0.5413,0.6667,0.5,0.3889,0.6364,0.6286,3~~6,none,0~~2~~7~~9,1~~4~~5~~8~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_7.0_7.0_Bin_0__p1_4.0_4.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~2~~8,3~~4~~6~~7~~9~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_7.0_7.0_Bin_0__x_9.0_9.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~2~~8,3~~4~~6~~7~~9~~10 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,x_10.0_10.0_Bin_0__p2_3.0_3.0_Bin_0,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -17c9,12 -< y2,0,0,1,0.45454545454545453,0.49792959773196915,x_11.0_11.0_Bin_0__p1_4.0_4.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,8,none,0~~1~~2~~5,3~~4~~6~~7~~9~~10 ---- -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_7.0_7.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,9,none,0~~2~~3~~6~~7,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_6.0_6.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,6.0:6.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,3,none,0~~2~~6~~7~~9,1~~4~~5~~8~~10 -> y1,0,0,1,0.5454545454545454,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,0,none,2~~3~~6~~7~~9,1~~4~~5~~8~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_7.0_7.0_Bin_0__x_9.0_9.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,x,9.0:9.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~2~~8,3~~4~~6~~7~~9~~10 -19,20c14,16 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,x_11.0_11.0_Bin_0__p2_6.0_6.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,6.0:6.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,3,none,0~~2~~6~~7~~9,1~~4~~5~~8~~10 -< y1,0,0,1,0.5454545454545454,0.49792959773196915,x_12.0_12.0_Bin_0__p2_7.0_7.0_Bin_0,x,12.0:12.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,7.0:7.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,1.8333333333333335,PSG,5.0,5.0,1.0,0.0,,,0.5834,1.8333,1.0,0.5207,0.5833,0.2858,0.2143,0.5455,0.5333,9,none,0~~2~~3~~6~~7,1~~4~~5~~8~~10 ---- -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_3.0_3.0_Bin_0,p2,3.0:3.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,x_10.0_10.0_Bin_0__p2_3.0_3.0_Bin_0,x,10.0:10.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_4.0_4.0_Bin_0,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,NA,,NA:NA,NA:NA,NA,NA,NA,NA,2.2,PSG,3.0,6.0,2.0,0.0,,,0.7,2.2,1.0,0.5496,0.7,0.5714,0.4762,0.7273,0.6749,5~~8,none,0~~1~~2,3~~4~~6~~7~~9~~10 -21a18,21 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,x_11.0_11.0_Bin_0__p1_4.0_4.0_Bin_0,x,11.0:11.0,1:1,1,9.0,12.0,10.363636363636363,0.9791208740244552,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,8,none,0~~1~~2~~5,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p2_7.0_7.0_Bin_0__p1_4.0_4.0_Bin_0,p2,7.0:7.0,1:1,1,3.0,8.0,5.454545454545454,1.6160353486028343,p1,4.0:4.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,5,none,0~~1~~2~~8,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_4.0_4.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,4.0:4.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,1,none,0~~2~~5~~8,3~~4~~6~~7~~9~~10 -> y2,0,0,1,0.45454545454545453,0.49792959773196915,p1_2.0_2.0_Bin_0__p2_3.0_3.0_Bin_0,p1,2.0:2.0,1:1,1,2.0,4.0,2.727272727272727,0.7496555682941201,p2,3.0:3.0,1:1,1,3,8,5.454545454545454,1.6160353486028343,,NA:NA,NA:NA,NA,NA,NA,NA,NA,2.2,PSG,4.0,6.0,1.0,0.0,,,0.6,2.2,1.0,0.5248,0.6,0.3333,0.2667,0.6364,0.5659,0,none,1~~2~~5~~8,3~~4~~6~~7~~9~~10 -=================== End of Test33_smlp_toy_num_resp_mult_features_ranking.csv diff report ================================ -=================== Diff report for: Test33_smlp_toy_num_resp_mult_ranking_resp_feat.csv ================================== -1,12c1,12 -< p1,p2,x,y1,y2 -< 2.0,3,10.0,1,1 -< 2.0,4,12.0,0,1 -< 3.0,4,10.0,1,1 -< 2.0,6,11.0,1,0 -< 2.0,8,10.0,0,0 -< 4.0,7,9.0,0,1 -< 3.0,6,9.0,1,0 -< 3.0,4,10.0,1,0 -< 4.0,4,11.0,0,1 -< 2.0,7,12.0,1,0 -< 3.0,7,10.0,0,0 ---- -> x,p2,p1,y1,y2 -> 10.0,3,2.0,1,1 -> 12.0,4,2.0,0,1 -> 10.0,4,3.0,1,1 -> 11.0,6,2.0,1,0 -> 10.0,8,2.0,0,0 -> 9.0,7,4.0,0,1 -> 9.0,6,3.0,1,0 -> 10.0,4,3.0,1,0 -> 11.0,4,4.0,0,1 -> 12.0,7,2.0,1,0 -> 10.0,7,3.0,0,0 -=================== End of Test33_smlp_toy_num_resp_mult_ranking_resp_feat.csv diff report ================================ +=================== Diff report for: Test47_test47_model_smlp_toy_pf_mult.txt ================================== +83c83 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test47_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test47_model_rerun_model_config.json +=================== End of Test47_test47_model_smlp_toy_pf_mult.txt diff report ================================ +=================== Diff report for: Test64_test63_model.txt ================================== +27c27 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test63_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test63_model_rerun_model_config.json +=================== End of Test64_test63_model.txt diff report ================================ =================== Diff report for: Test66_test65_model.txt ================================== 0a1,97 > @@ -427,7 +507,7 @@ diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test16_Test8_smlp_toy_n > smlp_logger - INFO - Executing run_smlp.py script: End =================== End of Test66_test65_model.txt diff report ================================ =================== Diff report for: Test66_test65_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test66_test65_model_verify_results.json: No such file or directory +diff: /app/smlp/regr_smlp/code/Test66_test65_model_verify_results.json: No such file or directory =================== End of Test66_test65_model_verify_results.json diff report ================================ =================== Diff report for: Test68_test67_model.txt ================================== 0a1,97 @@ -530,81 +610,14 @@ diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test66_test65_model_ver > smlp_logger - INFO - Executing run_smlp.py script: End =================== End of Test68_test67_model.txt diff report ================================ =================== Diff report for: Test68_test67_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test68_test67_model_verify_results.json: No such file or directory +diff: /app/smlp/regr_smlp/code/Test68_test67_model_verify_results.json: No such file or directory =================== End of Test68_test67_model_verify_results.json diff report ================================ -=================== Diff report for: Test69_smlp_toy_num_resp_mult_verify_results.json ================================== -6c6 -< "x": 8.601911912781919, ---- -> "x": 8.601911912575982, -9c9 -< "y2": 5.078784555196762 ---- -> "y2": 5.078784562647343 -=================== End of Test69_smlp_toy_num_resp_mult_verify_results.json diff report ================================ =================== Diff report for: Test70_test69_model.txt ================================== -25a26,82 -> -> smlp_logger - INFO - Creating model exploration base components: Start -> -> smlp_logger - INFO - Parsing the SPEC: Start -> -> smlp_logger - INFO - Parsing the SPEC: End -> -> smlp_logger - INFO - Variable domains (alpha): {'y2': {'range': 'float', 'interval': None}, 'x': {'range': 'float', 'interval': [0, 10]}, 'p1': {'range': 'float', 'interval': [0, 10]}, 'p2': {'range': 'float', 'interval': [3, 7]}} -> -> smlp_logger - INFO - Input bounds (alpha): {'x': {'min': 0, 'max': 10}, 'p1': {'min': 0, 'max': 10}, 'p2': {'min': 3, 'max': 7}} -> -> smlp_logger - INFO - Knob bounds (eta): {} -> -> smlp_logger - INFO - Knob grids (eta): {} -> -> smlp_logger - INFO - Alpha global constraints: (or (or (= p1 1) (= p1 4)) (= p1 7)) -> -> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x 0) (<= x 10))) (and (>= p1 0) (<= p1 10))) (and (>= p2 3) (<= p2 7))) -> -> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x 0) (<= x 10))) (and (>= p1 0) (<= p1 10))) (and (>= p2 3) (<= p2 7))) (or (or (= p1 1) (= p1 4)) (= p1 7))) -> -> smlp_logger - INFO - Beta global constraints: true -> -> smlp_logger - INFO - Eta ranges constraints: true -> -> smlp_logger - INFO - Eta grid constraints: true -> -> smlp_logger - INFO - Eta global constraints: true -> -> smlp_logger - INFO - Eta combined constraints: true -> -> smlp_logger - INFO - Creating model exploration base components: End -> -> smlp_logger - INFO - Input and knob interface constraints are consistent -> -> smlp_logger - INFO - Building model terms: Start -> -> smlp_logger - INFO - Model operator counts for y2: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} -> -> smlp_logger - INFO - Building model terms: End -> -> smlp_logger - INFO - Model interface constraints are consistent -> -> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: -> true -> -> smlp_logger - INFO - Input, knob and configuration constraints are consistent -> -> smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+p2)/2<6 -> -> smlp_logger - INFO - The configuration is consistent with assertion asrt1 -> -> smlp_logger - INFO - Completed with result: FAIL -> -> smlp_logger - INFO - Running SMLP in mode "verify": End -> -> smlp_logger - INFO - Executing run_smlp.py script: End +25c25 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test69_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test69_model_rerun_model_config.json =================== End of Test70_test69_model.txt diff report ================================ -=================== Diff report for: Test70_test69_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test70_test69_model_verify_results.json: No such file or directory -=================== End of Test70_test69_model_verify_results.json diff report ================================ =================== Diff report for: Test72_test71_model.txt ================================== 0a1,84 > @@ -693,7 +706,7 @@ diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test70_test69_model_ver > smlp_logger - INFO - Executing run_smlp.py script: End =================== End of Test72_test71_model.txt diff report ================================ =================== Diff report for: Test72_test71_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test72_test71_model_verify_results.json: No such file or directory +diff: /app/smlp/regr_smlp/code/Test72_test71_model_verify_results.json: No such file or directory =================== End of Test72_test71_model_verify_results.json diff report ================================ =================== Diff report for: Test77_test76_model.txt ================================== 0a1,110 @@ -809,8 +822,32 @@ diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test72_test71_model_ver > smlp_logger - INFO - Executing run_smlp.py script: End =================== End of Test77_test76_model.txt diff report ================================ =================== Diff report for: Test77_test76_model_verify_results.json ================================== -diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/Test77_test76_model_verify_results.json: No such file or directory +diff: /app/smlp/regr_smlp/code/Test77_test76_model_verify_results.json: No such file or directory =================== End of Test77_test76_model_verify_results.json diff report ================================ +=================== Diff report for: Test97_smlp_toy_num_resp_mult.txt ================================== +252c252 +< smlp_logger - INFO - Model operator counts for y2: {'add': 100, 'mul': 715, 'const': 2547, 'ite': 305, 'and': 408, 'prop': 713, 'sub': 713, 'var': 713} +--- +> smlp_logger - INFO - Model operator counts for y2: {'add': 100, 'mul': 716, 'const': 2550, 'ite': 305, 'and': 409, 'prop': 714, 'sub': 714, 'var': 714} +=================== End of Test97_smlp_toy_num_resp_mult.txt diff report ================================ +=================== Diff report for: Test102_test101_model.txt ================================== +38c38 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test101_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test101_model_rerun_model_config.json +=================== End of Test102_test101_model.txt diff report ================================ =================== Diff report for: test110_model_poly_sklearn_formula.txt ================================== -diff: /home/mdmitry/github/smlp_python313/regr_smlp/code/test110_model_poly_sklearn_formula.txt: No such file or directory +diff: /app/smlp/regr_smlp/code/test110_model_poly_sklearn_formula.txt: No such file or directory =================== End of test110_model_poly_sklearn_formula.txt diff report ================================ +=================== Diff report for: Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt ================================== +79c79 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test110_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test110_model_rerun_model_config.json +=================== End of Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt diff report ================================ +=================== Diff report for: Test112_test110_model_smlp_toy_basic_pred_unlabeled.txt ================================== +79c79 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test110_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test110_model_rerun_model_config.json +=================== End of Test112_test110_model_smlp_toy_basic_pred_unlabeled.txt diff report ================================ diff --git a/tests/smlp_regression/run_smlp_regression_whl_expected.log b/tests/smlp_regression/run_smlp_regression_whl_expected.log new file mode 100644 index 00000000..2d5c2748 --- /dev/null +++ b/tests/smlp_regression/run_smlp_regression_whl_expected.log @@ -0,0 +1,4283 @@ +Calling 8 workers for multiprocessing... +Initiating 0 worker... +Initiating 1 worker... +Initiating 2 worker... +Initiating 3 worker... +Initiating 4 worker... +Initiating 5 worker... +Initiating 6 worker... +Running test 3 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric response in training/test data only +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test3 -mode predict -resp y1 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_unlabeled.csv" + +Running test 10 test type: prediction, description: basic et_sklearn prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test10 -mode predict -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 15 -et_sklearn_bootstrap f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 18 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test18 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test19_model -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 24 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test24_model" -out_dir ./ -pref Test24 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 32 test type: unknown, description: test reusing saved model by using configuration file +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test32 -config ../models/test20_model_rerun_model_config.json -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 39 test type: doe, description: doe test with four levels with latin_hypercube +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test39 -mode doe -doe_algo latin_hypercube -doe_prob_distr Exponential -doe_samples 30 -log_time f + +Running test 47 test type: prediction, description: tests options -pos_val and -neg_val when re-using saved model +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test47_model" -out_dir ./ -pref Test47 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" + +Running test 55 test type: discretization, description: tests discretization options +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test55 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 68 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test67_model" -out_dir ./ -pref Test68 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -model_per_response t -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 80 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test80 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 86 test type: optimize, description: tests alpha +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test86 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -asrt_names asrt1,asrt2,asrt3 -asrt_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 94 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test94 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_witness.spec +specs_path ../specs +Running test 102 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test101_model" -out_dir ./ -pref Test102 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_stable_verify.spec +specs_path ../specs +Running test 105 test type: verify, description: basic dt_sklearn assertion verfication test with numeric labels and integer grid as domain +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test105 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_stable_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 110 test type: prediction, description: smlp toy basic example for predict mode from SMLP user manual +Running test 6 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test6 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 9 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test9 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -model_name test20_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -save_config t -save_model_config t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 16 test type: prediction, description: basic nn_keras prediction test from saved model on new data with numeric labels and two responses +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/Test8_smlp_toy_num_resp_mult" -out_dir ./ -pref Test16 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 25 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test25 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test26_model -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 35 test type: doe, description: doe test with four levels with plackett_burman +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test35 -mode doe -doe_algo plackett_burman -log_time f + +Running test 44 test type: doe, description: doe test with four levels with uniform_random_matrix +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test44 -mode doe -doe_algo uniform_random_matrix -doe_samples 20 -log_time f + +Running test 53 test type: discretization, description: tests discretization options +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test53 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult_y1_verify.spec +specs_path ../specs +Running test 64 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test63_model" -out_dir ./ -pref Test64 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 72 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test71_model" -out_dir ./ -pref Test72 -mode verify -resp y1,y2 -feat x0,x1,x2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_num_resp_mult_free_inps.spec +specs_path ../specs +Running test 83 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with numeric labels and integer grid as domain and with scaling objectives +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test83 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -beta "y1>7 and y2>6" -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1/2-y2;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 93 test type: optsyn, description: basic test for mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test93 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_witness.spec +specs_path ../specs +Running test 101 test type: certify, description: basic test in certify mode to test stability (theta) and guard (eta) constraint generation +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test101 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test101_model -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_witness.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_unsat_eta_verify.spec +specs_path ../specs +Running test 106 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test106 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_unsat_eta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_basic.spec +specs_path ../specs +Running test 113 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test113 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model t -model_name test113_model -save_model_config t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec + +spec_fn smlp_toy_system_stable_constant_query.spec +specs_path ../specs +Running test 7 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test7 -mode predict -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 11 test type: prediction, description: basic poly_sklearn prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test11 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 17 test type: prediction, description: basic poly_sklearn prediction test from saved model on new data with numeric labels and two responses +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/Test11_smlp_toy_num_resp_mult" -out_dir ./ -pref Test17 -mode predict -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 23 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels and saving model using name specified through model_name option - adapts Test6 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test23 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model t -use_model f -model_name test24_model -model_per_response t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 30 test type: subgroups, description: basic test for subgroup discovery for numric responses +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test30 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -feat x,p1,p2 -plots t -seed 10 -log_time f + +Running test 38 test type: doe, description: doe test with four levels with box_wilson +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test38 -mode doe -doe_algo box_wilson -doe_cc_face ccc -doe_cc_alpha r -doe_cc_center 2,3 -log_time f + +Running test 45 test type: doe, description: doe test with four levels with fractional_factorial +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels_real.csv" -out_dir ./ -pref Test45 -mode doe -doe_algo fractional_factorial -doe_resolution 5 -log_time f + +Running test 52 test type: discretization, description: tests discretization options +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test52 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult_y1_verify.spec +specs_path ../specs +Running test 63 test type: verify, description: basic dt_sklearn assertion verification test on data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test63 -mode verify -resp y1 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test63_model -spec ../specs/smlp_toy_num_resp_mult_y1_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x/2+y1>4.3;(y1+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 79 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test79 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult.spec -quer_names query1,query2,query3 -quer_exprs "(y2**3+p2)/2<6;y1>=9;y2<0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_query_vacuous.spec +specs_path ../specs +Running test 91 test type: query, description: test to detect contradictory constraints in optimization mode due to contradictory alpha global and alpha bounds constraints on FMAX_xyx +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test91 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 99 test type: optimize, description: testing that the response and feature names can be taken from spec file in model exploration modes when the responses and/or features are not specified in the command line +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test99 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_certify_witness.spec +specs_path ../specs +Running test 103 test type: certify, description: +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test103 -mode certify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -use_model f -model_name test103_model -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_certify_witness.spec -quer_names valid_candidate,grid_conflict,range_conflict -quer_exprs "True;True;True" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 111 test type: unknown, description: smlp toy basic test to rerun saved model using the model rerun config file saved during model training +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test111 -config ../models/test110_model_rerun_model_config.json -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" + +spec_fn smlp_toy_system_stable_constant_certify.spec +specs_path ../specs +Running test 117 test type: certify, description: certification test with knobs only where assertion is valid without stability and fails with stability +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test117 -mode certify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 123 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +Running test 8 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and two responses +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test8 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -nn_keras_epochs 20 -nn_keras_seq_api f -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 14 test type: train, description: EV-SI real life poly_sklearn prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test14 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 21 test type: prediction, description: test for illegal symbols in column names +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_metasymbol_mult_reg.csv" -out_dir ./ -pref Test21 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model t -use_model f -model_name test22_model -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" + +Running test 29 test type: subgroups, description: basic test for subgroup discovery for pass-fail responses +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_cls_metasymbol_colnames_mult.csv" -out_dir ./ -pref Test29 -mode subgroups -psg_dim 3 -psg_top 10 -resp "PF 1,PF#" -plots t -seed 10 -log_time f + +Running test 37 test type: doe, description: doe test with four levels with box_behnken +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_three_levels_real_nan.csv" -out_dir ./ -pref Test37 -mode doe -doe_algo box_behnken -log_time f + +Running test 46 test type: prediction, description: tests options -pos_val and -neg_val +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_pf_mult.csv" -out_dir ./ -pref Test46 -mode predict -resp "PF,PF1" -model poly_sklearn -save_model t -save_model_config f -use_model f -model_name test47_model -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -pos_val fail -neg_val pass -new_dat "../data/smlp_toy_pf_mult.csv" + +Running test 54 test type: discretization, description: tests discretization options +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test54 -mode discretize -resp "PF,PF1" -discr_algo ordinals -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_noknobs_verify.spec +specs_path ../specs +Running test 66 test type: verify, description: basic dt_sklearn assertion verification test on data with one numeric response +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test65_model" -out_dir ./ -pref Test66 -mode verify -resp y1,y2 -feat x0,x1,x2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model t -spec ../specs/smlp_toy_num_resp_noknobs_verify.spec -asrt_names asrt1,asrt2 -asrt_exprs "x0**2+y1>4.3;(y1+x2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 77 test type: unknown, description: verification test run using model_rerun config covering the case when mrmr selcts only a subset of features specified through the command line or config file +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test76_model" -out_dir ./ -pref Test77 -config ../models/test76_model_rerun_model_config.json + +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 85 test type: optimize, description: tests alpha and eta constraints specified in command line in optimization mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test85 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -data_scaler min_max -objv_names obj1,objv2 -objv_exprs "(y1+y2)/2;y1" -alpha "p2<5 and x==10 and x<12" -eta "p1==4" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_verify_vacuous.spec +specs_path ../specs +Running test 92 test type: verify, description: test to detect contradictory constraints in verification mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test92 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_verify_vacuous.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 100 test type: optimize, description: basic test for sat_threshold option enabing usage of objectve values in SAT assignments that prove optimization thresholds +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test100 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 104 test type: verify, description: assertion verfication test with wrong spec that does not assign a single value using a singleton grid or range with equal max and min +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test104 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_cannot_synthesize.spec +specs_path ../specs +Running test 109 test type: synthesize, description: basic test for mode synthesize where synthesis fails +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test109 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_cannot_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 121 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is feasible +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test121 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_configuration_verify.spec +specs_path ../specs +Running test 2 test type: prediction, description: basic rf_sklearn prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test2 -mode predict -resp y1 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 12 test type: train, description: EV-SI real life dt_sklearn predict test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test12 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 20 test type: prediction, description: basic dt_sklearn prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test20_model" -out_dir ./ -pref Test20 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -data_scaler none -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 27 test type: prediction, description: checks nn_keras prediction with nn_keras_seq_api t +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test27 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 34 test type: doe, description: doe test with four levels with full_factorial method +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test34 -mode doe -doe_algo full_factorial -log_time f + +Running test 42 test type: doe, description: doe test with four levels with maximin_reconstruction +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test42 -mode doe -doe_algo maximin_reconstruction -doe_samples 20 -log_time f + +Running test 50 test type: discretization, description: tests discretization options +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test50 -mode discretize -resp "PF,PF1" -discr_algo kmeans -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 59 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test59 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_num_resp_mult_free_inps.spec +specs_path ../specs +Running test 81 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test81 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_num_resp_mult_query.spec +specs_path ../specs +Running test 89 test type: query, description: basic test in query mode to test stability (theta) and guard (eta) constraint generation +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test89 -mode query -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_query.spec +specs_path ../specs +Running test 97 test type: query, description: basic test for rf_sklearn in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test97 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_beta_verify.spec +specs_path ../specs +Running test 107 test type: verify, description: test for verification mode to check that eta contraints are not contradictory and as otherwise verification problem is not well defined +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test107 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_beta_verify.spec -asrt_names asrt_y1,asrt_y2 -asrt_expr "y1*2+x<=5 and y1<=10;-2*y2-1<10-p2" -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 112 test type: prediction, description: smlp toy basic test from SMLP manual +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test110_model" -out_dir ./ -pref Test112 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -use_model t -save_model f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" + +spec_fn smlp_toy_system_stable_constant_verify.spec +specs_path ../specs +Running test 118 test type: verify, description: verification test with knobs only where assertion is valid without stability and fails with stability +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test118 -mode verify -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 125 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is feasible and optimization is performed +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test125 -mode optsyn -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system.spec +specs_path ../specs +Running test 5 test type: prediction, description: basic dt_caret prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test5 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model t -use_model f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 15 test type: prediction, description: basic dt_caret prediction test from saved model on new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/Test5_smlp_toy_num_resp_mult" -out_dir ./ -pref Test15 -mode predict -resp y1 -feat x,p1,p2 -model dt_caret -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 22 test type: prediction, description: test for illegal symbols in column names +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test22_model" -out_dir ./ -pref Test22 -mode predict -resp "PF ,|PF |" -model poly_sklearn -save_model f -use_model t -pred_plots t -resp_plots t -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_metasymbol_mult_reg_pred_labeled.csv" + +Running test 33 test type: unknown, description: testing -config option with subgroups mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test33 -config ../models/Test31_smlp_toy_num_resp_mult_args_config.json + +Running test 41 test type: doe, description: doe test with four levels with random_k_means +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test41 -mode doe -doe_algo random_k_means -doe_samples 20 -log_time f + +Running test 49 test type: discretization, description: tests discretization options +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test49 -mode discretize -resp "PF,PF1" -discr_algo quantile -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 58 test type: optimize, description: basic dt_sklearn optimization test with numeric labels and integer grid as domain and without scaling objectives +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test58 -mode optimize -pareto f -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 70 test type: verify, description: nn_keras verification test with re-using saved model_per_response trained model +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test69_model" -out_dir ./ -pref Test70 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model f -use_model t -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_num_resp_mult_alpha_asrt_verify.spec +specs_path ../specs +Running test 87 test type: verify, description: tests global alpha constraints and assertions specified in spec file +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test87 -mode verify -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_alpha_asrt_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 96 test type: optsyn, description: basic test for rf_sklearn in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test96 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_basic.spec +specs_path ../specs +Running test 114 test type: optimize, description: smlp toy basic test for mode optimize from SMLP manual without specifying resp and feat in command line +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test114 -mode optimize -pareto t -opt_strategy lazy -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -mrmr_pred 0 -epsilon 0.05 -delta_rel 0.01 -save_model f -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -spec ../specs/smlp_toy_basic.spec + +spec_fn smlp_toy_system_stable_constant_synth_fail.spec +specs_path ../specs +Running test 122 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test122 -mode optimize -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_verify.spec +specs_path ../specs +Running test 140 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test140 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -trace_prec 1 -trace_anonym t -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 147 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test147 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 154 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training +Running test 4 test type: prediction, description: basic nn_keras prediction test on labeled and new data with numeric labels and one response +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test4 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nn_keras_weights_precision 2 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 13 test type: train, description: EV-SI real life nn_keras prediction test on labeled and new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test13 -mode train -resp y1,y2 -feat x1,x2,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 28 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and functional API +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test28 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 36 test type: doe, description: doe test with four levels with sukharev_grid +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_four_levels_real.csv" -out_dir ./ -pref Test36 -mode doe -doe_algo sukharev_grid -doe_samples 125 -log_time f + +Running test 43 test type: doe, description: doe test with four levels with halton_sequence +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test43 -mode doe -doe_algo halton_sequence -doe_samples 20 -log_time f + +Running test 51 test type: discretization, description: tests discretization options +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test51 -mode discretize -resp "PF,PF1" -discr_algo jenks -discr_bins 6 -discr_labels f -discr_type integer -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 60 test type: verify, description: basic nn_keras assertion verification test for functional nn_keras model +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test60 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_num_resp_mult_free_inps.spec +specs_path ../specs +Running test 82 test type: optimize, description: basic dt_sklearn single objective optimization test with numeric labels and integer grid as domain and with scaling objectives +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test82 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps.spec -data_scaler min_max -objv_names obj1,objv2,objv3 -objv_exprs "(y1+y2)/2;y1;y2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_num_resp_mult_optsyn_vacuous.spec +specs_path ../specs +Running test 90 test type: optsyn, description: test to detect contradictory constraints in optsyn mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test90 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn_vacuous.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 98 test type: optsyn, description: basic test for et_caret in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test98 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system.spec +specs_path ../specs +Running test 115 test type: certify, description: basic test in certify mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test115 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_fail.spec +specs_path ../specs +Running test 124 test type: optsyn, description: optimized synthesis test with constant knob and no inputs where synthesis is not feasible because while beta constraint is feasible the assertion is not feasible therefore optimization is not performed +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test124 -mode optsyn -pareto f -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult.spec +specs_path ../specs +Running test 141 test type: optimize, description: basic test for compress_rules option for dt_sklearn in optimization mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test141 -mode optimize -opt_strategy lazy -pareto f -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -spec ../specs/smlp_toy_num_resp_mult.spec -objv_names objv_y1,objv_y2 -objv_exprs "y1;y2" -epsilon 0.01 -delta_rel 0.01 -data_scaler none -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 149 test type: prediction, description: tests the mae loss function MeanAbsoluteError and sample weoghts +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test149 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mae -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 155 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training +Running test 1 test type: train, description: basic dt_caret training and test on labeled data with single numeric response +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test1 -mode train -resp y1 -feat x,p1,p2 -model dt_caret -save_model_config f -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 19 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test19_model" -out_dir ./ -pref Test19 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 26 test type: prediction, description: basic dt_sklearn prediction test using a model saved under a name specified through model_name option on new data with numeric labels +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -model_name "../models/test26_model" -out_dir ./ -pref Test26 -mode predict -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model t -mrmr_pred 2 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 31 test type: subgroups, description: testing resp2b in subgroup discovery mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test31 -mode subgroups -psg_dim 3 -psg_top 10 -resp y1,y2 -resp2b "y1<6;y2>6" -feat x,p1,p2 -plots t -seed 10 -log_time f -save_config t + +Running test 40 test type: doe, description: doe test with four levels with latin_hypercube_space_filling +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -doe_spec "../grids/doe_two_levels.csv" -out_dir ./ -pref Test40 -mode doe -doe_algo latin_hypercube_sf -doe_samples 20 -log_time f + +Running test 48 test type: discretization, description: tests discretization options +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test48 -mode discretize -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 56 test type: discretization, description: tests discretization options +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test56 -mode discretize -resp "PF,PF1" -discr_algo ranks -discr_bins 6 -discr_labels f -discr_type object -data_scaler none -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 69 test type: verify, description: nn_keras verification test with model_per_response training +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test69 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model t -use_model f -model_name test69_model -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "(y2**3+p2)/2<6" -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 88 test type: optimize, description: basic dt_sklearn multi objective pareto optimization test with beta and objectives specified in spec file +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test88 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -tree_encoding nested -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 95 test type: optsyn, description: basic test for dt_caret in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test95 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -save_model f -use_model f -tree_encoding nested -compress_rules f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_synthesize.spec +specs_path ../specs +Running test 108 test type: synthesize, description: basic test for dt_sklearn in model exploration mode synthesize where synthesis succeeds +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test108 -mode synthesize -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -tree_encoding nested -compress_rules f -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_synthesize.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_fail.spec +specs_path ../specs +Running test 120 test type: synthesize, description: synthesis test with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test120 -mode synthesize -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_certify.spec +specs_path ../specs +Running test 127 test type: certify, description: certification example with knobs only and fictitious inputs with values fixed through their ranges +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test127 -mode certify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_certify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 145 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -out_dir ./ -pref Test145 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 151 test type: prediction, description: tests msle loss function MeanSquaredLogarithmicError and and sample weoghts +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test151 -mode predict -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss msle -sw_coef 3 -sw_exp 10 -sw_int 0 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_verify.spec +specs_path ../specs +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test110 -mode predict -resp y1,y2 -feat x1,x2,p1,p2 -model poly_sklearn -save_model t -model_name test110_model -save_model_config t -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -new_dat "../data/smlp_toy_basic_pred_unlabeled.csv" + +spec_fn smlp_toy_system.spec +specs_path ../specs +Running test 116 test type: certify, description: basic test in certify mode when system is specified and is used as the model; p2 rel-rad needs to be 0 or very close to it the witness to first query to be stable +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test116 -mode certify -resp y1,y2 -feat x1,x2,p1,p2 -model system -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_system.spec -quer_names query1,query2 -quer_exprs "y1>0;y2<=0" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_verify.spec +specs_path ../specs +Running test 126 test type: verify, description: verification example with knobs only and fictitious inputs that have no effect where proparty is valid without stability and fails with stability +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test126 -mode verify -model system -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_verify.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 142 test type: optsyn, description: basic test for compress_rules option for rf_sklearn in optsin mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test142 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 15 -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 153 test type: prediction, description: tests the logcosh loss function LogCosh and sample weights +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test153 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss logcosh -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 161 test type: prediction, description: tests nn keras tuner bayesian +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test161 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -nn_keras_metrics mape,logcosh -nn_keras_tuner random -nn_keras_lrates_grid "0.01,0.001" -nn_keras_batches_grid "32,64" -model_per_response f -sw_coef 4 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 170 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response f in model exploration mode optimize +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test170 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 173 test type: verify, description: basic test for nn_keras flat encoding for sequential api +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test173 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 182 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test182 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 191 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response t in model exploration mode optimize adapts test 169 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test191 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap t -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 194 test type: optsyn, description: basic branched tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test194 -mode optsyn -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 197 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when responses are not scaled modifies test 164 and test 182 +Running test 119 test type: query, description: query test with knobs only where query is satisfiable without stability and fails with stability +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test119 -mode query -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_query.spec -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_witness_certify.spec +specs_path ../specs +Running test 128 test type: certify, description: Basic regression test in certify mode covering all four possible outcomes when certifying a witness for a query: the witness is stable +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test128 -mode certify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -dt_sklearn_max_depth 15 -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_witness_certify.spec -quer_names query_stable_witness,query_grid_conflict,query_unstable_witness,query_infeasible_witness,query_poly_intercept_sensitive -quer_exprs "y2<=90;y1>=9;y1>=(-13);y1>9;y1>=(-10)" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 148 test type: prediction, description: checks nn_keras prediction with sw_coef 0.8 and sequential API +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test148 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_verify.spec +specs_path ../specs +Running test 156 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner for functional model training; adapts test 154 by consdering multiple responses +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test156 -mode verify -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 167 test type: optsyn, description: basic flat tree encoding test with model_per_response t for rf_sklearn in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test167 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 171 test type: optimize, description: basic test for et_caret with flat tree_encoding in model exploration mode optimize +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test171 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 176 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api f for nn_keras in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test176 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 179 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn when resposes are not scaled adapts test 174 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test179 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 181 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test181 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 183 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test183 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 192 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test192 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 129 test type: verify, description: verification example with demonstrating all basic result scenarious for assertions +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_ctg_num_resp.csv" -out_dir ./ -pref Test129 -mode verify -resp y1,y2 -feat x,p1,p2 -model poly_sklearn -save_model f -use_model f -model_per_response f -spec ../specs/smlp_toy_configuration_verify.spec -asrt_names assert_stable_config,assert_grid_conflict,assert_unstable_config,assert_infeasible -asrt_exprs "y2<=90;y1>=9;y1>=(-10);y1>20" -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 150 test type: prediction, description: tests the mape loss function MeanAbsolutePercentageError and sample weights +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test150 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -sw_coef 0.8 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 158 test type: prediction, description: tests the mape loss function and sample weights with model_per_response t +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test158 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -model_per_response t -sw_coef 8 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 166 test type: optsyn, description: basic flat tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test166 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding flat -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 169 test type: optimize, description: basic test for et_sklearn with flat tree_encoding and model_per_response t in model exploration mode optimize +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test169 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -rf_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding flat -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 175 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test175 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 178 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features are not scaled adapts test 177 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test178 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 188 test type: optsyn, description: basic branched tree encoding test for dt_caretin model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test188 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding branched -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 196 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features are not scaled modifies test 164 and test 181 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test196 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_no_input_beta.spec +specs_path ../specs +Running test 201 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and there are beta constraints +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test201 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input_beta.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 205 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 145 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -out_dir ./ -pref Test205 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -doe_spec ../grids/doe_two_levels_opt.csv -doe_algo latin_hypercube -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 217 test type: correlate, description: basic test for correlate mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test217 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type category -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test123 -mode optimize -pareto t -opt_strategy lazy -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_query.spec +specs_path ../specs +Running test 143 test type: query, description: basic test for compress_rules for et_sklearn in mode query +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test143 -mode query -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_bootstrap f -tree_encoding nested -compress_rules t -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_query.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 159 test type: prediction, description: tests the msle loss function and sample weights with model_per_response t +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test159 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_loss msle -model_per_response t -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae,cosine -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 165 test type: optsyn, description: basic flat tree encoding test for dt_caretin model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test165 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_caret -tree_encoding flat -save_model f -use_model f -compress_rules f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_y2_verify.spec +specs_path ../specs +Running test 172 test type: verify, description: basic test for nn_keras flat encoding for functional api +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test172 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 180 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api t for nn_keras in model exploration mode optsyn when features and responses are not scaled adapts test 175 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test180 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -scale_feat f -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 187 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization adapts test 164 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test187 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 195 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 192 by setting n_estimators 3 and then discrepancy between z3 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test195 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 3 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 198 test type: optimize, description: basic branched tree encoding test for dt_sklearn multi objective pareto optimization when features and responses are not scaled modifies test 164 and test 183 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test198 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -scale_feat f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 204 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 123 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test204 -mode optimize -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 215 test type: correlate, description: basic test for correlate mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test215 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 221 test type: correlate, description: basic test for correlate mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test221 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 226 test type: correlate, description: basic test for correlate mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test197 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding branched -scale_resp f -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat"" + +spec_fn smlp_toy_num_resp_mult_no_input.spec +specs_path ../specs +Running test 202 test type: optimize, description: basic dt_sklearn single objective optimization with the eager algorithm when there are no inputs and no beta constraints +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test202 -mode optimize -pareto t -opt_strategy eager -resp y1,y2 -feat p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -spec ../specs/smlp_toy_num_resp_mult_no_input.spec -data_scaler min_max -objv_names obj1 -objv_exprs "(y1+y2)/2" -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_feasible.spec +specs_path ../specs +Running test 206 test type: optsyn, description: optimized synthesis test with eager strategy and with constant knob and no inputs where synthesis is feasible and optimization is performed adapts test 125 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test206 -mode optsyn -pareto t -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_feasible.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 218 test type: correlate, description: basic test for correlate mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test218 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type ordered -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 223 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test223 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 146 test type: optimize, description: optimization test with constant knob and no inputs where synthesis is feasible and optimization is performed +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -out_dir ./ -pref Test146 -mode optimize -pareto t -opt_strategy lazy -model poly_sklearn -resp y1,y2 -feat p1,p2,x1,x2 -save_model t -use_model f -mrmr_pred 0 -model_per_response t -split 1 -spec ../specs/smlp_toy_system.spec -doe_spec ../grids/explore_doe_two_levels.csv -doe_algo latin_hypercube -epsilon 0.99999999 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 152 test type: prediction, description: tests the huber loss function Huber and sample weights +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test152 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss huber -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +Running test 160 test type: prediction, description: tests nn keras tuner bayesian +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test160 -mode predict -resp y1,y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_loss mape -nn_keras_metrics msle -nn_keras_tuner bayesian -nn_keras_layers_grid "2,3" -nn_keras_losses_grid "mse,mae,huber" -model_per_response f -sw_coef 8 -sw_exp 5 -sw_int 0.5 -new_dat "../data/smlp_toy_num_resp_mult_pred_labeled.csv" + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 177 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response t nn_keras_seq_api t for nn_keras in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test177 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api t -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response t -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 190 test type: optimize, description: basic test for rf_caret with branched tree_encoding and modelper_response in model exploration mode optimize adapts test 168 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test190 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding branched -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 199 test type: optimize, description: test to demonstrate that in pareto optimization and optsyn modes with multiple objectives when beta constraints are not present SMLP results are not consistent when different solvers are used; this is due to fact that when a subset of objectoves are exemined in pareto algo +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test199 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 219 test type: correlate, description: basic test for correlate mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test219 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type integer -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 224 test type: correlate, description: basic test for correlate mode and tests the Shannon mutual information +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test224 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method shannon -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 157 test type: verify, description: basic nn_keras assertion verification test that uses keras tuner with sequrntial models for model training; adapts test 155 by consdering multiple responses +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test157 -mode verify -resp y1,y2 -feat x,p1,p2 --model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics rmse,logcosh + +spec_fn smlp_toy_num_resp_mult_free_inps_beta_objv.spec +specs_path ../specs +Running test 164 test type: optimize, description: basic flat tree encoding test for dt_sklearn multi objective pareto optimization +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test164 -mode optimize -pareto t -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model dt_sklearn -dt_sklearn_max_depth 15 -compress_rules f -tree_encoding flat -spec ../specs/smlp_toy_num_resp_mult_free_inps_beta_objv.spec -data_scaler min_max -epsilon 0.05 -delta_rel 0.01 -save_model_config f -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 168 test type: optimize, description: basic test for rf_caret with flat tree_encoding and modelper_response in model exploration mode optimize +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test168 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_caret -model_per_response t -compress_rules t -tree_encoding flat -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 174 test type: optsyn, description: basic layered nn_keras encoding test with model_per_response f nn_keras_seq_api f for nn_keras in model exploration mode optsyn +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test174 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model nn_keras -nn_keras_epochs 20 -nn_keras_seq_api f -nnet_encoding layered -save_model f -use_model f -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 189 test type: optsyn, description: basic branched tree encoding test with model_per_response f for rf_sklearn in model exploration mode optsyn adapts test 166 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test189 -mode optsyn -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model rf_sklearn -rf_sklearn_max_depth 4 -rf_sklearn_n_estimators 3 -tree_encoding branched -compress_rules t -save_model f -use_model f -compress_rules t -mrmr_pred 2 -model_per_response f -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_num_resp_mult_optsyn.spec +specs_path ../specs +Running test 193 test type: optimize, description: basic test for et_caret with branched tree_encoding in model exploration mode optimize adapts test 171 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test193 -mode optimize -resp y1,y2 -feat x,p1,p2 -model et_caret -tree_encoding branched -model_per_response t -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0.05 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +spec_fn smlp_toy_system_stable_constant_synth_fail.spec +specs_path ../specs +Running test 203 test type: optimize, description: optimization test with eager strategy and with constant knob and no inputs where synthesis is not feasible because the assertion is not feasible but beta constraint is feasible therefore optimization is performed adapts test 122 +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test203 -mode optimize -pareto f -opt_strategy eager -model system -resp y1,y2 -feat p1,p2 -save_model f -use_model f -mrmr_pred 0 -model_per_response t -spec ../specs/smlp_toy_system_stable_constant_synth_fail.spec -epsilon 0.00000001 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +Running test 216 test type: correlate, description: basic test for correlate mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test216 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 220 test type: correlate, description: basic test for correlate mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test220 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 225 test type: correlate, description: basic test for correlate mode and tests the adjusted mutual information +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test225 -mode correlate -resp y1,y2 -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f + +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test226 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method correlation -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 222 test type: correlate, description: basic test for correlate mode +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_mult_discr.csv" -out_dir ./ -pref Test222 -mode correlate -resp "PF,PF1" -discr_algo uniform -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method adjusted -mrmr_pred 0 -plots f -seed 10 -log_time f -pos_val fail -neg_val pass + +Running test 227 test type: correlate, description: basic test for correlate mode and tests the normalized mutual information +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_basic.csv" -out_dir ./ -pref Test227 -mode correlate -resp y1,y2 -discr_algo uniform -discret_num t -discr_bins 6 -discr_labels t -discr_type object -data_scaler none -cont_est pearson,spearman,kendall -mi_method normalized -mrmr_pred 0 -plots f -seed 10 -log_time f + +Running test 200 test type: optimize, description: basic test for et_sklearn with branched tree_encoding and model_per_response f in model exploration mode optimize adapts test 170 !!!!!!!!! in this test z3 result differs from mathsat and yices results (the latter two give sma results +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test200 -mode optimize -opt_strategy lazy -resp y1,y2 -feat x,p1,p2 -model et_sklearn -et_sklearn_max_depth 2 -et_sklearn_n_estimators 100 -et_sklearn_bootstrap f -tree_encoding branched -model_per_response f -compress_rules t -save_model f -use_model f -mrmr_pred 2 -spec ../specs/smlp_toy_num_resp_mult_optsyn.spec -epsilon 0.1 -delta_rel 0 -solver_path ../../../external/mathsat-5.6.8-linux-x86_64-reentrant/bin/mathsat -plots f -pred_plots f -resp_plots f -seed 10 -log_time f + +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test154 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api f -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 + +/usr/local/lib/python3.11/dist-packages/smlp/run_smlp.py -data "../data/smlp_toy_num_resp_mult.csv" -out_dir ./ -pref Test155 -mode verify -resp y2 -feat x,p1,p2 -model nn_keras -nnet_encoding nested -mrmr_pred 0 -plots f -pred_plots f -resp_plots f -seed 10 -log_time f -nn_keras_epochs 20 -nn_keras_seq_api t -nn_keras_tuner hyperband -nn_keras_layers_grid "2,2;3,3,3" -save_model_config f -spec ../specs/smlp_toy_num_resp_mult_y2_verify.spec -asrt_names asrt1 -asrt_exprs "2*y2>1" -sw_coef 4 -sw_exp 5 -sw_int 0.5 -nn_keras_metrics mae + +Initiating 7 worker... +comparing Test1_smlp_toy_num_resp_mult_y1_dt_caret_tree_rules.txt to master +Passed! +comparing Test1_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test1_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +File master Test1_smlp_toy_num_resp_mult_eval_dt_caret_labeled-col-y1.png does not exist +File master Test1_smlp_toy_num_resp_mult_eval_dt_caret_test-col-y1.png does not exist +File master Test1_smlp_toy_num_resp_mult_eval_dt_caret_training-col-y1.png does not exist +comparing Test1_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test1_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test1_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test1_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test1_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +File master Test1_smlp_toy_num_resp_mult_resp-distr.png does not exist +comparing Test1_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test1_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +comparing Test1_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test1_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test2_smlp_toy_num_resp_mult_rf_sklearn_y1_tree_rules.txt does not exist +comparing Test2_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +File master Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_eval_rf_sklearn_labeled-col-y1.png does not exist +File master Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_eval_rf_sklearn_new-col-y1.png does not exist +File master Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_eval_rf_sklearn_test-col-y1.png does not exist +File master Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_eval_rf_sklearn_training-col-y1.png does not exist +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +File master Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_resp-distr.png does not exist +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test2_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_unlabeled.txt to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_poly_sklearn_formula.txt to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_unlabeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_unlabeled_labeled_predictions_summary.csv to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_unlabeled_missing_values_dict.json to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_unlabeled_new_predictions_summary.csv to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_unlabeled_test_prediction_precisions.csv to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_unlabeled_test_predictions_summary.csv to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_unlabeled_training_prediction_precisions.csv to master +Passed! +comparing Test3_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_unlabeled_training_predictions_summary.csv to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_model_gen.json to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +File master Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mse.png does not exist +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test4_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_y1_dt_caret_tree_rules.txt to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test6_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_rf_sklearn_tree_rules.txt to master +comparing Test7_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test7_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_model_gen.json to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +File master Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y1_mse.png does not exist +File master Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mse.png does not exist +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_args_config.json to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test9_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing test20_model_data_bounds.json to master +Passed! +comparing test20_model_dt_sklearn_tree_rules.txt to master +Passed! +comparing test20_model_model_features_dict.json to master +Passed! +comparing test20_model_model_levels_dict.json to master +Passed! +comparing test20_model_rerun_model_config.json to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt to master +comparing Test10_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test10_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_poly_sklearn_formula.txt to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test12_smlp_toy_basic.txt to master +Passed! +comparing Test12_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test12_smlp_toy_basic_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test12_smlp_toy_basic_labeled_prediction_precisions.csv to master +Passed! +comparing Test12_smlp_toy_basic_labeled_predictions_summary.csv to master +Passed! +comparing Test12_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test12_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test12_smlp_toy_basic_test_prediction_precisions.csv to master +Passed! +comparing Test12_smlp_toy_basic_test_predictions_summary.csv to master +Passed! +comparing Test12_smlp_toy_basic_training_prediction_precisions.csv to master +Passed! +comparing Test12_smlp_toy_basic_training_predictions_summary.csv to master +Passed! +comparing Test13_smlp_toy_basic.txt to master +Passed! +comparing Test13_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test13_smlp_toy_basic_labeled_prediction_precisions.csv to master +Passed! +comparing Test13_smlp_toy_basic_labeled_predictions_summary.csv to master +Passed! +comparing Test13_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test13_smlp_toy_basic_model_gen.json to master +Passed! +comparing Test13_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test13_smlp_toy_basic_test_prediction_precisions.csv to master +Passed! +comparing Test13_smlp_toy_basic_test_predictions_summary.csv to master +Passed! +File master Test13_smlp_toy_basic_train-reg_y1_mse.png does not exist +File master Test13_smlp_toy_basic_train-reg_y2_mse.png does not exist +comparing Test13_smlp_toy_basic_training_prediction_precisions.csv to master +Passed! +comparing Test13_smlp_toy_basic_training_predictions_summary.csv to master +Passed! +comparing Test14_smlp_toy_basic.txt to master +Passed! +comparing Test14_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test14_smlp_toy_basic_labeled_prediction_precisions.csv to master +Passed! +comparing Test14_smlp_toy_basic_labeled_predictions_summary.csv to master +Passed! +comparing Test14_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test14_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test14_smlp_toy_basic_poly_sklearn_formula.txt to master +Passed! +comparing Test14_smlp_toy_basic_test_prediction_precisions.csv to master +Passed! +comparing Test14_smlp_toy_basic_test_predictions_summary.csv to master +Passed! +comparing Test14_smlp_toy_basic_training_prediction_precisions.csv to master +Passed! +comparing Test14_smlp_toy_basic_training_predictions_summary.csv to master +Passed! +comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test18_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing test19_model_data_bounds.json to master +Passed! +comparing test19_model_dt_sklearn_tree_rules.txt to master +Passed! +comparing test19_model_model_features_dict.json to master +Passed! +comparing test19_model_model_levels_dict.json to master +Passed! +comparing test19_model_rerun_model_config.json to master +Passed! +comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled.txt to master +comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt to master +comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt to master +Passed! +File master Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_labeled-col-PF .png does not exist +File master Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_labeled-col-|PF |.png does not exist +File master Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_new-col-PF .png does not exist +File master Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_new-col-|PF |.png does not exist +File master Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_test-col-PF .png does not exist +File master Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_test-col-|PF |.png does not exist +File master Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_training-col-PF .png does not exist +File master Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_training-col-|PF |.png does not exist +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_new_predictions_summary.csv to master +Passed! +File master Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_resp-distr.png does not exist +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test21_smlp_toy_num_metasymbol_mult_reg_smlp_toy_num_metasymbol_mult_reg_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing test22_model_data_bounds.json to master +Passed! +comparing test22_model_model_features_dict.json to master +Passed! +comparing test22_model_model_levels_dict.json to master +Passed! +comparing test22_model_poly_sklearn_formula.txt to master +Passed! +comparing test22_model_rerun_model_config.json to master +Passed! +comparing Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt to master +File master Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_new-col-PF .png does not exist +File master Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_eval_poly_sklearn_new-col-|PF |.png does not exist +comparing Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_new_predictions_summary.csv to master +Passed! +File master Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled_resp-distr.png does not exist +File master test24_model_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test23_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing test24_model_data_bounds.json to master +Passed! +File master test24_model_dt_sklearn_y2_tree_rules.txt does not exist +comparing test24_model_model_features_dict.json to master +Passed! +comparing test24_model_model_levels_dict.json to master +Passed! +comparing test24_model_rerun_model_config.json to master +Passed! +comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled.txt to master +comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing test26_model_dt_sklearn_tree_rules.txt to master +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test25_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing test26_model_data_bounds.json to master +Passed! +comparing test26_model_model_features_dict.json to master +Passed! +comparing test26_model_model_levels_dict.json to master +Passed! +comparing test26_model_rerun_model_config.json to master +Passed! +comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled.txt to master +comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_model_gen.json to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +File master Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mse.png does not exist +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test27_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_model_gen.json to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv to master +Passed! +File master Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mse.png does not exist +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv to master +Passed! +comparing Test28_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv to master +Passed! +comparing Test29_smlp_toy_cls_metasymbol_colnames_mult.txt to master +comparing Test29_smlp_toy_cls_metasymbol_colnames_mult_features_ranking.csv to master +Passed! +comparing Test29_smlp_toy_cls_metasymbol_colnames_mult_missing_values_dict.json to master +Passed! +comparing Test29_smlp_toy_cls_metasymbol_colnames_mult_ranking_resp_feat.csv to master +Passed! +comparing Test30_smlp_toy_num_resp_mult.txt to master +comparing Test30_smlp_toy_num_resp_mult_features_ranking.csv to master +Passed! +comparing Test30_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test30_smlp_toy_num_resp_mult_ranking_resp_feat.csv to master +Passed! +comparing Test31_smlp_toy_num_resp_mult.txt to master +comparing Test31_smlp_toy_num_resp_mult_args_config.json to master +Passed! +comparing Test31_smlp_toy_num_resp_mult_features_ranking.csv to master +Passed! +comparing Test31_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test31_smlp_toy_num_resp_mult_ranking_resp_feat.csv to master +Passed! +comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt to master +comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled_args_config.json to master +Passed! +comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json to master +Passed! +comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv to master +Passed! +comparing Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv to master +Passed! +comparing Test33_smlp_toy_num_resp_mult.txt to master +comparing Test33_smlp_toy_num_resp_mult_features_ranking.csv to master +Passed! +comparing Test33_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test33_smlp_toy_num_resp_mult_ranking_resp_feat.csv to master +Passed! +comparing Test34_doe_four_levels_real.txt to master +Passed! +comparing Test34_doe_four_levels_real_doe.csv to master +Passed! +comparing Test35_doe_four_levels_real.txt to master +Passed! +comparing Test35_doe_four_levels_real_doe.csv to master +Passed! +comparing Test36_doe_four_levels_real.txt to master +Passed! +comparing Test36_doe_four_levels_real_doe.csv to master +Passed! +comparing Test37_doe_three_levels_real_nan.txt to master +Passed! +comparing Test37_doe_three_levels_real_nan_doe.csv to master +Passed! +comparing Test38_doe_two_levels.txt to master +Passed! +comparing Test38_doe_two_levels_doe.csv to master +Passed! +comparing Test39_doe_two_levels.txt to master +Passed! +comparing Test39_doe_two_levels_doe.csv to master +Passed! +comparing Test40_doe_two_levels.txt to master +Passed! +comparing Test40_doe_two_levels_doe.csv to master +Passed! +comparing Test41_doe_two_levels.txt to master +Passed! +comparing Test42_doe_two_levels.txt to master +Passed! +comparing Test43_doe_two_levels.txt to master +Passed! +comparing Test43_doe_two_levels_doe.csv to master +Passed! +comparing Test44_doe_two_levels.txt to master +Passed! +comparing Test44_doe_two_levels_doe.csv to master +Passed! +comparing Test45_doe_two_levels_real.txt to master +Passed! +comparing Test45_doe_two_levels_real_doe.csv to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult.txt to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult_missing_values_dict.json to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult_new_prediction_precisions.csv to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult_new_predictions_summary.csv to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult_test_prediction_precisions.csv to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult_test_predictions_summary.csv to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult_training_prediction_precisions.csv to master +Passed! +comparing Test46_smlp_toy_pf_mult_smlp_toy_pf_mult_training_predictions_summary.csv to master +Passed! +comparing test47_model_data_bounds.json to master +Passed! +comparing test47_model_model_features_dict.json to master +Passed! +comparing test47_model_model_levels_dict.json to master +Passed! +comparing test47_model_poly_sklearn_formula.txt to master +Passed! +comparing Test47_test47_model_smlp_toy_pf_mult.txt to master +comparing Test47_test47_model_smlp_toy_pf_mult_missing_values_dict.json to master +Passed! +comparing Test47_test47_model_smlp_toy_pf_mult_new_prediction_precisions.csv to master +Passed! +comparing Test47_test47_model_smlp_toy_pf_mult_new_predictions_summary.csv to master +Passed! +comparing Test48_smlp_toy_mult_discr.txt to master +Passed! +comparing Test48_smlp_toy_mult_discr_missing_values_dict.json to master +Passed! +comparing Test49_smlp_toy_mult_discr.txt to master +Passed! +comparing Test49_smlp_toy_mult_discr_missing_values_dict.json to master +Passed! +comparing Test50_smlp_toy_mult_discr.txt to master +Passed! +comparing Test50_smlp_toy_mult_discr_missing_values_dict.json to master +Passed! +comparing Test51_smlp_toy_mult_discr.txt to master +Passed! +comparing Test51_smlp_toy_mult_discr_missing_values_dict.json to master +Passed! +comparing Test52_smlp_toy_mult_discr.txt to master +Passed! +comparing Test52_smlp_toy_mult_discr_missing_values_dict.json to master +Passed! +comparing Test53_smlp_toy_mult_discr.txt to master +Passed! +comparing Test53_smlp_toy_mult_discr_missing_values_dict.json to master +Passed! +comparing Test54_smlp_toy_mult_discr.txt to master +Passed! +comparing Test54_smlp_toy_mult_discr_missing_values_dict.json to master +Passed! +comparing Test55_smlp_toy_mult_discr.txt to master +Passed! +comparing Test55_smlp_toy_mult_discr_missing_values_dict.json to master +Passed! +comparing Test56_smlp_toy_mult_discr.txt to master +Passed! +comparing Test56_smlp_toy_mult_discr_missing_values_dict.json to master +Passed! +Test 57 Failed: +Error in Build stage: +Data file does not exist +comparing Test58_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test58_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test58_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test58_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test58_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test58_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test58_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_model_gen.json to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test59_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test59_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +comparing Test59_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test59_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +File master Test59_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test59_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test60_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_model_gen.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test60_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test60_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +comparing Test60_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test60_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +File master Test60_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test60_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +Test 61 Failed: +Error in Build stage: +Data file does not exist +Test 62 Failed: +Error in Build stage: +Data file does not exist +File master test63_model_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test63_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test63_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test63_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test63_smlp_toy_num_resp_mult_verify_results.json to master +comparing test63_model_data_bounds.json to master +Passed! +comparing test63_model_model_features_dict.json to master +Passed! +comparing test63_model_model_levels_dict.json to master +Passed! +comparing test63_model_rerun_model_config.json to master +Passed! +File master test63_model_y1_smlp_full_model_term.json does not exist +File master test63_model_y1_smlp_model_term.json does not exist +comparing Test64_test63_model.txt to master +File master Test64_test63_model_trace.csv does not exist +comparing Test64_test63_model_verify_results.json to master +Test 65 Failed: +Error in Build stage: +Data file does not exist +comparing Test66_test65_model.txt to master +File new Test66_test65_model_verify_results.json does not exist +Test 67 Failed: +Error in Build stage: +Data file does not exist +comparing Test68_test67_model.txt to master +File new Test68_test67_model_verify_results.json does not exist +comparing Test69_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test69_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test69_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +comparing Test69_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test69_smlp_toy_num_resp_mult_verify_results.json to master +comparing test69_model_data_bounds.json to master +Passed! +comparing test69_model_model_features_dict.json to master +Passed! +comparing test69_model_model_gen.json to master +Passed! +comparing test69_model_model_levels_dict.json to master +Passed! +comparing test69_model_rerun_model_config.json to master +Passed! +File master test69_model_y2_smlp_full_model_term.json does not exist +File master test69_model_y2_smlp_model_term.json does not exist +comparing Test70_test69_model.txt to master +File master Test70_test69_model_trace.csv does not exist +comparing Test70_test69_model_verify_results.json to master +Test 71 Failed: +Error in Build stage: +Data file does not exist +comparing Test72_test71_model.txt to master +File new Test72_test71_model_verify_results.json does not exist +Test 73 Failed: +Error in Build stage: +Data file does not exist +Error in Build stage: +New data file does not exist +Test 74 Failed: +Error in Build stage: +New data file does not exist +Test 75 Failed: +Error in Build stage: +New data file does not exist +Test 76 Failed: +Error in Build stage: +Data file does not exist +comparing Test77_test76_model.txt to master +File new Test77_test76_model_verify_results.json does not exist +Test 78 Failed: +Error in Build stage: +Data file does not exist +comparing Test79_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test79_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +File master Test79_smlp_toy_num_resp_mult_dt_sklearn_y1_tree_rules.txt does not exist +File master Test79_smlp_toy_num_resp_mult_dt_sklearn_y2_tree_rules.txt does not exist +comparing Test79_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test79_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test79_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test79_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test79_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test79_smlp_toy_num_resp_mult_query_results.json to master +comparing Test79_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test79_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test79_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test79_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test79_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test79_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test79_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test79_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test79_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test80_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test80_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test80_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test80_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test80_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test80_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test80_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test81_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test81_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test81_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test81_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test81_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test81_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test81_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test82_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test82_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test82_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test82_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test82_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test82_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test82_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test83_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test83_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test83_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test83_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test83_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test83_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test83_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +Test 84 Failed: +Error in Build stage: +Data file does not exist +comparing Test85_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test85_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test85_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test85_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test85_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test85_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test85_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test85_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test86_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test86_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test86_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test86_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test86_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test86_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test86_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test87_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_rerun_model_config.json to master +Passed! +File master Test87_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test87_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test87_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test87_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test87_smlp_toy_num_resp_mult_verify_results.json to master +comparing Test88_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test88_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test88_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test88_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test88_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test88_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test88_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test89_smlp_toy_num_resp_mult_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test89_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test89_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +File master Test89_smlp_toy_num_resp_mult_dt_sklearn_y2_tree_rules.txt does not exist +comparing Test89_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test89_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test89_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test89_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test89_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test89_smlp_toy_num_resp_mult_query_results.json to master +comparing Test89_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test89_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test89_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test89_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test89_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test89_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test89_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test89_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test89_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test90_smlp_toy_num_resp_mult_dt_sklearn_y2_tree_rules.txt does not exist +comparing Test90_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test90_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +File master Test90_smlp_toy_num_resp_mult_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test90_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test90_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test90_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test90_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test90_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +File master Test90_smlp_toy_num_resp_mult_optimization_results.json does not exist +comparing Test90_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test90_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test90_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test90_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test90_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test90_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test90_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test90_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test90_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test91_smlp_toy_num_resp_mult_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test91_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test91_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +File master Test91_smlp_toy_num_resp_mult_dt_sklearn_y2_tree_rules.txt does not exist +comparing Test91_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test91_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test91_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test91_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test91_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test91_smlp_toy_num_resp_mult_query_results.json to master +Passed! +comparing Test91_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test91_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test91_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test91_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test91_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test92_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_rerun_model_config.json to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test92_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test92_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test92_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +File master Test93_smlp_toy_num_resp_mult_dt_sklearn_y2_tree_rules.txt does not exist +comparing Test93_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +File master Test93_smlp_toy_num_resp_mult_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test93_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test93_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_optimization_results.json to master +comparing Test93_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test93_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test93_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test93_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test93_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test93_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test93_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test93_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test94_smlp_toy_num_resp_mult_rf_sklearn_y2_tree_rules.txt does not exist +comparing Test94_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test94_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test94_smlp_toy_num_resp_mult_rf_sklearn_y1_tree_rules.txt does not exist +comparing Test94_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test94_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test94_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test94_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test94_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test94_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test94_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test94_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test95_smlp_toy_num_resp_mult_y2_dt_caret_tree_rules.txt to master +Passed! +comparing Test95_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test95_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_optimization_results.json to master +comparing Test95_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test95_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test95_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test95_smlp_toy_num_resp_mult_y1_dt_caret_tree_rules.txt to master +Passed! +File master Test95_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test95_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test95_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test95_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test96_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test96_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_optimization_results.json to master +comparing Test96_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test96_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test96_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test96_smlp_toy_num_resp_mult_y1_rf_caret_tree_rules.txt to master +Passed! +File master Test96_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test96_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +comparing Test96_smlp_toy_num_resp_mult_y2_rf_caret_tree_rules.txt to master +Passed! +File master Test96_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test96_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test97_smlp_toy_num_resp_mult.txt to master +comparing Test97_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +File master Test97_smlp_toy_num_resp_mult_et_sklearn_y1_tree_rules.txt does not exist +File master Test97_smlp_toy_num_resp_mult_et_sklearn_y2_tree_rules.txt does not exist +comparing Test97_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test97_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test97_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test97_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test97_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test97_smlp_toy_num_resp_mult_query_results.json to master +Passed! +comparing Test97_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test97_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test97_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test97_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test97_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test97_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test97_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test97_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test97_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test98_smlp_toy_num_resp_mult_y2_et_caret_tree_rules.txt to master +Passed! +comparing Test98_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test98_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_optimization_results.json to master +comparing Test98_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test98_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test98_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test98_smlp_toy_num_resp_mult_y1_et_caret_tree_rules.txt to master +Passed! +File master Test98_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test98_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test98_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test98_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test99_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test99_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test99_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test99_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test99_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test99_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test99_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test99_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test100_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_optimization_progress.csv to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_optimization_progress.json to master +comparing Test100_smlp_toy_num_resp_mult_optimization_results.csv to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_optimization_results.json to master +File master Test100_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test100_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test100_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test100_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test100_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master test101_model_dt_sklearn_y2_tree_rules.txt does not exist +comparing Test101_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test101_smlp_toy_num_resp_mult_certify_results.json to master +Passed! +comparing Test101_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test101_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test101_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test101_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test101_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test101_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test101_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test101_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing test101_model_data_bounds.json to master +Passed! +File master test101_model_dt_sklearn_y1_tree_rules.txt does not exist +comparing test101_model_model_features_dict.json to master +Passed! +comparing test101_model_model_levels_dict.json to master +Passed! +comparing test101_model_rerun_model_config.json to master +Passed! +File master test101_model_y1_smlp_full_model_term.json does not exist +File master test101_model_y1_smlp_model_term.json does not exist +File master test101_model_y2_smlp_full_model_term.json does not exist +File master test101_model_y2_smlp_model_term.json does not exist +comparing Test102_test101_model.txt to master +comparing Test102_test101_model_certify_results.json to master +Passed! +File master Test102_test101_model_trace.csv does not exist +comparing test103_model_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test103_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test103_smlp_toy_num_resp_mult_certify_results.json to master +Passed! +comparing Test103_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test103_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test103_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test103_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test103_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test103_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test103_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test103_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing test103_model_data_bounds.json to master +Passed! +comparing test103_model_model_features_dict.json to master +Passed! +comparing test103_model_model_levels_dict.json to master +Passed! +comparing test103_model_rerun_model_config.json to master +Passed! +File master test103_model_smlp_full_model_term.json does not exist +comparing Test104_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test104_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test104_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test104_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test104_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test104_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test104_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test104_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test104_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test104_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test104_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test104_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test104_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test105_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +File master Test105_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +comparing Test105_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test105_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test105_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test105_smlp_toy_num_resp_mult_verify_results.json to master +comparing Test106_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test106_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test106_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test106_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +comparing Test106_smlp_toy_num_resp_mult_verify_results.json to master +Passed! +comparing Test107_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test107_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test107_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test107_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test108_smlp_toy_num_resp_mult_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test108_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test108_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +File master Test108_smlp_toy_num_resp_mult_dt_sklearn_y2_tree_rules.txt does not exist +comparing Test108_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test108_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test108_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test108_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test108_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test108_smlp_toy_num_resp_mult_synthesize_results.json to master +Passed! +comparing Test108_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test108_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test108_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test108_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test108_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test108_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test108_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test108_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test108_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test109_smlp_toy_num_resp_mult_dt_sklearn_y2_tree_rules.txt does not exist +comparing Test109_smlp_toy_num_resp_mult.txt to master +Passed! +comparing Test109_smlp_toy_num_resp_mult_data_bounds.json to master +Passed! +File master Test109_smlp_toy_num_resp_mult_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test109_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv to master +Passed! +comparing Test109_smlp_toy_num_resp_mult_labeled_predictions_summary.csv to master +Passed! +comparing Test109_smlp_toy_num_resp_mult_missing_values_dict.json to master +Passed! +comparing Test109_smlp_toy_num_resp_mult_model_features_dict.json to master +Passed! +comparing Test109_smlp_toy_num_resp_mult_model_levels_dict.json to master +Passed! +comparing Test109_smlp_toy_num_resp_mult_synthesize_results.json to master +Passed! +comparing Test109_smlp_toy_num_resp_mult_test_prediction_precisions.csv to master +Passed! +comparing Test109_smlp_toy_num_resp_mult_test_predictions_summary.csv to master +Passed! +File master Test109_smlp_toy_num_resp_mult_trace.csv does not exist +comparing Test109_smlp_toy_num_resp_mult_training_prediction_precisions.csv to master +Passed! +comparing Test109_smlp_toy_num_resp_mult_training_predictions_summary.csv to master +Passed! +File master Test109_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test109_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test109_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test109_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +comparing Test110_smlp_toy_basic_smlp_toy_basic_pred_unlabeled.txt to master +Passed! +comparing Test110_smlp_toy_basic_smlp_toy_basic_pred_unlabeled_labeled_prediction_precisions.csv to master +Passed! +comparing Test110_smlp_toy_basic_smlp_toy_basic_pred_unlabeled_labeled_predictions_summary.csv to master +Passed! +comparing Test110_smlp_toy_basic_smlp_toy_basic_pred_unlabeled_new_predictions_summary.csv to master +Passed! +comparing Test110_smlp_toy_basic_smlp_toy_basic_pred_unlabeled_test_prediction_precisions.csv to master +Passed! +comparing Test110_smlp_toy_basic_smlp_toy_basic_pred_unlabeled_test_predictions_summary.csv to master +Passed! +comparing Test110_smlp_toy_basic_smlp_toy_basic_pred_unlabeled_training_prediction_precisions.csv to master +Passed! +comparing Test110_smlp_toy_basic_smlp_toy_basic_pred_unlabeled_training_predictions_summary.csv to master +Passed! +comparing test110_model_data_bounds.json to master +Passed! +comparing test110_model_model_features_dict.json to master +Passed! +comparing test110_model_model_levels_dict.json to master +Passed! +comparing test110_model_poly_sklearn_formula.txt to master +comparing test110_model_rerun_model_config.json to master +Passed! +comparing Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt to master +comparing Test111_test110_model_smlp_toy_basic_pred_unlabeled_new_predictions_summary.csv to master +Passed! +comparing Test112_test110_model_smlp_toy_basic_pred_unlabeled.txt to master +comparing Test112_test110_model_smlp_toy_basic_pred_unlabeled_new_predictions_summary.csv to master +Passed! +comparing test113_model_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test113_smlp_toy_basic.txt to master +Passed! +comparing Test113_smlp_toy_basic_labeled_prediction_precisions.csv to master +Passed! +comparing Test113_smlp_toy_basic_labeled_predictions_summary.csv to master +Passed! +comparing Test113_smlp_toy_basic_optimization_progress.csv to master +Passed! +comparing Test113_smlp_toy_basic_optimization_progress.json to master +comparing Test113_smlp_toy_basic_optimization_results.csv to master +Passed! +comparing Test113_smlp_toy_basic_optimization_results.json to master +comparing Test113_smlp_toy_basic_test_prediction_precisions.csv to master +Passed! +comparing Test113_smlp_toy_basic_test_predictions_summary.csv to master +Passed! +File master Test113_smlp_toy_basic_trace.csv does not exist +comparing Test113_smlp_toy_basic_training_prediction_precisions.csv to master +Passed! +comparing Test113_smlp_toy_basic_training_predictions_summary.csv to master +Passed! +comparing test113_model_data_bounds.json to master +Passed! +comparing test113_model_model_features_dict.json to master +Passed! +comparing test113_model_model_levels_dict.json to master +Passed! +comparing test113_model_rerun_model_config.json to master +Passed! +File master test113_model_smlp_full_model_term.json does not exist +comparing Test114_smlp_toy_basic_dt_sklearn_tree_rules.txt to master +Passed! +comparing Test114_smlp_toy_basic.txt to master +Passed! +comparing Test114_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test114_smlp_toy_basic_labeled_prediction_precisions.csv to master +Passed! +comparing Test114_smlp_toy_basic_labeled_predictions_summary.csv to master +Passed! +comparing Test114_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test114_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test114_smlp_toy_basic_optimization_progress.csv to master +Passed! +comparing Test114_smlp_toy_basic_optimization_progress.json to master +comparing Test114_smlp_toy_basic_optimization_results.csv to master +Passed! +comparing Test114_smlp_toy_basic_optimization_results.json to master +File master Test114_smlp_toy_basic_smlp_full_model_term.json does not exist +comparing Test114_smlp_toy_basic_test_prediction_precisions.csv to master +Passed! +comparing Test114_smlp_toy_basic_test_predictions_summary.csv to master +Passed! +File master Test114_smlp_toy_basic_trace.csv does not exist +comparing Test114_smlp_toy_basic_training_prediction_precisions.csv to master +Passed! +comparing Test114_smlp_toy_basic_training_predictions_summary.csv to master +Passed! +File master Test115_smlp_toy_basic_dt_sklearn_y2_tree_rules.txt does not exist +comparing Test115_smlp_toy_basic.txt to master +Passed! +comparing Test115_smlp_toy_basic_certify_results.json to master +Passed! +comparing Test115_smlp_toy_basic_data_bounds.json to master +Passed! +File master Test115_smlp_toy_basic_dt_sklearn_y1_tree_rules.txt does not exist +comparing Test115_smlp_toy_basic_labeled_prediction_precisions.csv to master +Passed! +comparing Test115_smlp_toy_basic_labeled_predictions_summary.csv to master +Passed! +comparing Test115_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test115_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test115_smlp_toy_basic_test_prediction_precisions.csv to master +Passed! +comparing Test115_smlp_toy_basic_test_predictions_summary.csv to master +Passed! +File master Test115_smlp_toy_basic_trace.csv does not exist +comparing Test115_smlp_toy_basic_training_prediction_precisions.csv to master +Passed! +comparing Test115_smlp_toy_basic_training_predictions_summary.csv to master +Passed! +File master Test115_smlp_toy_basic_y1_smlp_full_model_term.json does not exist +File master Test115_smlp_toy_basic_y1_smlp_model_term.json does not exist +File master Test115_smlp_toy_basic_y2_smlp_full_model_term.json does not exist +File master Test115_smlp_toy_basic_y2_smlp_model_term.json does not exist +comparing Test116_smlp_toy_basic.txt to master +Passed! +comparing Test116_smlp_toy_basic_certify_results.json to master +Passed! +comparing Test116_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test116_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test116_smlp_toy_basic_model_levels_dict.json to master +Passed! +File master Test116_smlp_toy_basic_trace.csv does not exist +comparing Test117_smlp_toy_basic.txt to master +Passed! +comparing Test117_smlp_toy_basic_certify_results.json to master +Passed! +comparing Test117_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test117_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test117_smlp_toy_basic_model_levels_dict.json to master +Passed! +File master Test117_smlp_toy_basic_trace.csv does not exist +comparing Test118_smlp_toy_basic.txt to master +Passed! +comparing Test118_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test118_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test118_smlp_toy_basic_model_levels_dict.json to master +Passed! +File master Test118_smlp_toy_basic_trace.csv does not exist +comparing Test118_smlp_toy_basic_verify_results.json to master +comparing Test119_smlp_toy_basic.txt to master +Passed! +comparing Test119_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test119_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test119_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test119_smlp_toy_basic_query_results.json to master +File master Test119_smlp_toy_basic_trace.csv does not exist +comparing Test120_smlp_toy_basic.txt to master +Passed! +comparing Test120_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test120_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test120_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test120_smlp_toy_basic_synthesize_results.json to master +Passed! +File master Test120_smlp_toy_basic_trace.csv does not exist +comparing Test121_smlp_toy_basic.txt to master +Passed! +comparing Test121_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test121_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test121_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test121_smlp_toy_basic_synthesize_results.json to master +File master Test121_smlp_toy_basic_trace.csv does not exist +comparing Test122_smlp_toy_basic.txt to master +Passed! +comparing Test122_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test122_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test122_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test122_smlp_toy_basic_optimization_progress.csv to master +Passed! +comparing Test122_smlp_toy_basic_optimization_progress.json to master +comparing Test122_smlp_toy_basic_optimization_results.json to master +File master Test122_smlp_toy_basic_trace.csv does not exist +comparing Test123_smlp_toy_basic.txt to master +Passed! +comparing Test123_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test123_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test123_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test123_smlp_toy_basic_optimization_progress.csv to master +Passed! +comparing Test123_smlp_toy_basic_optimization_progress.json to master +comparing Test123_smlp_toy_basic_optimization_results.csv to master +Passed! +comparing Test123_smlp_toy_basic_optimization_results.json to master +File master Test123_smlp_toy_basic_sampling_prediction_precisions.csv does not exist +File master Test123_smlp_toy_basic_sampling_predictions_summary.csv does not exist +File master Test123_smlp_toy_basic_trace.csv does not exist +comparing Test124_smlp_toy_basic.txt to master +Passed! +comparing Test124_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test124_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test124_smlp_toy_basic_model_levels_dict.json to master +Passed! +File master Test124_smlp_toy_basic_optimization_results.json does not exist +File master Test124_smlp_toy_basic_trace.csv does not exist +comparing Test125_smlp_toy_basic.txt to master +Passed! +comparing Test125_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test125_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test125_smlp_toy_basic_model_levels_dict.json to master +Passed! +comparing Test125_smlp_toy_basic_optimization_progress.csv to master +Passed! +comparing Test125_smlp_toy_basic_optimization_progress.json to master +comparing Test125_smlp_toy_basic_optimization_results.csv to master +Passed! +comparing Test125_smlp_toy_basic_optimization_results.json to master +File master Test125_smlp_toy_basic_trace.csv does not exist +comparing Test126_smlp_toy_basic.txt to master +Passed! +comparing Test126_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test126_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test126_smlp_toy_basic_model_levels_dict.json to master +Passed! +File master Test126_smlp_toy_basic_trace.csv does not exist +comparing Test126_smlp_toy_basic_verify_results.json to master +comparing Test127_smlp_toy_basic.txt to master +Passed! +comparing Test127_smlp_toy_basic_certify_results.json to master +Passed! +comparing Test127_smlp_toy_basic_data_bounds.json to master +Passed! +comparing Test127_smlp_toy_basic_model_features_dict.json to master +Passed! +comparing Test127_smlp_toy_basic_model_levels_dict.json to master +Passed! +File master Test127_smlp_toy_basic_trace.csv does not exist +comparing Test128_smlp_toy_ctg_num_resp.txt to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_certify_results.json to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_data_bounds.json to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_labeled_prediction_precisions.csv to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_labeled_predictions_summary.csv to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_missing_values_dict.json to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_model_features_dict.json to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_model_levels_dict.json to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_poly_sklearn_formula.txt to master +Passed! +File master Test128_smlp_toy_ctg_num_resp_smlp_full_model_term.json does not exist +comparing Test128_smlp_toy_ctg_num_resp_test_prediction_precisions.csv to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_test_predictions_summary.csv to master +Passed! +File master Test128_smlp_toy_ctg_num_resp_trace.csv does not exist +comparing Test128_smlp_toy_ctg_num_resp_training_prediction_precisions.csv to master +Passed! +comparing Test128_smlp_toy_ctg_num_resp_training_predictions_summary.csv to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp.txt to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_data_bounds.json to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_labeled_prediction_precisions.csv to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_labeled_predictions_summary.csv to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_missing_values_dict.json to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_model_features_dict.json to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_model_levels_dict.json to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_poly_sklearn_formula.txt to master +Passed! +File master Test129_smlp_toy_ctg_num_resp_smlp_full_model_term.json does not exist +comparing Test129_smlp_toy_ctg_num_resp_test_prediction_precisions.csv to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_test_predictions_summary.csv to master +Passed! +File master Test129_smlp_toy_ctg_num_resp_trace.csv does not exist +comparing Test129_smlp_toy_ctg_num_resp_training_prediction_precisions.csv to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_training_predictions_summary.csv to master +Passed! +comparing Test129_smlp_toy_ctg_num_resp_verify_results.json to master +Test 130 Failed: +Error in Build stage: +Data file does not exist +Test 131 Failed: +Error in Build stage: +Data file does not exist +Test 132 Failed: +Error in Build stage: +Data file does not exist +Test 133 Failed: +Error in Build stage: +Data file does not exist +Test 134 Failed: +Error in Build stage: +Data file does not exist +Test 135 Failed: +Error in Build stage: +Data file does not exist +Test 136 Failed: +Error in Build stage: +Data file does not exist +Test 137 Failed: +Error in Build stage: +Data file does not exist +Error in Build stage: +New data file does not exist +Test 138 Failed: +Error in Build stage: +Data file does not exist +Test 139 Failed: +Error in Build stage: +Data file does not exist +Error in Build stage: +New data file does not exist +File master Test140_smlp_toy_basic.txt does not exist +File master Test140_smlp_toy_basic_data_bounds.json does not exist +File master Test140_smlp_toy_basic_features_scaler.pkl does not exist +File master Test140_smlp_toy_basic_model_features_dict.json does not exist +File master Test140_smlp_toy_basic_model_levels_dict.json does not exist +File master Test140_smlp_toy_basic_responses_scaler.pkl does not exist +File master Test140_smlp_toy_basic_trace.csv does not exist +File master Test140_smlp_toy_basic_verify_results.json does not exist +File master Test141_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt does not exist +File master Test141_smlp_toy_num_resp_mult.txt does not exist +File master Test141_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test141_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist +File master Test141_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test141_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test141_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test141_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test141_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test141_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test141_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test141_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test141_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test141_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test141_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test141_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test141_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test141_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test141_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test141_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test141_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test142_smlp_toy_num_resp_mult_rf_sklearn_y1_tree_rules.txt does not exist +File master Test142_smlp_toy_num_resp_mult.txt does not exist +File master Test142_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test142_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test142_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test142_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test142_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test142_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test142_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test142_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test142_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test142_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test142_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test142_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test142_smlp_toy_num_resp_mult_rf_sklearn_y2_tree_rules.txt does not exist +File master Test142_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test142_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test142_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test142_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test142_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test142_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test142_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test142_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test142_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test143_smlp_toy_num_resp_mult_et_sklearn_y2_tree_rules.txt does not exist +File master Test143_smlp_toy_num_resp_mult.txt does not exist +File master Test143_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test143_smlp_toy_num_resp_mult_et_sklearn_y1_tree_rules.txt does not exist +File master Test143_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test143_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test143_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test143_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test143_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test143_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test143_smlp_toy_num_resp_mult_query_results.json does not exist +File master Test143_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test143_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test143_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test143_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test143_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test143_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test143_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test143_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test143_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test143_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +Test 144 Failed: +Error in Build stage: +Data file does not exist +File master Test145_doe_two_levels_opt.txt does not exist +File master Test145_doe_two_levels_opt_trace.csv does not exist +File master Test146_explore_doe_two_levels.txt does not exist +File master Test146_explore_doe_two_levels_trace.csv does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test147_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test147_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test147_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test147_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test147_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test147_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test147_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test147_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mse.png does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test147_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test148_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test148_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test148_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test148_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test148_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test148_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test148_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test148_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_all_responses_mse.png does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test148_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test149_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test149_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test149_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test149_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test149_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test149_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test149_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test149_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mse.png does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test149_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test150_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test150_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test150_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test150_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test150_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test150_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test150_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test150_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y1_mse.png does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mse.png does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test150_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test151_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test151_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test151_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test151_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test151_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test151_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test151_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test151_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mse.png does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test151_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test152_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test152_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test152_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test152_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test152_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test152_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test152_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test152_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y1_mse.png does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mse.png does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test152_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test153_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test153_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test153_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test153_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test153_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test153_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test153_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test153_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_all_responses_mse.png does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test153_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test154_smlp_toy_num_resp_mult.txt does not exist +File master Test154_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test154_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test154_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test154_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test154_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test154_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test154_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test154_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test154_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test154_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test154_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test154_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test154_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +File master Test154_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test154_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test154_smlp_toy_num_resp_mult_verify_results.json does not exist +File master Test154_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test154_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test155_smlp_toy_num_resp_mult.txt does not exist +File master Test155_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test155_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test155_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test155_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test155_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test155_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test155_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test155_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test155_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test155_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test155_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test155_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test155_smlp_toy_num_resp_mult_train-reg_y2_mae.png does not exist +File master Test155_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test155_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test155_smlp_toy_num_resp_mult_verify_results.json does not exist +File master Test155_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test155_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test156_smlp_toy_num_resp_mult.txt does not exist +File master Test156_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test156_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test156_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test156_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test156_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test156_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test156_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test156_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test156_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test156_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test156_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test156_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test156_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test156_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test156_smlp_toy_num_resp_mult_train-reg_y1_rmse.png does not exist +File master Test156_smlp_toy_num_resp_mult_train-reg_y2_rmse.png does not exist +File master Test156_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test156_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test156_smlp_toy_num_resp_mult_verify_results.json does not exist +File master Test157_smlp_toy_num_resp_mult.txt does not exist +File master Test157_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test157_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test157_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test157_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test157_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test157_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test157_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test157_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test157_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test157_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test157_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test157_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test157_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test157_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test157_smlp_toy_num_resp_mult_train-reg_all_responses_logcosh.png does not exist +File master Test157_smlp_toy_num_resp_mult_train-reg_all_responses_rmse.png does not exist +File master Test157_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test157_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test157_smlp_toy_num_resp_mult_verify_results.json does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test158_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test158_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test158_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test158_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test158_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test158_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test158_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y1_rmse.png does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_rmse.png does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test158_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test158_smlp_toy_num_resp_mult_y1_nn_keras_model_complete.h5 does not exist +File master Test158_smlp_toy_num_resp_mult_y2_nn_keras_model_complete.h5 does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test159_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test159_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test159_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test159_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test159_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test159_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test159_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y1_cosine.png does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y1_mae.png does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_cosine.png does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_mae.png does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test159_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test159_smlp_toy_num_resp_mult_y1_nn_keras_model_complete.h5 does not exist +File master Test159_smlp_toy_num_resp_mult_y2_nn_keras_model_complete.h5 does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test160_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test160_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test160_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test160_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test160_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test160_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y1_msle.png does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_y2_msle.png does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test160_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt does not exist +File master Test161_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test161_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test161_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test161_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test161_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test161_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_prediction_precisions.csv does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_labeled_predictions_summary.csv does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_missing_values_dict.json does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_prediction_precisions.csv does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_new_predictions_summary.csv does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_prediction_precisions.csv does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_test_predictions_summary.csv does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_all_responses_logcosh.png does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_train-reg_all_responses_mape.png does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_prediction_precisions.csv does not exist +File master Test161_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled_training_predictions_summary.csv does not exist +Test 162 Failed: +Error in Build stage: +Data file does not exist +Error in Build stage: +New data file does not exist +Test 163 Failed: +Error in Build stage: +Data file does not exist +Error in Build stage: +New data file does not exist +File master Test164_smlp_toy_num_resp_mult.txt does not exist +File master Test164_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test164_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist +File master Test164_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt does not exist +File master Test164_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test164_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test164_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test164_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test164_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test164_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test164_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test164_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test164_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test164_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test164_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test164_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test164_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test164_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test164_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test164_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test164_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test164_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test165_smlp_toy_num_resp_mult_y2_dt_caret_tree_rules.txt does not exist +File master Test165_smlp_toy_num_resp_mult.txt does not exist +File master Test165_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test165_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test165_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test165_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test165_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test165_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test165_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test165_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test165_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test165_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test165_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test165_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test165_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test165_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test165_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test165_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test165_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test165_smlp_toy_num_resp_mult_y1_dt_caret_tree_rules.txt does not exist +File master Test165_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test165_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test165_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test165_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test166_smlp_toy_num_resp_mult_rf_sklearn_tree_rules.txt does not exist +File master Test166_smlp_toy_num_resp_mult.txt does not exist +File master Test166_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test166_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test166_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test166_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test166_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test166_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test166_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test166_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test166_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test166_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test166_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test166_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test166_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test166_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test166_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test166_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test166_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test166_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test166_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test167_smlp_toy_num_resp_mult_rf_sklearn_y1_tree_rules.txt does not exist +File master Test167_smlp_toy_num_resp_mult.txt does not exist +File master Test167_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test167_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test167_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test167_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test167_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test167_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test167_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test167_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test167_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test167_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test167_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test167_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test167_smlp_toy_num_resp_mult_rf_sklearn_y2_tree_rules.txt does not exist +File master Test167_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test167_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test167_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test167_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test167_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test167_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test167_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test167_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test167_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test168_smlp_toy_num_resp_mult_y2_rf_caret_tree_rules.txt does not exist +File master Test168_smlp_toy_num_resp_mult.txt does not exist +File master Test168_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test168_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test168_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test168_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test168_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test168_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test168_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test168_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test168_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test168_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test168_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test168_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test168_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test168_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test168_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test168_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test168_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test168_smlp_toy_num_resp_mult_y1_rf_caret_tree_rules.txt does not exist +File master Test168_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test168_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test168_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test168_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test169_smlp_toy_num_resp_mult_et_sklearn_y2_tree_rules.txt does not exist +File master Test169_smlp_toy_num_resp_mult.txt does not exist +File master Test169_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test169_smlp_toy_num_resp_mult_et_sklearn_y1_tree_rules.txt does not exist +File master Test169_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test169_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test169_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test169_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test169_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test169_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test169_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test169_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test169_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test169_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test169_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test169_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test169_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test169_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test169_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test169_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test169_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test169_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test169_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test169_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test170_smlp_toy_num_resp_mult.txt does not exist +File master Test170_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test170_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt does not exist +File master Test170_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test170_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test170_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test170_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test170_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test170_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test170_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test170_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test170_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test170_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test170_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test170_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test170_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test170_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test170_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test170_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test170_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test170_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test171_smlp_toy_num_resp_mult_y2_et_caret_tree_rules.txt does not exist +File master Test171_smlp_toy_num_resp_mult.txt does not exist +File master Test171_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test171_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test171_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test171_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test171_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test171_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test171_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test171_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test171_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test171_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test171_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test171_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test171_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test171_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test171_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test171_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test171_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test171_smlp_toy_num_resp_mult_y1_et_caret_tree_rules.txt does not exist +File master Test171_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test171_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test171_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test171_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test172_smlp_toy_num_resp_mult.txt does not exist +File master Test172_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test172_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test172_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test172_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test172_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test172_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test172_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test172_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test172_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test172_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test172_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test172_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test172_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +File master Test172_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test172_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test172_smlp_toy_num_resp_mult_verify_results.json does not exist +File master Test172_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test172_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test173_smlp_toy_num_resp_mult.txt does not exist +File master Test173_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test173_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test173_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test173_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test173_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test173_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test173_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test173_smlp_toy_num_resp_mult_nn_keras_model_complete.h5 does not exist +File master Test173_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test173_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test173_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test173_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test173_smlp_toy_num_resp_mult_train-reg_y2_mae.png does not exist +File master Test173_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test173_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test173_smlp_toy_num_resp_mult_verify_results.json does not exist +File master Test173_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test173_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test174_smlp_toy_num_resp_mult.txt does not exist +File master Test174_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test174_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test174_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test174_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test174_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test174_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test174_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test174_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test174_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test174_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test174_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test174_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test174_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test174_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test174_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test174_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test174_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test174_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test174_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test174_smlp_toy_num_resp_mult_train-reg_y1_mse.png does not exist +File master Test174_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +File master Test174_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test174_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test175_smlp_toy_num_resp_mult.txt does not exist +File master Test175_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test175_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test175_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test175_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test175_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test175_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test175_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test175_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test175_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test175_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test175_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test175_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test175_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test175_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test175_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test175_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test175_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test175_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test175_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test175_smlp_toy_num_resp_mult_train-reg_all_responses_mse.png does not exist +File master Test175_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test175_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test176_smlp_toy_num_resp_mult.txt does not exist +File master Test176_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test176_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test176_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test176_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test176_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test176_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test176_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test176_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test176_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test176_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test176_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test176_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test176_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test176_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test176_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test176_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test176_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test176_smlp_toy_num_resp_mult_train-reg_y1_mse.png does not exist +File master Test176_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +File master Test176_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test176_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test176_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test176_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test176_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test176_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test177_smlp_toy_num_resp_mult.txt does not exist +File master Test177_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test177_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test177_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test177_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test177_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test177_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test177_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test177_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test177_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test177_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test177_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test177_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test177_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test177_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test177_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test177_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test177_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test177_smlp_toy_num_resp_mult_train-reg_y1_mse.png does not exist +File master Test177_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +File master Test177_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test177_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test177_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test177_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test177_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test177_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test178_smlp_toy_num_resp_mult.txt does not exist +File master Test178_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test178_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test178_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test178_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test178_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test178_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test178_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test178_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test178_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test178_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test178_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test178_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test178_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test178_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test178_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test178_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test178_smlp_toy_num_resp_mult_train-reg_y1_mse.png does not exist +File master Test178_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +File master Test178_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test178_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test178_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test178_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test178_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test178_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test179_smlp_toy_num_resp_mult.txt does not exist +File master Test179_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test179_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test179_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test179_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test179_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test179_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test179_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test179_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test179_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test179_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test179_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test179_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test179_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test179_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test179_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test179_smlp_toy_num_resp_mult_train-reg_y1_mse.png does not exist +File master Test179_smlp_toy_num_resp_mult_train-reg_y2_mse.png does not exist +File master Test179_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test179_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test180_smlp_toy_num_resp_mult.txt does not exist +File master Test180_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test180_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test180_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test180_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test180_smlp_toy_num_resp_mult_model_checkpoint.h5 does not exist +File master Test180_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test180_smlp_toy_num_resp_mult_model_gen.json does not exist +File master Test180_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test180_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test180_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test180_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test180_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test180_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test180_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test180_smlp_toy_num_resp_mult_train-reg_all_responses_mse.png does not exist +File master Test180_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test180_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test181_smlp_toy_num_resp_mult.txt does not exist +File master Test181_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test181_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist +File master Test181_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt does not exist +File master Test181_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test181_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test181_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test181_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test181_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test181_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test181_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test181_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test181_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test181_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test181_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test181_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test181_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test181_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test181_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test181_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test181_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test182_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt does not exist +File master Test182_smlp_toy_num_resp_mult.txt does not exist +File master Test182_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test182_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist +File master Test182_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test182_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test182_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test182_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test182_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test182_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test182_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test182_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test182_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test182_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test182_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test182_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test182_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test182_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test182_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test182_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test182_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test183_smlp_toy_num_resp_mult.txt does not exist +File master Test183_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test183_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist +File master Test183_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt does not exist +File master Test183_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test183_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test183_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test183_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test183_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test183_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test183_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test183_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test183_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test183_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test183_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test183_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test183_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test183_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test183_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test183_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +Test 184 Failed: +Error in Build stage: +Data file does not exist +Error in Build stage: +New data file does not exist +Test 185 Failed: +Error in Build stage: +Data file does not exist +Error in Build stage: +New data file does not exist +Test 186 Failed: +Error in Build stage: +Data file does not exist +Error in Build stage: +New data file does not exist +File master Test187_smlp_toy_num_resp_mult.txt does not exist +File master Test187_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test187_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist +File master Test187_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt does not exist +File master Test187_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test187_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test187_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test187_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test187_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test187_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test187_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test187_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test187_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test187_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test187_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test187_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test187_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test187_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test187_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test187_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test187_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test187_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test188_smlp_toy_num_resp_mult_y2_dt_caret_tree_rules.txt does not exist +File master Test188_smlp_toy_num_resp_mult.txt does not exist +File master Test188_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test188_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test188_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test188_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test188_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test188_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test188_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test188_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test188_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test188_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test188_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test188_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test188_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test188_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test188_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test188_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test188_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test188_smlp_toy_num_resp_mult_y1_dt_caret_tree_rules.txt does not exist +File master Test188_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test188_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test188_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test188_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test189_smlp_toy_num_resp_mult.txt does not exist +File master Test189_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test189_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test189_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test189_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test189_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test189_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test189_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test189_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test189_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test189_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test189_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test189_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test189_smlp_toy_num_resp_mult_rf_sklearn_tree_rules.txt does not exist +File master Test189_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test189_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test189_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test189_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test189_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test189_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test189_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test190_smlp_toy_num_resp_mult_y2_rf_caret_tree_rules.txt does not exist +File master Test190_smlp_toy_num_resp_mult.txt does not exist +File master Test190_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test190_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test190_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test190_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test190_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test190_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test190_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test190_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test190_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test190_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test190_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test190_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test190_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test190_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test190_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test190_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test190_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test190_smlp_toy_num_resp_mult_y1_rf_caret_tree_rules.txt does not exist +File master Test190_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test190_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test190_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test190_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test191_smlp_toy_num_resp_mult_et_sklearn_y1_tree_rules.txt does not exist +File master Test191_smlp_toy_num_resp_mult.txt does not exist +File master Test191_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test191_smlp_toy_num_resp_mult_et_sklearn_y2_tree_rules.txt does not exist +File master Test191_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test191_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test191_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test191_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test191_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test191_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test191_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test191_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test191_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test191_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test191_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test191_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test191_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test191_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test191_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test191_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test191_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test191_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test191_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test191_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test192_smlp_toy_num_resp_mult.txt does not exist +File master Test192_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test192_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt does not exist +File master Test192_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test192_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test192_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test192_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test192_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test192_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test192_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test192_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test192_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test192_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test192_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test192_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test192_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test192_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test192_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test192_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test192_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test192_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test193_smlp_toy_num_resp_mult_y2_et_caret_tree_rules.txt does not exist +File master Test193_smlp_toy_num_resp_mult.txt does not exist +File master Test193_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test193_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test193_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test193_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test193_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test193_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test193_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test193_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test193_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test193_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test193_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test193_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test193_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test193_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test193_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test193_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test193_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test193_smlp_toy_num_resp_mult_y1_et_caret_tree_rules.txt does not exist +File master Test193_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test193_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test193_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test193_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test194_smlp_toy_num_resp_mult_rf_sklearn_y1_tree_rules.txt does not exist +File master Test194_smlp_toy_num_resp_mult.txt does not exist +File master Test194_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test194_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test194_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test194_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test194_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test194_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test194_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test194_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test194_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test194_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test194_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test194_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test194_smlp_toy_num_resp_mult_rf_sklearn_y2_tree_rules.txt does not exist +File master Test194_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test194_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test194_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test194_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test194_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test194_smlp_toy_num_resp_mult_y1_smlp_full_model_term.json does not exist +File master Test194_smlp_toy_num_resp_mult_y1_smlp_model_term.json does not exist +File master Test194_smlp_toy_num_resp_mult_y2_smlp_full_model_term.json does not exist +File master Test194_smlp_toy_num_resp_mult_y2_smlp_model_term.json does not exist +File master Test195_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt does not exist +File master Test195_smlp_toy_num_resp_mult.txt does not exist +File master Test195_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test195_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test195_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test195_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test195_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test195_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test195_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test195_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test195_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test195_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test195_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test195_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test195_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test195_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test195_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test195_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test195_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test195_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test195_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test196_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt does not exist +File master Test196_smlp_toy_num_resp_mult.txt does not exist +File master Test196_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test196_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist +File master Test196_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test196_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test196_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test196_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test196_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test196_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test196_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test196_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test196_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test196_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test196_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test196_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test196_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test196_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test196_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test196_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test196_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test197_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt does not exist +File master Test197_smlp_toy_num_resp_mult.txt does not exist +File master Test197_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test197_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist +File master Test197_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test197_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test197_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test197_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test197_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test197_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test197_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test197_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test197_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test197_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test197_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test197_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test197_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test197_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test197_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test197_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test197_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test198_smlp_toy_num_resp_mult.txt does not exist +File master Test198_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test198_smlp_toy_num_resp_mult_dt_sklearn_model_complete.pkl does not exist +File master Test198_smlp_toy_num_resp_mult_dt_sklearn_tree_rules.txt does not exist +File master Test198_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test198_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test198_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test198_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test198_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test198_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test198_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test198_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test198_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test198_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test198_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test198_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test198_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test198_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test198_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test198_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test199_smlp_toy_num_resp_mult.txt does not exist +File master Test199_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test199_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt does not exist +File master Test199_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test199_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test199_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test199_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test199_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test199_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test199_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test199_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test199_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test199_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test199_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test199_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test199_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test199_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test199_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test199_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test199_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test199_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test200_smlp_toy_num_resp_mult.txt does not exist +File master Test200_smlp_toy_num_resp_mult_data_bounds.json does not exist +File master Test200_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt does not exist +File master Test200_smlp_toy_num_resp_mult_features_scaler.pkl does not exist +File master Test200_smlp_toy_num_resp_mult_labeled_prediction_precisions.csv does not exist +File master Test200_smlp_toy_num_resp_mult_labeled_predictions_summary.csv does not exist +File master Test200_smlp_toy_num_resp_mult_missing_values_dict.json does not exist +File master Test200_smlp_toy_num_resp_mult_model_features_dict.json does not exist +File master Test200_smlp_toy_num_resp_mult_model_levels_dict.json does not exist +File master Test200_smlp_toy_num_resp_mult_optimization_progress.csv does not exist +File master Test200_smlp_toy_num_resp_mult_optimization_progress.json does not exist +File master Test200_smlp_toy_num_resp_mult_optimization_results.csv does not exist +File master Test200_smlp_toy_num_resp_mult_optimization_results.json does not exist +File master Test200_smlp_toy_num_resp_mult_responses_scaler.pkl does not exist +File master Test200_smlp_toy_num_resp_mult_smlp_full_model_term.json does not exist +File master Test200_smlp_toy_num_resp_mult_smlp_model_term.json does not exist +File master Test200_smlp_toy_num_resp_mult_test_prediction_precisions.csv does not exist +File master Test200_smlp_toy_num_resp_mult_test_predictions_summary.csv does not exist +File master Test200_smlp_toy_num_resp_mult_trace.csv does not exist +File master Test200_smlp_toy_num_resp_mult_training_prediction_precisions.csv does not exist +File master Test200_smlp_toy_num_resp_mult_training_predictions_summary.csv does not exist +File master Test201_smlp_toy_num_resp_mult.txt does not exist +File master Test202_smlp_toy_num_resp_mult.txt does not exist +File master Test203_smlp_toy_basic.txt does not exist +File master Test203_smlp_toy_basic_data_bounds.json does not exist +File master Test203_smlp_toy_basic_features_scaler.pkl does not exist +File master Test203_smlp_toy_basic_model_features_dict.json does not exist +File master Test203_smlp_toy_basic_model_levels_dict.json does not exist +File master Test203_smlp_toy_basic_optimization_progress.csv does not exist +File master Test203_smlp_toy_basic_optimization_progress.json does not exist +File master Test203_smlp_toy_basic_optimization_results.json does not exist +File master Test203_smlp_toy_basic_responses_scaler.pkl does not exist +File master Test203_smlp_toy_basic_trace.csv does not exist +File master Test204_smlp_toy_basic.txt does not exist +File master Test204_smlp_toy_basic_data_bounds.json does not exist +File master Test204_smlp_toy_basic_features_scaler.pkl does not exist +File master Test204_smlp_toy_basic_model_features_dict.json does not exist +File master Test204_smlp_toy_basic_model_levels_dict.json does not exist +File master Test204_smlp_toy_basic_optimization_progress.csv does not exist +File master Test204_smlp_toy_basic_optimization_progress.json does not exist +File master Test204_smlp_toy_basic_optimization_results.csv does not exist +File master Test204_smlp_toy_basic_optimization_results.json does not exist +File master Test204_smlp_toy_basic_responses_scaler.pkl does not exist +File master Test204_smlp_toy_basic_sampling_prediction_precisions.csv does not exist +File master Test204_smlp_toy_basic_sampling_predictions_summary.csv does not exist +File master Test204_smlp_toy_basic_trace.csv does not exist +File master Test205_doe_two_levels_opt.txt does not exist +File master Test205_doe_two_levels_opt_trace.csv does not exist +File master Test206_smlp_toy_basic.txt does not exist +File master Test206_smlp_toy_basic_data_bounds.json does not exist +File master Test206_smlp_toy_basic_features_scaler.pkl does not exist +File master Test206_smlp_toy_basic_model_features_dict.json does not exist +File master Test206_smlp_toy_basic_model_levels_dict.json does not exist +File master Test206_smlp_toy_basic_optimization_progress.csv does not exist +File master Test206_smlp_toy_basic_optimization_progress.json does not exist +File master Test206_smlp_toy_basic_optimization_results.csv does not exist +File master Test206_smlp_toy_basic_optimization_results.json does not exist +File master Test206_smlp_toy_basic_responses_scaler.pkl does not exist +File master Test206_smlp_toy_basic_trace.csv does not exist +Test 207 Failed: +Error in Build stage: +Data file does not exist +Test 208 Failed: +Error in Build stage: +Data file does not exist +Test 209 Failed: +Error in Build stage: +Data file does not exist +Test 210 Failed: +Error in Build stage: +Data file does not exist +Test 211 Failed: +Error in Build stage: +Data file does not exist +Test 212 Failed: +Error in Build stage: +Data file does not exist +Test 213 Failed: +Error in Build stage: +Data file does not exist +Test 214 Failed: +Error in Build stage: +Data file does not exist +File master Test215_smlp_toy_mult_discr.txt does not exist +File master Test215_smlp_toy_mult_discr_features_summary.csv does not exist +File master Test215_smlp_toy_mult_discr_missing_values_dict.json does not exist +File master Test216_smlp_toy_basic.txt does not exist +File master Test216_smlp_toy_basic_features_summary.csv does not exist +File master Test217_smlp_toy_mult_discr.txt does not exist +File master Test217_smlp_toy_mult_discr_features_summary.csv does not exist +File master Test217_smlp_toy_mult_discr_missing_values_dict.json does not exist +File master Test218_smlp_toy_mult_discr.txt does not exist +File master Test218_smlp_toy_mult_discr_features_summary.csv does not exist +File master Test218_smlp_toy_mult_discr_missing_values_dict.json does not exist +File master Test219_smlp_toy_mult_discr.txt does not exist +File master Test219_smlp_toy_mult_discr_features_summary.csv does not exist +File master Test219_smlp_toy_mult_discr_missing_values_dict.json does not exist +File master Test220_smlp_toy_mult_discr.txt does not exist +File master Test220_smlp_toy_mult_discr_features_summary.csv does not exist +File master Test220_smlp_toy_mult_discr_missing_values_dict.json does not exist +File master Test221_smlp_toy_mult_discr.txt does not exist +File master Test221_smlp_toy_mult_discr_features_summary.csv does not exist +File master Test221_smlp_toy_mult_discr_missing_values_dict.json does not exist +File master Test222_smlp_toy_mult_discr.txt does not exist +File master Test222_smlp_toy_mult_discr_features_summary.csv does not exist +File master Test222_smlp_toy_mult_discr_missing_values_dict.json does not exist +File master Test223_smlp_toy_basic.txt does not exist +File master Test223_smlp_toy_basic_features_summary.csv does not exist +File master Test224_smlp_toy_basic.txt does not exist +File master Test224_smlp_toy_basic_features_summary.csv does not exist +File master Test225_smlp_toy_basic.txt does not exist +File master Test225_smlp_toy_basic_features_summary.csv does not exist +File master Test226_smlp_toy_basic.txt does not exist +File master Test226_smlp_toy_basic_features_summary.csv does not exist +File master Test227_smlp_toy_basic.txt does not exist +File master Test227_smlp_toy_basic_features_summary.csv does not exist +master log file does not exist! +Do you wish to copy the new log file to master? +(yes/no|y/n): No new tests crashed (not in the masters) +Time: 32.69679698944092 minutes +End of regression diff --git a/tests/smlp_regression/run_smlp_regression_whl_expected_diff_report.log b/tests/smlp_regression/run_smlp_regression_whl_expected_diff_report.log new file mode 100644 index 00000000..38cac8eb --- /dev/null +++ b/tests/smlp_regression/run_smlp_regression_whl_expected_diff_report.log @@ -0,0 +1,3293 @@ +=================== Diff report for: Test7_smlp_toy_num_resp_mult_rf_sklearn_tree_rules.txt ================================== +94d93 +< if (p2 > 0.4000000134110451) and (p1 <= 0.75) and (p2 > 0.7000000178813934) and (x <= 0.6666666716337204) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +96c95 +< if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.4000000134110451) and (p1 <= 0.75) and (p2 > 0.7000000178813934) and (x <= 0.6666666716337204) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +97a97 +> if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +=================== End of Test7_smlp_toy_num_resp_mult_rf_sklearn_tree_rules.txt diff report ================================ +=================== Diff report for: Test10_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt ================================== +6d5 +< if (p1 > 0.7673577288013687) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +7a7 +> if (p1 > 0.7673577288013687) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +21d20 +< if (p2 > 0.565498446377692) and (p1 > 0.21566598080828134) and (p2 > 0.7262518305173236) and (x > 0.03081251758592215) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +22a22 +> if (p2 > 0.565498446377692) and (p1 > 0.21566598080828134) and (p2 > 0.7262518305173236) and (x > 0.03081251758592215) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +44d43 +< if (p2 > 0.05282566885129813) and (p1 > 0.9621611074368288) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +45a45 +> if (p2 > 0.05282566885129813) and (p1 > 0.9621611074368288) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +66d65 +< if (p2 > 0.10769168804757841) and (p2 > 0.9843916629018533) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +67a67 +> if (p2 > 0.10769168804757841) and (p2 > 0.9843916629018533) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +73d72 +< if (p1 <= 0.9643084043470717) and (p2 > 0.7106753814549537) and (x <= 0.7383051325780686) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +75c74 +< if (p1 > 0.9643084043470717) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.9643084043470717) and (p2 > 0.7106753814549537) and (x <= 0.7383051325780686) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +76a76 +> if (p1 > 0.9643084043470717) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +88d87 +< if (p2 > 0.37253817607301204) and (p1 > 0.5069297996847564) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +89a89 +> if (p2 > 0.37253817607301204) and (p1 > 0.5069297996847564) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +96d95 +< if (p1 > 0.8097833990164955) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +97a97 +> if (p1 > 0.8097833990164955) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +103d102 +< if (p2 > 0.30979522099243256) and (p2 > 0.7974478444662528) and (x <= 0.7718040145802331) and (p1 <= 0.9832575130198419) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +105c104 +< if (p2 > 0.30979522099243256) and (p2 > 0.7974478444662528) and (x > 0.7718040145802331) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +--- +> if (p2 > 0.30979522099243256) and (p2 > 0.7974478444662528) and (x <= 0.7718040145802331) and (p1 <= 0.9832575130198419) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +106a106 +> if (p2 > 0.30979522099243256) and (p2 > 0.7974478444662528) and (x > 0.7718040145802331) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +118d117 +< if (p1 > 0.6478692095949636) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +119a119 +> if (p1 > 0.6478692095949636) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +126d125 +< if (p2 > 0.5083941302766997) and (p1 > 0.9604900148513215) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +127a127 +> if (p2 > 0.5083941302766997) and (p1 > 0.9604900148513215) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +134d133 +< if (p2 > 0.38305688667253446) and (p1 > 0.2547155522064844) and (p1 > 0.7231216464299344) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +135a135 +> if (p2 > 0.38305688667253446) and (p1 > 0.2547155522064844) and (p1 > 0.7231216464299344) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +141d140 +< if (p1 <= 0.5519522472992318) and (p2 > 0.33294736609465786) and (p2 > 0.7309142946144038) and (x <= 0.6016799023753295) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +143c142 +< if (p1 > 0.5519522472992318) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.5519522472992318) and (p2 > 0.33294736609465786) and (p2 > 0.7309142946144038) and (x <= 0.6016799023753295) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +144a144 +> if (p1 > 0.5519522472992318) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +155d154 +< if (p1 <= 0.5757871147132418) and (p2 > 0.39527962049249543) and (p2 > 0.6458938131907435) and (x <= 0.6096057855068803) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +157c156 +< if (p1 > 0.5757871147132418) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.5757871147132418) and (p2 > 0.39527962049249543) and (p2 > 0.6458938131907435) and (x <= 0.6096057855068803) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +158a158 +> if (p1 > 0.5757871147132418) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +204d203 +< if (p1 <= 0.8026946730924279) and (p2 > 0.05688018773537138) and (p2 > 0.4364986121396114) and (p2 > 0.6777798791052577) and (x <= 0.6717608829535293) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +206c205 +< if (p1 > 0.8026946730924279) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.8026946730924279) and (p2 > 0.05688018773537138) and (p2 > 0.4364986121396114) and (p2 > 0.6777798791052577) and (x <= 0.6717608829535293) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +207a207 +> if (p1 > 0.8026946730924279) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +218d217 +< if (p2 > 0.5397191641571555) and (p1 <= 0.7132398065706901) and (p2 > 0.634045960603359) and (x <= 0.6156377025174744) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +220c219 +< if (p2 > 0.5397191641571555) and (p1 > 0.7132398065706901) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.5397191641571555) and (p1 <= 0.7132398065706901) and (p2 > 0.634045960603359) and (x <= 0.6156377025174744) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +221a221 +> if (p2 > 0.5397191641571555) and (p1 > 0.7132398065706901) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +233d232 +< if (p1 > 0.6806756624396312) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +234a234 +> if (p1 > 0.6806756624396312) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +241d240 +< if (p2 > 0.4943563777461445) and (p1 > 0.920448066629678) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +242a242 +> if (p2 > 0.4943563777461445) and (p1 > 0.920448066629678) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +264d263 +< if (p1 > 0.7209493553829144) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +265a265 +> if (p1 > 0.7209493553829144) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +286d285 +< if (p2 > 0.9499151226831205) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +287a287 +> if (p2 > 0.9499151226831205) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +300d299 +< if (p2 > 0.4160567286499109) and (p1 <= 0.7821608059025187) and (p2 > 0.7500101945124135) and (x <= 0.8482964849267818) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +302c301 +< if (p2 > 0.4160567286499109) and (p1 > 0.7821608059025187) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.4160567286499109) and (p1 <= 0.7821608059025187) and (p2 > 0.7500101945124135) and (x <= 0.8482964849267818) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +303a303 +> if (p2 > 0.4160567286499109) and (p1 > 0.7821608059025187) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +357d356 +< if (p1 <= 0.6838882522116826) and (p2 > 0.0626814738207876) and (p2 > 0.3422835304420513) and (p2 > 0.6023659933821865) and (x <= 0.7909361110756394) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +359c358 +< if (p1 > 0.6838882522116826) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.6838882522116826) and (p2 > 0.0626814738207876) and (p2 > 0.3422835304420513) and (p2 > 0.6023659933821865) and (x <= 0.7909361110756394) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +360a360 +> if (p1 > 0.6838882522116826) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +364d363 +< if (p2 > 0.40336603038169727) and (p1 <= 0.7296289325364069) and (p2 > 0.6471116276536257) and (x <= 0.4249370103517018) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +366c365 +< if (p2 > 0.40336603038169727) and (p1 > 0.7296289325364069) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.40336603038169727) and (p1 <= 0.7296289325364069) and (p2 > 0.6471116276536257) and (x <= 0.4249370103517018) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +367a367 +> if (p2 > 0.40336603038169727) and (p1 > 0.7296289325364069) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +379d378 +< if (p1 > 0.5139580394672034) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +380a380 +> if (p1 > 0.5139580394672034) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +387d386 +< if (p2 > 0.8305540819794657) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +388a388 +> if (p2 > 0.8305540819794657) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +395d394 +< if (p2 > 0.9449580902908733) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +396a396 +> if (p2 > 0.9449580902908733) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +410d409 +< if (p1 > 0.5120007179267708) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +411a411 +> if (p1 > 0.5120007179267708) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +417d416 +< if (p1 <= 0.610545704891228) and (p2 > 0.6501177091384854) and (x <= 0.7877861639695688) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +419c418 +< if (p1 > 0.610545704891228) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p1 <= 0.610545704891228) and (p2 > 0.6501177091384854) and (x <= 0.7877861639695688) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +420a420 +> if (p1 > 0.610545704891228) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +424d423 +< if (p2 > 0.45482464342137086) and (p1 <= 0.6847267941034989) and (p2 > 0.7332769920461923) and (x <= 0.7843162575827582) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +426c425 +< if (p2 > 0.45482464342137086) and (p1 > 0.6847267941034989) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.45482464342137086) and (p1 <= 0.6847267941034989) and (p2 > 0.7332769920461923) and (x <= 0.7843162575827582) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +427a427 +> if (p2 > 0.45482464342137086) and (p1 > 0.6847267941034989) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +446d445 +< if (p2 > 0.054852516881587224) and (p1 > 0.8053342741007611) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +447a447 +> if (p2 > 0.054852516881587224) and (p1 > 0.8053342741007611) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +461d460 +< if (p2 > 0.35838276676758324) and (p1 > 0.6469177723149386) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +462a462 +> if (p2 > 0.35838276676758324) and (p1 > 0.6469177723149386) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +476d475 +< if (p1 > 0.6586678422329332) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +477a477 +> if (p1 > 0.6586678422329332) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +483d482 +< if (p2 > 0.6300749409152544) and (p1 <= 0.9968296801656623) and (x <= 0.37232690124052253) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +485c484 +< if (p2 > 0.6300749409152544) and (p1 > 0.9968296801656623) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.6300749409152544) and (p1 <= 0.9968296801656623) and (x <= 0.37232690124052253) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +486a486 +> if (p2 > 0.6300749409152544) and (p1 > 0.9968296801656623) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +497d496 +< if (p2 > 0.37996283470651265) and (p1 <= 0.5025284804881217) and (p2 > 0.7656687748432474) and (x <= 0.5092783321373655) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +499c498 +< if (p2 > 0.37996283470651265) and (p1 > 0.5025284804881217) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.37996283470651265) and (p1 <= 0.5025284804881217) and (p2 > 0.7656687748432474) and (x <= 0.5092783321373655) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +500a500 +> if (p2 > 0.37996283470651265) and (p1 > 0.5025284804881217) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +505d504 +< if (p2 > 0.22414377714700243) and (p1 > 0.904138504017209) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +506a506 +> if (p2 > 0.22414377714700243) and (p1 > 0.904138504017209) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +513d512 +< if (p1 > 0.8937446875002909) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +514a514 +> if (p1 > 0.8937446875002909) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +535d534 +< if (p2 > 0.07422745696931493) and (p2 > 0.545370601481947) and (p2 > 0.9300510326789317) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +536a536 +> if (p2 > 0.07422745696931493) and (p2 > 0.545370601481947) and (p2 > 0.9300510326789317) then (y1 = 1.0) and (y2 = 0.0) | based on 1 samples +542d541 +< if (p2 > 0.7889165751584417) and (x <= 0.9135558618761394) and (p1 <= 0.8191675724550931) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +544c543 +< if (p2 > 0.7889165751584417) and (x > 0.9135558618761394) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +--- +> if (p2 > 0.7889165751584417) and (x <= 0.9135558618761394) and (p1 <= 0.8191675724550931) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +545a545 +> if (p2 > 0.7889165751584417) and (x > 0.9135558618761394) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +549d548 +< if (p2 > 0.09689574087825406) and (p2 > 0.33636798963203285) and (p1 <= 0.8197608151565123) and (p2 > 0.6876566959859812) and (x <= 0.6647538383146583) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +551c550 +< if (p2 > 0.09689574087825406) and (p2 > 0.33636798963203285) and (p1 > 0.8197608151565123) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.09689574087825406) and (p2 > 0.33636798963203285) and (p1 <= 0.8197608151565123) and (p2 > 0.6876566959859812) and (x <= 0.6647538383146583) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +552a552 +> if (p2 > 0.09689574087825406) and (p2 > 0.33636798963203285) and (p1 > 0.8197608151565123) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +556d555 +< if (p2 > 0.16337258050375272) and (p2 > 0.20940975581022525) and (x <= 0.792321881648303) and (p1 <= 0.574456778622445) and (p2 > 0.7807007138854876) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +558c557 +< if (p2 > 0.16337258050375272) and (p2 > 0.20940975581022525) and (x > 0.792321881648303) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +--- +> if (p2 > 0.16337258050375272) and (p2 > 0.20940975581022525) and (x <= 0.792321881648303) and (p1 <= 0.574456778622445) and (p2 > 0.7807007138854876) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +559a559 +> if (p2 > 0.16337258050375272) and (p2 > 0.20940975581022525) and (x > 0.792321881648303) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +563d562 +< if (p2 > 0.7007179663985585) and (x <= 0.47267748344348626) and (p1 <= 0.5654538877147501) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +565c564 +< if (p2 > 0.7007179663985585) and (x > 0.47267748344348626) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +--- +> if (p2 > 0.7007179663985585) and (x <= 0.47267748344348626) and (p1 <= 0.5654538877147501) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +566a566 +> if (p2 > 0.7007179663985585) and (x > 0.47267748344348626) then (y1 = 0.0) and (y2 = 0.0) | based on 1 samples +570d569 +< if (p2 > 0.30978463977099613) and (p1 <= 0.6320834037065894) and (p2 > 0.6127691589194908) and (x <= 0.6760619000417248) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +572c571 +< if (p2 > 0.30978463977099613) and (p1 > 0.6320834037065894) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.30978463977099613) and (p1 <= 0.6320834037065894) and (p2 > 0.6127691589194908) and (x <= 0.6760619000417248) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +573a573 +> if (p2 > 0.30978463977099613) and (p1 > 0.6320834037065894) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +578d577 +< if (p1 > 0.6819941814439344) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +579a579 +> if (p1 > 0.6819941814439344) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +600d599 +< if (p1 > 0.5931505284240239) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +601a601 +> if (p1 > 0.5931505284240239) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +631d630 +< if (p2 > 0.3774126001528523) and (p1 <= 0.6553700527434098) and (p2 > 0.7353104082940054) and (x <= 0.4155191624469929) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +633c632 +< if (p2 > 0.3774126001528523) and (p1 > 0.6553700527434098) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.3774126001528523) and (p1 <= 0.6553700527434098) and (p2 > 0.7353104082940054) and (x <= 0.4155191624469929) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +634a634 +> if (p2 > 0.3774126001528523) and (p1 > 0.6553700527434098) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +638d637 +< if (p2 > 0.23018040425618197) and (p1 <= 0.7102524464532046) and (p2 > 0.7248063887234734) and (x <= 0.7892120132227867) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +640c639 +< if (p2 > 0.23018040425618197) and (p1 > 0.7102524464532046) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.23018040425618197) and (p1 <= 0.7102524464532046) and (p2 > 0.7248063887234734) and (x <= 0.7892120132227867) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +641a641 +> if (p2 > 0.23018040425618197) and (p1 > 0.7102524464532046) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +653d652 +< if (p2 > 0.21158053456413584) and (p1 > 0.5586915479780601) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +654a654 +> if (p2 > 0.21158053456413584) and (p1 > 0.5586915479780601) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +661d660 +< if (p1 > 0.6795984351446844) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +662a662 +> if (p1 > 0.6795984351446844) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +677d676 +< if (p1 > 0.8126054242312002) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +678a678 +> if (p1 > 0.8126054242312002) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +684d683 +< if (p2 > 0.26886312862339573) and (p1 <= 0.8950548846717248) and (p2 > 0.6824239181038759) and (x <= 0.7910962169102163) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +686c685 +< if (p2 > 0.26886312862339573) and (p1 > 0.8950548846717248) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.26886312862339573) and (p1 <= 0.8950548846717248) and (p2 > 0.6824239181038759) and (x <= 0.7910962169102163) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +687a687 +> if (p2 > 0.26886312862339573) and (p1 > 0.8950548846717248) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +698d697 +< if (p2 > 0.25803846924474394) and (p1 <= 0.85781003667871) and (p2 > 0.6174925668996046) and (x <= 0.4784058184220548) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +700c699 +< if (p2 > 0.25803846924474394) and (p1 > 0.85781003667871) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.25803846924474394) and (p1 <= 0.85781003667871) and (p2 > 0.6174925668996046) and (x <= 0.4784058184220548) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +701a701 +> if (p2 > 0.25803846924474394) and (p1 > 0.85781003667871) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +705d704 +< if (p2 > 0.16944414757631912) and (p1 <= 0.7608029128801092) and (p2 > 0.31309513934344707) and (p2 > 0.7650359471516853) and (x <= 0.6112039253981838) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +707c706 +< if (p2 > 0.16944414757631912) and (p1 > 0.7608029128801092) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +--- +> if (p2 > 0.16944414757631912) and (p1 <= 0.7608029128801092) and (p2 > 0.31309513934344707) and (p2 > 0.7650359471516853) and (x <= 0.6112039253981838) then (y1 = 1.0) and (y2 = 0.0) | based on 2 samples +708a708 +> if (p2 > 0.16944414757631912) and (p1 > 0.7608029128801092) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +728d727 +< if (p2 > 0.29657478970316) and (p1 > 0.8279177280272906) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +729a729 +> if (p2 > 0.29657478970316) and (p1 > 0.8279177280272906) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +736d735 +< if (p2 > 0.30543398172847647) and (p1 > 0.7651434488432218) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +737a737 +> if (p2 > 0.30543398172847647) and (p1 > 0.7651434488432218) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +=================== End of Test10_smlp_toy_num_resp_mult_et_sklearn_tree_rules.txt diff report ================================ +=================== Diff report for: Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/Test5_smlp_toy_num_resp_mult_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/Test5_smlp_toy_num_resp_mult_rerun_model_config.json +=================== End of Test15_Test5_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/Test8_smlp_toy_num_resp_mult_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/Test8_smlp_toy_num_resp_mult_rerun_model_config.json +=================== End of Test16_Test8_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/Test11_smlp_toy_num_resp_mult_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/Test11_smlp_toy_num_resp_mult_rerun_model_config.json +=================== End of Test17_Test11_smlp_toy_num_resp_mult_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test19_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test19_model_rerun_model_config.json +=================== End of Test19_test19_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +75c75 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test20_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test20_model_rerun_model_config.json +=================== End of Test20_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt ================================== +75c75 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test22_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test22_model_rerun_model_config.json +=================== End of Test22_test22_model_smlp_toy_num_metasymbol_mult_reg_pred_labeled.txt diff report ================================ +=================== Diff report for: Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test24_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test24_model_rerun_model_config.json +=================== End of Test24_test24_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: test26_model_dt_sklearn_tree_rules.txt ================================== +6d5 +< if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +7a7 +> if (p2 > 0.4000000134110451) and (p1 > 0.75) then (y1 = 1.0) and (y2 = 1.0) | based on 1 samples +=================== End of test26_model_dt_sklearn_tree_rules.txt diff report ================================ +=================== Diff report for: Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +87c87 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test26_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test26_model_rerun_model_config.json +=================== End of Test26_test26_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test29_smlp_toy_cls_metasymbol_colnames_mult.txt ================================== +95,96d94 +< smlp_logger - WARNING - Range plots are not supported in this version of SMLP +< +=================== End of Test29_smlp_toy_cls_metasymbol_colnames_mult.txt diff report ================================ +=================== Diff report for: Test30_smlp_toy_num_resp_mult.txt ================================== +95,96d94 +< smlp_logger - WARNING - Range plots are not supported in this version of SMLP +< +=================== End of Test30_smlp_toy_num_resp_mult.txt diff report ================================ +=================== Diff report for: Test31_smlp_toy_num_resp_mult.txt ================================== +95,96d94 +< smlp_logger - WARNING - Range plots are not supported in this version of SMLP +< +=================== End of Test31_smlp_toy_num_resp_mult.txt diff report ================================ +=================== Diff report for: Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt ================================== +75c75 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test20_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test20_model_rerun_model_config.json +=================== End of Test32_test20_model_smlp_toy_num_resp_mult_pred_labeled.txt diff report ================================ +=================== Diff report for: Test33_smlp_toy_num_resp_mult.txt ================================== +95,96d94 +< smlp_logger - WARNING - Range plots are not supported in this version of SMLP +< +=================== End of Test33_smlp_toy_num_resp_mult.txt diff report ================================ +=================== Diff report for: Test47_test47_model_smlp_toy_pf_mult.txt ================================== +83c83 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test47_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test47_model_rerun_model_config.json +=================== End of Test47_test47_model_smlp_toy_pf_mult.txt diff report ================================ +=================== Diff report for: Test58_smlp_toy_num_resp_mult_optimization_progress.json ================================== +17,19d16 +< "p1": { +< "value_in_config": 2.0 +< }, +23,24c20,21 +< "y1": { +< "value_in_config": 5.0 +--- +> "p1": { +> "value_in_config": 2.0 +27a25,27 +> }, +> "y1": { +> "value_in_config": 5.0 +=================== End of Test58_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test58_smlp_toy_num_resp_mult_optimization_results.json ================================== +3d2 +< "p1": 7.0, +5c4 +< "y1": 9.0, +--- +> "p1": 7.0, +6a6 +> "y1": 9.0, +15d14 +< "p1": 7.0, +17c16 +< "y1": 5.0, +--- +> "p1": 7.0, +18a18 +> "y1": 5.0, +=================== End of Test58_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test63_smlp_toy_num_resp_mult_verify_results.json ================================== +12d11 +< "p1": 1.0, +14,15c13,15 +< "y1": 9.0, +< "p2": 7.0 +--- +> "p1": 1.0, +> "p2": 7.0, +> "y1": 9.0 +=================== End of Test63_smlp_toy_num_resp_mult_verify_results.json diff report ================================ +=================== Diff report for: Test64_test63_model.txt ================================== +27c27 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test63_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test63_model_rerun_model_config.json +=================== End of Test64_test63_model.txt diff report ================================ +=================== Diff report for: Test64_test63_model_verify_results.json ================================== +12d11 +< "p1": 1.0, +14,15c13,15 +< "y1": 9.0, +< "p2": 7.0 +--- +> "p1": 1.0, +> "p2": 7.0, +> "y1": 9.0 +=================== End of Test64_test63_model_verify_results.json diff report ================================ +=================== Diff report for: Test66_test65_model.txt ================================== +0a1,97 +> +> smlp_logger - INFO - Model exploration specification: +> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} +> +> smlp_logger - INFO - Executing run_smlp.py script: Start +> +> smlp_logger - INFO - Running SMLP in mode "verify": Start +> +> smlp_logger - INFO - Computed spec global constraint expressions: +> +> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 +> +> smlp_logger - INFO - Global beta : None +> +> smlp_logger - INFO - Radii theta : {} +> +> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} +> +> smlp_logger - INFO - Assertion asrt1: x0**2+y1>4.3 +> +> smlp_logger - INFO - Assertion asrt2: (y1+x2)/2<6 +> +> smlp_logger - INFO - PREPARE DATA FOR MODELING +> +> smlp_logger - INFO - LOAD TRAINED MODEL +> +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test65_model_rerun_model_config.json +> +> smlp_logger - INFO - Creating model exploration base components: Start +> +> smlp_logger - INFO - Parsing the SPEC: Start +> +> smlp_logger - INFO - Parsing the SPEC: End +> +> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} +> +> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} +> +> smlp_logger - INFO - Knob bounds (eta): {} +> +> smlp_logger - INFO - Knob grids (eta): {} +> +> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) +> +> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) +> +> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) +> +> smlp_logger - INFO - Beta global constraints: true +> +> smlp_logger - INFO - Eta ranges constraints: true +> +> smlp_logger - INFO - Eta grid constraints: true +> +> smlp_logger - INFO - Eta global constraints: true +> +> smlp_logger - INFO - Eta combined constraints: true +> +> smlp_logger - INFO - Creating model exploration base components: End +> +> smlp_logger - INFO - Input and knob interface constraints are consistent +> +> smlp_logger - INFO - Building model terms: Start +> +> smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} +> +> smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 15, 'ite': 5, 'and': 9, 'prop': 14, 'const': 50, 'sub': 14, 'var': 14} +> +> smlp_logger - INFO - Building model terms: End +> +> smlp_logger - INFO - Model interface constraints are consistent +> +> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: +> true +> +> smlp_logger - INFO - Input, knob and configuration constraints are consistent +> +> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: +> true +> +> smlp_logger - INFO - Input, knob and configuration constraints are consistent +> +> smlp_logger - INFO - Verifying assertion asrt1 <-> x0**2+y1>4.3 +> +> smlp_logger - INFO - The configuration is consistent with assertion asrt1 +> +> smlp_logger - INFO - Completed with result: PASS +> +> smlp_logger - INFO - Verifying assertion asrt2 <-> (y1+x2)/2<6 +> +> smlp_logger - INFO - The configuration is consistent with assertion asrt2 +> +> smlp_logger - INFO - Completed with result: FAIL +> +> smlp_logger - INFO - Running SMLP in mode "verify": End +> +> smlp_logger - INFO - Executing run_smlp.py script: End +=================== End of Test66_test65_model.txt diff report ================================ +=================== Diff report for: Test66_test65_model_verify_results.json ================================== +diff: /app/smlp/regr_smlp/code/Test66_test65_model_verify_results.json: No such file or directory +=================== End of Test66_test65_model_verify_results.json diff report ================================ +=================== Diff report for: Test68_test67_model.txt ================================== +0a1,97 +> +> smlp_logger - INFO - Model exploration specification: +> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} +> +> smlp_logger - INFO - Executing run_smlp.py script: Start +> +> smlp_logger - INFO - Running SMLP in mode "verify": Start +> +> smlp_logger - INFO - Computed spec global constraint expressions: +> +> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 +> +> smlp_logger - INFO - Global beta : None +> +> smlp_logger - INFO - Radii theta : {} +> +> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} +> +> smlp_logger - INFO - Assertion asrt1: x0**2+y1>4.3 +> +> smlp_logger - INFO - Assertion asrt2: (y1+x2)/2<6 +> +> smlp_logger - INFO - PREPARE DATA FOR MODELING +> +> smlp_logger - INFO - LOAD TRAINED MODEL +> +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test67_model_rerun_model_config.json +> +> smlp_logger - INFO - Creating model exploration base components: Start +> +> smlp_logger - INFO - Parsing the SPEC: Start +> +> smlp_logger - INFO - Parsing the SPEC: End +> +> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} +> +> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} +> +> smlp_logger - INFO - Knob bounds (eta): {} +> +> smlp_logger - INFO - Knob grids (eta): {} +> +> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) +> +> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) +> +> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) +> +> smlp_logger - INFO - Beta global constraints: true +> +> smlp_logger - INFO - Eta ranges constraints: true +> +> smlp_logger - INFO - Eta grid constraints: true +> +> smlp_logger - INFO - Eta global constraints: true +> +> smlp_logger - INFO - Eta combined constraints: true +> +> smlp_logger - INFO - Creating model exploration base components: End +> +> smlp_logger - INFO - Input and knob interface constraints are consistent +> +> smlp_logger - INFO - Building model terms: Start +> +> smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 7, 'ite': 3, 'and': 3, 'prop': 6, 'const': 24, 'sub': 6, 'var': 6} +> +> smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 4, 'ite': 2, 'and': 1, 'prop': 3, 'const': 14, 'sub': 3, 'var': 3} +> +> smlp_logger - INFO - Building model terms: End +> +> smlp_logger - INFO - Model interface constraints are consistent +> +> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: +> true +> +> smlp_logger - INFO - Input, knob and configuration constraints are consistent +> +> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: +> true +> +> smlp_logger - INFO - Input, knob and configuration constraints are consistent +> +> smlp_logger - INFO - Verifying assertion asrt1 <-> x0**2+y1>4.3 +> +> smlp_logger - INFO - The configuration is consistent with assertion asrt1 +> +> smlp_logger - INFO - Completed with result: PASS +> +> smlp_logger - INFO - Verifying assertion asrt2 <-> (y1+x2)/2<6 +> +> smlp_logger - INFO - The configuration is consistent with assertion asrt2 +> +> smlp_logger - INFO - Completed with result: FAIL +> +> smlp_logger - INFO - Running SMLP in mode "verify": End +> +> smlp_logger - INFO - Executing run_smlp.py script: End +=================== End of Test68_test67_model.txt diff report ================================ +=================== Diff report for: Test68_test67_model_verify_results.json ================================== +diff: /app/smlp/regr_smlp/code/Test68_test67_model_verify_results.json: No such file or directory +=================== End of Test68_test67_model_verify_results.json diff report ================================ +=================== Diff report for: Test69_smlp_toy_num_resp_mult_verify_results.json ================================== +6d5 +< "p1": 1.0, +8,9c7,9 +< "y2": 5.078784562647343, +< "p2": 7.0 +--- +> "p1": 1.0, +> "p2": 7.0, +> "y2": 5.078784562647343 +=================== End of Test69_smlp_toy_num_resp_mult_verify_results.json diff report ================================ +=================== Diff report for: Test70_test69_model.txt ================================== +25c25 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test69_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test69_model_rerun_model_config.json +=================== End of Test70_test69_model.txt diff report ================================ +=================== Diff report for: Test70_test69_model_verify_results.json ================================== +6d5 +< "p1": 1.0, +8,9c7,9 +< "y2": 5.078784562647343, +< "p2": 7.0 +--- +> "p1": 1.0, +> "p2": 7.0, +> "y2": 5.078784562647343 +=================== End of Test70_test69_model_verify_results.json diff report ================================ +=================== Diff report for: Test72_test71_model.txt ================================== +0a1,84 +> +> smlp_logger - INFO - Model exploration specification: +> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} +> +> smlp_logger - INFO - Executing run_smlp.py script: Start +> +> smlp_logger - INFO - Running SMLP in mode "verify": Start +> +> smlp_logger - INFO - Computed spec global constraint expressions: +> +> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 +> +> smlp_logger - INFO - Global beta : None +> +> smlp_logger - INFO - Radii theta : {} +> +> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} +> +> smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 +> +> smlp_logger - INFO - PREPARE DATA FOR MODELING +> +> smlp_logger - INFO - LOAD TRAINED MODEL +> +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test71_model_rerun_model_config.json +> +> smlp_logger - INFO - Creating model exploration base components: Start +> +> smlp_logger - INFO - Parsing the SPEC: Start +> +> smlp_logger - INFO - Parsing the SPEC: End +> +> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} +> +> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} +> +> smlp_logger - INFO - Knob bounds (eta): {} +> +> smlp_logger - INFO - Knob grids (eta): {} +> +> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) +> +> smlp_logger - INFO - Alpha ranges constraints: (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) +> +> smlp_logger - INFO - Alpha combined constraints: (and (and (and (and true (and (>= x0 0) (<= x0 10))) (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) +> +> smlp_logger - INFO - Beta global constraints: true +> +> smlp_logger - INFO - Eta ranges constraints: true +> +> smlp_logger - INFO - Eta grid constraints: true +> +> smlp_logger - INFO - Eta global constraints: true +> +> smlp_logger - INFO - Eta combined constraints: true +> +> smlp_logger - INFO - Creating model exploration base components: End +> +> smlp_logger - INFO - Input and knob interface constraints are consistent +> +> smlp_logger - INFO - Building model terms: Start +> +> smlp_logger - INFO - Model operator counts for y1: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} +> +> smlp_logger - INFO - Model operator counts for y2: {'add': 256, 'mul': 472, 'ite': 39, 'prop': 39, 'const': 846, 'sub': 216, 'var': 216} +> +> smlp_logger - INFO - Building model terms: End +> +> smlp_logger - INFO - Model interface constraints are consistent +> +> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: +> true +> +> smlp_logger - INFO - Input, knob and configuration constraints are consistent +> +> smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 +> +> smlp_logger - INFO - The configuration is consistent with assertion asrt1 +> +> smlp_logger - INFO - Completed with result: FAIL +> +> smlp_logger - INFO - Running SMLP in mode "verify": End +> +> smlp_logger - INFO - Executing run_smlp.py script: End +=================== End of Test72_test71_model.txt diff report ================================ +=================== Diff report for: Test72_test71_model_verify_results.json ================================== +diff: /app/smlp/regr_smlp/code/Test72_test71_model_verify_results.json: No such file or directory +=================== End of Test72_test71_model_verify_results.json diff report ================================ +=================== Diff report for: Test77_test76_model.txt ================================== +0a1,110 +> +> smlp_logger - INFO - Model exploration specification: +> {'version': '1.1', 'spec': [{'label': 'y1', 'type': 'response', 'range': 'float'}, {'label': 'y2', 'type': 'response', 'range': 'float'}, {'label': 'x0', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x1', 'type': 'input', 'range': 'float', 'bounds': [0, 10]}, {'label': 'x2', 'type': 'input', 'range': 'float', 'bounds': [3, 7]}], 'alpha': 'x1==1 or x1==4 or x1==7'} +> +> smlp_logger - INFO - Executing run_smlp.py script: Start +> +> smlp_logger - INFO - Running SMLP in mode "verify": Start +> +> smlp_logger - INFO - Computed spec global constraint expressions: +> +> smlp_logger - INFO - Global alpha : x1==1 or x1==4 or x1==7 +> +> smlp_logger - INFO - Global beta : None +> +> smlp_logger - INFO - Radii theta : {} +> +> smlp_logger - INFO - Delta const : {'delta_abs': 0.0, 'delta_rel': 0.01} +> +> smlp_logger - INFO - Assertion asrt1: (y2**3+x2)/2<6 +> +> smlp_logger - INFO - Assertion asrt2: y1>=9 +> +> smlp_logger - INFO - Assertion asrt3: y2<0 +> +> smlp_logger - INFO - PREPARE DATA FOR MODELING +> +> smlp_logger - INFO - LOAD TRAINED MODEL +> +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test76_model_rerun_model_config.json +> +> smlp_logger - INFO - Creating model exploration base components: Start +> +> smlp_logger - INFO - Parsing the SPEC: Start +> +> smlp_logger - INFO - Parsing the SPEC: End +> +> smlp_logger - INFO - Variable domains (alpha): {'y1': {'range': 'float', 'interval': None}, 'y2': {'range': 'float', 'interval': None}, 'x0': {'range': 'float', 'interval': [0, 10]}, 'x1': {'range': 'float', 'interval': [0, 10]}, 'x2': {'range': 'float', 'interval': [3, 7]}} +> +> smlp_logger - INFO - Input bounds (alpha): {'x0': {'min': 0, 'max': 10}, 'x1': {'min': 0, 'max': 10}, 'x2': {'min': 3, 'max': 7}} +> +> smlp_logger - INFO - Knob bounds (eta): {} +> +> smlp_logger - INFO - Knob grids (eta): {} +> +> smlp_logger - INFO - Alpha global constraints: (or (or (= x1 1) (= x1 4)) (= x1 7)) +> +> smlp_logger - INFO - Alpha ranges constraints: (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) +> +> smlp_logger - INFO - Alpha combined constraints: (and (and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 3) (<= x2 7))) (or (or (= x1 1) (= x1 4)) (= x1 7))) +> +> smlp_logger - INFO - Beta global constraints: true +> +> smlp_logger - INFO - Eta ranges constraints: true +> +> smlp_logger - INFO - Eta grid constraints: true +> +> smlp_logger - INFO - Eta global constraints: true +> +> smlp_logger - INFO - Eta combined constraints: true +> +> smlp_logger - INFO - Creating model exploration base components: End +> +> smlp_logger - INFO - Input and knob interface constraints are consistent +> +> smlp_logger - INFO - Building model terms: Start +> +> smlp_logger - INFO - Model operator counts for y1: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} +> +> smlp_logger - INFO - Model operator counts for y2: {'add': 1, 'mul': 21, 'ite': 6, 'and': 14, 'prop': 20, 'const': 69, 'sub': 20, 'var': 20} +> +> smlp_logger - INFO - Building model terms: End +> +> smlp_logger - INFO - Model interface constraints are consistent +> +> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt1: +> true +> +> smlp_logger - INFO - Input, knob and configuration constraints are consistent +> +> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt2: +> true +> +> smlp_logger - INFO - Input, knob and configuration constraints are consistent +> +> smlp_logger - INFO - Verifying consistency of configuration for assertion asrt3: +> true +> +> smlp_logger - INFO - Input, knob and configuration constraints are consistent +> +> smlp_logger - INFO - Verifying assertion asrt1 <-> (y2**3+x2)/2<6 +> +> smlp_logger - INFO - The configuration is inconsistent with assertion asrt1 +> +> smlp_logger - INFO - Completed with result: FAIL +> +> smlp_logger - INFO - Verifying assertion asrt2 <-> y1>=9 +> +> smlp_logger - INFO - The configuration is consistent with assertion asrt2 +> +> smlp_logger - INFO - Completed with result: FAIL +> +> smlp_logger - INFO - Verifying assertion asrt3 <-> y2<0 +> +> smlp_logger - INFO - The configuration is inconsistent with assertion asrt3 +> +> smlp_logger - INFO - Completed with result: FAIL +> +> smlp_logger - INFO - Running SMLP in mode "verify": End +> +> smlp_logger - INFO - Executing run_smlp.py script: End +=================== End of Test77_test76_model.txt diff report ================================ +=================== Diff report for: Test77_test76_model_verify_results.json ================================== +diff: /app/smlp/regr_smlp/code/Test77_test76_model_verify_results.json: No such file or directory +=================== End of Test77_test76_model_verify_results.json diff report ================================ +=================== Diff report for: Test79_smlp_toy_num_resp_mult_query_results.json ================================== +14a15 +> "p1": 7.0, +16d16 +< "y2": 9.0, +18c18 +< "p1": 7.0 +--- +> "y2": 9.0 +=================== End of Test79_smlp_toy_num_resp_mult_query_results.json diff report ================================ +=================== Diff report for: Test80_smlp_toy_num_resp_mult_optimization_progress.json ================================== +10,12d9 +< "p1": { +< "value_in_config": 4.0 +< }, +16,17c13,14 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +20a18,20 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test80_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test80_smlp_toy_num_resp_mult_optimization_results.json ================================== +3d2 +< "p1": 7.0, +5c4 +< "y1": 9.0, +--- +> "p1": 7.0, +6a6 +> "y1": 9.0, +=================== End of Test80_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test81_smlp_toy_num_resp_mult_optimization_progress.json ================================== +10,12d9 +< "p1": { +< "value_in_config": 4.0 +< }, +16,17c13,14 +< "y1": { +< "value_in_config": 5.0 +--- +> "p1": { +> "value_in_config": 4.0 +20a18,20 +> }, +> "y1": { +> "value_in_config": 5.0 +=================== End of Test81_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test81_smlp_toy_num_resp_mult_optimization_results.json ================================== +3d2 +< "p1": 4.0, +5c4 +< "y1": 9.0, +--- +> "p1": 4.0, +6a6 +> "y1": 9.0, +=================== End of Test81_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test82_smlp_toy_num_resp_mult_optimization_progress.json ================================== +24,26d23 +< "p1": { +< "value_in_config": 4.0 +< }, +30,31c27,28 +< "y1": { +< "value_in_config": 5.0 +--- +> "p1": { +> "value_in_config": 4.0 +35a33,35 +> "y1": { +> "value_in_config": 5.0 +> }, +62,64d61 +< "p1": { +< "value_in_config": 4.0 +< }, +68,69c65,66 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +73a71,73 +> "y1": { +> "value_in_config": 9.0 +> }, +121,123d120 +< "p1": { +< "value_in_config": 4.0 +< }, +127,128c124,125 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +131a129,131 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test82_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test82_smlp_toy_num_resp_mult_optimization_results.json ================================== +23,25d22 +< "p1": { +< "value_in_config": 4.0 +< }, +29,30c26,27 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +33a31,33 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test82_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test83_smlp_toy_num_resp_mult_optimization_progress.json ================================== +24,26d23 +< "p1": { +< "value_in_config": 4.0 +< }, +30,31c27,28 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +35a33,35 +> "y1": { +> "value_in_config": 9.0 +> }, +62,64d61 +< "p1": { +< "value_in_config": 4.0 +< }, +68,69c65,66 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +73a71,73 +> "y1": { +> "value_in_config": 9.0 +> }, +100,102d99 +< "p1": { +< "value_in_config": 4.0 +< }, +106,107c103,104 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +111a109,111 +> "y1": { +> "value_in_config": 9.0 +> }, +138,140d137 +< "p1": { +< "value_in_config": 4.0 +< }, +144,145c141,142 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +149a147,149 +> "y1": { +> "value_in_config": 9.0 +> }, +197,199d196 +< "p1": { +< "value_in_config": 4.0 +< }, +203,204c200,201 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +208a206,208 +> "y1": { +> "value_in_config": 9.0 +> }, +235,237d234 +< "p1": { +< "value_in_config": 7.0 +< }, +241,242c238,239 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +246a244,246 +> "y1": { +> "value_in_config": 9.0 +> }, +273,275d272 +< "p1": { +< "value_in_config": 7.0 +< }, +279,280c276,277 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +284a282,284 +> "y1": { +> "value_in_config": 9.0 +> }, +332,334d331 +< "p1": { +< "value_in_config": 7.0 +< }, +338,339c335,336 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +342a340,342 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test83_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test83_smlp_toy_num_resp_mult_optimization_results.json ================================== +23,25d22 +< "p1": { +< "value_in_config": 7.0 +< }, +29,30c26,27 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +33a31,33 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test83_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test85_smlp_toy_num_resp_mult_optimization_progress.json ================================== +17,19d16 +< "p1": { +< "value_in_config": 4.0 +< }, +23,24c20,21 +< "y1": { +< "value_in_config": 5.0 +--- +> "p1": { +> "value_in_config": 4.0 +27a25,27 +> }, +> "y1": { +> "value_in_config": 5.0 +=================== End of Test85_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test85_smlp_toy_num_resp_mult_optimization_results.json ================================== +3d2 +< "p1": 4.0, +5c4 +< "y1": 5.0, +--- +> "p1": 4.0, +6a6 +> "y1": 5.0, +18d17 +< "p1": 4.0, +20c19 +< "y1": 5.0, +--- +> "p1": 4.0, +21a21 +> "y1": 5.0, +=================== End of Test85_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test86_smlp_toy_num_resp_mult_optimization_progress.json ================================== +17,19d16 +< "p1": { +< "value_in_config": 4.0 +< }, +23,24c20,21 +< "y1": { +< "value_in_config": 5.0 +--- +> "p1": { +> "value_in_config": 4.0 +27a25,27 +> }, +> "y1": { +> "value_in_config": 5.0 +=================== End of Test86_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test86_smlp_toy_num_resp_mult_optimization_results.json ================================== +3d2 +< "p1": 4.0, +5c4 +< "y1": 5.0, +--- +> "p1": 4.0, +6a6 +> "y1": 5.0, +18d17 +< "p1": 4.0, +20c19 +< "y1": 5.0, +--- +> "p1": 4.0, +21a21 +> "y1": 5.0, +=================== End of Test86_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test87_smlp_toy_num_resp_mult_verify_results.json ================================== +6d5 +< "p1": 2.5, +8c7 +< "y1": 9.0, +--- +> "p1": 2.5, +9a9 +> "y1": 9.0, +24d23 +< "p1": 2.5, +26c25 +< "y1": 9.0, +--- +> "p1": 2.5, +27a27 +> "y1": 9.0, +=================== End of Test87_smlp_toy_num_resp_mult_verify_results.json diff report ================================ +=================== Diff report for: Test88_smlp_toy_num_resp_mult_optimization_progress.json ================================== +24,26d23 +< "p1": { +< "value_in_config": 4.0 +< }, +30,31c27,28 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +35a33,35 +> "y1": { +> "value_in_config": 9.0 +> }, +62,64d61 +< "p1": { +< "value_in_config": 4.0 +< }, +68,69c65,66 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +73a71,73 +> "y1": { +> "value_in_config": 9.0 +> }, +100,102d99 +< "p1": { +< "value_in_config": 4.0 +< }, +106,107c103,104 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +111a109,111 +> "y1": { +> "value_in_config": 9.0 +> }, +138,140d137 +< "p1": { +< "value_in_config": 4.0 +< }, +144,145c141,142 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +149a147,149 +> "y1": { +> "value_in_config": 9.0 +> }, +197,199d196 +< "p1": { +< "value_in_config": 4.0 +< }, +203,204c200,201 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +208a206,208 +> "y1": { +> "value_in_config": 9.0 +> }, +235,237d234 +< "p1": { +< "value_in_config": 7.0 +< }, +241,242c238,239 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +246a244,246 +> "y1": { +> "value_in_config": 9.0 +> }, +273,275d272 +< "p1": { +< "value_in_config": 7.0 +< }, +279,280c276,277 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +284a282,284 +> "y1": { +> "value_in_config": 9.0 +> }, +332,334d331 +< "p1": { +< "value_in_config": 7.0 +< }, +338,339c335,336 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +342a340,342 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test88_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test88_smlp_toy_num_resp_mult_optimization_results.json ================================== +23,25d22 +< "p1": { +< "value_in_config": 7.0 +< }, +29,30c26,27 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +33a31,33 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test88_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test89_smlp_toy_num_resp_mult_query_results.json ================================== +14a15 +> "p1": 7.0, +16d16 +< "y2": 9.0, +18c18 +< "p1": 7.0 +--- +> "y2": 9.0 +=================== End of Test89_smlp_toy_num_resp_mult_query_results.json diff report ================================ +=================== Diff report for: Test93_smlp_toy_num_resp_mult_optimization_progress.json ================================== +17,19d16 +< "p1": { +< "value_in_config": 2.0 +< }, +23,24c20,21 +< "y1": { +< "value_in_config": 5.0 +--- +> "p1": { +> "value_in_config": 2.0 +28a26,28 +> "y1": { +> "value_in_config": 5.0 +> }, +48,50d47 +< "p1": { +< "value_in_config": 2.0 +< }, +54,55c51,52 +< "y1": { +< "value_in_config": 5.0 +--- +> "p1": { +> "value_in_config": 2.0 +59a57,59 +> "y1": { +> "value_in_config": 5.0 +> }, +100,102d99 +< "p1": { +< "value_in_config": 2.0 +< }, +106,107c103,104 +< "y1": { +< "value_in_config": 5.0 +--- +> "p1": { +> "value_in_config": 2.0 +110a108,110 +> }, +> "y1": { +> "value_in_config": 5.0 +=================== End of Test93_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test93_smlp_toy_num_resp_mult_optimization_results.json ================================== +16,18d15 +< "p1": { +< "value_in_config": 2.0 +< }, +22,23c19,20 +< "y1": { +< "value_in_config": 5.0 +--- +> "p1": { +> "value_in_config": 2.0 +26a24,26 +> }, +> "y1": { +> "value_in_config": 5.0 +=================== End of Test93_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test94_smlp_toy_num_resp_mult_optimization_progress.json ================================== +17,19d16 +< "p1": { +< "value_in_config": 4.0 +< }, +23,24c20,21 +< "y1": { +< "value_in_config": 7.48 +--- +> "p1": { +> "value_in_config": 4.0 +28a26,28 +> "y1": { +> "value_in_config": 7.48 +> }, +48,50d47 +< "p1": { +< "value_in_config": 4.0 +< }, +54,55c51,52 +< "y1": { +< "value_in_config": 7.48 +--- +> "p1": { +> "value_in_config": 4.0 +59a57,59 +> "y1": { +> "value_in_config": 7.48 +> }, +79,81d78 +< "p1": { +< "value_in_config": 4.0 +< }, +85,86c82,83 +< "y1": { +< "value_in_config": 7.48 +--- +> "p1": { +> "value_in_config": 4.0 +90a88,90 +> "y1": { +> "value_in_config": 7.48 +> }, +110,112d109 +< "p1": { +< "value_in_config": 4.0 +< }, +116,117c113,114 +< "y1": { +< "value_in_config": 7.48 +--- +> "p1": { +> "value_in_config": 4.0 +121a119,121 +> "y1": { +> "value_in_config": 7.48 +> }, +162,164d161 +< "p1": { +< "value_in_config": 4.0 +< }, +168,169c165,166 +< "y1": { +< "value_in_config": 7.48 +--- +> "p1": { +> "value_in_config": 4.0 +173a171,173 +> "y1": { +> "value_in_config": 7.48 +> }, +193,195d192 +< "p1": { +< "value_in_config": 4.0 +< }, +199,200c196,197 +< "y1": { +< "value_in_config": 7.48 +--- +> "p1": { +> "value_in_config": 4.0 +204a202,204 +> "y1": { +> "value_in_config": 7.48 +> }, +224,226d223 +< "p1": { +< "value_in_config": 4.0 +< }, +230,231c227,228 +< "y1": { +< "value_in_config": 7.48 +--- +> "p1": { +> "value_in_config": 4.0 +235a233,235 +> "y1": { +> "value_in_config": 7.48 +> }, +276,278d275 +< "p1": { +< "value_in_config": 4.0 +< }, +282,283c279,280 +< "y1": { +< "value_in_config": 7.48 +--- +> "p1": { +> "value_in_config": 4.0 +286a284,286 +> }, +> "y1": { +> "value_in_config": 7.48 +=================== End of Test94_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test94_smlp_toy_num_resp_mult_optimization_results.json ================================== +16,18d15 +< "p1": { +< "value_in_config": 4.0 +< }, +22,23c19,20 +< "y1": { +< "value_in_config": 7.48 +--- +> "p1": { +> "value_in_config": 4.0 +26a24,26 +> }, +> "y1": { +> "value_in_config": 7.48 +=================== End of Test94_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test95_smlp_toy_num_resp_mult_optimization_progress.json ================================== +17,19d16 +< "p1": { +< "value_in_config": 2.0 +< }, +23,24c20,21 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +28a26,28 +> "y1": { +> "value_in_config": 7.0 +> }, +48,50d47 +< "p1": { +< "value_in_config": 2.0 +< }, +54,55c51,52 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +59a57,59 +> "y1": { +> "value_in_config": 7.0 +> }, +79,81d78 +< "p1": { +< "value_in_config": 2.0 +< }, +85,86c82,83 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +90a88,90 +> "y1": { +> "value_in_config": 7.0 +> }, +131,133d130 +< "p1": { +< "value_in_config": 2.0 +< }, +137,138c134,135 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +141a139,141 +> }, +> "y1": { +> "value_in_config": 7.0 +=================== End of Test95_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test95_smlp_toy_num_resp_mult_optimization_results.json ================================== +16,18d15 +< "p1": { +< "value_in_config": 2.0 +< }, +22,23c19,20 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +26a24,26 +> }, +> "y1": { +> "value_in_config": 7.0 +=================== End of Test95_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test96_smlp_toy_num_resp_mult_optimization_progress.json ================================== +17,19d16 +< "p1": { +< "value_in_config": 4.0 +< }, +23,24c20,21 +< "y1": { +< "value_in_config": 6.44 +--- +> "p1": { +> "value_in_config": 4.0 +28a26,28 +> "y1": { +> "value_in_config": 6.44 +> }, +48,50d47 +< "p1": { +< "value_in_config": 4.0 +< }, +54,55c51,52 +< "y1": { +< "value_in_config": 6.44 +--- +> "p1": { +> "value_in_config": 4.0 +59a57,59 +> "y1": { +> "value_in_config": 6.44 +> }, +79,81d78 +< "p1": { +< "value_in_config": 4.0 +< }, +85,86c82,83 +< "y1": { +< "value_in_config": 6.44 +--- +> "p1": { +> "value_in_config": 4.0 +90a88,90 +> "y1": { +> "value_in_config": 6.44 +> }, +131,133d130 +< "p1": { +< "value_in_config": 4.0 +< }, +137,138c134,135 +< "y1": { +< "value_in_config": 6.44 +--- +> "p1": { +> "value_in_config": 4.0 +142a140,142 +> "y1": { +> "value_in_config": 6.44 +> }, +162,164d161 +< "p1": { +< "value_in_config": 4.0 +< }, +168,169c165,166 +< "y1": { +< "value_in_config": 6.44 +--- +> "p1": { +> "value_in_config": 4.0 +173a171,173 +> "y1": { +> "value_in_config": 6.44 +> }, +193,195d192 +< "p1": { +< "value_in_config": 4.0 +< }, +199,200c196,197 +< "y1": { +< "value_in_config": 6.44 +--- +> "p1": { +> "value_in_config": 4.0 +204a202,204 +> "y1": { +> "value_in_config": 6.44 +> }, +245,247d244 +< "p1": { +< "value_in_config": 4.0 +< }, +251,252c248,249 +< "y1": { +< "value_in_config": 6.44 +--- +> "p1": { +> "value_in_config": 4.0 +255a253,255 +> }, +> "y1": { +> "value_in_config": 6.44 +=================== End of Test96_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test96_smlp_toy_num_resp_mult_optimization_results.json ================================== +16,18d15 +< "p1": { +< "value_in_config": 4.0 +< }, +22,23c19,20 +< "y1": { +< "value_in_config": 6.44 +--- +> "p1": { +> "value_in_config": 4.0 +26a24,26 +> }, +> "y1": { +> "value_in_config": 6.44 +=================== End of Test96_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test97_smlp_toy_num_resp_mult.txt ================================== +252c252 +< smlp_logger - INFO - Model operator counts for y2: {'add': 100, 'mul': 715, 'const': 2547, 'ite': 305, 'and': 408, 'prop': 713, 'sub': 713, 'var': 713} +--- +> smlp_logger - INFO - Model operator counts for y2: {'add': 100, 'mul': 716, 'const': 2550, 'ite': 305, 'and': 409, 'prop': 714, 'sub': 714, 'var': 714} +=================== End of Test97_smlp_toy_num_resp_mult.txt diff report ================================ +=================== Diff report for: Test98_smlp_toy_num_resp_mult_optimization_progress.json ================================== +17,19d16 +< "p1": { +< "value_in_config": 2.0 +< }, +23,24c20,21 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +28a26,28 +> "y1": { +> "value_in_config": 7.0 +> }, +48,50d47 +< "p1": { +< "value_in_config": 2.0 +< }, +54,55c51,52 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +59a57,59 +> "y1": { +> "value_in_config": 7.0 +> }, +79,81d78 +< "p1": { +< "value_in_config": 2.0 +< }, +85,86c82,83 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +90a88,90 +> "y1": { +> "value_in_config": 7.0 +> }, +131,133d130 +< "p1": { +< "value_in_config": 2.0 +< }, +137,138c134,135 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +141a139,141 +> }, +> "y1": { +> "value_in_config": 7.0 +=================== End of Test98_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test98_smlp_toy_num_resp_mult_optimization_results.json ================================== +16,18d15 +< "p1": { +< "value_in_config": 2.0 +< }, +22,23c19,20 +< "y1": { +< "value_in_config": 7.0 +--- +> "p1": { +> "value_in_config": 2.0 +26a24,26 +> }, +> "y1": { +> "value_in_config": 7.0 +=================== End of Test98_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test99_smlp_toy_num_resp_mult_optimization_progress.json ================================== +24,26d23 +< "p1": { +< "value_in_config": 4.0 +< }, +30,31c27,28 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +35a33,35 +> "y1": { +> "value_in_config": 9.0 +> }, +62,64d61 +< "p1": { +< "value_in_config": 4.0 +< }, +68,69c65,66 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +73a71,73 +> "y1": { +> "value_in_config": 9.0 +> }, +100,102d99 +< "p1": { +< "value_in_config": 4.0 +< }, +106,107c103,104 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +111a109,111 +> "y1": { +> "value_in_config": 9.0 +> }, +138,140d137 +< "p1": { +< "value_in_config": 4.0 +< }, +144,145c141,142 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +149a147,149 +> "y1": { +> "value_in_config": 9.0 +> }, +197,199d196 +< "p1": { +< "value_in_config": 4.0 +< }, +203,204c200,201 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +208a206,208 +> "y1": { +> "value_in_config": 9.0 +> }, +235,237d234 +< "p1": { +< "value_in_config": 7.0 +< }, +241,242c238,239 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +246a244,246 +> "y1": { +> "value_in_config": 9.0 +> }, +273,275d272 +< "p1": { +< "value_in_config": 7.0 +< }, +279,280c276,277 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +284a282,284 +> "y1": { +> "value_in_config": 9.0 +> }, +332,334d331 +< "p1": { +< "value_in_config": 7.0 +< }, +338,339c335,336 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +342a340,342 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test99_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test99_smlp_toy_num_resp_mult_optimization_results.json ================================== +23,25d22 +< "p1": { +< "value_in_config": 7.0 +< }, +29,30c26,27 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 7.0 +33a31,33 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test99_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test100_smlp_toy_num_resp_mult_optimization_progress.json ================================== +24,26d23 +< "p1": { +< "value_in_config": 4.0 +< }, +30,31c27,28 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +35a33,35 +> "y1": { +> "value_in_config": 9.0 +> }, +62,64d61 +< "p1": { +< "value_in_config": 4.0 +< }, +68,69c65,66 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +73a71,73 +> "y1": { +> "value_in_config": 9.0 +> }, +100,102d99 +< "p1": { +< "value_in_config": 4.0 +< }, +106,107c103,104 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +111a109,111 +> "y1": { +> "value_in_config": 9.0 +> }, +138,140d137 +< "p1": { +< "value_in_config": 4.0 +< }, +144,145c141,142 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +149a147,149 +> "y1": { +> "value_in_config": 9.0 +> }, +197,199d196 +< "p1": { +< "value_in_config": 4.0 +< }, +203,204c200,201 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +208a206,208 +> "y1": { +> "value_in_config": 9.0 +> }, +235,237d234 +< "p1": { +< "value_in_config": 4.0 +< }, +241,242c238,239 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +246a244,246 +> "y1": { +> "value_in_config": 9.0 +> }, +273,275d272 +< "p1": { +< "value_in_config": 4.0 +< }, +279,280c276,277 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +284a282,284 +> "y1": { +> "value_in_config": 9.0 +> }, +332,334d331 +< "p1": { +< "value_in_config": 4.0 +< }, +338,339c335,336 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +342a340,342 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test100_smlp_toy_num_resp_mult_optimization_progress.json diff report ================================ +=================== Diff report for: Test100_smlp_toy_num_resp_mult_optimization_results.json ================================== +23,25d22 +< "p1": { +< "value_in_config": 4.0 +< }, +29,30c26,27 +< "y1": { +< "value_in_config": 9.0 +--- +> "p1": { +> "value_in_config": 4.0 +33a31,33 +> }, +> "y1": { +> "value_in_config": 9.0 +=================== End of Test100_smlp_toy_num_resp_mult_optimization_results.json diff report ================================ +=================== Diff report for: Test102_test101_model.txt ================================== +38c38 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test101_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test101_model_rerun_model_config.json +=================== End of Test102_test101_model.txt diff report ================================ +=================== Diff report for: Test105_smlp_toy_num_resp_mult_verify_results.json ================================== +6d5 +< "p1": 4.0, +8c7 +< "y1": 9.0, +--- +> "p1": 4.0, +9a9 +> "y1": 9.0, +=================== End of Test105_smlp_toy_num_resp_mult_verify_results.json diff report ================================ +=================== Diff report for: test110_model_poly_sklearn_formula.txt ================================== +diff: /app/smlp/regr_smlp/code/test110_model_poly_sklearn_formula.txt: No such file or directory +=================== End of test110_model_poly_sklearn_formula.txt diff report ================================ +=================== Diff report for: Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt ================================== +79c79 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test110_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test110_model_rerun_model_config.json +=================== End of Test111_test110_model_smlp_toy_basic_pred_unlabeled.txt diff report ================================ +=================== Diff report for: Test112_test110_model_smlp_toy_basic_pred_unlabeled.txt ================================== +79c79 +< smlp_logger - INFO - Seving model rerun configuration in file ../models/test110_model_rerun_model_config.json +--- +> smlp_logger - INFO - Seving model rerun configuration in file ./../models/test110_model_rerun_model_config.json +=================== End of Test112_test110_model_smlp_toy_basic_pred_unlabeled.txt diff report ================================ +=================== Diff report for: Test113_smlp_toy_basic_optimization_progress.json ================================== +17,18c17,21 +< "x2": { +< "value_in_config": 0.0 +--- +> "x1": { +> "value_in_config": 10.0 +> }, +> "p2": { +> "value_in_config": 4.0 +23,24c26,27 +< "x1": { +< "value_in_config": 10.0 +--- +> "x2": { +> "value_in_config": 0.0 +31,33d33 +< }, +< "p2": { +< "value_in_config": 4.0 +51,52c51,55 +< "x2": { +< "value_in_config": 0.0 +--- +> "x1": { +> "value_in_config": 10.0 +> }, +> "p2": { +> "value_in_config": 4.0 +57,58c60,61 +< "x1": { +< "value_in_config": 10.0 +--- +> "x2": { +> "value_in_config": 0.0 +66,68d68 +< "p2": { +< "value_in_config": 4.0 +< }, +106,107c106,110 +< "x2": { +< "value_in_config": 0.0 +--- +> "x1": { +> "value_in_config": 10.0 +> }, +> "p2": { +> "value_in_config": 4.0 +112,113c115,116 +< "x1": { +< "value_in_config": 10.0 +--- +> "x2": { +> "value_in_config": 0.0 +120,122d122 +< }, +< "p2": { +< "value_in_config": 4.0 +=================== End of Test113_smlp_toy_basic_optimization_progress.json diff report ================================ +=================== Diff report for: Test113_smlp_toy_basic_optimization_results.json ================================== +16,17c16,20 +< "x2": { +< "value_in_config": 0.0 +--- +> "x1": { +> "value_in_config": 10.0 +> }, +> "p2": { +> "value_in_config": 4.0 +22,23c25,26 +< "x1": { +< "value_in_config": 10.0 +--- +> "x2": { +> "value_in_config": 0.0 +30,32d32 +< }, +< "p2": { +< "value_in_config": 4.0 +=================== End of Test113_smlp_toy_basic_optimization_results.json diff report ================================ +=================== Diff report for: Test114_smlp_toy_basic_optimization_progress.json ================================== +17,18c17,21 +< "x2": { +< "value_in_config": 0.0 +--- +> "x1": { +> "value_in_config": 10.0 +> }, +> "p2": { +> "value_in_config": 4.0 +23,24c26,27 +< "x1": { +< "value_in_config": 10.0 +--- +> "x2": { +> "value_in_config": 0.0 +31,33d33 +< }, +< "p2": { +< "value_in_config": 4.0 +51,52c51,55 +< "x2": { +< "value_in_config": 0.0 +--- +> "x1": { +> "value_in_config": 10.0 +> }, +> "p2": { +> "value_in_config": 4.0 +57,58c60,61 +< "x1": { +< "value_in_config": 10.0 +--- +> "x2": { +> "value_in_config": 0.0 +66,68d68 +< "p2": { +< "value_in_config": 4.0 +< }, +106,107c106,110 +< "x2": { +< "value_in_config": 0.0 +--- +> "x1": { +> "value_in_config": 10.0 +> }, +> "p2": { +> "value_in_config": 4.0 +112,113c115,116 +< "x1": { +< "value_in_config": 10.0 +--- +> "x2": { +> "value_in_config": 0.0 +120,122d122 +< }, +< "p2": { +< "value_in_config": 4.0 +=================== End of Test114_smlp_toy_basic_optimization_progress.json diff report ================================ +=================== Diff report for: Test114_smlp_toy_basic_optimization_results.json ================================== +16,17c16,20 +< "x2": { +< "value_in_config": 0.0 +--- +> "x1": { +> "value_in_config": 10.0 +> }, +> "p2": { +> "value_in_config": 4.0 +22,23c25,26 +< "x1": { +< "value_in_config": 10.0 +--- +> "x2": { +> "value_in_config": 0.0 +30,32d32 +< }, +< "p2": { +< "value_in_config": 4.0 +=================== End of Test114_smlp_toy_basic_optimization_results.json diff report ================================ +=================== Diff report for: Test118_smlp_toy_basic_verify_results.json ================================== +5a6 +> "p1": 0.2, +7d7 +< "y2": 0.0, +9c9 +< "p1": 0.2 +--- +> "y2": 0.0 +=================== End of Test118_smlp_toy_basic_verify_results.json diff report ================================ +=================== Diff report for: Test119_smlp_toy_basic_query_results.json ================================== +14a15 +> "p1": 0.0, +16d16 +< "y2": 0.0, +18c18 +< "p1": 0.0 +--- +> "y2": 0.0 +=================== End of Test119_smlp_toy_basic_query_results.json diff report ================================ +=================== Diff report for: Test121_smlp_toy_basic_synthesize_results.json ================================== +9,10c9,10 +< "p2": 0.0, +< "p1": 0.0 +--- +> "p1": 0.0, +> "p2": 0.0 +=================== End of Test121_smlp_toy_basic_synthesize_results.json diff report ================================ +=================== Diff report for: Test122_smlp_toy_basic_optimization_progress.json ================================== +16a17,19 +> "p1": { +> "value_in_config": 0.0 +> }, +20c23 +< "y2": { +--- +> "y1": { +24c27 +< "y1": { +--- +> "y2": { +27,29d29 +< }, +< "p1": { +< "value_in_config": 0.0 +=================== End of Test122_smlp_toy_basic_optimization_progress.json diff report ================================ +=================== Diff report for: Test122_smlp_toy_basic_optimization_results.json ================================== +2a3 +> "p1": 0.0, +4d4 +< "y2": 0.0, +6c6 +< "p1": 0.0, +--- +> "y2": 0.0, +16a17 +> "p1": 0.0, +18d18 +< "y2": 0.0, +20c20 +< "p1": 0.0, +--- +> "y2": 0.0, +=================== End of Test122_smlp_toy_basic_optimization_results.json diff report ================================ +=================== Diff report for: Test123_smlp_toy_basic_optimization_progress.json ================================== +16a17,19 +> "p1": { +> "value_in_config": 0.0 +> }, +20c23 +< "y2": { +--- +> "y1": { +24c27 +< "y1": { +--- +> "y2": { +27,29d29 +< }, +< "p1": { +< "value_in_config": 0.0 +46a47,49 +> "p1": { +> "value_in_config": 0.0 +> }, +50c53 +< "y2": { +--- +> "y1": { +54c57 +< "y1": { +--- +> "y2": { +57,59d59 +< }, +< "p1": { +< "value_in_config": 0.0 +76a77,79 +> "p1": { +> "value_in_config": 0.0 +> }, +80c83 +< "y2": { +--- +> "y1": { +84c87 +< "y1": { +--- +> "y2": { +87,89d89 +< }, +< "p1": { +< "value_in_config": 0.0 +106a107,109 +> "p1": { +> "value_in_config": 0.0 +> }, +110c113 +< "y2": { +--- +> "y1": { +114c117 +< "y1": { +--- +> "y2": { +117,119d119 +< }, +< "p1": { +< "value_in_config": 0.0 +136a137,139 +> "p1": { +> "value_in_config": 0.0 +> }, +140c143 +< "y2": { +--- +> "y1": { +144c147 +< "y1": { +--- +> "y2": { +147,149d149 +< }, +< "p1": { +< "value_in_config": 0.0 +166a167,169 +> "p1": { +> "value_in_config": 0.0 +> }, +170c173 +< "y2": { +--- +> "y1": { +174c177 +< "y1": { +--- +> "y2": { +177,179d179 +< }, +< "p1": { +< "value_in_config": 0.0 +196a197,199 +> "p1": { +> "value_in_config": 0.0 +> }, +200c203 +< "y2": { +--- +> "y1": { +204c207 +< "y1": { +--- +> "y2": { +207,209d209 +< }, +< "p1": { +< "value_in_config": 0.0 +226a227,229 +> "p1": { +> "value_in_config": 0.0 +> }, +230c233 +< "y2": { +--- +> "y1": { +234c237 +< "y1": { +--- +> "y2": { +237,239d239 +< }, +< "p1": { +< "value_in_config": 0.0 +256a257,259 +> "p1": { +> "value_in_config": 0.0 +> }, +260c263 +< "y2": { +--- +> "y1": { +264c267 +< "y1": { +--- +> "y2": { +267,269d269 +< }, +< "p1": { +< "value_in_config": 0.0 +286a287,289 +> "p1": { +> "value_in_config": 0.0 +> }, +290c293 +< "y2": { +--- +> "y1": { +294c297 +< "y1": { +--- +> "y2": { +297,299d299 +< }, +< "p1": { +< "value_in_config": 0.0 +316a317,319 +> "p1": { +> "value_in_config": 0.0 +> }, +320c323 +< "y2": { +--- +> "y1": { +324c327 +< "y1": { +--- +> "y2": { +327,329d329 +< }, +< "p1": { +< "value_in_config": 0.0 +346a347,349 +> "p1": { +> "value_in_config": 0.0 +> }, +350c353 +< "y2": { +--- +> "y1": { +354c357 +< "y1": { +--- +> "y2": { +357,359d359 +< }, +< "p1": { +< "value_in_config": 0.0 +376a377,379 +> "p1": { +> "value_in_config": 0.0 +> }, +380c383 +< "y2": { +--- +> "y1": { +384c387 +< "y1": { +--- +> "y2": { +388,390d390 +< "p1": { +< "value_in_config": 0.0 +< }, +427a428,430 +> "p1": { +> "value_in_config": 0.0 +> }, +431c434 +< "y2": { +--- +> "y1": { +435c438 +< "y1": { +--- +> "y2": { +438,440d440 +< }, +< "p1": { +< "value_in_config": 0.0 +457a458,460 +> "p1": { +> "value_in_config": 0.0 +> }, +461c464 +< "y2": { +--- +> "y1": { +465c468 +< "y1": { +--- +> "y2": { +468,470d470 +< }, +< "p1": { +< "value_in_config": 0.0 +487a488,490 +> "p1": { +> "value_in_config": 0.0 +> }, +491c494 +< "y2": { +--- +> "y1": { +495c498 +< "y1": { +--- +> "y2": { +498,500d500 +< }, +< "p1": { +< "value_in_config": 0.0 +517a518,520 +> "p1": { +> "value_in_config": 0.0 +> }, +521c524 +< "y2": { +--- +> "y1": { +525c528 +< "y1": { +--- +> "y2": { +528,530d530 +< }, +< "p1": { +< "value_in_config": 0.0 +547a548,550 +> "p1": { +> "value_in_config": 0.0 +> }, +551c554 +< "y2": { +--- +> "y1": { +555c558 +< "y1": { +--- +> "y2": { +558,560d560 +< }, +< "p1": { +< "value_in_config": 0.0 +577a578,580 +> "p1": { +> "value_in_config": 0.0 +> }, +581c584 +< "y2": { +--- +> "y1": { +585c588 +< "y1": { +--- +> "y2": { +588,590d590 +< }, +< "p1": { +< "value_in_config": 0.0 +607a608,610 +> "p1": { +> "value_in_config": 0.0 +> }, +611c614 +< "y2": { +--- +> "y1": { +615c618 +< "y1": { +--- +> "y2": { +618,620d620 +< }, +< "p1": { +< "value_in_config": 0.0 +637a638,640 +> "p1": { +> "value_in_config": 0.0 +> }, +641c644 +< "y2": { +--- +> "y1": { +645c648 +< "y1": { +--- +> "y2": { +648,650d650 +< }, +< "p1": { +< "value_in_config": 0.0 +667a668,670 +> "p1": { +> "value_in_config": 0.0 +> }, +671c674 +< "y2": { +--- +> "y1": { +675c678 +< "y1": { +--- +> "y2": { +678,680d680 +< }, +< "p1": { +< "value_in_config": 0.0 +697a698,700 +> "p1": { +> "value_in_config": 0.0 +> }, +701c704 +< "y2": { +--- +> "y1": { +705c708 +< "y1": { +--- +> "y2": { +708,710d710 +< }, +< "p1": { +< "value_in_config": 0.0 +727a728,730 +> "p1": { +> "value_in_config": 0.0 +> }, +731c734 +< "y2": { +--- +> "y1": { +735c738 +< "y1": { +--- +> "y2": { +738,740d740 +< }, +< "p1": { +< "value_in_config": 0.0 +757a758,760 +> "p1": { +> "value_in_config": 0.0 +> }, +761c764 +< "y2": { +--- +> "y1": { +765c768 +< "y1": { +--- +> "y2": { +768,770d770 +< }, +< "p1": { +< "value_in_config": 0.0 +787a788,790 +> "p1": { +> "value_in_config": 0.0 +> }, +791c794 +< "y2": { +--- +> "y1": { +795c798 +< "y1": { +--- +> "y2": { +799,801d801 +< "p1": { +< "value_in_config": 0.0 +< }, +838a839,841 +> "p1": { +> "value_in_config": 0.0 +> }, +842c845 +< "y2": { +--- +> "y1": { +846c849 +< "y1": { +--- +> "y2": { +849,851d851 +< }, +< "p1": { +< "value_in_config": 0.0 +=================== End of Test123_smlp_toy_basic_optimization_progress.json diff report ================================ +=================== Diff report for: Test123_smlp_toy_basic_optimization_results.json ================================== +15a16,18 +> "p1": { +> "value_in_config": 0.0 +> }, +19c22 +< "y2": { +--- +> "y1": { +23c26 +< "y1": { +--- +> "y2": { +26,28d28 +< }, +< "p1": { +< "value_in_config": 0.0 +=================== End of Test123_smlp_toy_basic_optimization_results.json diff report ================================ +=================== Diff report for: Test125_smlp_toy_basic_optimization_progress.json ================================== +16a17,19 +> "p1": { +> "value_in_config": 0.0 +> }, +20c23 +< "y2": { +--- +> "y1": { +24c27 +< "y1": { +--- +> "y2": { +27,29d29 +< }, +< "p1": { +< "value_in_config": 0.0 +46a47,49 +> "p1": { +> "value_in_config": 0.0 +> }, +50c53 +< "y2": { +--- +> "y1": { +54c57 +< "y1": { +--- +> "y2": { +57,59d59 +< }, +< "p1": { +< "value_in_config": 0.0 +76a77,79 +> "p1": { +> "value_in_config": 0.0 +> }, +80c83 +< "y2": { +--- +> "y1": { +84c87 +< "y1": { +--- +> "y2": { +87,89d89 +< }, +< "p1": { +< "value_in_config": 0.0 +106a107,109 +> "p1": { +> "value_in_config": 0.0 +> }, +110c113 +< "y2": { +--- +> "y1": { +114c117 +< "y1": { +--- +> "y2": { +117,119d119 +< }, +< "p1": { +< "value_in_config": 0.0 +136a137,139 +> "p1": { +> "value_in_config": 0.0 +> }, +140c143 +< "y2": { +--- +> "y1": { +144c147 +< "y1": { +--- +> "y2": { +147,149d149 +< }, +< "p1": { +< "value_in_config": 0.0 +166a167,169 +> "p1": { +> "value_in_config": 0.0 +> }, +170c173 +< "y2": { +--- +> "y1": { +174c177 +< "y1": { +--- +> "y2": { +177,179d179 +< }, +< "p1": { +< "value_in_config": 0.0 +196a197,199 +> "p1": { +> "value_in_config": 0.0 +> }, +200c203 +< "y2": { +--- +> "y1": { +204c207 +< "y1": { +--- +> "y2": { +207,209d209 +< }, +< "p1": { +< "value_in_config": 0.0 +226a227,229 +> "p1": { +> "value_in_config": 0.0 +> }, +230c233 +< "y2": { +--- +> "y1": { +234c237 +< "y1": { +--- +> "y2": { +237,239d239 +< }, +< "p1": { +< "value_in_config": 0.0 +256a257,259 +> "p1": { +> "value_in_config": 0.0 +> }, +260c263 +< "y2": { +--- +> "y1": { +264c267 +< "y1": { +--- +> "y2": { +267,269d269 +< }, +< "p1": { +< "value_in_config": 0.0 +286a287,289 +> "p1": { +> "value_in_config": 0.0 +> }, +290c293 +< "y2": { +--- +> "y1": { +294c297 +< "y1": { +--- +> "y2": { +297,299d299 +< }, +< "p1": { +< "value_in_config": 0.0 +316a317,319 +> "p1": { +> "value_in_config": 0.0 +> }, +320c323 +< "y2": { +--- +> "y1": { +324c327 +< "y1": { +--- +> "y2": { +327,329d329 +< }, +< "p1": { +< "value_in_config": 0.0 +346a347,349 +> "p1": { +> "value_in_config": 0.0 +> }, +350c353 +< "y2": { +--- +> "y1": { +354c357 +< "y1": { +--- +> "y2": { +357,359d359 +< }, +< "p1": { +< "value_in_config": 0.0 +376a377,379 +> "p1": { +> "value_in_config": 0.0 +> }, +380c383 +< "y2": { +--- +> "y1": { +384c387 +< "y1": { +--- +> "y2": { +388,390d390 +< "p1": { +< "value_in_config": 0.0 +< }, +427a428,430 +> "p1": { +> "value_in_config": 0.0 +> }, +431c434 +< "y2": { +--- +> "y1": { +435c438 +< "y1": { +--- +> "y2": { +438,440d440 +< }, +< "p1": { +< "value_in_config": 0.0 +457a458,460 +> "p1": { +> "value_in_config": 0.0 +> }, +461c464 +< "y2": { +--- +> "y1": { +465c468 +< "y1": { +--- +> "y2": { +468,470d470 +< }, +< "p1": { +< "value_in_config": 0.0 +487a488,490 +> "p1": { +> "value_in_config": 0.0 +> }, +491c494 +< "y2": { +--- +> "y1": { +495c498 +< "y1": { +--- +> "y2": { +498,500d500 +< }, +< "p1": { +< "value_in_config": 0.0 +517a518,520 +> "p1": { +> "value_in_config": 0.0 +> }, +521c524 +< "y2": { +--- +> "y1": { +525c528 +< "y1": { +--- +> "y2": { +528,530d530 +< }, +< "p1": { +< "value_in_config": 0.0 +547a548,550 +> "p1": { +> "value_in_config": 0.0 +> }, +551c554 +< "y2": { +--- +> "y1": { +555c558 +< "y1": { +--- +> "y2": { +558,560d560 +< }, +< "p1": { +< "value_in_config": 0.0 +577a578,580 +> "p1": { +> "value_in_config": 0.0 +> }, +581c584 +< "y2": { +--- +> "y1": { +585c588 +< "y1": { +--- +> "y2": { +588,590d590 +< }, +< "p1": { +< "value_in_config": 0.0 +607a608,610 +> "p1": { +> "value_in_config": 0.0 +> }, +611c614 +< "y2": { +--- +> "y1": { +615c618 +< "y1": { +--- +> "y2": { +618,620d620 +< }, +< "p1": { +< "value_in_config": 0.0 +637a638,640 +> "p1": { +> "value_in_config": 0.0 +> }, +641c644 +< "y2": { +--- +> "y1": { +645c648 +< "y1": { +--- +> "y2": { +648,650d650 +< }, +< "p1": { +< "value_in_config": 0.0 +667a668,670 +> "p1": { +> "value_in_config": 0.0 +> }, +671c674 +< "y2": { +--- +> "y1": { +675c678 +< "y1": { +--- +> "y2": { +678,680d680 +< }, +< "p1": { +< "value_in_config": 0.0 +697a698,700 +> "p1": { +> "value_in_config": 0.0 +> }, +701c704 +< "y2": { +--- +> "y1": { +705c708 +< "y1": { +--- +> "y2": { +708,710d710 +< }, +< "p1": { +< "value_in_config": 0.0 +727a728,730 +> "p1": { +> "value_in_config": 0.0 +> }, +731c734 +< "y2": { +--- +> "y1": { +735c738 +< "y1": { +--- +> "y2": { +738,740d740 +< }, +< "p1": { +< "value_in_config": 0.0 +757a758,760 +> "p1": { +> "value_in_config": 0.0 +> }, +761c764 +< "y2": { +--- +> "y1": { +765c768 +< "y1": { +--- +> "y2": { +768,770d770 +< }, +< "p1": { +< "value_in_config": 0.0 +787a788,790 +> "p1": { +> "value_in_config": 0.0 +> }, +791c794 +< "y2": { +--- +> "y1": { +795c798 +< "y1": { +--- +> "y2": { +799,801d801 +< "p1": { +< "value_in_config": 0.0 +< }, +838a839,841 +> "p1": { +> "value_in_config": 0.0 +> }, +842c845 +< "y2": { +--- +> "y1": { +846c849 +< "y1": { +--- +> "y2": { +849,851d851 +< }, +< "p1": { +< "value_in_config": 0.0 +=================== End of Test125_smlp_toy_basic_optimization_progress.json diff report ================================ +=================== Diff report for: Test125_smlp_toy_basic_optimization_results.json ================================== +15a16,18 +> "p1": { +> "value_in_config": 0.0 +> }, +19c22 +< "y2": { +--- +> "y1": { +23c26 +< "y1": { +--- +> "y2": { +26,28d28 +< }, +< "p1": { +< "value_in_config": 0.0 +=================== End of Test125_smlp_toy_basic_optimization_results.json diff report ================================ +=================== Diff report for: Test126_smlp_toy_basic_verify_results.json ================================== +6,7d5 +< "x2": 0.0, +< "y1": 0.2, +8a7,9 +> "p2": 0.0, +> "y1": 0.2, +> "x2": 0.0, +10,11c11 +< "y2": 0.0, +< "p2": 0.0 +--- +> "y2": 0.0 +=================== End of Test126_smlp_toy_basic_verify_results.json diff report ================================ +=================== Diff report for: Test129_smlp_toy_ctg_num_resp_verify_results.json ================================== +18d17 +< "p1": 7.5, +20c19 +< "y1": -61.895667856765954, +--- +> "p1": 7.5, +21a21 +> "y1": -61.895667856765954, +30d29 +< "p1": 7.5, +32c31 +< "y1": -61.895667856765954, +--- +> "p1": 7.5, +33a33 +> "y1": -61.895667856765954, +=================== End of Test129_smlp_toy_ctg_num_resp_verify_results.json diff report ================================ From a447efc62242b978e4bbb82b6c8034299a6c3e76 Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Wed, 25 Mar 2026 17:11:56 +0200 Subject: [PATCH 05/11] Updated Docker wheel test images after synch with master branch --- scripts/bin/README.md | 8 ++++---- .../Dockerfile.smlp-test-build-almalinux_9-python311 | 2 +- .../Dockerfile.smlp-test-build-opensuse_15.5-python311 | 2 +- .../Dockerfile.smlp-test-build-ubuntu_22.04-python311 | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/bin/README.md b/scripts/bin/README.md index 17baede0..7221b041 100644 --- a/scripts/bin/README.md +++ b/scripts/bin/README.md @@ -15,10 +15,10 @@ All `enter*` commands mentioned below create `$HOME/shared` directory and mount | Repository | Tag | Image Index Digest | |--------------------------------------------------|-------|-------------| -| mdmitry1/smlp-test-build-opensuse\_15.5-python311 | latest | sha256:ca6725b46ce9 | -| mdmitry1/smlp-test-build-almalinux\_9-python311 | latest | sha256:82ae0eeb8ac8 | -| mdmitry1/smlp-test-build-ubuntu\_22.04-python311 | latest | sha256:11ab93a137dc | -| mdmitry1/python311-dev | latest | sha256:887dc7086b5d | +| mdmitry1/smlp-test-build-almalinux\_9-python311 | latest | sha256:4c17ef42dc8c | +| mdmitry1/smlp-test-build-opensuse\_15.5-python311 | latest | sha256:6f27caef60b0 | +| mdmitry1/smlp-test-build-ubuntu\_22.04-python311 | latest | sha256:bbd984c96074 | +| mdmitry1/python311-dev | latest | sha256:dcb64c8194fb | GUI and data sharing is supported for container `mdmitry1/python311-dev:latest` only. Virtual display is supported for all containers. diff --git a/scripts/docker/Dockerfile.smlp-test-build-almalinux_9-python311 b/scripts/docker/Dockerfile.smlp-test-build-almalinux_9-python311 index 30b88622..1f6cf47c 100644 --- a/scripts/docker/Dockerfile.smlp-test-build-almalinux_9-python311 +++ b/scripts/docker/Dockerfile.smlp-test-build-almalinux_9-python311 @@ -53,7 +53,7 @@ RUN cd /app && ./run_git_clone $GIT_BRANCH # --------------------------------------------------------------------------- # 5. Install smlp from wheel # --------------------------------------------------------------------------- -RUN python3.11 -m pip install /app/smlp/scripts/dist/smlp-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl +RUN python3.11 -m pip install /app/smlp/scripts/dist/smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl # --------------------------------------------------------------------------- # 6. Configure UTF-8 locale diff --git a/scripts/docker/Dockerfile.smlp-test-build-opensuse_15.5-python311 b/scripts/docker/Dockerfile.smlp-test-build-opensuse_15.5-python311 index 1bc379fa..4515cf52 100644 --- a/scripts/docker/Dockerfile.smlp-test-build-opensuse_15.5-python311 +++ b/scripts/docker/Dockerfile.smlp-test-build-opensuse_15.5-python311 @@ -53,7 +53,7 @@ RUN cd /app && ./run_git_clone $GIT_BRANCH # --------------------------------------------------------------------------- # 5. Install smlp from wheel # --------------------------------------------------------------------------- -RUN python3.11 -m pip install /app/smlp/scripts/dist/smlp-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl +RUN python3.11 -m pip install /app/smlp/scripts/dist/smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl # --------------------------------------------------------------------------- # 6. Configure UTF-8 locale diff --git a/scripts/docker/Dockerfile.smlp-test-build-ubuntu_22.04-python311 b/scripts/docker/Dockerfile.smlp-test-build-ubuntu_22.04-python311 index 905fc2c5..e9750c94 100644 --- a/scripts/docker/Dockerfile.smlp-test-build-ubuntu_22.04-python311 +++ b/scripts/docker/Dockerfile.smlp-test-build-ubuntu_22.04-python311 @@ -47,7 +47,7 @@ RUN cd /app && ./run_git_clone $GIT_BRANCH # --------------------------------------------------------------------------- # 5. Install smlp from wheel # --------------------------------------------------------------------------- -RUN python3.11 -m pip install /app/smlp/scripts/dist/smlp-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl +RUN python3.11 -m pip install /app/smlp/scripts/dist/smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl # --------------------------------------------------------------------------- # 6. Install UTF-8 fonts From 1cd41f96d7f40730921575ece39be341b95a047b Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Wed, 25 Mar 2026 18:26:18 +0200 Subject: [PATCH 06/11] Changed package name to smlptech in the installation test --- tests/install/test_container_install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/install/test_container_install b/tests/install/test_container_install index fbdb22cc..b12df926 100755 --- a/tests/install/test_container_install +++ b/tests/install/test_container_install @@ -8,7 +8,7 @@ log=$(realpath $0 | xargs basename).$(echo $image | tr / _ | cut -d: -f1).log success=0 if [[ "$image" == *wheel* ]]; then docker run --rm $image /bin/bash -c \ - 'auditwheel show dist/smlp-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl \ + 'auditwheel show dist/smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl \ | grep -B1 -A1 consistent |& tr "\012" " "' &> $log echo "" >> $log if [ "1" -eq $(grep -c consistent $log) ]; then From e551a7b7b19045bd0a62cb7eb5dca875c97f0017 Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Thu, 26 Mar 2026 14:49:06 +0200 Subject: [PATCH 07/11] Added readme, which will be displayed in pypi.org --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b0f803ad..6efac9c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ build-backend = "setuptools.build_meta" name = "smlptech" version = "1.0.1" description = "SMLP - The Symbolic Machine Learning Prover" +readme = "README.md" requires-python = "==3.11.*" dependencies = [ "blosc2", From 220e882d17fbfef80e8cb075b6ece0bbca31bfa9 Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Thu, 26 Mar 2026 15:27:00 +0200 Subject: [PATCH 08/11] Added support of running regression on locally installed wheel --- tests/smlp_regression/run_smlp_regression | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/smlp_regression/run_smlp_regression b/tests/smlp_regression/run_smlp_regression index df2b6fa1..764b141b 100755 --- a/tests/smlp_regression/run_smlp_regression +++ b/tests/smlp_regression/run_smlp_regression @@ -1,4 +1,5 @@ #!/usr/bin/tcsh -f +onintr cleanup set script_path=`realpath $0 | xargs dirname` set script_name=`realpath $0 | xargs basename` set log=$PWD/${script_name}.log @@ -22,5 +23,14 @@ echo "Log file: $log" if(! $?DISPLAY ) then setenv DISPLAY :99 endif -echo n | env CUDA_VISIBLE_DEVICES=-1 ./smlp_regr.py -w 8 -def n -t all -tol 7 -g |& tee $log +set python3_version=`python3 -c 'from sys import version_info; print(version_info.minor)'` +set local_script=`echo smlp_regr.py | sed 's/regr/regr_local/'` +if( $python3_version != 11 ) then + \rm -f $local_script >& /dev/null + sed 's@../../src/run_smlp.py@smlp@' smlp_regr.py > $local_script + chmod +x $local_script +endif +echo n | env CUDA_VISIBLE_DEVICES=-1 ./${local_script} -w 8 -def n -t all -tol 7 -g |& tee $log ${script_path}/create_diff_report >& $diff_report +cleanup: + \rm -f $local_script >& /dev/null From 3e6629a2a6a7043cfa397efc30ac5e62c2da8c59 Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Thu, 26 Mar 2026 15:54:20 +0200 Subject: [PATCH 09/11] Incremented package version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6efac9c5..704e3d6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" [project] name = "smlptech" -version = "1.0.1" +version = "1.0.2" description = "SMLP - The Symbolic Machine Learning Prover" readme = "README.md" requires-python = "==3.11.*" From 1280407c8ad575e4609674f42ffeb0dfad13f39f Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Thu, 26 Mar 2026 17:54:39 +0200 Subject: [PATCH 10/11] Added matplotlib default backend test --- tests/install/test_matplotlib_backend | 11 +++++++++++ tests/install/test_matplotlib_in_virtual_display | 2 ++ tests/install/test_matplotlib_in_vnc | 2 ++ 3 files changed, 15 insertions(+) create mode 100755 tests/install/test_matplotlib_backend create mode 100755 tests/install/test_matplotlib_in_virtual_display create mode 100755 tests/install/test_matplotlib_in_vnc diff --git a/tests/install/test_matplotlib_backend b/tests/install/test_matplotlib_backend new file mode 100755 index 00000000..27edc2bf --- /dev/null +++ b/tests/install/test_matplotlib_backend @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +script_path=$(realpath $0 | xargs dirname) +i=1 +for image in $(docker images --format table | awk '{print $1}' | egrep "smlp|dev" | grep python311 | egrep -v "wheel|test\-build"); do + echo $i $image matplotlib backend for VNC server: $(docker run -e DISPLAY=:99 -p 5900:5900 --rm -v $script_path/test_matplotlib_in_vnc:/test_matplotlib $image /test_matplotlib) + (( i++ )) +done +for image in $(docker images --format table | awk '{print $1}' | egrep "smlp|dev" | grep python311 | grep -v wheel); do + echo $i $image matplotlib backend for virtual display: $(docker run --rm -v $script_path/test_matplotlib_in_virtual_display:/test_matplotlib $image /test_matplotlib) + (( i++ )) +done diff --git a/tests/install/test_matplotlib_in_virtual_display b/tests/install/test_matplotlib_in_virtual_display new file mode 100755 index 00000000..8fae489d --- /dev/null +++ b/tests/install/test_matplotlib_in_virtual_display @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +./open_virtual_display && env DISPLAY=:99 python3 -c "import matplotlib; print(matplotlib.get_backend())" diff --git a/tests/install/test_matplotlib_in_vnc b/tests/install/test_matplotlib_in_vnc new file mode 100755 index 00000000..0b23d0f6 --- /dev/null +++ b/tests/install/test_matplotlib_in_vnc @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +./start_vnc && python3 -c "import matplotlib; print(matplotlib.get_backend())" From 5de51fa7a182c3e153c63ff9a3ed381b2ac662bf Mon Sep 17 00:00:00 2001 From: Dmitry Messerman Date: Thu, 26 Mar 2026 18:10:52 +0200 Subject: [PATCH 11/11] Synched Makefile with pyproject.toml --- scripts/docker/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index 9ae420f8..1c579bee 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -5,7 +5,7 @@ DOCKERFILE := Dockerfile.$(IMAGE) GIT_BRANCH := $(shell git branch --show-current) SMLP_COMMIT := $(shell git ls-remote https://github.com/SMLP-Systems/smlp.git refs/heads/$(GIT_BRANCH) | cut -f1) -WHEEL_SRC := /app/smlp/dist/smlptech-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl +WHEEL_SRC := /app/smlp/dist/smlptech-1.0.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl WHEEL_DST := ../dist define copy_out