From 7bba3ed637c956325a568f6c9c7381bd5d2cd18c Mon Sep 17 00:00:00 2001 From: William Grzybowski Date: Mon, 12 Oct 2026 10:13:11 -0300 Subject: [PATCH 1/5] Add truenas-appliance-runner: boot the TrueNAS appliance a GARM runner's pool attached, under qemu inside the runner --- .../__pycache__/tnctl.cpython-314.pyc | Bin 0 -> 27102 bytes .../truenas-appliance-runner/action.yml | 124 ++++++ .../truenas-appliance-runner/appliance.sh | 215 ++++++++++ .../actions/truenas-appliance-runner/lib.sh | 74 ++++ .../truenas-appliance-runner/test_tnctl.py | 184 ++++++++ .../actions/truenas-appliance-runner/tnctl.py | 395 ++++++++++++++++++ .github/workflows/ci.yml | 20 + README.md | 47 +++ 8 files changed, 1059 insertions(+) create mode 100644 .github/actions/truenas-appliance-runner/__pycache__/tnctl.cpython-314.pyc create mode 100644 .github/actions/truenas-appliance-runner/action.yml create mode 100755 .github/actions/truenas-appliance-runner/appliance.sh create mode 100644 .github/actions/truenas-appliance-runner/lib.sh create mode 100644 .github/actions/truenas-appliance-runner/test_tnctl.py create mode 100755 .github/actions/truenas-appliance-runner/tnctl.py diff --git a/.github/actions/truenas-appliance-runner/__pycache__/tnctl.cpython-314.pyc b/.github/actions/truenas-appliance-runner/__pycache__/tnctl.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3cca8a11a21a4aa071e3d13fe61e5b74f530de85 GIT binary patch literal 27102 zcmd6Q33OZ6ncjQA!%7e!L5iCs64XM85*IDhM#+*XYF{V{q#(zk2}1x#LLvd^0n~!T z4wGgkq^($#?Ky%Idjw~xs2QiGr!%!Ylc{AV)3|4*b07n<=oj|H)27XFPp3z=G{=r} zI@9mJZvjX_w%gO`={@4bd-vUY-(CLw|L@akd$E;6x;FbaC%?Xd)Edm7uc^MZeYL0xRL#u;wJWMj+@!9C2nEA)_9Ry z7q>0o>N&T*l5-1>6}Rc-cgF3k)PPcBn@%Znurd?M%y~6SSg8f2R;^~KyQq?jJKZ+q zW$uOU;!19qkC(d_x$W#~f!l#6748z`mF`mHRcO(x~q`axvP=89;zLLbrd@u5g~eLK1jT0`Mj z+&4TN6j#JhH9%#qkx(Eo96ak2gD5=EbEK=;-QDS0*RsYHIm1SAHh3Zy@t+FDn^+qf z&Wf1z=;3a)!e}^j4&$IIExGx$#D-iL{%{QA+sfV)8u1MVT_;9E!vR+y5{$Xxkq9Q{ zaY$cBM>~_1lJ9>JKA93%Zb%}>Yf|1d#|!yleOIeCP^>eJ6&4%`pJxeRcEe(Z=0h!*zIf@4h2ge}*I79qx|9J%G2| z76ci%yL>S=wI;UMVlXy3jAfKD3n=S~#RJ&7mi+h8(lrCuLIdXuytdGO%PUCGBZD?U z6R?+-E9{GTmA+2+V!`21ILKBwc#Z(RJrCX0O&3FP;39S}0w@BG55xhr`8y%wJxB|{ zAUx>u5Bb7_!GP;T5a>ZLrFIk<1!A8Fy2M~KA_72Y+Hv1V6f;A!0WmT{03XJC=I-C= zk*KTZNaw-by{^t9hYxpjsf*m<-pfE-qYp#5noDB7Gr=<>=r$M|YE}?%i87QUm^2Y6 ze`I6?bgsz-JodSUMsedBiG)LO+$+y8Br!5NI7Ia^ECLJw9-}5hkyyN)rXu5W)CYpP z67+?U1zLQK2sOD{ebG>>e^eBK9`xKw&vT@;kvjDS0$7xxpcrhib|sdr+pu|e$DWQ2 z&AYd(ZEJ4Zw5y|e%ZAR5=FSZrJJ;;$+`M7u+O>X#o3J17V3zLI;Af0`jEnPb4jg!W zT<_-N0ywF`tzXJD3X-vNI1&p65*57|g+{+=41*v~E!ViNMx$i*dc(evpw}x|z21>X zV04(SZC>x`QQxrg#NqWa2qQ2GNATF>^#&q-TrBW*%@WK`^M%6U}EM;;8|mGOaCgNvqEv z4@JVURt21!H5S<{v$j?h9z;D}#Ip6ch>^7N``mvNTw;-0K4<+DDT2G(jmN(H72Fip zS8x$2oKS4*9fX&D_i`ua%!U0!oB%! z7ArW$^{M4tx4L3l3CDG-GGShshi_A0GphIKyCLk%Pi}7WSx89^l9Ku?>QfJI!IO^KP43EB9N^p?7=kUcEwTiF%#;-O8^}ZU!;CMO+=%-@RVv(YbY-lzDl$ zur`*ya^lx)Vpp%zj>*i>n~Afq5gI+o>vLCs&p6?Fy`9(pWi=bH?64a z{+jCC0+{(CNX~_8*|>GO!@S$jl-EJj;NiNRc?HU4_tN{z+{Ru5_Q-_Z3#>bynDOLG zu03j7JOXm-I!pKR``@LmjkJ%G4{jt)Ex*jkAEL(~pNW$U%z&yYF_W zV=CC620>A?2YOOd9-I}dU)rxz-ygN~wW{OtSk$-l>o#Mb zYRlZO`Z{&>3PFWhCtB>Wbw8AAt6sS))YhKjJSsYO&noj_*j=?0t6#^B>Z3(bYlv&C zmUL%hiRyO7LKJQSZV(%d%97eOUNR|4g~Yc?{0@mr2zw!qB}!fUdV9ND*S4%Jw(cW& z$R+XZiS5PKW6?p;7YMfJ_*}8IGXl+%G4FPl^1RsE6ZAK0jIsG(@O--~Q8&NX14#iq zzumQVeX+IJ${2WzWC^+paiB1X#n57yRO1SaGJ%pLEQq3k*pTm3Fku>oFcNNFo6r?o zCEg|J*REOnnGS7Ksfm`8co)IiU8wBb5^#huu9w^60-i&X^6*K#5d|hq61xxykv+$xsNyMBwfoIi64#u7q|wONUTk= zY#R)QgXf~+c4@h`J}q8v2;!;N+p>)*|FP{YTD2auimgKSA%>maHhk0LO&@iq{R8Qf zr*8UB{nUH?=$psiIDVt&o%%Pu*S#5!_qQ=Ks(;I}il3}m#kYS*)9LC;ltYVZ)}+E_ zpWh#h#uGK|t`ok1YwmV+uxDM3b&_#ZgdBH5GDLl1EGU_A%O)ooV~{>Zf|Awa8y*er z7R88&GA0H|rLx}Xh0>+~hGZtGnxsofAlQ=xswx>tSJ~JmSx#(h3x@rXKv2>}Vv=c8 z#0E!3Bx3-I9|{E$!X?A$(V%!QFlD(O#*4oLdpU>F+# z88s>yAZG?e$u#7P4bfx-z|LAp*n4dMF1#WH2`nz@2ZQlOJ01mx(HPSdiC0SesH8s? zJTI=H8(|0FN#+Z-%Cs_`Tmm`d&_P6hMV>c8T+Pj{Y**eW#c8|<>JZW z^~(-8tDd*^sVU zJ=O8*folh*woRYDUixOm8x?OZy1wYfy6g5koNh0_8)ix?CQtuC)#chtwJCk-Xr^?5n(3R;8<(GtQQ@t>yh~!)eb~(|!H#dA!+e!&&FZ&27W4?V9eGTzh%b zrA^OmxxC}jj!fAydD(s#dbg|<6~;|Bog=f31-Bh_?>g#!WV&kmfo;nFr@MY~@^5$a!(=#MX3WbEc?edRew;qcXg- zt36%4HRIftwr%@J&)GKp_O5{|3-d9O%->nI`QS36pGc9Kh|n?xzzGTg1wbQv5V!K{ zDfKmW#FKatIG_k-)XTioZJJkrTQ{e+k3tyN1T)aYeAU3MsT4!Jhug{*8iz-hFRG2| z!rHhG`a!-6j@_;E@caXwd3d*u2je`wCfAdP6l(p49LIBs(i|C6SW#D_K_odsgyEbM zi8hPG%^3-56vXYgmiVyPNxyoSeHcq4G9Y8W!Aq7d$&rWpR6xur7DNWTx0g;XyS(z! z%BM$?zO2129}6~Q${STINH1QKsaTsXSqt#LxbLZbNy9DM;*0xcsuo=_zhF)|Q}sz( zDsXk^<)Nw5Q(|g(ym(xH;gK1Kb6n=NbM!$AUd_P4ip}>K@&TfM%)>Y3ZEB0QO;LRx z+Q`YHX>`zYg#ITPm@HyoqR#;r`VtEFmA|b~{ow(R;5M*6AVWe2$nWDHb0g@}0I-k- z3J(MC5mb3OU$&d~w$?fQ|I`0Q|GF>Je;=`Ambx`vS1ln4v|musBSLx_i3;;lH*}c! z$)oR?qvPdP*<)DGlOe;*ivMlGO?=pv6~0_-dc8}e#$q!3@mHy zpN;<6>Q^Iw6v-Uu`|VvCz`tl{ zjFAOsLJ;|a;n9(x2xE1lQ6>(?NR-Jnrl{}yaKsl7cT$Tk%KDHog3aWHhgFI})0jgs zm=HvD{~Xz;AQ_JG$w1Qc2P2c-jAQw@Fl%#6nw~zyUS0i1tEU6kJ2I=b-zwjJ%eH-D*9WZe11pU0 znst<3OgxoH)<6BkJc%H=ELHx}>Q@Ze=Ewe~G2Q#s?18>a^JAHnkKL+y>^*Zo6Zz+G zB@)&d8QO!3F{0TTeMfLj=TH+jB5P_akT$Bl&)eBZ0>{VVzu`6s{D#Q1t!c_sk7!8eMbdK@iX8ktT9# zW-~$2{8VTbIl~&p&qZ|$X%%Q)<|P;cl(k{l*5=XoJDT!V3q<%E`S)VbV+2_SqY1!l z0KThXWw%D2a<5>d^9$aquJc^8Mbvn>4R87edlQi1OT0<gu(?`sO-|aD~?~a;!%v!&7Y(8W>Wyxa(p@p#+gtk~KGpQ}_C$un2 z8T!p^WkCC|G8$QT8`M^ieGL5;RtF%*W}y*r^&K47Z}pheRjK24^9L6|`Z4rdSv!!d zMO22-9$zts)#(5w1y^iPZ3BJSqw6lqy-B@NO%}uyWMg|T9*M`0b^X8Q3*vElHHOTI zUqM!g4biB^pv#sm5Vm~S; z$`xb8iSy*tf|po?*R9dYsJ&!`3s09MnF7Y#^&Po6CRi9I5lAl@DFjana|Bv=hwq0(& z)SmP`|Il>X_^z3Xn#(6IolKQJH!{BWSB|pTn!0S=k)Q6rF__)fm8mh zmR)!PT3>qc`b?JaT#Rvy%l@b1?U?8A&Fks_^!rc zkw~scyhS9PL1Yslm=&Qjax!Mne8doCOmm_kizFCJrCzUp*cXchLjJfvI6Un2!b(}3 zR|p3r%~gSWJ2j+i6aHf0HQcA)!4yBz=?t!W<{HC8A8$4p%06~l4PO4^D&DZ-p0U!f z@{@H2L(LrynM}s0141+{sk?KR1UMh(M006PlR9g!Fu?(kyO^WiBw0) zjBpyZS#xwOS`P+g0bFcDuY}KXf}qSjj+JA+3GR!unw%`3Z>6u}j=REuCOMI15_?pQ zJ7}yhhK<4}&-OqPPR1fIj|E0YqA|&U(uk-kuEvodLWct4@1vPyQ5GJHPTfK#Rpe~i zNvTwwqh`N+7RAh1NfN2Kc*1nkT$S9D+?-nXqrlaZKR7wPAk(lZy?k@JdP~;aKC__a z8Cel+`kHfek^2NCaDh1{pjs!*P1_4SF^0+rtWIe3MEkyF^cj&Hk1^;}d|! z!hhh9$$L*V3hzCU@HzhC%X9q2q+Rjwy$k*4TH$NMosRs*ZX@oDeaeXqoU9>ODXk%Hs7n$P?h>w3 zoZwPU{5mcfTgJY82iKAjmSf*YY>tR+-ZtzTIT7$}muj&X8ua9GXyy4!=z&4r823g% zBZds=OQ>^QC?Hw1bx}ZA+(Bc2{KtvUtL*`~3Uy?VYs-J__o&ixWPn-c!enR0*$BI# zX~qdVBz~%w+^+6SS9jiAv2$E_#x6q*^{%Z2tphCqu-klh7Ar);K8#l=nxEi})5Js$ zMyIe=!3{GMyuJ`7!Bw=<+(oONyvHk)0>!7UZIMUw)REPs0kyt7Z~7wJ1ne6M@Tk9a zT%V@67xED2;vr$I%T0lc8o*Z#U$$)%p}dl!Mko~A=60wqSbB)fnn29xrO~4!fAuvAM4js8J{4cq>kJ0SBQh^}Om2PtJ08%prjDp%fY0mi zAg)?P7dn(o#6rZVKcPp^C_{DeIh2c&$P$irP%s3v#NcR*{27F#%94AYdqlPPbhzf3xz9%Cx;>T>q=0(%D6>RK@oXjdxC(Ck~V7 zZJID$v`yHa@4fu!vyZ0q&v_w$&z4nQ-u~?NOxe=Z;H|ReaWe_yWeZ;nTnT?KoT+WO zRn`Il*-&LI}zi~9ZbZ@3&U%F%;_0j%pdoplk@P)xE!!Ha^ zwPkA8WXjgEmKV=WoJ$H%Kbb1OTKRHi*0FM?tR@+Fe#dOhlGMg&(@!4C)NDzYZ<(oF zc=_C=b18GCvN2uKIOC{Hw!P=5BjMV9$)1d-_PzAvRQ%O**UnuxXIeYnE7{3t;D?WZ z(#@fPgmIvOw!3%OS{23tU>@JXd-O;64aRy-;G#lK7A_=BKbV$$>2|Ue8DyTR(h4oZ zShb|kUZF(~yRX9Gh~j9}76cV-L4oq|#x2abGcfE1T`Ex1m|t@#deVA^+*+Xi+@6aV zrC35W`>?)pI$kk%G~Fa&&r5F25TVWrbWxcfVy*J%tJ)BjXw9|{eE-)RmuJ?XxGgh_ z<*_h&Yh9LL?@X<#2vzECroX@F`4dv6e)!A0eeJ z@W6V}C?=Bz{vmST!e}a)dX9B=?(XRkU%)6C8A%x1se4zVm;@yjd`(nFOriqZ?#4YG z`w#8jC7C*Q9&z{X-qmOle@IQgPuYuOD2*iaPcR28vA$1#<&mc&WoX^LP=xVzA{yjv9BE0--q+Fs1trFoUzxY zs@s$7{aSvj{xx~U^u)iGO9n>;hs@#C{szxMLirZ;6)Y|T_`OP6e$CGGF21Cytp zIXr8xNe(0*NqMF^r#4I%Umv{wl^eeF!hIS0{A zrvj^jB4k8H`_G~)@f@=I)L08GD^p@q!|$uLdxePQoCL2uZ_-?5e&I{B&2 zdmR1VX(%!r;HTE#65{|U zMfQvB6YZ1!r*|a#ZxyXh3#;Xk=C2>Y0vqWJYm@V#<#{$W=QZSPl|R78S;#2whWeyQ z^19YseM7GyPu0R3N!4oQ3R#PCreL>`#eQJ}z$Ju2G(Lw`eVS1kQB4MQoRAy|9!!a1 zI>yn0eGP@Z!E+F;f)vspij7Ei^^}RX`%uUJE=2jjuE~OR%u$$P5UL%JiXpfmI5`&d z#?Qlya1Cn(BQQe>T6CGf5=E8qhzKm6(Bz@qURHRUcQ*E6zqHzJnTn zjlWm`85t3ZE_6=npWZh)m@(Imb zkTjvg2XO@^7Lg#(kpEK zDO3yK00agND0(@tZ@!n9xQ@bm*J%QNuVF}+*Egxx^HM$R1gi2whDx_SYS>{o7=y~g zu0p(KxD*mS1L2j8I5tGbedv&ye0ZvfMZk|n+376hB$INkXDgEoVD*DMK&B`wf)vTR zjzbGB(HFvzhL(hdG>SNJ`Gh65)kT%tU0o3Nb4>jV1})px8G9#|E|OX=zJd%VAB3|w zYA1uBZUEM=AWZk?WzNnA?}tZ>8;}1MZL|1`eI3~tchAU`lwW-EsV6gzy3~qWj>fTl zGgkW@oxxK3e%XTO+QtulI$KitV#SZDUfK53nzUorJx*__opqF{Z8m2et4W#??7r%z5;-ZSlLd#lO2_LKmMg@8cC?1NSdoI~VBRGIj_% zOZ0D-7;rO(KWpS{7=#L@O9L5ZD!GUuN7&qaPr)b|56H%P1j>_9vmXZ`HYsE>$^(-5 zg2YZ!cvoO~CWG=Klp|#zM1}j%gvk5JV+&$HkLgBrEfiJ^NIt#9ZORv=yL1O( zPGNz-`lv2<#DD_SRRUPYVYUTC-MqJz)0kn-&zV^n+sD)2bnHf6Nk{fN*omS`AA83LhT}^IXF{so!9YU3ID{eTx zu3&FQs7wljSC1rPMa(d+;IAY2ci(RUOHHLY3GP$VhN1O)qH57B$V3R$i{1tWEAj z=t$|UlBMGv@E#S{%vRSXPhS~*VKnt9A}#CIXR6z#W4Ef?p|Vu%xPCfa(J`q9sVpv; zv?SMGHN0#{g{GYuSKE8G4IdSAw&qXonYjhEFDBAutKeiRu9>N*epU}RQ*q5_F|zEv z)X=e%duwS~#}@rtn|b7kVqyr|$vZ#lXo>ckL0Ug9w(0W4=fXHrkdkR4-ds_%zqW^0 zAt5K0>TwpQ=BE_fOx>#tA_CZPHmPT`*e)^)1}kC z*MqMGUmt#L_{RFo#=Y+>zqRp5`e;x3*kjpsa>xwaet$!~f>nBbfk+hRE)(nKERlTl zkUeq1TkQk)#2G|HFw&_sobY9*Z6zxi;JkDg88OQw@+Hc(MRvu4I3dcObtP)%cFrEb z#79ph0~Kd7xF%dotZOqo%E z;Qx@#vWFdO-hD0VP)EtdGZSZSJ2s{r8?OdoP~`t`<#hSDK<;qre@n*Mp0>5C zlJD6I4zwOWlI+czmyqAq$^_!`$xW%=)SjsW*M;=Ay5?Bg`}l3|&`s~qxG`%E zQLC=3dEp1LEc}e+w;xw?r4RF;-l?KaARd3$@>y&v*5k(&yXv{0E5moho~GQKz+_jm=9klpbDi0#E|}T`E!Jx{Q1lofyrRWUrIG`P9VTI%}j2%V?AajUNd|p47%{ld$P7;AK>ZUiM`M4|8%^AF21|}vl!_(KWOY&%)PbP z+0m*`*oj!>j$w$F}-V>JGZ!FPd z_7|}}klFR%Vdzy(pRaWSk#cB3l5PRC$b~BO(=k}cTa+R=VyXrSGf6AgkC2m;`mKfg z{IcUi;CLR;=U=OcHE4u=~pH>H61~zqVXHs&k|C7d3CyWSb5#KxN1?XR>-U zVv{|D3nHT!HaeT1a>l(C$|FTzqV_E1pZ1Vr3_i2ve}N)VOe#MUR9yHh3B~XJ5Dl5! z3kx6-qP!VP23e%3ynoK@ft@C-h*%2KoebZ0EPvOr9DFwE%{ZIWwq_ihDUri@#gEZH zr1P}{L>hBu290XKr|3H;o4fuEs$ySA5@B8oF)rT4wLC=HXviRn1Y%Z)Cw_wRA{x{L zE+Br8-rx09SH`wzY&U{u#zV={E7dPlXO|sH_a4idJ!2jJfY_CmJ7&GXNzcPC`cq4A zuqgC$=ug{T-~K1tGwVBV1aGZBc&p)1ruOhVqv^iKZkhWPeh=H1V7btHvHz+5O!4BB z;a2hTls|1-^`6l5=^cHZ1YpBxehTZt7VhmW&d#E_ES;ze?cd!74BW^2botUYn3tkE zp@x=FS^(5hT7XiipdNw6!OekTY8so?9c)-CGbT&cI(b1QK{I?jV|4%&*c2M~YwgoBL#O7G-N1KJP@ zscd`=uJ2b>>QxX`2dFV|m3S;Bu%$Hu@kpJ%dgkRb>6RTg%Cjr>Fq)Bg;5TW`;tQLlxl7{hN3!o>kk$Etu={INko4i;+55m&GEi?Z8F1A|6Dy&b{3R=q2Usk{U+e<<_GoEHSa70)kIN$($T!LiF>=rxpVtmuqF+Nkn!%< z|33t4x85z}Kxi(=;UH>Z27_oaw<*6J1HD2(jo40nIVkT9tY!i672goDp)QQL7!46~ zBvw}wX5})<^B9do0qD=Ir2%w__&?D+2g%G}*ee6>KjMj0j*p;uRhOPu{xDhq+y|!^ zLjGWj4CrtS$A;)|J$*JU;ZPsq6l`3K3`>Td-VS#!9LHcR;rJesjUI>$nqM8ZGLns$ zJcQHrp}<@$uoD)@FZAN&q6LuDLYVL}8htS)*c}%KCI%+^V2mt->1^P&0Xh-Cbo-43 z?+7=FGxn}`;aOfNBlmWrbLZB%=s{b@&?EKFiXJ(QE$2lnL{BOFs&L3A z=x+(eIxkW9L7@Q3JiiRM1bi==me*ZAuM6Dbtr~}b&t{?Qvl)elfrU0ucmN%)o0nR7 zfrDuV^aXI4jqLl{H&8fO?fM{z;$CV^l4 zHf6s@*$*lE4rOG57ypnlCuPiLk_Y_aZIsRhex?vmn4w528GZC9HjYZ45B9POa3OH< zGhefnP=erxm1)ra1# zemw0te%s^4S43{&I&?B?Kb18PYf$~$j}2VYVg6I4fGmQGF_@oh-MN)}Yh~Tejn21C zi*fz-Mg!$rot?${Lc!?V-9wm&&*(GFF=e2u`3c|H(uxYH59+3q!{lYO6IaM`+s6X{wM+7{;WAvC*m*_B0@yk7?oJ${XG|)F})iKKVd13d$=bJJZ zb^n4o8fY3My?FM7Ut26(5M5zf!85oZO&r7171G63%!mThzKk~hg~v~{CB-Lnz(xsb zm9XPZgdOwHJ@mZ^D}CZoJ}}3;?cq2+QfMY+oMLcgS3Sca!WRT4AsQIPS8yZo0}M%A zs(BG7pzz6rQgoQJWKhvwLYH>5fFi=;XUpk$*>l?vFjrQI6JyD~OvS2n$*Ngr#pPAc zu1fZ1oJ+<#XB-tt|9g)5naY|;4jjf?9RU+;bx6bC3ozSpkVeZ3QJ($iZopP8R64V1!D% zxhA0;$CG?9TvZ++#_I}MqH_`M*kXfZ3Z4u4@eN4H;u8mP$Q|Fjxr;U9tXi};eK@z# zBpFEY3+7SG^{6HO8D-z3>>*_6PY*#r63|yn`~?-4qcd>AFW8k{p+@ndVoxjJoIO%|lliP6bNp8iJKT+6gx zsRZEZ_4}f}6Cr&6DTFTsd>uo9hpK@_YwFnJJ=V4V5i!i}l&^|N<`I0yinMf@SD58b z4Vjehl!~Op%B&=bW^$B#ZdJ@EqoXabE-B_EJ;op|N23l}3SvJ@`?X&zeyaHSfy?1b z;f#4X;-mz_cMg5?&}3y+sHW@v-`qbQ`1TQ+JDqLckTF-o+$5L`3qNrh4Rv=o%1WRq zzi+l(Fd|DA*JaJF_Yt1HH{)2Lv5P!an3RUMR#ogQ<=!rJ?p!)oDk2mk>UH;jAp5_o z3v1|B&$L%sB9;m{Um%whwnJg@C?Vp5Q|(2tv}jruV$6UoiA;VaPcas0)0q6$e4MR1Pno7gw$ zyBxX{nm+ydnb*%upZQVw)tVpF;LA*VuIsbmHF-37I%};j z=%LVT7yp&LAm_Zp`sI^OJf%cIka$vnEasH4Xg+D&Lwg0ul;C1sQc0S&uPtm}HRAXA zt#SmD@hSAJL?~$_Fc{57uBJv@Tz`}YuEI2Ko-QIPh~gDTiVD_N|DCWg{gtVMJjRzrI2 z{*3j&Sm$p{*74ZI^AqPUe(kBRk%`KGF*Ff+8j@TMA~z;`E<8la|Iz1Q(5|B|K~;Z* z??#dZBO&WHiC|M$<3)!JINe{Sd|9 zZc=cf;B2eiHSUG;X1l0Qv#a4hF{o9|4F%iRxn1|QukTx=&33VSiP|3XvxM7I4z1Y$ zb%d~QUPACT^^AlgIDi?7Z!BEXorjMwTJgEqzXzy7kk&V8eY3!DeZr#s*8((Y3h$Oo zf#84-pRk9!;IA&(e3BlY>6dKqs(Y12F_?sYg;uPoylTYR zBwY-VX=nxm)98u3V>F7xk~CwRxq>+tqO5sO@q|0{@GGz5L&F%(Y}<$c9JMBkyo2?h555{UnV zo)cEts73$zz&sm)#IM3}21A(1x`pHbzy}P;#2AR1D5p7G%4z;e$&7~bH#zMcis;?V zc1I+}C@pmR&hE+q`M%(AbdL(=2*JYsVGrc(jIlT}@KRZlCwLamP)2NzArl09oiEBvAF>(nKhP6q=^#4nLQk5LL$~*wyt(IO)^ciW=Zw{M+q&?kbzy43ba~pk zFk{^~)_F&OaXhvD6G*Jik4iX$aV+%w!enr&H!G~WE$p}{?6_f0yL;2Zj;wI(eZep` z^1L?{obpURniU?V`+*BXNkj5XDuD)bp1)IeOE^p=L(f~2yHgvd+NVde!b4Ot{Crnx zH_m~!XNB!^`k3B)qbDouzb$xf3ZC?FZ+h@#TJU6rQ&cN>;rQfG%90UQ-4-_86gFKq zA`o2Ilobx-o*J{l3IbuAb$Z%C-YUIe4N2+3gfM>e8Plh;wyGCb{%FrDd(%a22&FWv znYC9XH>T>ZE`NDBESI{ewNp#e)h!u&YueoUJA_yo)_fML!m9tgzI0a`_w%-eyIb{t z+OnzUrSqwMPC!+EEf(iS#utVelnBOwd9EStPAe7FwI>iio)Q zIAyO>Ms9P-CVxb>MTWO}D&I=wmfh#*8?+P!$e@OpvZl+GU}6RBAK_z|NFsq@gOSW9~XNQ8J39W zKN9MB;V$5iFZm@`{7cUMORngboP}NE*2-?nJ}@Vh}?Z-6>dFRL5Ed0`s ztBUyMj~AJ6TW&^H!7>ByyjQXax7&57Vm0!vkDb;0nme00!8V@A>Z{)u%)h?}1W=Ga HX!5@RWWJ+a literal 0 HcmV?d00001 diff --git a/.github/actions/truenas-appliance-runner/action.yml b/.github/actions/truenas-appliance-runner/action.yml new file mode 100644 index 0000000..93f2244 --- /dev/null +++ b/.github/actions/truenas-appliance-runner/action.yml @@ -0,0 +1,124 @@ +name: TrueNAS appliance +description: >- + Boot the TrueNAS appliance attached to this runner under qemu, and hand + the job its address and a credential of its own. The appliance lives + and dies with the runner, so there is nothing to release. + +# The runner comes from a GARM pool on a TrueNAS box whose extra specs +# attach the appliance's disk: +# +# {"disks": ["truenas_appliance:nightly"]} +# +# and whose flavor has room for it beside the job (the appliance's memory, +# 6 GB by default, plus the browser and the build). The pool, the provider +# and the appliance image are iXsystems/github-truenas-runner's (its +# docs/e2e-appliances.md). +# +# These scripts are the one copy: that repository's image build fetches +# them from here, at the last commit that changed this directory, to +# install the appliance image and test-boot it. A change here is a new +# appliance image there. What the two sides share is in lib.sh. +# +# Usage: +# - uses: iXsystems/ux-github-workflows/.github/actions/truenas-appliance-runner@master +# id: appliance +# - run: curl -sk "https://$TN_HOST/api/v2.0/system/info" -u "$TN_USERNAME:$TN_PASSWORD" +# # between tests, back to the baseline in seconds: +# - run: truenas-appliance revert fresh-install +# +# The TN_* variables are the e2e suite's contract, as the lab's claim +# action sets them; truenas-appliance (appliance.sh) is on PATH for +# snapshot, revert, probe, call and console. + +inputs: + baseline: + description: Snapshot taken once the appliance is up, to revert to between tests. Empty takes none. + default: fresh-install + memory-mb: + description: Appliance memory. + default: '6144' + vcpus: + description: Appliance vCPUs. + default: '4' + data-disk-count: + description: Data disks, sparse. + default: '10' + data-disk-gb: + description: Size of each data disk. + default: '10' + ssh: + description: '`true` enables ssh for the admin, on TN_SSH.' + default: 'false' + disk: + description: The appliance disk, if not the one the pool attached. + default: '' + export-env: + description: '`true` writes TN_* to the job environment, for suites that read them from there.' + default: 'true' + +outputs: + host: + description: host:port of the appliance over https. + value: ${{ steps.start.outputs.host }} + host-http: + description: host:port of the appliance over http. + value: ${{ steps.start.outputs.host-http }} + username: + description: Admin user. + value: ${{ steps.start.outputs.username }} + password: + description: Admin password, made for this job and masked. + value: ${{ steps.start.outputs.password }} + ssh: + description: host:port of its ssh, when asked for. + value: ${{ steps.start.outputs.ssh }} + boot-seconds: + description: From qemu starting to the API taking a login. + value: ${{ steps.start.outputs.boot-seconds }} + snapshot-seconds: + description: What taking the baseline took. + value: ${{ steps.start.outputs.snapshot-seconds }} + +runs: + using: composite + steps: + - id: start + shell: bash + env: + APPLIANCE_BASELINE: ${{ inputs.baseline }} + APPLIANCE_MEMORY_MB: ${{ inputs.memory-mb }} + APPLIANCE_VCPUS: ${{ inputs.vcpus }} + APPLIANCE_DATA_DISKS: ${{ inputs.data-disk-count }} + APPLIANCE_DATA_DISK_GB: ${{ inputs.data-disk-gb }} + APPLIANCE_SSH: ${{ inputs.ssh }} + APPLIANCE_DISK_INPUT: ${{ inputs.disk }} + EXPORT_ENV: ${{ inputs.export-env }} + run: | + set -euo pipefail + lib="$GITHUB_ACTION_PATH" + [ -n "$APPLIANCE_DISK_INPUT" ] && export APPLIANCE_DISK="$APPLIANCE_DISK_INPUT" + export APPLIANCE_DIR="$RUNNER_TEMP/truenas-appliance" + # The password is made here, so it is masked before anything can + # print it. + APPLIANCE_PASSWORD="$(head -c 15 /dev/urandom | base64 | tr -d '/+=')" + echo "::add-mask::$APPLIANCE_PASSWORD" + export APPLIANCE_PASSWORD + out="$("$lib/appliance.sh" start)" + get() { sed -n "s/^$1=//p" <<<"$out"; } + { + echo "host=$(get TN_HOST)" + echo "host-http=$(get TN_HOST_HTTP)" + echo "username=$(get TN_USERNAME)" + echo "password=$(get TN_PASSWORD)" + echo "ssh=$( [ "$APPLIANCE_SSH" = true ] && get TN_SSH || true)" + echo "boot-seconds=$(get APPLIANCE_BOOT_SECONDS)" + echo "snapshot-seconds=$(get APPLIANCE_SNAPSHOT_SECONDS)" + } >> "$GITHUB_OUTPUT" + if [ "$EXPORT_ENV" = true ]; then + grep '^TN_' <<<"$out" >> "$GITHUB_ENV" + fi + echo "APPLIANCE_DIR=$APPLIANCE_DIR" >> "$GITHUB_ENV" + mkdir -p "$RUNNER_TEMP/truenas-appliance-bin" + ln -sf "$lib/appliance.sh" "$RUNNER_TEMP/truenas-appliance-bin/truenas-appliance" + echo "$RUNNER_TEMP/truenas-appliance-bin" >> "$GITHUB_PATH" + echo "Appliance $(get TN_DOMAIN) at $(get TN_HOST): API up in $(get APPLIANCE_BOOT_SECONDS)s, baseline '${APPLIANCE_BASELINE:-none}' in $(get APPLIANCE_SNAPSHOT_SECONDS)s" diff --git a/.github/actions/truenas-appliance-runner/appliance.sh b/.github/actions/truenas-appliance-runner/appliance.sh new file mode 100755 index 0000000..10ec7a0 --- /dev/null +++ b/.github/actions/truenas-appliance-runner/appliance.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# A TrueNAS appliance for this job, booted under qemu inside the runner. +# +# The runner's pool attaches the appliance's disk ("disks" in the pool's +# extra specs: a clone of truenas_appliance: on the box), and the +# appliance lives and dies with the runner: nothing to claim from anyone, +# nothing to release, nothing to leak. See iXsystems/github-truenas-runner's +# docs/e2e-appliances.md. +# +# appliance.sh start boot it, wait for its API, give it this +# claim's admin password, and snapshot it +# as the baseline; KEY=value on stdout +# appliance.sh snapshot save the running appliance, memory and +# disks, as +# appliance.sh revert back to , in seconds, no boot +# appliance.sh probe [baseline] change it, revert, check the change went +# appliance.sh call [params...] +# a middleware API call, as the admin, its +# result as JSON on stdout +# appliance.sh snapshots the names there are +# appliance.sh console [lines] the end of its serial console +# appliance.sh stop +# +# Settings, environment variables, all optional: +# +# APPLIANCE_DISK the appliance's disk: a device or a qcow2/raw file +# (default /dev/disk/by-id/virtio-truenas_appliance, +# the disk the pool attached) +# APPLIANCE_DIR its state: overlay, data disks, sockets, console +# (default $RUNNER_TEMP/truenas-appliance) +# APPLIANCE_MEMORY_MB 6144 APPLIANCE_VCPUS 4 +# APPLIANCE_DATA_DISKS 10 APPLIANCE_DATA_DISK_GB 10 (sparse) +# APPLIANCE_HTTP_PORT 8080 APPLIANCE_HTTPS_PORT 8443 +# APPLIANCE_SSH_PORT 2222 APPLIANCE_SSH false +# APPLIANCE_BASELINE fresh-install: the snapshot start takes; empty +# takes none +# APPLIANCE_PASSWORD the claim's admin password (default: generated) +# APPLIANCE_BOOT_TIMEOUT 900 seconds +# APPLIANCE_ACCEL kvm; tcg runs without KVM, too slowly for +# anything but testing this script +# +# start prints TN_PROFILE, TN_HOST (host:https port), TN_HOST_HTTP, +# TN_USERNAME, TN_PASSWORD, TN_DOMAIN, TN_BASELINE (the suite's contract, +# as the lab's claim action gives it), TN_SSH, and what it measured: +# APPLIANCE_BOOT_SECONDS, APPLIANCE_SNAPSHOT_SECONDS. + +set -euo pipefail +. "$(dirname "$(realpath "$0")")/lib.sh" + +DIR="${APPLIANCE_DIR:-${RUNNER_TEMP:-/tmp}/truenas-appliance}" +QMP="$DIR/qmp.sock" +STATE="$DIR/state.env" + +hmp() { "${TNCTL[@]}" hmp "$QMP" "$*"; } +# resume: a failed loadvm leaves the VM paused, and later loadvms leave it +# so, however they go; cont is a no-op on one that runs. +resume() { "${TNCTL[@]}" qmp "$QMP" cont >/dev/null; } +url() { echo "ws://127.0.0.1:${APPLIANCE_HTTP_PORT:-8080}"; } +running() { [ -f "$DIR/qemu.pid" ] && kill -0 "$(cat "$DIR/qemu.pid")" 2>/dev/null; } +# shellcheck source=/dev/null +state() { [ -r "$STATE" ] || die "no appliance started here ($STATE)"; . "$STATE"; } + +start() { + local disk="${APPLIANCE_DISK:-/dev/disk/by-id/virtio-truenas_appliance}" + local memory="${APPLIANCE_MEMORY_MB:-6144}" vcpus="${APPLIANCE_VCPUS:-4}" + local ndata="${APPLIANCE_DATA_DISKS:-10}" datagb="${APPLIANCE_DATA_DISK_GB:-10}" + local http="${APPLIANCE_HTTP_PORT:-8080}" https="${APPLIANCE_HTTPS_PORT:-8443}" sshp="${APPLIANCE_SSH_PORT:-2222}" + local baseline="${APPLIANCE_BASELINE-fresh-install}" + local password="${APPLIANCE_PASSWORD:-$(head -c 15 /dev/urandom | base64 | tr -d '/+=')}" + local name="appliance-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${GITHUB_JOB:-job}" + + running && die "an appliance is running already ($DIR)" + needs qemu-system-x86_64 qemu-system-x86 qemu-img qemu-utils python3 python3 + ovmf + local accel="${APPLIANCE_ACCEL:-kvm}" cpu=host + if [ "$accel" = kvm ]; then + [ -w /dev/kvm ] || die "no /dev/kvm this user can use: the runner needs nested virtualization (cpu_mode HOST-PASSTHROUGH) and to be in the kvm group" + else + cpu=max + fi + if [ ! -e "$disk" ]; then + echo "appliance: no appliance disk at $disk. The disks this runner has:" >&2 + ls -l /dev/disk/by-id/ >&2 || true + die "give the runner's pool {\"disks\": [\"truenas_appliance:nightly\"]} in its extra specs, or set APPLIANCE_DISK" + fi + disk="$(realpath "$disk")" + # Read only: the appliance writes to an overlay of it. + if [ ! -r "$disk" ]; then + sudo -n chown "$(id -u)" "$disk" || die "$disk is not readable here, and sudo cannot change that" + fi + local format=raw + [ -f "$disk" ] && format="$(qemu-img info --output=json "$disk" | python3 -c 'import json,sys; print(json.load(sys.stdin)["format"])')" + + rm -rf "$DIR" + mkdir -p "$DIR" + qemu-img create -q -f qcow2 -F "$format" -b "$disk" "$DIR/os.qcow2" + vars_qcow2 "$DIR/vars.qcow2" + local drives=() i + for i in $(seq 1 "$ndata"); do + qemu-img create -q -f qcow2 "$DIR/data$i.qcow2" "${datagb}G" + drives+=(-drive "file=$DIR/data$i.qcow2,if=none,id=data$i,format=qcow2,discard=unmap" + -device "virtio-blk-pci,drive=data$i,serial=data$i") + done + + local booted + say "booting $disk as $name: ${memory} MB, $vcpus vCPUs, $ndata data disks of ${datagb}G" + qemu-system-x86_64 -name "$name" -machine "q35,accel=$accel" -cpu "$cpu" -smp "$vcpus" -m "$memory" \ + -smbios "type=1,serial=$name" \ + -drive "if=pflash,format=raw,readonly=on,file=$OVMF_CODE" \ + -drive "if=pflash,format=qcow2,file=$DIR/vars.qcow2" \ + -drive "file=$DIR/os.qcow2,if=none,id=os,format=qcow2,discard=unmap" \ + -device "virtio-blk-pci,drive=os,serial=os,bootindex=0" \ + "${drives[@]}" \ + -nic "user,model=virtio-net-pci$(hostfwd 80 "$http")$(hostfwd 443 "$https")$(hostfwd 22 "$sshp")" \ + -device virtio-rng-pci \ + -qmp "unix:$QMP,server=on,wait=off" \ + -display none -serial "file:$DIR/console.log" \ + -daemonize -pidfile "$DIR/qemu.pid" + + if ! booted=$("${TNCTL[@]}" wait "$(url)" "$APPLIANCE_USER" "$APPLIANCE_TEMPLATE_PASSWORD" \ + --timeout "${APPLIANCE_BOOT_TIMEOUT:-900}"); then + say "--- its console, last 60 lines" + tail -n 60 "$DIR/console.log" >&2 || true + die "the appliance's API never came up" + fi + say "API up ${booted}s after qemu started" + "${TNCTL[@]}" rotate "$(url)" "$APPLIANCE_USER" "$APPLIANCE_TEMPLATE_PASSWORD" "$password" + if [ "${APPLIANCE_SSH:-false}" = true ]; then + "${TNCTL[@]}" enable-ssh "$(url)" "$APPLIANCE_USER" "$password" + say "ssh on 127.0.0.1:$sshp" + fi + { + echo "APPLIANCE_PASSWORD_NOW=$(printf %q "$password")" + echo "APPLIANCE_HTTP_PORT=$http" + } > "$STATE" + + local snapped="" + if [ -n "$baseline" ]; then + local s0 + s0=$(date +%s.%N) + hmp savevm "$baseline" >&2 + snapped=$(awk -v a="$s0" -v b="$(date +%s.%N)" 'BEGIN {printf "%.1f", b - a}') + say "snapshot $baseline taken in ${snapped}s" + fi + + cat <&2 || rc=$? + resume + return "$rc" +} + +# revert: loadvm restores memory and disks as they were, so the API is up +# the moment it returns; its connections from before are gone, and its +# clock is the snapshot's until it catches up. +revert() { + local name="${1:?revert }" + running || die "no appliance running" + state + local t0 + t0=$(date +%s.%N) + if ! hmp loadvm "$name" >&2; then + resume + die "no snapshot $name to revert to; the appliance runs on as it was" + fi + resume + "${TNCTL[@]}" wait "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" --timeout 120 >/dev/null + awk -v a="$t0" -v b="$(date +%s.%N)" 'BEGIN {printf "%.1f\n", b - a}' +} + +# probe : that revert works and what it costs. A group is made +# over the API, the appliance reverted to , and the group must +# be gone; KEY=value on stdout. +probe() { + local baseline="${1:-fresh-install}" took left + state + call() { "${TNCTL[@]}" call "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" "$@"; } + call group.create '{"name": "e2e_probe"}' >/dev/null + [ "$(call group.query '[["name", "=", "e2e_probe"]]' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')" = 1 ] \ + || die "probe: the group made over the API is not there" + took="$(revert "$baseline")" + left="$(call group.query '[["name", "=", "e2e_probe"]]' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')" + [ "$left" = 0 ] || die "probe: after reverting to $baseline the group is still there" + say "revert to $baseline took ${took}s, and undid the change" + echo "APPLIANCE_REVERT_SECONDS=$took" +} + +case "${1:-}" in +start) start ;; +call) shift; [ $# -gt 0 ] || die "call [params...]"; state + "${TNCTL[@]}" call "$(url)" "$APPLIANCE_USER" "$APPLIANCE_PASSWORD_NOW" "$@" ;; +probe) shift; probe "$@" ;; +snapshot) shift; snapshot "$@" ;; +revert) shift; revert "$@" ;; +snapshots) running || die "no appliance running"; hmp info snapshots ;; +console) tail -n "${2:-100}" "$DIR/console.log" ;; +stop) running && "${TNCTL[@]}" qmp "$QMP" quit >/dev/null || true ;; +*) die "usage: appliance.sh start | snapshot | revert | probe [baseline] | call [params...] | snapshots | console [lines] | stop" ;; +esac diff --git a/.github/actions/truenas-appliance-runner/lib.sh b/.github/actions/truenas-appliance-runner/lib.sh new file mode 100644 index 0000000..c1be3ec --- /dev/null +++ b/.github/actions/truenas-appliance-runner/lib.sh @@ -0,0 +1,74 @@ +# shellcheck shell=bash disable=SC2034 +# Shared by appliance.sh, which boots an appliance in a runner, and by the +# image build in iXsystems/github-truenas-runner, which installs one and +# fetches this from here: how a TrueNAS appliance runs under qemu, and the +# admin it is built with. Sourced, not run. + +# The admin the image is built with, and its password until a claim +# rotates it. Not a secret: the image is booted only by qemu inside a +# runner, its ports bound there alone, and every claim changes the +# password before handing the appliance out. +APPLIANCE_USER=truenas_admin +APPLIANCE_TEMPLATE_PASSWORD=truenas-appliance-template + +APPLIANCE_LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TNCTL=(python3 "$APPLIANCE_LIB/tnctl.py") + +die() { echo "appliance: $*" >&2; exit 1; } +say() { echo "appliance: $*" >&2; } + +# ovmf: the UEFI firmware, as Debian and Ubuntu install it. +ovmf() { + local d try + for try in 1 2; do + for d in /usr/share/OVMF /usr/share/ovmf /usr/share/qemu; do + if [ -r "$d/OVMF_CODE_4M.fd" ] && [ -r "$d/OVMF_VARS_4M.fd" ]; then + OVMF_CODE="$d/OVMF_CODE_4M.fd" OVMF_VARS="$d/OVMF_VARS_4M.fd" + return + fi + done + [ "$try" = 1 ] && needs __no_such_command__ ovmf + done + die "no OVMF firmware (OVMF_CODE_4M.fd), even after installing ovmf" +} + +# needs ...: install what is missing, when this +# runner can (passwordless sudo and apt). +needs() { + local missing=() pkg cmd + while [ $# -gt 0 ]; do + cmd="$1" pkg="$2"; shift 2 + command -v "$cmd" >/dev/null || missing+=("$pkg") + done + [ ${#missing[@]} -eq 0 ] && return + if ! command -v apt-get >/dev/null || ! sudo -n true 2>/dev/null; then + die "missing ${missing[*]}, and no sudo and apt here to install them" + fi + say "installing ${missing[*]}" + sudo -n apt-get update -qq + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends "${missing[@]}" >/dev/null +} + +# vars_qcow2 : a fresh UEFI variable store, as qcow2. savevm snapshots +# every writable disk, and refuses one that is not qcow2: the variable +# store is one. +vars_qcow2() { + qemu-img convert -q -f raw -O qcow2 "$OVMF_VARS" "$1" +} + +# binds: where the appliance's forwarded ports listen. This machine's +# loopback, and Docker's bridge when there is one, so that a container +# reaches it as host.docker.internal; never the runner's network. +binds() { + echo 127.0.0.1 + ip -4 -o addr show dev docker0 2>/dev/null | awk '{sub(/\/.*/, "", $4); print $4}' +} + +# hostfwd : qemu's forward of it, on every bind. +hostfwd() { + local a out="" + for a in $(binds); do + out+=",hostfwd=tcp:$a:$2-:$1" + done + echo "$out" +} diff --git a/.github/actions/truenas-appliance-runner/test_tnctl.py b/.github/actions/truenas-appliance-runner/test_tnctl.py new file mode 100644 index 0000000..9dae252 --- /dev/null +++ b/.github/actions/truenas-appliance-runner/test_tnctl.py @@ -0,0 +1,184 @@ +"""tnctl against a fake installer, middleware and QMP: python3 +test_tnctl.py. Standard library only, like tnctl.""" + +import base64 +import hashlib +import json +import os +import socket +import struct +import sys +import tempfile +import threading +import unittest +from types import SimpleNamespace + +sys.path.insert(0, os.path.dirname(__file__)) +import tnctl # noqa: E402 + + +class FakeServer: + """A websocket server answering JSON-RPC from handlers[method].""" + + def __init__(self, handlers): + self.handlers = handlers + self.calls = [] + self.paths = [] + self.sock = socket.socket() + self.sock.bind(("127.0.0.1", 0)) + self.sock.listen() + self.url = f"ws://127.0.0.1:{self.sock.getsockname()[1]}" + threading.Thread(target=self.serve, daemon=True).start() + + def serve(self): + while True: + try: + conn, _ = self.sock.accept() + except OSError: + return + threading.Thread(target=self.handle, args=(conn,), daemon=True).start() + + def handle(self, conn): + f = conn.makefile("rb") + request = f.readline().decode() + self.paths.append(request.split()[1]) + key = "" + while (line := f.readline().decode().strip()): + if line.lower().startswith("sec-websocket-key:"): + key = line.split(":", 1)[1].strip() + accept = base64.b64encode(hashlib.sha1((key + tnctl.GUID).encode()).digest()).decode() + conn.sendall(f"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n" + f"Sec-WebSocket-Accept: {accept}\r\n\r\n".encode()) + + def send(obj, opcode=1): + payload = json.dumps(obj).encode() if opcode == 1 else obj + n = len(payload) + head = bytes([0x80 | opcode]) + head += bytes([n]) if n < 126 else (bytes([126]) + struct.pack("!H", n) if n < 1 << 16 else bytes([127]) + struct.pack("!Q", n)) + conn.sendall(head + payload) + + def read(): + b0, b1 = f.read(2) + n = b1 & 0x7F + if n == 126: + n = struct.unpack("!H", f.read(2))[0] + elif n == 127: + n = struct.unpack("!Q", f.read(8))[0] + mask = f.read(4) + return b0 & 0x0F, bytes(b ^ mask[i % 4] for i, b in enumerate(f.read(n))) + + try: + while True: + opcode, payload = read() + if opcode == 0x8: + return + if opcode != 1: + continue + msg = json.loads(payload) + self.calls.append((msg["method"], msg.get("params", []))) + # A ping before every answer: the client must pong and go on. + send(b"hi", opcode=0x9) + result = self.handlers[msg["method"]](*msg.get("params", []), notify=lambda m, p: send({"jsonrpc": "2.0", "method": m, "params": p})) + if isinstance(result, tnctl.RPCError): + send({"jsonrpc": "2.0", "id": msg["id"], "error": result.error}) + else: + send({"jsonrpc": "2.0", "id": msg["id"], "result": result}) + except (OSError, ValueError): + return + finally: + conn.close() + + def close(self): + self.sock.close() + + +def args(**kw): + return SimpleNamespace(timeout=10, **kw) + + +class TestTnctl(unittest.TestCase): + def test_install(self): + notes = [] + + def install(opts, notify): + notify("installation_progress", [{"progress": 0.5, "message": "half"}]) + notes.append(opts) + return None + + srv = FakeServer({ + "is_adopted": lambda notify: False, + "list_disks": lambda notify: [{"name": "vda"}, {"name": "vdb"}], + "adopt": lambda notify: None, + "install": install, + }) + self.addCleanup(srv.close) + tnctl.cmd_install(args(url=srv.url, disk="")) + self.assertEqual(srv.paths, ["/ws"]) + self.assertEqual(notes, [{"disks": ["vda"], "set_pmbr": True, "authentication": None}]) + + def test_wait_until_ready(self): + ready = iter([False, False, True]) + srv = FakeServer({ + "auth.login_ex": lambda creds, notify: {"response_type": "SUCCESS" if creds["password"] == "pw" else "AUTH_ERR"}, + "system.ready": lambda notify: next(ready), + # Bigger than one short frame length can say, and than two. + "system.info": lambda notify: {"blob": "x" * 70000}, + }) + self.addCleanup(srv.close) + tnctl.cmd_wait(args(url=srv.url, user="truenas_admin", password="pw")) + self.assertEqual([m for m, _ in srv.calls], ["auth.login_ex", "system.ready", "system.ready", "system.ready"]) + self.assertEqual(srv.paths, ["/api/current"]) + c = tnctl.login(srv.url, "truenas_admin", "pw") + self.assertEqual(len(c.call("system.info")["blob"]), 70000) + c.close() + with self.assertRaises(tnctl.RPCError): + tnctl.login(srv.url, "truenas_admin", "wrong") + + def test_errors_and_jobs(self): + states = iter(["RUNNING", "SUCCESS"]) + srv = FakeServer({ + "auth.login_ex": lambda creds, notify: {"response_type": "SUCCESS"}, + "user.query": lambda *a, notify: {"id": 1} if len(a) > 1 else [{"id": 1}], + "user.update": lambda *a, notify: tnctl.RPCError("user.update", {"code": 22, "message": "nope"}), + "service.update": lambda *a, notify: None, + "service.control": lambda *a, notify: 42, + "core.get_jobs": lambda *a, notify: [{"id": 42, "state": next(states), "result": True}], + }) + self.addCleanup(srv.close) + with self.assertRaises(tnctl.RPCError): + tnctl.cmd_rotate(args(url=srv.url, user="truenas_admin", password="pw", new="new")) + c = tnctl.login(srv.url, "u", "p") + self.assertTrue(c.job("service.control", "START", "ssh")) + c.close() + + def test_qmp(self): + path = os.path.join(tempfile.mkdtemp(), "qmp") + srv = socket.socket(socket.AF_UNIX) + srv.bind(path) + srv.listen() + seen = [] + + def serve(): + conn, _ = srv.accept() + f = conn.makefile("rwb") + f.write(b'{"QMP": {"version": {}}}\n') + f.flush() + for line in f: + msg = json.loads(line) + seen.append(msg) + f.write(b'{"event": "STOP"}\n') + out = {"return": {}} + if msg["execute"] == "human-monitor-command": + out = {"return": "Error: no snapshot" if "nope" in msg["arguments"]["command-line"] else ""} + f.write(json.dumps(out).encode() + b"\n") + f.flush() + conn.close() + + threading.Thread(target=serve, daemon=True).start() + self.addCleanup(srv.close) + self.assertEqual(tnctl.qmp(path, "human-monitor-command", {"command-line": "savevm fresh-install"}), "") + self.assertEqual([m["execute"] for m in seen], ["qmp_capabilities", "human-monitor-command"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/actions/truenas-appliance-runner/tnctl.py b/.github/actions/truenas-appliance-runner/tnctl.py new file mode 100755 index 0000000..6cc6a2e --- /dev/null +++ b/.github/actions/truenas-appliance-runner/tnctl.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +"""tnctl: drive a TrueNAS appliance that runs under a local qemu. + +Standard library only, so it runs on any runner with a python3: the +installer's and the middleware's JSON-RPC 2.0 over a websocket, and qemu's +QMP over a unix socket. appliance.sh calls it; the image build does too. + + tnctl.py install URL [--disk NAME] + Drive the installer ISO at ws://.../ws: install to the first disk it + lists (or NAME), with the admin left for setup-admin. Returns when the + install is done; the caller powers the VM off and boots the disk. + + tnctl.py setup-admin URL USER PASSWORD [--timeout S] + Once the installed system is up: create its admin. Only a system + with no admin yet allows this, unauthenticated. + + tnctl.py wait URL USER PASSWORD [--timeout S] + Until USER can log in and system.ready is true; prints the seconds. + + tnctl.py rotate URL USER PASSWORD NEW + tnctl.py enable-ssh URL USER PASSWORD + tnctl.py call URL USER PASSWORD METHOD [JSON-PARAMS...] + One call as USER, its result as JSON on stdout. + tnctl.py wait-verify URL USER PASSWORD [--timeout S] + Until the first boot's truenas_verify baseline log exists: middleware + writes it once, after system.ready, and anything changed before it + would be reported as tampering from then on. + + tnctl.py qmp SOCKET COMMAND [JSON-ARGS] + tnctl.py hmp SOCKET "savevm fresh-install" + One QMP command, or a human monitor command through QMP. + +URL is ws://host:port; the path (/ws for the installer, /api/current for +the middleware) is added here. +""" + +import argparse +import base64 +import hashlib +import json +import os +import socket +import struct +import sys +import time +import urllib.parse + +GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + +class Closed(Exception): + """The websocket is gone.""" + + +class WebSocket: + """A websocket client, enough for JSON-RPC: text messages, ping and + pong, close. ws:// only: the appliance is reached through qemu's port + forward on this machine.""" + + def __init__(self, url, timeout=30): + u = urllib.parse.urlsplit(url) + if u.scheme != "ws": + raise ValueError(f"{url}: only ws:// is supported") + self.sock = socket.create_connection((u.hostname, u.port or 80), timeout=timeout) + key = base64.b64encode(os.urandom(16)).decode() + path = (u.path or "/") + (f"?{u.query}" if u.query else "") + self.sock.sendall( + f"GET {path} HTTP/1.1\r\nHost: {u.hostname}:{u.port or 80}\r\n" + "Upgrade: websocket\r\nConnection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n".encode() + ) + head = b"" + while b"\r\n\r\n" not in head: + chunk = self.sock.recv(4096) + if not chunk: + raise Closed(f"{url}: closed during the handshake") + head += chunk + head, self.buf = head.split(b"\r\n\r\n", 1) + lines = head.decode("latin-1").split("\r\n") + status = lines[0].split(" ", 2) + if len(status) < 2 or status[1] != "101": + raise Closed(f"{url}: {lines[0]}") + headers = {k.strip().lower(): v.strip() for k, v in (l.split(":", 1) for l in lines[1:] if ":" in l)} + want = base64.b64encode(hashlib.sha1((key + GUID).encode()).digest()).decode() + if headers.get("sec-websocket-accept") != want: + raise Closed(f"{url}: bad Sec-WebSocket-Accept") + + def _read(self, n): + while len(self.buf) < n: + chunk = self.sock.recv(65536) + if not chunk: + raise Closed("connection closed") + self.buf += chunk + out, self.buf = self.buf[:n], self.buf[n:] + return out + + def _send(self, opcode, payload): + head = bytes([0x80 | opcode]) + n = len(payload) + if n < 126: + head += bytes([0x80 | n]) + elif n < 1 << 16: + head += bytes([0x80 | 126]) + struct.pack("!H", n) + else: + head += bytes([0x80 | 127]) + struct.pack("!Q", n) + mask = os.urandom(4) + self.sock.sendall(head + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) + + def send(self, text): + self._send(0x1, text.encode()) + + def recv(self, timeout=None): + self.sock.settimeout(timeout) + message = b"" + while True: + b0, b1 = self._read(2) + opcode, n = b0 & 0x0F, b1 & 0x7F + if n == 126: + n = struct.unpack("!H", self._read(2))[0] + elif n == 127: + n = struct.unpack("!Q", self._read(8))[0] + mask = self._read(4) if b1 & 0x80 else None + payload = self._read(n) + if mask: + payload = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + if opcode == 0x8: + raise Closed("closed by the server") + if opcode == 0x9: + self._send(0xA, payload) + continue + if opcode == 0xA: + continue + message += payload + if b0 & 0x80: + return message.decode() + + def close(self): + try: + self._send(0x8, b"") + except OSError: + pass + self.sock.close() + + +class RPCError(Exception): + def __init__(self, method, error): + super().__init__(f"{method}: {json.dumps(error)}") + self.error = error + + +class RPC: + """JSON-RPC 2.0 over a websocket; notifications go to on_notify.""" + + def __init__(self, url, timeout=30, on_notify=None): + self.ws = WebSocket(url, timeout) + self.id = 0 + self.on_notify = on_notify or (lambda method, params: None) + + def call(self, method, *params, timeout=120): + self.id += 1 + msg = {"jsonrpc": "2.0", "method": method, "id": self.id} + if params: + msg["params"] = list(params) + self.ws.send(json.dumps(msg)) + deadline = time.monotonic() + timeout + while True: + left = deadline - time.monotonic() + if left <= 0: + raise TimeoutError(f"{method}: no answer in {timeout}s") + data = json.loads(self.ws.recv(left)) + if data.get("id") == self.id: + if "error" in data: + raise RPCError(method, data["error"]) + return data.get("result") + if "method" in data: + self.on_notify(data["method"], data.get("params")) + + def job(self, method, *params, timeout=600): + """A middleware job: its id, then its result once it ends.""" + jid = self.call(method, *params) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + jobs = self.call("core.get_jobs", [["id", "=", jid]]) + if jobs and jobs[0]["state"] == "SUCCESS": + return jobs[0]["result"] + if jobs and jobs[0]["state"] in ("FAILED", "ABORTED"): + raise RPCError(method, jobs[0].get("error")) + time.sleep(1) + raise TimeoutError(f"{method}: job {jid} not done in {timeout}s") + + def close(self): + self.ws.close() + + +def log(*a): + print("tnctl:", *a, file=sys.stderr, flush=True) + + +def api(url): + return url.rstrip("/") + "/api/current" + + +def login(url, user, password, timeout=30): + c = RPC(api(url), timeout) + r = c.call("auth.login_ex", {"mechanism": "PASSWORD_PLAIN", "username": user, "password": password}) + if (r or {}).get("response_type") != "SUCCESS": + c.close() + raise RPCError("auth.login_ex", r) + return c + + +def retry(what, timeout, fn, every=3): + """fn() until it returns, for up to timeout seconds; its last error + is the one raised.""" + deadline = time.monotonic() + timeout + while True: + try: + return fn() + except (OSError, Closed, RPCError, TimeoutError, ValueError) as e: + if time.monotonic() >= deadline: + raise TimeoutError(f"{what}: not in {timeout}s; last: {e}") from e + time.sleep(every) + + +def cmd_install(a): + def progress(method, params): + if method == "installation_progress": + p = (params or [{}])[0] + try: + pct = f"{float(p.get('progress', 0)) * 100:.0f}%" + except (TypeError, ValueError): + pct = "?" + log("install", pct, p.get("message", "")) + + # nginx answers on 80 before the installer behind it does. + c = retry("the installer", a.timeout, lambda: RPC(a.url.rstrip("/") + "/ws", 30, progress), every=5) + try: + if c.call("is_adopted"): + raise SystemExit("the installer is adopted by another client already") + disks = [d["name"] for d in c.call("list_disks") or []] + log("installer sees disks", disks) + disk = a.disk or (disks[0] if disks else None) + if not disk: + raise SystemExit("the installer sees no disk") + c.call("adopt") + log("installing to", disk) + # authentication None: the admin is made after the first boot, by + # setup-admin, as the web UI's first-run page would. + c.call("install", {"disks": [disk], "set_pmbr": True, "authentication": None}, timeout=a.timeout) + log("install done") + finally: + c.close() + + +def cmd_setup_admin(a): + def setup(): + c = RPC(api(a.url), 30) + try: + c.call("user.setup_local_administrator", a.user, a.password, timeout=120) + finally: + c.close() + + retry("the middleware", a.timeout, setup, every=5) + log("admin", a.user, "set up") + + +def cmd_wait(a): + start = time.monotonic() + c = retry("a login", a.timeout, lambda: login(a.url, a.user, a.password), every=2) + def ready(): + if not c.call("system.ready"): + raise ValueError("not ready yet") + + try: + retry("system.ready", max(10, a.timeout - (time.monotonic() - start)), ready, every=2) + finally: + c.close() + print(f"{time.monotonic() - start:.1f}") + + +def cmd_wait_verify(a): + c = login(a.url, a.user, a.password) + try: + path = f"/var/log/audit/truenas_verify.{c.call('system.version')}.log" + retry(path, a.timeout, lambda: c.call("filesystem.stat", path)) + log("truenas_verify baseline written:", path) + finally: + c.close() + + +def cmd_rotate(a): + c = login(a.url, a.user, a.password) + try: + users = c.call("user.query", [["username", "=", a.user]]) + if not users: + raise SystemExit(f"no user {a.user}") + c.call("user.update", users[0]["id"], {"password": a.new}) + finally: + c.close() + + +def cmd_enable_ssh(a): + c = login(a.url, a.user, a.password) + try: + uid = c.call("user.query", [["username", "=", a.user]], {"get": True})["id"] + c.call("user.update", uid, {"ssh_password_enabled": True}) + c.call("service.update", "ssh", {"enable": True}) + c.job("service.control", "START", "ssh", {"silent": False}) + finally: + c.close() + + +def cmd_call(a): + c = login(a.url, a.user, a.password) + try: + print(json.dumps(c.call(a.method, *[json.loads(p) for p in a.params], timeout=a.timeout), indent=2)) + finally: + c.close() + + +def qmp(path, command, arguments=None, timeout=600): + s = socket.socket(socket.AF_UNIX) + s.settimeout(timeout) + s.connect(path) + f = s.makefile("rwb") + + def read(): + while True: + line = f.readline() + if not line: + raise Closed("qmp closed") + msg = json.loads(line) + if "event" not in msg: + return msg + + def send(cmd, args=None): + f.write(json.dumps({"execute": cmd, **({"arguments": args} if args else {})}).encode() + b"\n") + f.flush() + r = read() + if "error" in r: + raise RPCError(cmd, r["error"]) + return r.get("return") + + try: + read() # the greeting + send("qmp_capabilities") + return send(command, arguments) + finally: + s.close() + + +def cmd_qmp(a): + r = qmp(a.socket, a.command, json.loads(a.args) if a.args else None) + print(json.dumps(r, indent=2)) + + +def cmd_hmp(a): + out = qmp(a.socket, "human-monitor-command", {"command-line": a.line}) + # HMP reports its own errors as text, savevm and loadvm included. + if out and out.strip(): + print(out.strip()) + if "rror" in out: + raise SystemExit(1) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + + def p(name, fn, *args, timeout=600): + sp = sub.add_parser(name) + for arg in args: + sp.add_argument(arg) + sp.add_argument("--timeout", type=int, default=timeout) + sp.set_defaults(fn=fn) + return sp + + p("install", cmd_install, "url", timeout=3600).add_argument("--disk", default="") + p("setup-admin", cmd_setup_admin, "url", "user", "password") + p("wait", cmd_wait, "url", "user", "password") + p("wait-verify", cmd_wait_verify, "url", "user", "password", timeout=300) + p("rotate", cmd_rotate, "url", "user", "password", "new") + p("enable-ssh", cmd_enable_ssh, "url", "user", "password") + p("call", cmd_call, "url", "user", "password", "method", timeout=120).add_argument("params", nargs="*") + p("qmp", cmd_qmp, "socket", "command").add_argument("args", nargs="?") + p("hmp", cmd_hmp, "socket", "line") + a = ap.parse_args() + try: + a.fn(a) + except (TimeoutError, RPCError, Closed, OSError) as e: + raise SystemExit(f"tnctl {a.cmd}: {e}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4218673..e2819b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,6 +153,26 @@ jobs: sed -E 's/^\{\{file:(.*)\}\}$/\1/' | sort -u) exit "$missing" + # The runner-side appliance action's scripts. Nothing in this repo runs + # them, and github-truenas-runner's appliance image build fetches them + # from master: a broken one is a failed image build there, or an e2e job + # that cannot boot its appliance. + appliance-runner: + name: Check the appliance runner scripts + runs-on: ubuntu-latest + defaults: + run: + working-directory: .github/actions/truenas-appliance-runner + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: tnctl against a fake installer, middleware and QMP + run: python3 test_tnctl.py + + - name: shellcheck + run: shellcheck -x appliance.sh lib.sh + # Smoke test: this repo calls its own reusable workflow, so a change to # check-member.yml is executed before it can be merged rather than after. # diff --git a/README.md b/README.md index 4ecb22b..27c0c7d 100644 --- a/README.md +++ b/README.md @@ -523,6 +523,53 @@ What the runner has to provide, the timings, and the lessons behind every rule in the script are in `truenas/webui`'s `e2e/docs/05-ci.md`, which is where this was built and first used. +### `.github/actions/truenas-appliance-runner` + +The other way to give a job an appliance, being prototyped: **the appliance +is inside the runner**. The job runs on an ephemeral GARM runner from +`iXsystems/github-truenas-runner`'s TrueNAS provider, whose pool attaches a +copy-on-write clone of a prebuilt TrueNAS image as a second disk; this +action boots that disk under qemu, inside the runner, rotates its admin +password, snapshots it as `fresh-install`, and exports the same `TN_*` +variables as the claim above. There is nothing to claim and nothing to +release: picking the runner label is the claim, and GARM deleting the +runner is the release. Capacity is the pool's size; jobs queue for a runner +instead of a concurrency group, and sharding is a matrix. + +```yaml +jobs: + e2e: + runs-on: [self-hosted, truenas-appliance] + steps: + - uses: iXsystems/ux-github-workflows/.github/actions/truenas-appliance-runner@master + id: appliance + - run: yarn e2e + # between tests, back to the baseline in seconds, memory included: + - run: truenas-appliance revert fresh-install +``` + +| Input | Default | Notes | +|---|---|---| +| `baseline` | `fresh-install` | Snapshot (`savevm`) taken once it is up; empty takes none | +| `memory-mb`, `vcpus` | `6144`, `4` | The appliance's; the runner's flavor needs room for it and the job | +| `data-disk-count`, `data-disk-gb` | `10`, `10` | Sparse qcow2 files | +| `ssh` | `'false'` | `true` enables ssh for the admin, on `TN_SSH` | +| `disk` | the attached one | `/dev/disk/by-id/virtio-truenas_appliance` by default | +| `export-env` | `'true'` | Write `TN_*` to the job environment | + +Outputs: `host`, `host-http`, `username`, `password` (masked), `ssh`, +`boot-seconds`, `snapshot-seconds`. `truenas-appliance` is put on `PATH`: +`snapshot `, `revert `, `call [params...]`, +`snapshots`, `console [lines]`, `probe`. + +These scripts are the one copy. github-truenas-runner's image pipeline +fetches them from here, at the last commit that changed this directory, to +install the appliance image from the nightly ISO and test-boot it the way a +runner will; a change here is therefore a new appliance image there. What +the two must agree on, the admin the image is built with and how qemu boots +it, is `lib.sh`. The design, the pool setup and the numbers to measure are +in github-truenas-runner's `docs/e2e-appliances.md`. + ## Adoption status | Repo | `check-ticket` | `pr-title` | `check-member` | `prepare` | review | From 91c6ebed63a1b307ec0500ef23c3066c02604d52 Mon Sep 17 00:00:00 2001 From: William Grzybowski Date: Mon, 12 Oct 2026 10:14:08 -0300 Subject: [PATCH 2/5] Ignore __pycache__ from running the appliance scripts' tests --- .../__pycache__/tnctl.cpython-314.pyc | Bin 27102 -> 0 bytes .gitignore | 1 + 2 files changed, 1 insertion(+) delete mode 100644 .github/actions/truenas-appliance-runner/__pycache__/tnctl.cpython-314.pyc diff --git a/.github/actions/truenas-appliance-runner/__pycache__/tnctl.cpython-314.pyc b/.github/actions/truenas-appliance-runner/__pycache__/tnctl.cpython-314.pyc deleted file mode 100644 index 3cca8a11a21a4aa071e3d13fe61e5b74f530de85..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27102 zcmd6Q33OZ6ncjQA!%7e!L5iCs64XM85*IDhM#+*XYF{V{q#(zk2}1x#LLvd^0n~!T z4wGgkq^($#?Ky%Idjw~xs2QiGr!%!Ylc{AV)3|4*b07n<=oj|H)27XFPp3z=G{=r} zI@9mJZvjX_w%gO`={@4bd-vUY-(CLw|L@akd$E;6x;FbaC%?Xd)Edm7uc^MZeYL0xRL#u;wJWMj+@!9C2nEA)_9Ry z7q>0o>N&T*l5-1>6}Rc-cgF3k)PPcBn@%Znurd?M%y~6SSg8f2R;^~KyQq?jJKZ+q zW$uOU;!19qkC(d_x$W#~f!l#6748z`mF`mHRcO(x~q`axvP=89;zLLbrd@u5g~eLK1jT0`Mj z+&4TN6j#JhH9%#qkx(Eo96ak2gD5=EbEK=;-QDS0*RsYHIm1SAHh3Zy@t+FDn^+qf z&Wf1z=;3a)!e}^j4&$IIExGx$#D-iL{%{QA+sfV)8u1MVT_;9E!vR+y5{$Xxkq9Q{ zaY$cBM>~_1lJ9>JKA93%Zb%}>Yf|1d#|!yleOIeCP^>eJ6&4%`pJxeRcEe(Z=0h!*zIf@4h2ge}*I79qx|9J%G2| z76ci%yL>S=wI;UMVlXy3jAfKD3n=S~#RJ&7mi+h8(lrCuLIdXuytdGO%PUCGBZD?U z6R?+-E9{GTmA+2+V!`21ILKBwc#Z(RJrCX0O&3FP;39S}0w@BG55xhr`8y%wJxB|{ zAUx>u5Bb7_!GP;T5a>ZLrFIk<1!A8Fy2M~KA_72Y+Hv1V6f;A!0WmT{03XJC=I-C= zk*KTZNaw-by{^t9hYxpjsf*m<-pfE-qYp#5noDB7Gr=<>=r$M|YE}?%i87QUm^2Y6 ze`I6?bgsz-JodSUMsedBiG)LO+$+y8Br!5NI7Ia^ECLJw9-}5hkyyN)rXu5W)CYpP z67+?U1zLQK2sOD{ebG>>e^eBK9`xKw&vT@;kvjDS0$7xxpcrhib|sdr+pu|e$DWQ2 z&AYd(ZEJ4Zw5y|e%ZAR5=FSZrJJ;;$+`M7u+O>X#o3J17V3zLI;Af0`jEnPb4jg!W zT<_-N0ywF`tzXJD3X-vNI1&p65*57|g+{+=41*v~E!ViNMx$i*dc(evpw}x|z21>X zV04(SZC>x`QQxrg#NqWa2qQ2GNATF>^#&q-TrBW*%@WK`^M%6U}EM;;8|mGOaCgNvqEv z4@JVURt21!H5S<{v$j?h9z;D}#Ip6ch>^7N``mvNTw;-0K4<+DDT2G(jmN(H72Fip zS8x$2oKS4*9fX&D_i`ua%!U0!oB%! z7ArW$^{M4tx4L3l3CDG-GGShshi_A0GphIKyCLk%Pi}7WSx89^l9Ku?>QfJI!IO^KP43EB9N^p?7=kUcEwTiF%#;-O8^}ZU!;CMO+=%-@RVv(YbY-lzDl$ zur`*ya^lx)Vpp%zj>*i>n~Afq5gI+o>vLCs&p6?Fy`9(pWi=bH?64a z{+jCC0+{(CNX~_8*|>GO!@S$jl-EJj;NiNRc?HU4_tN{z+{Ru5_Q-_Z3#>bynDOLG zu03j7JOXm-I!pKR``@LmjkJ%G4{jt)Ex*jkAEL(~pNW$U%z&yYF_W zV=CC620>A?2YOOd9-I}dU)rxz-ygN~wW{OtSk$-l>o#Mb zYRlZO`Z{&>3PFWhCtB>Wbw8AAt6sS))YhKjJSsYO&noj_*j=?0t6#^B>Z3(bYlv&C zmUL%hiRyO7LKJQSZV(%d%97eOUNR|4g~Yc?{0@mr2zw!qB}!fUdV9ND*S4%Jw(cW& z$R+XZiS5PKW6?p;7YMfJ_*}8IGXl+%G4FPl^1RsE6ZAK0jIsG(@O--~Q8&NX14#iq zzumQVeX+IJ${2WzWC^+paiB1X#n57yRO1SaGJ%pLEQq3k*pTm3Fku>oFcNNFo6r?o zCEg|J*REOnnGS7Ksfm`8co)IiU8wBb5^#huu9w^60-i&X^6*K#5d|hq61xxykv+$xsNyMBwfoIi64#u7q|wONUTk= zY#R)QgXf~+c4@h`J}q8v2;!;N+p>)*|FP{YTD2auimgKSA%>maHhk0LO&@iq{R8Qf zr*8UB{nUH?=$psiIDVt&o%%Pu*S#5!_qQ=Ks(;I}il3}m#kYS*)9LC;ltYVZ)}+E_ zpWh#h#uGK|t`ok1YwmV+uxDM3b&_#ZgdBH5GDLl1EGU_A%O)ooV~{>Zf|Awa8y*er z7R88&GA0H|rLx}Xh0>+~hGZtGnxsofAlQ=xswx>tSJ~JmSx#(h3x@rXKv2>}Vv=c8 z#0E!3Bx3-I9|{E$!X?A$(V%!QFlD(O#*4oLdpU>F+# z88s>yAZG?e$u#7P4bfx-z|LAp*n4dMF1#WH2`nz@2ZQlOJ01mx(HPSdiC0SesH8s? zJTI=H8(|0FN#+Z-%Cs_`Tmm`d&_P6hMV>c8T+Pj{Y**eW#c8|<>JZW z^~(-8tDd*^sVU zJ=O8*folh*woRYDUixOm8x?OZy1wYfy6g5koNh0_8)ix?CQtuC)#chtwJCk-Xr^?5n(3R;8<(GtQQ@t>yh~!)eb~(|!H#dA!+e!&&FZ&27W4?V9eGTzh%b zrA^OmxxC}jj!fAydD(s#dbg|<6~;|Bog=f31-Bh_?>g#!WV&kmfo;nFr@MY~@^5$a!(=#MX3WbEc?edRew;qcXg- zt36%4HRIftwr%@J&)GKp_O5{|3-d9O%->nI`QS36pGc9Kh|n?xzzGTg1wbQv5V!K{ zDfKmW#FKatIG_k-)XTioZJJkrTQ{e+k3tyN1T)aYeAU3MsT4!Jhug{*8iz-hFRG2| z!rHhG`a!-6j@_;E@caXwd3d*u2je`wCfAdP6l(p49LIBs(i|C6SW#D_K_odsgyEbM zi8hPG%^3-56vXYgmiVyPNxyoSeHcq4G9Y8W!Aq7d$&rWpR6xur7DNWTx0g;XyS(z! z%BM$?zO2129}6~Q${STINH1QKsaTsXSqt#LxbLZbNy9DM;*0xcsuo=_zhF)|Q}sz( zDsXk^<)Nw5Q(|g(ym(xH;gK1Kb6n=NbM!$AUd_P4ip}>K@&TfM%)>Y3ZEB0QO;LRx z+Q`YHX>`zYg#ITPm@HyoqR#;r`VtEFmA|b~{ow(R;5M*6AVWe2$nWDHb0g@}0I-k- z3J(MC5mb3OU$&d~w$?fQ|I`0Q|GF>Je;=`Ambx`vS1ln4v|musBSLx_i3;;lH*}c! z$)oR?qvPdP*<)DGlOe;*ivMlGO?=pv6~0_-dc8}e#$q!3@mHy zpN;<6>Q^Iw6v-Uu`|VvCz`tl{ zjFAOsLJ;|a;n9(x2xE1lQ6>(?NR-Jnrl{}yaKsl7cT$Tk%KDHog3aWHhgFI})0jgs zm=HvD{~Xz;AQ_JG$w1Qc2P2c-jAQw@Fl%#6nw~zyUS0i1tEU6kJ2I=b-zwjJ%eH-D*9WZe11pU0 znst<3OgxoH)<6BkJc%H=ELHx}>Q@Ze=Ewe~G2Q#s?18>a^JAHnkKL+y>^*Zo6Zz+G zB@)&d8QO!3F{0TTeMfLj=TH+jB5P_akT$Bl&)eBZ0>{VVzu`6s{D#Q1t!c_sk7!8eMbdK@iX8ktT9# zW-~$2{8VTbIl~&p&qZ|$X%%Q)<|P;cl(k{l*5=XoJDT!V3q<%E`S)VbV+2_SqY1!l z0KThXWw%D2a<5>d^9$aquJc^8Mbvn>4R87edlQi1OT0<gu(?`sO-|aD~?~a;!%v!&7Y(8W>Wyxa(p@p#+gtk~KGpQ}_C$un2 z8T!p^WkCC|G8$QT8`M^ieGL5;RtF%*W}y*r^&K47Z}pheRjK24^9L6|`Z4rdSv!!d zMO22-9$zts)#(5w1y^iPZ3BJSqw6lqy-B@NO%}uyWMg|T9*M`0b^X8Q3*vElHHOTI zUqM!g4biB^pv#sm5Vm~S; z$`xb8iSy*tf|po?*R9dYsJ&!`3s09MnF7Y#^&Po6CRi9I5lAl@DFjana|Bv=hwq0(& z)SmP`|Il>X_^z3Xn#(6IolKQJH!{BWSB|pTn!0S=k)Q6rF__)fm8mh zmR)!PT3>qc`b?JaT#Rvy%l@b1?U?8A&Fks_^!rc zkw~scyhS9PL1Yslm=&Qjax!Mne8doCOmm_kizFCJrCzUp*cXchLjJfvI6Un2!b(}3 zR|p3r%~gSWJ2j+i6aHf0HQcA)!4yBz=?t!W<{HC8A8$4p%06~l4PO4^D&DZ-p0U!f z@{@H2L(LrynM}s0141+{sk?KR1UMh(M006PlR9g!Fu?(kyO^WiBw0) zjBpyZS#xwOS`P+g0bFcDuY}KXf}qSjj+JA+3GR!unw%`3Z>6u}j=REuCOMI15_?pQ zJ7}yhhK<4}&-OqPPR1fIj|E0YqA|&U(uk-kuEvodLWct4@1vPyQ5GJHPTfK#Rpe~i zNvTwwqh`N+7RAh1NfN2Kc*1nkT$S9D+?-nXqrlaZKR7wPAk(lZy?k@JdP~;aKC__a z8Cel+`kHfek^2NCaDh1{pjs!*P1_4SF^0+rtWIe3MEkyF^cj&Hk1^;}d|! z!hhh9$$L*V3hzCU@HzhC%X9q2q+Rjwy$k*4TH$NMosRs*ZX@oDeaeXqoU9>ODXk%Hs7n$P?h>w3 zoZwPU{5mcfTgJY82iKAjmSf*YY>tR+-ZtzTIT7$}muj&X8ua9GXyy4!=z&4r823g% zBZds=OQ>^QC?Hw1bx}ZA+(Bc2{KtvUtL*`~3Uy?VYs-J__o&ixWPn-c!enR0*$BI# zX~qdVBz~%w+^+6SS9jiAv2$E_#x6q*^{%Z2tphCqu-klh7Ar);K8#l=nxEi})5Js$ zMyIe=!3{GMyuJ`7!Bw=<+(oONyvHk)0>!7UZIMUw)REPs0kyt7Z~7wJ1ne6M@Tk9a zT%V@67xED2;vr$I%T0lc8o*Z#U$$)%p}dl!Mko~A=60wqSbB)fnn29xrO~4!fAuvAM4js8J{4cq>kJ0SBQh^}Om2PtJ08%prjDp%fY0mi zAg)?P7dn(o#6rZVKcPp^C_{DeIh2c&$P$irP%s3v#NcR*{27F#%94AYdqlPPbhzf3xz9%Cx;>T>q=0(%D6>RK@oXjdxC(Ck~V7 zZJID$v`yHa@4fu!vyZ0q&v_w$&z4nQ-u~?NOxe=Z;H|ReaWe_yWeZ;nTnT?KoT+WO zRn`Il*-&LI}zi~9ZbZ@3&U%F%;_0j%pdoplk@P)xE!!Ha^ zwPkA8WXjgEmKV=WoJ$H%Kbb1OTKRHi*0FM?tR@+Fe#dOhlGMg&(@!4C)NDzYZ<(oF zc=_C=b18GCvN2uKIOC{Hw!P=5BjMV9$)1d-_PzAvRQ%O**UnuxXIeYnE7{3t;D?WZ z(#@fPgmIvOw!3%OS{23tU>@JXd-O;64aRy-;G#lK7A_=BKbV$$>2|Ue8DyTR(h4oZ zShb|kUZF(~yRX9Gh~j9}76cV-L4oq|#x2abGcfE1T`Ex1m|t@#deVA^+*+Xi+@6aV zrC35W`>?)pI$kk%G~Fa&&r5F25TVWrbWxcfVy*J%tJ)BjXw9|{eE-)RmuJ?XxGgh_ z<*_h&Yh9LL?@X<#2vzECroX@F`4dv6e)!A0eeJ z@W6V}C?=Bz{vmST!e}a)dX9B=?(XRkU%)6C8A%x1se4zVm;@yjd`(nFOriqZ?#4YG z`w#8jC7C*Q9&z{X-qmOle@IQgPuYuOD2*iaPcR28vA$1#<&mc&WoX^LP=xVzA{yjv9BE0--q+Fs1trFoUzxY zs@s$7{aSvj{xx~U^u)iGO9n>;hs@#C{szxMLirZ;6)Y|T_`OP6e$CGGF21Cytp zIXr8xNe(0*NqMF^r#4I%Umv{wl^eeF!hIS0{A zrvj^jB4k8H`_G~)@f@=I)L08GD^p@q!|$uLdxePQoCL2uZ_-?5e&I{B&2 zdmR1VX(%!r;HTE#65{|U zMfQvB6YZ1!r*|a#ZxyXh3#;Xk=C2>Y0vqWJYm@V#<#{$W=QZSPl|R78S;#2whWeyQ z^19YseM7GyPu0R3N!4oQ3R#PCreL>`#eQJ}z$Ju2G(Lw`eVS1kQB4MQoRAy|9!!a1 zI>yn0eGP@Z!E+F;f)vspij7Ei^^}RX`%uUJE=2jjuE~OR%u$$P5UL%JiXpfmI5`&d z#?Qlya1Cn(BQQe>T6CGf5=E8qhzKm6(Bz@qURHRUcQ*E6zqHzJnTn zjlWm`85t3ZE_6=npWZh)m@(Imb zkTjvg2XO@^7Lg#(kpEK zDO3yK00agND0(@tZ@!n9xQ@bm*J%QNuVF}+*Egxx^HM$R1gi2whDx_SYS>{o7=y~g zu0p(KxD*mS1L2j8I5tGbedv&ye0ZvfMZk|n+376hB$INkXDgEoVD*DMK&B`wf)vTR zjzbGB(HFvzhL(hdG>SNJ`Gh65)kT%tU0o3Nb4>jV1})px8G9#|E|OX=zJd%VAB3|w zYA1uBZUEM=AWZk?WzNnA?}tZ>8;}1MZL|1`eI3~tchAU`lwW-EsV6gzy3~qWj>fTl zGgkW@oxxK3e%XTO+QtulI$KitV#SZDUfK53nzUorJx*__opqF{Z8m2et4W#??7r%z5;-ZSlLd#lO2_LKmMg@8cC?1NSdoI~VBRGIj_% zOZ0D-7;rO(KWpS{7=#L@O9L5ZD!GUuN7&qaPr)b|56H%P1j>_9vmXZ`HYsE>$^(-5 zg2YZ!cvoO~CWG=Klp|#zM1}j%gvk5JV+&$HkLgBrEfiJ^NIt#9ZORv=yL1O( zPGNz-`lv2<#DD_SRRUPYVYUTC-MqJz)0kn-&zV^n+sD)2bnHf6Nk{fN*omS`AA83LhT}^IXF{so!9YU3ID{eTx zu3&FQs7wljSC1rPMa(d+;IAY2ci(RUOHHLY3GP$VhN1O)qH57B$V3R$i{1tWEAj z=t$|UlBMGv@E#S{%vRSXPhS~*VKnt9A}#CIXR6z#W4Ef?p|Vu%xPCfa(J`q9sVpv; zv?SMGHN0#{g{GYuSKE8G4IdSAw&qXonYjhEFDBAutKeiRu9>N*epU}RQ*q5_F|zEv z)X=e%duwS~#}@rtn|b7kVqyr|$vZ#lXo>ckL0Ug9w(0W4=fXHrkdkR4-ds_%zqW^0 zAt5K0>TwpQ=BE_fOx>#tA_CZPHmPT`*e)^)1}kC z*MqMGUmt#L_{RFo#=Y+>zqRp5`e;x3*kjpsa>xwaet$!~f>nBbfk+hRE)(nKERlTl zkUeq1TkQk)#2G|HFw&_sobY9*Z6zxi;JkDg88OQw@+Hc(MRvu4I3dcObtP)%cFrEb z#79ph0~Kd7xF%dotZOqo%E z;Qx@#vWFdO-hD0VP)EtdGZSZSJ2s{r8?OdoP~`t`<#hSDK<;qre@n*Mp0>5C zlJD6I4zwOWlI+czmyqAq$^_!`$xW%=)SjsW*M;=Ay5?Bg`}l3|&`s~qxG`%E zQLC=3dEp1LEc}e+w;xw?r4RF;-l?KaARd3$@>y&v*5k(&yXv{0E5moho~GQKz+_jm=9klpbDi0#E|}T`E!Jx{Q1lofyrRWUrIG`P9VTI%}j2%V?AajUNd|p47%{ld$P7;AK>ZUiM`M4|8%^AF21|}vl!_(KWOY&%)PbP z+0m*`*oj!>j$w$F}-V>JGZ!FPd z_7|}}klFR%Vdzy(pRaWSk#cB3l5PRC$b~BO(=k}cTa+R=VyXrSGf6AgkC2m;`mKfg z{IcUi;CLR;=U=OcHE4u=~pH>H61~zqVXHs&k|C7d3CyWSb5#KxN1?XR>-U zVv{|D3nHT!HaeT1a>l(C$|FTzqV_E1pZ1Vr3_i2ve}N)VOe#MUR9yHh3B~XJ5Dl5! z3kx6-qP!VP23e%3ynoK@ft@C-h*%2KoebZ0EPvOr9DFwE%{ZIWwq_ihDUri@#gEZH zr1P}{L>hBu290XKr|3H;o4fuEs$ySA5@B8oF)rT4wLC=HXviRn1Y%Z)Cw_wRA{x{L zE+Br8-rx09SH`wzY&U{u#zV={E7dPlXO|sH_a4idJ!2jJfY_CmJ7&GXNzcPC`cq4A zuqgC$=ug{T-~K1tGwVBV1aGZBc&p)1ruOhVqv^iKZkhWPeh=H1V7btHvHz+5O!4BB z;a2hTls|1-^`6l5=^cHZ1YpBxehTZt7VhmW&d#E_ES;ze?cd!74BW^2botUYn3tkE zp@x=FS^(5hT7XiipdNw6!OekTY8so?9c)-CGbT&cI(b1QK{I?jV|4%&*c2M~YwgoBL#O7G-N1KJP@ zscd`=uJ2b>>QxX`2dFV|m3S;Bu%$Hu@kpJ%dgkRb>6RTg%Cjr>Fq)Bg;5TW`;tQLlxl7{hN3!o>kk$Etu={INko4i;+55m&GEi?Z8F1A|6Dy&b{3R=q2Usk{U+e<<_GoEHSa70)kIN$($T!LiF>=rxpVtmuqF+Nkn!%< z|33t4x85z}Kxi(=;UH>Z27_oaw<*6J1HD2(jo40nIVkT9tY!i672goDp)QQL7!46~ zBvw}wX5})<^B9do0qD=Ir2%w__&?D+2g%G}*ee6>KjMj0j*p;uRhOPu{xDhq+y|!^ zLjGWj4CrtS$A;)|J$*JU;ZPsq6l`3K3`>Td-VS#!9LHcR;rJesjUI>$nqM8ZGLns$ zJcQHrp}<@$uoD)@FZAN&q6LuDLYVL}8htS)*c}%KCI%+^V2mt->1^P&0Xh-Cbo-43 z?+7=FGxn}`;aOfNBlmWrbLZB%=s{b@&?EKFiXJ(QE$2lnL{BOFs&L3A z=x+(eIxkW9L7@Q3JiiRM1bi==me*ZAuM6Dbtr~}b&t{?Qvl)elfrU0ucmN%)o0nR7 zfrDuV^aXI4jqLl{H&8fO?fM{z;$CV^l4 zHf6s@*$*lE4rOG57ypnlCuPiLk_Y_aZIsRhex?vmn4w528GZC9HjYZ45B9POa3OH< zGhefnP=erxm1)ra1# zemw0te%s^4S43{&I&?B?Kb18PYf$~$j}2VYVg6I4fGmQGF_@oh-MN)}Yh~Tejn21C zi*fz-Mg!$rot?${Lc!?V-9wm&&*(GFF=e2u`3c|H(uxYH59+3q!{lYO6IaM`+s6X{wM+7{;WAvC*m*_B0@yk7?oJ${XG|)F})iKKVd13d$=bJJZ zb^n4o8fY3My?FM7Ut26(5M5zf!85oZO&r7171G63%!mThzKk~hg~v~{CB-Lnz(xsb zm9XPZgdOwHJ@mZ^D}CZoJ}}3;?cq2+QfMY+oMLcgS3Sca!WRT4AsQIPS8yZo0}M%A zs(BG7pzz6rQgoQJWKhvwLYH>5fFi=;XUpk$*>l?vFjrQI6JyD~OvS2n$*Ngr#pPAc zu1fZ1oJ+<#XB-tt|9g)5naY|;4jjf?9RU+;bx6bC3ozSpkVeZ3QJ($iZopP8R64V1!D% zxhA0;$CG?9TvZ++#_I}MqH_`M*kXfZ3Z4u4@eN4H;u8mP$Q|Fjxr;U9tXi};eK@z# zBpFEY3+7SG^{6HO8D-z3>>*_6PY*#r63|yn`~?-4qcd>AFW8k{p+@ndVoxjJoIO%|lliP6bNp8iJKT+6gx zsRZEZ_4}f}6Cr&6DTFTsd>uo9hpK@_YwFnJJ=V4V5i!i}l&^|N<`I0yinMf@SD58b z4Vjehl!~Op%B&=bW^$B#ZdJ@EqoXabE-B_EJ;op|N23l}3SvJ@`?X&zeyaHSfy?1b z;f#4X;-mz_cMg5?&}3y+sHW@v-`qbQ`1TQ+JDqLckTF-o+$5L`3qNrh4Rv=o%1WRq zzi+l(Fd|DA*JaJF_Yt1HH{)2Lv5P!an3RUMR#ogQ<=!rJ?p!)oDk2mk>UH;jAp5_o z3v1|B&$L%sB9;m{Um%whwnJg@C?Vp5Q|(2tv}jruV$6UoiA;VaPcas0)0q6$e4MR1Pno7gw$ zyBxX{nm+ydnb*%upZQVw)tVpF;LA*VuIsbmHF-37I%};j z=%LVT7yp&LAm_Zp`sI^OJf%cIka$vnEasH4Xg+D&Lwg0ul;C1sQc0S&uPtm}HRAXA zt#SmD@hSAJL?~$_Fc{57uBJv@Tz`}YuEI2Ko-QIPh~gDTiVD_N|DCWg{gtVMJjRzrI2 z{*3j&Sm$p{*74ZI^AqPUe(kBRk%`KGF*Ff+8j@TMA~z;`E<8la|Iz1Q(5|B|K~;Z* z??#dZBO&WHiC|M$<3)!JINe{Sd|9 zZc=cf;B2eiHSUG;X1l0Qv#a4hF{o9|4F%iRxn1|QukTx=&33VSiP|3XvxM7I4z1Y$ zb%d~QUPACT^^AlgIDi?7Z!BEXorjMwTJgEqzXzy7kk&V8eY3!DeZr#s*8((Y3h$Oo zf#84-pRk9!;IA&(e3BlY>6dKqs(Y12F_?sYg;uPoylTYR zBwY-VX=nxm)98u3V>F7xk~CwRxq>+tqO5sO@q|0{@GGz5L&F%(Y}<$c9JMBkyo2?h555{UnV zo)cEts73$zz&sm)#IM3}21A(1x`pHbzy}P;#2AR1D5p7G%4z;e$&7~bH#zMcis;?V zc1I+}C@pmR&hE+q`M%(AbdL(=2*JYsVGrc(jIlT}@KRZlCwLamP)2NzArl09oiEBvAF>(nKhP6q=^#4nLQk5LL$~*wyt(IO)^ciW=Zw{M+q&?kbzy43ba~pk zFk{^~)_F&OaXhvD6G*Jik4iX$aV+%w!enr&H!G~WE$p}{?6_f0yL;2Zj;wI(eZep` z^1L?{obpURniU?V`+*BXNkj5XDuD)bp1)IeOE^p=L(f~2yHgvd+NVde!b4Ot{Crnx zH_m~!XNB!^`k3B)qbDouzb$xf3ZC?FZ+h@#TJU6rQ&cN>;rQfG%90UQ-4-_86gFKq zA`o2Ilobx-o*J{l3IbuAb$Z%C-YUIe4N2+3gfM>e8Plh;wyGCb{%FrDd(%a22&FWv znYC9XH>T>ZE`NDBESI{ewNp#e)h!u&YueoUJA_yo)_fML!m9tgzI0a`_w%-eyIb{t z+OnzUrSqwMPC!+EEf(iS#utVelnBOwd9EStPAe7FwI>iio)Q zIAyO>Ms9P-CVxb>MTWO}D&I=wmfh#*8?+P!$e@OpvZl+GU}6RBAK_z|NFsq@gOSW9~XNQ8J39W zKN9MB;V$5iFZm@`{7cUMORngboP}NE*2-?nJ}@Vh}?Z-6>dFRL5Ed0`s ztBUyMj~AJ6TW&^H!7>ByyjQXax7&57Vm0!vkDb;0nme00!8V@A>Z{)u%)h?}1W=Ga HX!5@RWWJ+a diff --git a/.gitignore b/.gitignore index 214f1ae..e76f916 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ # CI installs js-yaml into the workspace to run scripts/check-input-refs.js node_modules/ +__pycache__/ From 1ba3df96c693ec5b8b7d6c9b8976ffcb281b3694 Mon Sep 17 00:00:00 2001 From: William Grzybowski Date: Mon, 12 Oct 2026 13:27:46 -0300 Subject: [PATCH 3/5] truenas-appliance-runner: find the attached disk by its boot-pool, since TrueNAS replaces the serial; truenas-appliance on PATH before start --- .../truenas-appliance-runner/action.yml | 12 +++--- .../truenas-appliance-runner/appliance.sh | 39 ++++++++++++++++--- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/actions/truenas-appliance-runner/action.yml b/.github/actions/truenas-appliance-runner/action.yml index 93f2244..8ae0c2f 100644 --- a/.github/actions/truenas-appliance-runner/action.yml +++ b/.github/actions/truenas-appliance-runner/action.yml @@ -50,7 +50,7 @@ inputs: description: '`true` enables ssh for the admin, on TN_SSH.' default: 'false' disk: - description: The appliance disk, if not the one the pool attached. + description: The appliance disk, if not the one the pool attached (found by its boot-pool). default: '' export-env: description: '`true` writes TN_* to the job environment, for suites that read them from there.' @@ -103,6 +103,12 @@ runs: APPLIANCE_PASSWORD="$(head -c 15 /dev/urandom | base64 | tr -d '/+=')" echo "::add-mask::$APPLIANCE_PASSWORD" export APPLIANCE_PASSWORD + # On PATH before it starts, so that a failed start's console is + # there for a later step to show. + echo "APPLIANCE_DIR=$APPLIANCE_DIR" >> "$GITHUB_ENV" + mkdir -p "$RUNNER_TEMP/truenas-appliance-bin" + ln -sf "$lib/appliance.sh" "$RUNNER_TEMP/truenas-appliance-bin/truenas-appliance" + echo "$RUNNER_TEMP/truenas-appliance-bin" >> "$GITHUB_PATH" out="$("$lib/appliance.sh" start)" get() { sed -n "s/^$1=//p" <<<"$out"; } { @@ -117,8 +123,4 @@ runs: if [ "$EXPORT_ENV" = true ]; then grep '^TN_' <<<"$out" >> "$GITHUB_ENV" fi - echo "APPLIANCE_DIR=$APPLIANCE_DIR" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/truenas-appliance-bin" - ln -sf "$lib/appliance.sh" "$RUNNER_TEMP/truenas-appliance-bin/truenas-appliance" - echo "$RUNNER_TEMP/truenas-appliance-bin" >> "$GITHUB_PATH" echo "Appliance $(get TN_DOMAIN) at $(get TN_HOST): API up in $(get APPLIANCE_BOOT_SECONDS)s, baseline '${APPLIANCE_BASELINE:-none}' in $(get APPLIANCE_SNAPSHOT_SECONDS)s" diff --git a/.github/actions/truenas-appliance-runner/appliance.sh b/.github/actions/truenas-appliance-runner/appliance.sh index 10ec7a0..7067222 100755 --- a/.github/actions/truenas-appliance-runner/appliance.sh +++ b/.github/actions/truenas-appliance-runner/appliance.sh @@ -24,8 +24,8 @@ # Settings, environment variables, all optional: # # APPLIANCE_DISK the appliance's disk: a device or a qcow2/raw file -# (default /dev/disk/by-id/virtio-truenas_appliance, -# the disk the pool attached) +# (default: the disk the pool attached, found by +# attached_disk) # APPLIANCE_DIR its state: overlay, data disks, sockets, console # (default $RUNNER_TEMP/truenas-appliance) # APPLIANCE_MEMORY_MB 6144 APPLIANCE_VCPUS 4 @@ -60,8 +60,34 @@ running() { [ -f "$DIR/qemu.pid" ] && kill -0 "$(cat "$DIR/qemu.pid")" 2>/dev/nu # shellcheck source=/dev/null state() { [ -r "$STATE" ] || die "no appliance started here ($STATE)"; . "$STATE"; } +# attached_disk: the appliance disk the runner's pool attached. By its +# serial when the box kept the one the provider gave it; TrueNAS gives +# every VM disk a random serial of its own instead, so otherwise by what is +# on it: the one disk, not the one / is on, that holds a ZFS pool named +# boot-pool (a TrueNAS install). +attached_disk() { + local serial=/dev/disk/by-id/virtio-truenas_appliance + [ -e "$serial" ] && { realpath "$serial"; return; } + lsblk -J -p -o NAME,PKNAME,FSTYPE,LABEL,MOUNTPOINT | python3 -c ' +import json, sys +devs = [] +def walk(ds): + for d in ds: + devs.append(d) + walk(d.get("children") or []) +walk(json.load(sys.stdin)["blockdevices"]) +root = {d["pkname"] for d in devs if d.get("mountpoint") == "/"} +found = sorted({d["pkname"] for d in devs + if d.get("fstype") == "zfs_member" and d.get("label") == "boot-pool" + and d.get("pkname") and d["pkname"] not in root}) +if len(found) != 1: + sys.exit("found %d disks with a TrueNAS boot-pool: %s" % (len(found), " ".join(found) or "none")) +print(found[0]) +' +} + start() { - local disk="${APPLIANCE_DISK:-/dev/disk/by-id/virtio-truenas_appliance}" + local disk="${APPLIANCE_DISK:-}" local memory="${APPLIANCE_MEMORY_MB:-6144}" vcpus="${APPLIANCE_VCPUS:-4}" local ndata="${APPLIANCE_DATA_DISKS:-10}" datagb="${APPLIANCE_DATA_DISK_GB:-10}" local http="${APPLIANCE_HTTP_PORT:-8080}" https="${APPLIANCE_HTTPS_PORT:-8443}" sshp="${APPLIANCE_SSH_PORT:-2222}" @@ -78,11 +104,12 @@ start() { else cpu=max fi - if [ ! -e "$disk" ]; then - echo "appliance: no appliance disk at $disk. The disks this runner has:" >&2 - ls -l /dev/disk/by-id/ >&2 || true + if [ -z "$disk" ] && ! disk="$(attached_disk)"; then + echo "appliance: no appliance disk attached to this runner. Its disks:" >&2 + lsblk -p -o NAME,SIZE,SERIAL,FSTYPE,LABEL,MOUNTPOINT >&2 || true die "give the runner's pool {\"disks\": [\"truenas_appliance:nightly\"]} in its extra specs, or set APPLIANCE_DISK" fi + [ -e "$disk" ] || die "no appliance disk at $disk" disk="$(realpath "$disk")" # Read only: the appliance writes to an overlay of it. if [ ! -r "$disk" ]; then From 2c671e5894047594050fd226d4d505f4128d5d0c Mon Sep 17 00:00:00 2001 From: William Grzybowski Date: Mon, 12 Oct 2026 14:41:53 -0300 Subject: [PATCH 4/5] truenas-appliance-runner: shellcheck-clean stop and source, TN_SSH only when ssh is on, README on how the disk is found --- .github/actions/truenas-appliance-runner/action.yml | 2 +- .../actions/truenas-appliance-runner/appliance.sh | 12 +++++++++--- README.md | 7 ++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/actions/truenas-appliance-runner/action.yml b/.github/actions/truenas-appliance-runner/action.yml index 8ae0c2f..b00d59b 100644 --- a/.github/actions/truenas-appliance-runner/action.yml +++ b/.github/actions/truenas-appliance-runner/action.yml @@ -116,7 +116,7 @@ runs: echo "host-http=$(get TN_HOST_HTTP)" echo "username=$(get TN_USERNAME)" echo "password=$(get TN_PASSWORD)" - echo "ssh=$( [ "$APPLIANCE_SSH" = true ] && get TN_SSH || true)" + echo "ssh=$(get TN_SSH)" echo "boot-seconds=$(get APPLIANCE_BOOT_SECONDS)" echo "snapshot-seconds=$(get APPLIANCE_SNAPSHOT_SECONDS)" } >> "$GITHUB_OUTPUT" diff --git a/.github/actions/truenas-appliance-runner/appliance.sh b/.github/actions/truenas-appliance-runner/appliance.sh index 7067222..8b4a30f 100755 --- a/.github/actions/truenas-appliance-runner/appliance.sh +++ b/.github/actions/truenas-appliance-runner/appliance.sh @@ -41,10 +41,12 @@ # # start prints TN_PROFILE, TN_HOST (host:https port), TN_HOST_HTTP, # TN_USERNAME, TN_PASSWORD, TN_DOMAIN, TN_BASELINE (the suite's contract, -# as the lab's claim action gives it), TN_SSH, and what it measured: +# as the lab's claim action gives it, less its TN_GUEST_ISO), TN_SSH when +# APPLIANCE_SSH is true, and what it measured: # APPLIANCE_BOOT_SECONDS, APPLIANCE_SNAPSHOT_SECONDS. set -euo pipefail +# shellcheck source=lib.sh . "$(dirname "$(realpath "$0")")/lib.sh" DIR="${APPLIANCE_DIR:-${RUNNER_TEMP:-/tmp}/truenas-appliance}" @@ -178,10 +180,14 @@ TN_USERNAME=$APPLIANCE_USER TN_PASSWORD=$password TN_DOMAIN=$name TN_BASELINE=${baseline:-none} -TN_SSH=127.0.0.1:$sshp APPLIANCE_BOOT_SECONDS=$booted APPLIANCE_SNAPSHOT_SECONDS=$snapped EOF + # Only when it is there to use: a suite may take TN_SSH being set to + # mean it can ssh in. + if [ "${APPLIANCE_SSH:-false}" = true ]; then + echo "TN_SSH=127.0.0.1:$sshp" + fi } snapshot() { @@ -237,6 +243,6 @@ snapshot) shift; snapshot "$@" ;; revert) shift; revert "$@" ;; snapshots) running || die "no appliance running"; hmp info snapshots ;; console) tail -n "${2:-100}" "$DIR/console.log" ;; -stop) running && "${TNCTL[@]}" qmp "$QMP" quit >/dev/null || true ;; +stop) if running; then "${TNCTL[@]}" qmp "$QMP" quit >/dev/null || true; fi ;; *) die "usage: appliance.sh start | snapshot | revert | probe [baseline] | call [params...] | snapshots | console [lines] | stop" ;; esac diff --git a/README.md b/README.md index 27c0c7d..048062c 100644 --- a/README.md +++ b/README.md @@ -530,8 +530,9 @@ is inside the runner**. The job runs on an ephemeral GARM runner from `iXsystems/github-truenas-runner`'s TrueNAS provider, whose pool attaches a copy-on-write clone of a prebuilt TrueNAS image as a second disk; this action boots that disk under qemu, inside the runner, rotates its admin -password, snapshots it as `fresh-install`, and exports the same `TN_*` -variables as the claim above. There is nothing to claim and nothing to +password, snapshots it as `fresh-install`, and exports the suite's `TN_*` +variables as the claim above does, less `TN_GUEST_ISO` (the appliance is an +image, not an ISO), and with `TN_SSH` when `ssh` is `true`. There is nothing to claim and nothing to release: picking the runner label is the claim, and GARM deleting the runner is the release. Capacity is the pool's size; jobs queue for a runner instead of a concurrency group, and sharding is a matrix. @@ -554,7 +555,7 @@ jobs: | `memory-mb`, `vcpus` | `6144`, `4` | The appliance's; the runner's flavor needs room for it and the job | | `data-disk-count`, `data-disk-gb` | `10`, `10` | Sparse qcow2 files | | `ssh` | `'false'` | `true` enables ssh for the admin, on `TN_SSH` | -| `disk` | the attached one | `/dev/disk/by-id/virtio-truenas_appliance` by default | +| `disk` | the attached one | Found by its serial, `virtio-truenas_appliance`, where the box keeps it; TrueNAS replaces it, so otherwise the one disk besides the root disk that holds a ZFS `boot-pool` | | `export-env` | `'true'` | Write `TN_*` to the job environment | Outputs: `host`, `host-http`, `username`, `password` (masked), `ssh`, From 10f92626262a61cf9f4205e39213723f8724b4e4 Mon Sep 17 00:00:00 2001 From: William Grzybowski Date: Mon, 12 Oct 2026 14:45:04 -0300 Subject: [PATCH 5/5] truenas-appliance-runner: test that an HMP error exits non-zero, which revert relies on --- .../truenas-appliance-runner/test_tnctl.py | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/actions/truenas-appliance-runner/test_tnctl.py b/.github/actions/truenas-appliance-runner/test_tnctl.py index 9dae252..ffe7708 100644 --- a/.github/actions/truenas-appliance-runner/test_tnctl.py +++ b/.github/actions/truenas-appliance-runner/test_tnctl.py @@ -159,25 +159,36 @@ def test_qmp(self): seen = [] def serve(): - conn, _ = srv.accept() - f = conn.makefile("rwb") - f.write(b'{"QMP": {"version": {}}}\n') - f.flush() - for line in f: - msg = json.loads(line) - seen.append(msg) - f.write(b'{"event": "STOP"}\n') - out = {"return": {}} - if msg["execute"] == "human-monitor-command": - out = {"return": "Error: no snapshot" if "nope" in msg["arguments"]["command-line"] else ""} - f.write(json.dumps(out).encode() + b"\n") + # One connection per call, as tnctl makes them. + while True: + try: + conn, _ = srv.accept() + except OSError: + return + f = conn.makefile("rwb") + f.write(b'{"QMP": {"version": {}}}\n') f.flush() - conn.close() + for line in f: + msg = json.loads(line) + seen.append(msg) + f.write(b'{"event": "STOP"}\n') + out = {"return": {}} + if msg["execute"] == "human-monitor-command": + out = {"return": "Error: no snapshot" if "nope" in msg["arguments"]["command-line"] else ""} + f.write(json.dumps(out).encode() + b"\n") + f.flush() + conn.close() threading.Thread(target=serve, daemon=True).start() self.addCleanup(srv.close) self.assertEqual(tnctl.qmp(path, "human-monitor-command", {"command-line": "savevm fresh-install"}), "") self.assertEqual([m["execute"] for m in seen], ["qmp_capabilities", "human-monitor-command"]) + # HMP reports its own failures as text: cmd_hmp has to turn one into + # a non-zero exit, or revert would take a missing snapshot as done. + tnctl.cmd_hmp(args(socket=path, line="savevm fresh-install")) + with self.assertRaises(SystemExit) as e: + tnctl.cmd_hmp(args(socket=path, line="loadvm nope")) + self.assertEqual(e.exception.code, 1) if __name__ == "__main__":