From 344646210efe5a1de96afda0823ff50530162b81 Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Mon, 11 May 2026 12:55:05 -0700 Subject: [PATCH 1/7] feat: add external host driver runtime --- docs/api-reference.md | 46 +- docs/img/external-host-cowork-reporter.png | Bin 0 -> 71836 bytes package-lock.json | 32 +- package.json | 5 + src/assertions/validators/toolCalls.test.ts | 4 +- src/assertions/validators/toolCalls.ts | 12 +- src/assertions/validators/validators.test.ts | 12 + src/evals/datasetTypes.test.ts | 62 + src/evals/datasetTypes.ts | 26 +- src/evals/evalRunner.externalHost.test.ts | 362 +++++ src/evals/evalRunner.test.ts | 4 +- src/evals/evalRunner.ts | 334 ++++- src/evals/externalHost/builtinCapabilities.ts | 22 + .../anthropicClaude.integration.test.ts | 70 + .../builtins/anthropicClaude.test.ts | 835 +++++++++++ .../externalHost/builtins/anthropicClaude.ts | 1331 +++++++++++++++++ .../builtins/macosDesktop.test.ts | 38 + .../externalHost/builtins/macosDesktop.ts | 441 ++++++ src/evals/externalHost/capabilities.test.ts | 24 + src/evals/externalHost/capabilities.ts | 18 + .../externalHost/capabilityRuntime.test.ts | 155 ++ src/evals/externalHost/capabilityRuntime.ts | 336 +++++ src/evals/externalHost/driverIdentity.ts | 77 + src/evals/externalHost/hostRegistry.test.ts | 88 ++ src/evals/externalHost/hostRegistry.ts | 85 ++ src/evals/externalHost/index.ts | 69 + src/evals/externalHost/runtime.test.ts | 56 + src/evals/externalHost/runtime.ts | 139 ++ src/evals/externalHost/schema.test.ts | 68 + src/evals/externalHost/schema.ts | 281 ++++ src/evals/externalHost/types.ts | 274 ++++ src/index.ts | 34 + src/mcp/response.ts | 5 + src/reporters/mcpReporter.test.ts | 71 + .../ui-src/components/Results/DetailModal.tsx | 961 +++++++++++- .../components/Results/ResultsTable.tsx | 139 +- src/types/reporter.ts | 50 + vitest.config.mts | 7 +- vitest.external-host.config.mts | 10 + 39 files changed, 6488 insertions(+), 95 deletions(-) create mode 100644 docs/img/external-host-cowork-reporter.png create mode 100644 src/evals/evalRunner.externalHost.test.ts create mode 100644 src/evals/externalHost/builtinCapabilities.ts create mode 100644 src/evals/externalHost/builtins/anthropicClaude.integration.test.ts create mode 100644 src/evals/externalHost/builtins/anthropicClaude.test.ts create mode 100644 src/evals/externalHost/builtins/anthropicClaude.ts create mode 100644 src/evals/externalHost/builtins/macosDesktop.test.ts create mode 100644 src/evals/externalHost/builtins/macosDesktop.ts create mode 100644 src/evals/externalHost/capabilities.test.ts create mode 100644 src/evals/externalHost/capabilities.ts create mode 100644 src/evals/externalHost/capabilityRuntime.test.ts create mode 100644 src/evals/externalHost/capabilityRuntime.ts create mode 100644 src/evals/externalHost/driverIdentity.ts create mode 100644 src/evals/externalHost/hostRegistry.test.ts create mode 100644 src/evals/externalHost/hostRegistry.ts create mode 100644 src/evals/externalHost/index.ts create mode 100644 src/evals/externalHost/runtime.test.ts create mode 100644 src/evals/externalHost/runtime.ts create mode 100644 src/evals/externalHost/schema.test.ts create mode 100644 src/evals/externalHost/schema.ts create mode 100644 src/evals/externalHost/types.ts create mode 100644 vitest.external-host.config.mts diff --git a/docs/api-reference.md b/docs/api-reference.md index 36f6400..97e8ca4 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1043,7 +1043,12 @@ interface MCPConformanceResult { ### `EvalExpectBlock` -```typescript snippet=src/evals/datasetTypes.ts#L186-L277 +```typescript snippet=src/evals/datasetTypes.ts#L190-L288 +/** + * Unified expectation block for eval cases + * + * Mirrors the Playwright matcher API for consistency. + */ export interface EvalExpectBlock { /** * Exact response match (toMatchToolResponse) @@ -1102,8 +1107,9 @@ export interface EvalExpectBlock { }; /** - * Asserts which tools the LLM called during a mcp_host simulation. - * Only meaningful for mcp_host mode — direct mode has no tool call trace. + * Asserts which tools the LLM called during a host simulation. + * Only meaningful for mcp_host or external_host runs with high-confidence + * structured tool evidence — direct mode has no tool call trace. */ toolsTriggered?: { /** Expected tool calls */ @@ -1125,7 +1131,8 @@ export interface EvalExpectBlock { }; /** - * Asserts the number of tool calls made during a mcp_host simulation. + * Asserts the number of tool calls made during a host simulation. + * External-host runs require high-confidence structured tool evidence. */ toolCallCount?: { /** Minimum number of tool calls */ @@ -1140,7 +1147,14 @@ export interface EvalExpectBlock { ### `EvalCase` -````typescript snippet=src/evals/datasetTypes.ts#L27-L139 +````typescript snippet=src/evals/datasetTypes.ts#L23-L148 +/** + * A single eval test case + * + * For 'direct' mode: toolName and args are required + * For 'mcp_host' mode: scenario and mcpHostConfig are required + * For 'external_host' mode: scenario and externalHost are required + */ export interface EvalCase { /** * Unique identifier for this test case @@ -1155,7 +1169,8 @@ export interface EvalCase { /** * Evaluation mode * - 'direct': Direct API calls to MCP tools (default) - * - 'mcp_host': LLM-driven tool selection via natural language + * - 'mcp_host': SDK/CLI host simulation via natural language + * - 'external_host': Real external MCP host driven by configured capabilities * * @default 'direct' */ @@ -1172,7 +1187,7 @@ export interface EvalCase { args?: Record; /** - * Natural language scenario for LLM to execute (optional, required for 'mcp_host' mode) + * Natural language scenario for LLM to execute (required for 'mcp_host' and 'external_host' modes) * * @example "Get the weather for London and tell me if I need an umbrella" */ @@ -1185,6 +1200,11 @@ export interface EvalCase { */ mcpHostConfig?: MCPHostConfig; + /** + * External host configuration (required for 'external_host' mode) + */ + externalHost?: ExternalHostConfig; + /** * Additional metadata for this test case * @@ -1256,18 +1276,6 @@ export interface EvalCase { } ```` -### `EvalDataset` - -```typescript -interface EvalDataset { - name: string; - description?: string; - cases: EvalCase[]; - metadata?: Record; - schemas?: Record; // Zod schemas for toMatchToolSchema assertions -} -``` - ## Next Steps - See the [Authentication Guide](./authentication.md) for OAuth and token auth diff --git a/docs/img/external-host-cowork-reporter.png b/docs/img/external-host-cowork-reporter.png new file mode 100644 index 0000000000000000000000000000000000000000..fb279b13b06bfaa4ec9fce6f1c097f4eab26cd03 GIT binary patch literal 71836 zcmdRWWmH>j*CzF+1qT=Z_(oJR$LR@tt|x#6n9N=3GNms?xaX?g1buy z5CU^b-*0Aq&7WD{nl=17D>>)n-22{g?d#fyAVqme!UxYE;NjsBN_`Yl#>2bw6A$m! z`ad^;Pjblyv+(f##gh{Ipz4;gJ$uJ%bd{>>U{o(3k2@>7vbLe2Ve58h3b=nq*swnl z`^@jH3GkUr)hI6O%7ENu%j)UtN|2E4u5-yuG(#P2JSD~}THq~Wa8Moi0pRaP zV*a0>!o&Nt{Lj1VgU>@W^VbJ>65*e3T^~IYy!r6@;M4K{o0I4qF52@kxpK`O+>4ML zX(=f|mkSdQ5093l=b@{z>vh}P+g~$-6PY+UIhU8o+|{KeV92z8cqvFpOJ*J=zB%4@ zhwB@7RV|X9&YXp4X=y!>Ccbg+&x+=A)(pHx`g4p>U7)!ut7rV?yR9{IxbCqr36Aej z^NR)TvH;bID$i!V>ff!6DzmkZ#v86)n-g6H4Dj*RVHLbC{D1r)s0v%YqoCk}9LZHL zItP0dj9)pOy9V@xBr?5yYq`H4$gEnRRbuZcS+l94@6&LaAeuV@Ki=f$h(4F&tM$B7 zwKY?cS4zJv?N;JLV=7*M^1|qK%&!dRKva1|3tGExnbRD>>@;OW@=oAF8_`7bf=Lyo zm76Vqp%UQX;NY8EwjQ_TvGKxh=nd6zI$IwpLH?GcP)k?-)P%e^2_#e2&o$&@SXo)8T{qa>zcuLGY0Ha=8U5<-p6lT3 zOor(m;dj|d6<%8#>2j6x@mMV3T$8=pl&l)uvi07d4R00{vcG6bwVtj4pB-B74W=E% z^A+Ufb3Gr3cobf!RdW`CKnZ(%>SJaXVqF;#x-3tq!d~=I31JtpAbs!M{-9Y--!l&c z$}e2rL!N@$^)jmIR+ny_gId5bcpNmFo$x5nvMg;1XL`Mf5OA2UdK8+Bp&FyKj zxXe23Qn1Cw8)B#ztTmvtL^W3q9-o;M*5~zI#uzl~SD7t|K1pMAQu6b=Y5LGW?&{ZM z&qfs}x9RZQGeGJ7{h*Ne;z6CWE~t4=MSgzJ;ira@8(oXpFv_&kLNTrKJ(3^!e~gA_ z%1pm1fn;hDy^H1+xPWP>Nt&9PQt){-njX=E78i|-mriU9W^*+(+P?XdtfoqrjO*y= zz)vq!{O%qXKT^cSk~%&=@x)LlJCu+`uBSs%=P`5cRrZ&`RdhSm#y&5c&?}4x$|WT+OPe^4J525 zuRCv@u1}p^yFBCc4$0b}K_S^XduC%JQ%^>3PF$1-sq_9Xl;6FR$aeFCuXBY(MWig+ zX|CHCVVv1BA+P1r9u@PauY>Cx4A!0M?ZXA=A`?PGL%qIPkAcqtk87wdE+p1fx%Rgh zqPJzB0yQ*trH0p+RadpYSssn zT`srP=;@|JN2?slGutwJ@T)Y3x}9hLt{^pO?V|s2bW+TcNqg%7Z#i69T!C(lEBFINf!0i{q z^jF76w%Kp5#uth8JPBtjO#}I)1t-%1m*bHB`ortmP6dtr@omKd2mLa{%pS5n=WpE^ z-MRyadmOUI}?NWoLIlkK((BA*gy4#v6%ec?aQr zNB8!cisQOlVH*JvPMPzxxrmh5pXrX_ErDS##y2aW@>j*iHD98l@{Ikg#v&vjO;U%f zrKfYr?0;h4SfNpowX|m>+Q9qZjKRQTfic@6Ki|^8=g85?sgN4?2=3`pq}bxTT3E-y zlHLhDA`K_!oOx4SXu*|~*(Cy*`L5DCC@AbF`l{xUF{MtW$qLB8=QY)GfZX(@cI`#( zo-jGTRd*NE8+O;BCPfO!`^v)u`~4#$@K1Mw%9m+00#j2diV6!4!(zXg&Rw>}x3v!R z%O-P!H9HWXx#hA2W!^Lj@?9e7~T##-ebtZz+LHkeJwk zp0PGIX5F~1i1RIqCGtfDYF@d%=V^6yXLU!UW~}KUzHNU} z#e4Wa**`@{HMP5ZS+#}CMkCDR$aiL{_M#EZp7%!5GFH3P7E@Wqm4)Th2tn)ELc#4tu0( zvma7TH4H_-$UZ`bGZA*qZYU~YLbB2EH&(0L?zWeEqN2t00s=E>o@UCZ6r05->@2#C zul5L55SjLw9*5=^=gbH#P0iQ-J-xjiCuifnsb{&jUk9`X5N`I~eA>*n5_q9jqTf(f z`yex*|6;{@Wp^9p_);d?)XYl#V$$eO)b!5uW;+tJ-MBgcx za3Ucpj>Sk3n3OECN#+XblbQ;eRdm3*S`L?Qr8hU`CT=r>a$7w0sRW|W)qv&fL<^Xn?P2sZuwWDoW41 zvkNEJTx2RVp98>$aD3_MXr7_GEk*f<(RgzaL|?_Z3lQwhKv0aMQnW8_b}`U)S2*rY zzF;DaPl0JKEb-+-K0ndnP)P$c5im)$9TxAwNl^KGgooY>6@$2E4BO`-LLAOt0UtvDen7-Wvh2+xJbV$L5;lE<+b zsFdbxCoNIdB~tc~6F#5lnuZOJ?KIHMM!W-$QQ^~~`dr~7eJW|YV^yL?Mu)v>Tcm7S zkqHTvM#ys_;WO=%A}>yZX8YuJTWN^w>4+<|XP20a40m8Y_fell#Tz%v?&`bP=KT^T zY&JU3z{jpxD=YGv>>0mD9<^)X;#%gPU37c%?%hPgJAUuu4Ny)_4tfiji0<;@)xH>A z3V+SJX1eB*J(Q6}>XdgV$89qzW=T9fJ-stg=PC)=OXj%Pl;H)6=^>@}c3q?~10faP zd`O9JRFvf&TjnAFGXZ6}m?3$Tj>Lvi7+|lZzGXY#Xje!?#DH_of6_G%j-#fkMZDVLS zRH8sc2ZH>mP2Aw&pQm?re;3CB2&;tis!+M3Mk_i10JB&}l#zl0CXN-t&Z8_1kIksD6_?vL4XQ~ZI%Yc#VSW-uQSL#<}YO3IFWbSh2`Sy}4)Z40+ z%9kxB{xQ2$eVO^(7XX}^Z=r!~dqT4$dCBKE81cp17k3b{3lA)sF6ExXeS1J>-&sy< zjfAVC*DgBz(@YLdRoJb#p3`Cap|(vIimZiX&wf|h`_Z#HqCS$a82lP42KTvuwR_*s zp=O`+6k?Y-dG)p-lhqZUeo^25&bNO;&R*cl`MAKF8+WVc=vA?#{zH6_6Z!{KAhIHqNeJ&LK#%w6-U#aWw&8TAbUI zaJ;sh&q65k0xx{q>k7?qXl{4U5mjeuimK}N9Owp6-{@!2rsszceL9AL$$G=3F#vy0 zPH6U>Y)rx(QO)mzoSV;2;`JIZ{wV@9zIx8q_sGuK;=WlcLFBU_fnP+s`aTz?PN@X7 zryMHv9(J{NR$C$)y&Kd4n9P$>&*O2iWNEo^h+*$zH!RquEYjhLZIt~3Bkd6VskqEa z#2l1IuV)A3)a$6wZTN|>c z38NC`c03W_?PT$X`W`}#%t9f&$dtL}_|BaAN*9-E+vc|`8w`}O3(n-#Ug#YJLP(H$ z?$D+;HTF%v^bc}Q*_&bX!}2yWKNFno9UOEUjMBv33=gY1cn0cn`ARHF@a8&q^EI7K z9R(zbvzFQ*;QV9VH=9pZRW08nB_>t@#r$Dta`K#1n^*hq8|;n8CwU8g(heE#^F1#+ zKy9t9+4osC0vdk=heeu)H?uP`G1CX|55$lpE_{Bc{O}ioL)qhsaCwG}&gEPxa>2`>JNWV7=~~&TWfXF=C&%4kxXpj?sDa9t z^SFh&-g%?c<4kxw>!Y^)JA|{k71ayLaDOfMiTo>_hxfAEm2a;j74V!1`MD{D3@hDR$EXib*2!X%honFIy(cx;J4Ut< zt6P#o4)r-6bVSHTjYjZWCd-HD2W1cP8^s#&VjX9eQuqJPi6SJBK_V zpiCMiPZ8nDUORGe(YAIEV-pdWMJ^5>PyBrp1?hgD9h`PHOgPk}$Lu^mO=UJ(Ft-{z zKBrS|Q)Tk+S%Dz-V$13-wf(PQ|B4&{>-zb5+iVkBPOmi#N^|@pEDT)`y?4AfE!re@ zmuS6cIqgN6=`>qaYYr>_D3{gg?@Lz!^A(TNlFdU<+3%Ka+Z$?^O+_2MS63xlV}f| zuINP^&#;r|$95V7P|G>Il*{>scd@weJDWSUz0!;Z8Lt9Y=w=hncder!h0~=g`Q`@s zda%`1=MaQJ?L^u1dnu66`8+Eb-A3oknarpT%x1R98o=sKJ2OpceK|ELz9XH4Ca%KH ze5yP%TP3n_kI|^+ovG@=xM{!Q4<@RWe2UDNZ;8{+E4@fdD&O>nHDHx)xG*hyL;w!p zqY9P-)SzBzBP~Db>fCc5g6ZX6%+ABGfs7W*g=~J1qtnD$4;{v_CT;hG^0s#`_*vb(+bw#s9i1x@J43!w2_6xl{8O2~ zQ5>q7*1lZd+VfE>(MjZ9BCl!=Pn+fEwnD8e=W{x)vZWl(JaLa-fM3CrO0ZOaA`PSB z;1K*Gv3a(8elUlL7@%wPcHHK?u`7)1&4LRDYvvcnrl{3it`w;E z0$TjY?CCaB)o854FKX%z6ovIDbG*JgPSDcriW{RL<*Mzvo;k81lz4LDh>%xqSbLL2 zE_nVU?~x)HXhLH5Sv$iiE{9=z0OU^N^2koab4C%si?d$m&1{lG{l{C~#^^*KLq4 zJ~h>Q{pt~lDuns*7su?IaeO+BF>R2Umqv=aD80G1yAcxCymKre%{b| zzI*6jZbBso{u`0zlNOJUr*PiQ)@LuE?uSZ$BH{gHzUGjq;4d#Kz_Jb|dM$#+@t#pl zvB>76M1`qimymxjjd0E7>HWRwWAbh!QjL1(7m#!}}9H|z%CDCVPt6ww0h)8t^2AM5K0MOcF zVR40mAJf^6icp99`Qd)vfD^1I#Yc8uW)X^ceX`YT$0(+Lnog2#INHPxY0+lWny{12TGDn;-WC1Q70ma4;$$ z^qhbO$jnUL?Ub7ib4=*ol*}8S((bV7PLh%5>VBNA#P7P3%I_ARM71k zR_sLX+%^Cy7BN;{38$c-I1KY%+=vgOL|7Ull5r|1$h(`?r>=Wb)EK=Pohpl|EPL;* zX4?W?0%m6Bs;*fLeXma8S}nli)M0{l`*|gZ0bY>p*!pDdCY<56N(5>? zEtJoCKHQ#kRfPX8zM8kkx)#ASYShymPtXU-M@e5RV!v!6FP{Ff;QPAe`A>TW@PdU} zPyNv6W7AERbzH&m#hNrbTPI*09*e1-w%LKsPKi3quHmnPp30i!s$C(kV`BiE6{n`| z%r6kJ>orL{+-`nzc0ib1Ep@9l7>bBPhs@3}y{@smn0WNqu0OR|46eQi^k(>D`DD)& z8`R~bKma{T<>?@e3OH`U1tJ?9Zr{7Jx@F=7ge%!@hVlkBqEUU4 zm7CKoNka_Kx{;vyh{ySbE-IsiQf&Z4Vdmvk)f2@CZ18fJoY5`(hx$XdOLjE@@&Lds zt|K#LTFPRXHM|zf0!`jCr3g5$9x0$yc0-h#LZtPFg`h`ROaq3}Vxmyx(+&xURq{OkByq9$Zwb) zK&T3WrQWUS`_kU%S~Iw`12_e;^#p#Mw?Ie67Pz82>bQN{Qy)6jXS2m+mAiVaZrM@M z=4KId^3ZU~#Cv)UDBIHrF$%nrsN?tWK8cKQ+?3<#>s`)e=v~-XTB0Np^lZxG%asod zdv>G=aOJx?&B2&`f`@dNSNKv?W5sh(v9ZWZ9R}V(_KnWZ)Jt$UvrN5GSw^6PWdFcG zSJ?9cl!DX6#Dch*HNd%KoHa`&ykgx1h_fD=y@{akACci=bt($79&SB{+=hk=K!(~H z+cNkNl*L3P@2x^{xrH#8a@r)P^w}BeO7jFF-&AT;2f+g#-X98-m_5`pxz!{L_OP-V zoJ-En-vF9R!pvQVLAvAJX?B_8m-qWzg)dKV3Y>t7Lnj;Q05B<*d(G6h5DICP;4X;! z7SM+FbTQr9ywCE1+uG?AD_;*ZSNy&UQ3Q=Zr{b;)pk|i?xv?FJavD6-HX`g^b7;k1qN2u9YD%}&tfl8Wu9(#0}Fzo zds+9F(%c=sW|2GWHv?&jccZh@)HMZ?H>xXIoli^Ap}IAkKf8FRc?GU0Qf)Kv>ob6- z_p!D@b;$uvXe+g_?P)HfiO-F*zTE_L1`}{_l}EDX!qQTeA==;;7USUR8XY4!bv2xT z*!XI4HWQ`mNBj0mmeHmiwwm<_QrVE=cwZ``%;J*Z>^2Uj5&Kzjn6O97WXA=fMNO3# zV(kLOyAD(;n;*1>T64@se&3J|(#VV~#THlH_f`jB-Ct8q|ILWafd_0gKY z<>Z{LU$gL#P>-(;MP0acR%$SOe0796))M6BHmG+AOolnm6|>5e3uMbY)M&=-b^$tL z6iLl<_HJsh$c7C9QvHYiUJ&q1uxe?m5g7ALu~T9jMWo z2+I|i!N))b-PH=$mHOR5Q69JNK|cHQ{TCN-hpEHCu9RK#bzMMu&9z8h>$>s<-?@7wFYq z(8yXz;Y@8V?U&Y>?BwM$Ts&pa_h6j;N$`0UZe=A+lmTp0DC7nHy3As5QC|9i)tA>I zH6h01-SW%`5s{sjpF?rtpt@;Mq|X&AI#pA9<*&^RCS@WM@rIUOKwe!HcSv|2f+<2oKgN zH|xqt4NT_whl3MgDP(SDMT`E3=zj2UKbgZ)v*|p(SXbEdz^bV=lYyOJXQXAQMSVEJ zjL>g%v;17K#2dfMTdKM>wRp0Be5@S$>61EDWfiAZpFV^5EJ{E&ifJej7IHXl zPxHP=zZlH;VAW=qbPFka+YaKm#aKEQ0c`OnW94g~v@1i(DcFWO`liV8Ce2?YG zndb0kUCE6=x9y=atV{9ER2}?!P~;b`+^{P>htO(jdUl(*HS?>ZiBn3tM(_7-a`zI= zp8nBoUMFda>dH5^%It5JWe1C>DR#hARMWQS(+XVq21k7cB&Ili#b<(feeFu{qqvDF z&E%w{kNxRQvL74Wx!d;e$B_A2=u8kLb{!p*7Ul7@cHnzmqLw((xL?=csk>)yvb2N! z>P0Met|P#{Ztg~4)HHI_yFrTJ+Ljw+TGN;pTT{hB=~+>^$w4)q7&X#xfv(<%X^B`6=-<&+;_MgCv?{akZ${Qa_W z0|Qsi%vzS%vAo?1mo1~EW6Ghkk=vzO=VpP2ri+C;DI7-0~nJW2w%+6F$DZ9V@rBi`ZUm2I%_=57JF~o{mnj zVq@?1|Cqa)sdaB~z&d%*(c!9vNfX6#HLi_QYukHD5LOCP!q)&d*zt;FBDJ-&M)OuV zfhvM|Am1zKxVgQurDP8zIJF;N+Xv+b_48Tyg2fLN$_eSqBo7W zST{t!)M%?~Zou1c_;>JbaWDK}e?Kj_%-w786qv{0Y%J6tP^e?);>-T6FP^!GR`7;C zT{iaVvt;f6k74}E1v;A?+9>aNCUcRK^fQAZNZ%1ktqp5Mn zxT0y(gp27!ne1|cj{%3Xdwrv=qm@6j@W-2vFNyZNE*ts{SW_z8mP#BoU~LT#zcn5l zlzbkD98e&D4R^LLFVP(WBhsTcE2S7MB^A<27FCo2 zouXUc%E%I8p0bYZIqi=577-{V8NFwyTmDBDFV4D4zZlz;2>c~AqM)| z7nEdVPu5f>MH%l+RYQDEX8e&`JO%9WEPAnVY5L6X_Gj~<-k7C+26{mO^z1l*G`Bz& zHcj_4GmJcr$@H0D+YMdJY4g}}cpbM#=MFwavjIj}X=z%;X=^gpi2P^7&9;|i{-dn4 zoV*llS&Q1BE=D)>d!$>s538@EW5rS~yL5W{i}6fbt|9{!BA3kY*Y!jat&(a6cJp;pjz9mPx^rfITc0Urkwh=?v5a^0E9#OekA`Ui`9AYaWUN znnscAY(3|MJ4yLW^ZFM%rwYvz=}!BTMI_$`B%neWX^Pz;l>`A=ittAz6!fGnG`i&6 zE>BTT+RGcR!)4oOg}z(WT2MtE_;AzO$cO_N?N|z|7P{Kb>Y}8;UMzRfHGU5XLGZ(9 zy5xb0tH93C_0jZ!aRXWYN>^iX#48%|(B0z44~oLb|BkEeMssZWwRKJnw~I+3y8AWK zfD~?ROqW?$#IiM7=yGxFA6E4IV?VWj%NMm`$=Tml&J9H0#JW~1r;%GKKoqKDxAS&2 zCYSekoaKBC?s@7Y{)S~F_tVw}KvRpHQJ{JUR_U~w9o~B5IeW5qE}LI(vjz3!qhbZJ zRy&x->E_>Xasn*(!4G%qtXBWhe|m&s%75o-j9+@&x=j3|bw4vGNnU#j>@w%Emz!Cf#N*(* zRk!*F`pe6?ly`Smx3?|WiF&vJptrudm^TLE#^v&|wuNqvzVq4qDxWLt%u^_Z`{`wy zH3+zXQfh3Y#@-;Sti!F__a)f^#J}nP6My51NuM!)<1Oeh?cqO%7`0vzczspZkLMXe z65%fjNgk3-favI)v4e1c%E?nlmPjq#RX%MWVMmj_IbSb(A}=3CxH-=Fj+vBn&97m+ z`Dm;UAd=-g4#ltL&C<;gr0AwqGgo4TnU&@1937*pg%ZzBGRVeB4t|pqBj>A*5LoQ3 zTKAdN&S&4cs{!KV6!bW3Y=5C3ETAae$Xq9bH&p8Kek+H!2(xWNCrBe&9wo9iSNUN8 z>={l0WFq9o=J+83G&T;*66o+0ujsdE_0&9i6)UmV*4Nc( zz2`2Wz=rkpC0&|2-jz&eQa`qst40c(cx*oUGD;~ZTsf~oZ5z)ol1l{H*#(<% zXTMr#&T@vWs!mQ^02(?$U@)~=v*E)uFdf&3>kRR4^7l`SO1?Qu4N1SoNI)Gbj~=3i zPxJFn`tmz&tm;X!>D4v4^j#XP73FX0iK7k66}Zm2;+C=k%iNXABM>_gsGks&aZ!h#rQv=&b>^jnb`jx zROdd`SDKDnW7n-V-}`_*e`;-QEh!lYXyE}V0u?T}RYcQi(|jQbtCr=~HfD*v0vK`Q zcRnv%rfB)_MI2r-IlDt)W?JWXU9KrvOx^DW!czxEhzJuJY>o_=YpkmHn3+Rw20Gu8EbJuPPFmb7M*^CvxBrxghK_1I zl-wQyAr8*Z9N)^!G!_(`CDe-}5g!h%HGw-|btz;A>csx2Ab57oF#!Ht(b6_^kv`wq zp*`WDuU1cK`VTPr=bfGPh88cGq}0jNCr>(BI@_nBoUUiw?F=5-{S*UW5icvhjLrVQwT{`}fK{;7W4Z)$U;vYo}ld<~Cl>u*CczfA>KO!N#?~fxA@>fjNGD`L)MiZ z-Y(VGc?FNsCWjt?-#H{CX7VoTIXT(BP5x3Y`FH>2s2M#j{I*QM;A!pXs3PMCyh9-W z_^*q8$Q{|JsN^eBkB^AjSo{PwH?}k~%7{rDCU|F}-jknLUJDs?`Sb1upn4b@Evgi< z3{cns75VG~jI~PWoixLIM6Ecq$WBu8`VvKv$298| z6@BA$On+}-`u)G5Wpm6HkQ&Z1+blQv)zUgR)nER}f156spYYMSj9$!-USQ6+sI`wpTC3ql-z*m?txuSO!@13;8!Er$kw02i6QG zR4r0D@5dv*4An8HUkJ{CcaU)RuDDWdp~odDGWATid`^f784M;8vVYONb5PaMK zP--WeqZtz1wrwdZJ0^qRBqTTY+iUFsuY(E8`7S_XPau2uV~A11vMM4ig64=AtV`&R za!K3#wehWVx*!gbz5I_HWRA+}TcmkwGCR^FE1;sp&wuAE%UXubHrW{>CoAD)s#7Cf z&71n8$$nnaotOTn=X5YS)?sfy$k9+EGv+FzLoa8x95m@jTJl`vc5Egbj0hW$l?x$w z6cwcc+TnWT9HqOPP5$xKZj<#_B@3C-kRRn6 zQ(p-5)UmyULlD?qyES} zCmkfknCwac(PdW*5cU9P!1V@s`TbHT^u6{1PuopM83JMB#uJ&D@XP*ZsqzfB z{7H6sy@p3-P3`tl6mxZ|@vLz1()!^E9uqC@?b*|fJQ z1tKSGd+-D9KTy3TDPiLyg%}~Pw>Tx83d_303 ztm+s^vDpN;owu8q6TtBvh&Fg?^h#--Q1QZTaK*2C30SO70nh_o6zFU83@a;rDXc2-|kFt zl$T1_dx#ce6$1!xd{^GYKVnodN7B+#YS+b9oSC#Hh&3%b+gm(2h519q@RGAT$M^4} z&zxVrsaZk*nfdkl-r@JkiN=-jOCPO~$#KPX>op)?u=|^-w)!a!`?>4Kr%IQzQyRUN zVM1`Mqx%+gy0N@*t4lqHgnbuBFPE&;tz*WlRCW*;2K6>ROt2SiVAm|;-qs%Erh$QW zz^FH;(I6^Y)!fc)aq=u$L7JoR4)V0*HzrfePk^-YB$r((J&Rm#n&her&R$ue8h5 zU`Y5;>uDNA5>CKe98f0?p)LR;@>~sW>&qv;!!BxS2+0}QD6gm>qQp-rnF%XQJ43|O}(yjO%e2>IX1b-rr-VGFE3OoNzQPO z-@Ou|y#~kzKKg0lVcXeZR%376HtJKQih@t{ddlriJo!|1Tj%{6^_;%J;^K4V~Fz?MzPiyy! zI;W9y@ICU;hp#K&l-8G|7RlAPoRRjxGO)4MN-f+Mmp)yu(IU_O_~Cn{A2jp-xa60k z&ke~E9@Q($JsCB0e!0Gfm+=iLmZNWMYL)&lIVs^T-x*~Hzb`AR*(}m~wlP`Cu$+FU zL?#(u-XbDPDzzMh8Dt!QI+u3W&Aj9j)K5bu_dUOxF|riXnvj^JZf9SwH8OB^k!{WK zQWC8Jo+5`Pu%rj*Ls{tnQ9ARU^BMS3J5+)>U-D|Vh^2p^i)f1tykWZKio`Dv`YB1? zBXl@8IXE`;#tRU$07a7KmGMA-KcH^{uTWS<#H9g%=w$a;Yg6XF+~!?1jmw5%&hlb0 zv3sfw(VXv!^}@9pOnn2@35&%$i)Tu6CA(5AdRhio50QD5$FfrCBt~Fsazk`PQ8ZL$QXC<0z+PB3M1Z3B3 zZEi;-W}jIO_A*8Xs9|6!2{GqWlFfPCI$Q;GM3x!=@#ZtIit}!$E?m{wK;}vSHIh}Y z=BVb(THnj7R{7f;X`!`hO=m^;!zh83&~P7ar?rCB7d&HQW9o5x@Z(YeW}?sdhU2sZ z4lb3=ewNc*Fi&2A!wpbVWo4zs*vnd(=@dt{a~Es$j;#zsGcMHTSL7`|3L$&>A^|hS zc(S8$n`zJf(b3Un=@q40{G@8@dOf(vM}mL|cYrmp&Ta%()ay-g#T^3LyJSbZH+n0h zzYwSfmpeM6;h{3uuNvWrbB2A+%#ic9)sc%NcqJN-Zbg+4telYWG*@~f?bsyl!*E_^ zTJ>y4ZZl`#@$i#j2~Pe~h(k&fO2X)D!I8Dzo3*w|NvljXJ+(37704F^2mi{HD?L&wD4 zDk*eLH0>osrACX0iFFoFHSVPw&cd1M)zC$>W#8pmi_<{RjRpB{6x)UGj$~)saKIYq1FlueovFT7uuovu(-`4eW*Q`n+%>V%a%56-e^>Z$lz zm$)IbceVb>+}PHf)82{7^49o+!owB~$CT9VD(%@*!`avj^#3k}%7|r<@L2zQ+&824 zNU01V_lyH#QL|^pHrs?rkbwPxBOY^cR|OTMZPQ#uu$epLTO#t#jGJjJ{&$~a6)v*6k{z)7TkL@Vb)+bDwVp$=UB=P7W{_GJB5Ps^F~Px}&qo2S@cEubM?ZC?{}N zpsM9uPzrLbAOc9^no4^>sYm;b_I-u)9gL8gF5Oed`~~*e(%N5RXSWCXr58|<5+rsP zpl3H9Uh_hq?>)N_Xdf+?77??!=)?fmyUuR=P^I7-fkVeF2L=EG@*hq^F?hFn`Y)OE zndASOFpAawWoVTFtn=?3f8M5*h$m&TjEt#di3te_$%z(~ zDJ|FKA5U5=`ib4g3`IV3`xtFyBU|IhgsAju(1jQC9^hK8FRS8w2h5pmgZlx#=C4Nl z*{GvE_uWK-OoC$5;Cw{(TW;>EifS&<0R7V6i#sCLf6b_tRaO6ao|_NrMbK zlpL2X)$^NoQb#KxS!>ng?M%>LE&X1qd@f>mlS><*cmm*6iCN^UtGxMNl%=IP=OeT> z(C5i-C3Dna@!5#6_^Ivdr~h2W!L8KaFWWCCXH(QaYLWR=VNOvRG_UBu`H*{6F`w=) z&*WB`uNAN{Co3x}{dQJST5w)5|81a1{a*uI=FmUSrUcKV@RD8AeRywNj`99p_x~M_ z_kZ%ISj=b95`@rW2R43weuJ}~ig<4AA_sl|o0|_BDDA2`NGI{RTWNRJ$&CM%g%u$& z@t>FKYMMhsWw;s}|0Ov%G3PXWNyqKo5ipy-L3Xx@NT|`4V5|WqMBT?)ojX)Dn{l6{ zwIh!j9aueYs?+ zuRBP*pBId{&r91Kc7OnplS3cy7%qgpSb=dDoV~75fQ`V<4t`jc$VV&yeTxQUA~h;W zsMLJAk$9B_<+eR~hX}OxL2&wxD!LaGNN4T3JLTIt)>$vvl@rh!ogN=wG)GJd9nFJ; zF5vi*5)x!_o8eh8l?pxyI9;KFL}h{k7RdM7U*WZR`wSZ?I>W{4?bA z8^DFPJFH!{1)ZeTt7WD;Q>VH9@{I4bUa!cg0!=`PIm?;zF4KMP(Z^&F@yf5I#12LR zB*hFG9rV0`;`Y4PwtY>v{xR7TOjicF?cBu+>wgfHo|S`Yv{)%2)&Y_)Kl_ip$@EEt zqk2{jZp$cc_MQSIALaStQXAC6;uoOnjJ8hHw$G6ilFh(&bP%D@3u?(H(hDKc0lX8F z0!RBl*>;}yX%|k_x&rl8#lT0kHKa-hBXcKIo%#MpRAS&aeIakQZCpH+`&myU_8~M( zLH2s;7w^v+KDQ0GUifzfQ1Ux31WxwRO+^F}bjvVufjF6crhp07tIKgdbeYFdlb5kV zfIdVsSHg}^SkiOa(3^kVWUTH{?-%2AQEa7aX-yHi?rd)P{~+!?qncd5c3+n+JHoPo zbd}zv_n@HkfKsI^ReA^ME)|g8LhrqV&>?iBLugV%=q+>zhTg)tS?j;wan9LiydU-$ zdmp|yhCmXYazFE)^SXXlr*FxRdR1z1w?^>5P1)w8vFR5V6;NdYde)INjhCQP1L|2& zN=1qYE7+KSG&D41VEbUosBEi9vuM!HW>)oTw6*vRS?$mGcrA($0bz!|QOrD$z10Z` z|L1C9hY=W9)je62;`ug*X_tKtPgUI9md_r$vG^jB?C{^4ZHkoEOO_s$lNJq#3Wx~I zwD!HBuBt4pDzA+ByJC;1n`|Z-qQIdnD<2`2k(W<&ZG}^R+Gj5~!7|J24z!UWg35R_ zN8E!X8P$c7SCr>xSBE^P)+G+}#06>fD-kDa^M9JGPx-f3j%pEKGgey&UZJHtIaa=F z{Qib~q|H<$(4>L=jYv}VU_org_kCs8}Q|EqMn0uDnxw9w-^Q}!O%l#WzdFS_Oq@Cfo+ z4i^?>enZG@dueyv+d7!;&7M4@8UEyqNEkUNT1s&(;IyVNAP!5yZy$m-2el4=aN~V0 zU$9{W9gjpHUCm+4QGwX+^-*RzebVwdXxT?UHxfmfS2zz0lFgj=cT5z#;<(rt6uhIC z1@7KNS6YnMP8QM4n@9F1|IP|b+Ue_l3_nCMCF>Utud{P-;G@k<&FPO`@UoqKhZVAi z#0LxXNeFO=507=Q+6Gt=VVfTA#2vLZ1@!fI0FtB$raUy4C3OW&0|x|V}Ze7PzOs_5p4kQ6<9?t%qcnh z@#O%9hLS!RSFVWVDJp_d(&a4h zt%f{Ds1GuK9$dAXTAn*RFiI8558X5l!R;lBt(k z&m#XlGyls_E)-<$R?yggswmg=;A^4Nx#iM|3gfQQng71@6pLfFB2b+5I9f=%G`}-o zm$Rh!G{ZBl!eYt7lFz15RQGqr}72Ozw{oSaHE0F87n<*e6H;%sLz zf=(v-A|qMU#ruWxL{V}bOqAbwy`p|n47YfS^Le}7HO=+-%3O@bcF#C1$_)n8aZPE# z19QxN((rMuS#`p~_Iyf0g8Q*UoEWEGW4+z{RfV2O2rWH?ldu|*>2l!G8Ae^{TB7;( z2@tqCPnw#US@H=>!0shZfQsvnD^tEku@$sjvNezLPq)oGPe$4CA$ql z>SHe>_b7@O=rx?3!{uMtDzu#u8J@rqHkq(=o{#hUPDOH0{QUfIR#-%Md$P_JQ-%di z8`AsF1}eXPlM@X5jql;|HMkemnXmdx-X<@-5!Trj@bJ-6ng#!?s#16uzg4RH~*0q@w!8?gYm^m%h9!=?8%Ws ziLS0{w`aX*1E1k^FPZLFGw!=vH||k*5iomPjYkeGnGa3WWV__Z_t%(GyX`bI#yV+z zhUzp^e^R`n^9xJhH4B~JqVaOdP!o088<$$7B`(miL>K9ax@^KC4`;l5FheFhZ+TxF zRcg-uvZ(YS9GjKraAbwD%gM_Zh}00(6DAT7I+X~Gz9tW33a{x|P7D(hLr=xG8^ppM z2b;g1wVs_@I&F@-ogPs89X2?v;e%B5Eif6Cg(GD0_cVdG?C9^ zKN_9!JTq(Q8wmu4OyRezO7B0#LDi9wDu(DHeGe>F)1=>dV=R?#EnvFNb%k2gb*4Uo zPP(UKbF}=>pVk}fg~8oRa1mEJ+%;bL8&P;SG;-DF=;rNwatxfrxa(!xy@p3A4Fz6X z$fXq9HfUUmkdIDnx`T?vLB(`RXL($kA;+a#l8R3_FRd1|BfSAtZ(dG?A5mN=F12LumGT&SwN zAuE?_u;zNb+N7IcYLP)?Zb^3$(umh__#AAbCbPjFg6~}3qm(;2>=a!(Z*uM_WzUyz z`+^9oJ5K6bi}I1xJ*k2VYm$RvXz?S}ADz?T0>?uPJ;rJ8Y=TaFS-5htvpMw3O4Q*R zc!#rUe+`5-NBT=}%Z#4Lb$5FlY)6h&pOl&O#3YM4-!8mgZH0N@R^zfIA}46P*#V^l zyGJQ0sXCW6!ZM}pJu0D>&q|D%E4Gt7__U%xGm)Vt}{$KY%M!C928{v;Q8_?pUAn8b4Z&KWG8*G%rGeNS}X zM(-N$#uy@Bvq@h~%_){s?+`cHL2Y(QGUX zyj~|{kv2pWd^>fxrs!Co{^ZlC@_{BHY&(aJAX6Jt{lWT%hK7bE8WDkb7BrFVhwS38S&+g1MOBEFrHFzBMcXfp`)<*7n3>v!@$wxa-tjmPa z@G1&<9=(vNefKMZPFT|?m|RqQ=pOkkA|k8V#$;pW&QNN>5x0~0T)gx6V4*qhJ}jJ+ zBcZ3f*I;6o=U0W6s#|DiV2l})S?*zOHpO zDQ{CuR9Z6u<2bHI?pr-Myw|c>7_>0;?)%#;YkrXre|;7^`z6I= zhaWMwwA@G$cmGs5VLgdd=+1#AVrvYV8s!7>bv8>o8E16s@6#C{44B*nKU7zQ1hmFvX0O0rXjpKW4fyvru$2+*vjU3R^k#*N>D`&hQ-FcJ})78kg{bfT9_Q- zfX+HU!;%T3z2XuQ1JvrFhV>hj`fJ#V6NCk*^ zF89WVQj6v>fQzD6FxovisP+Zk%EUUmvDeiVkn{-Rq^YD{`&+yj5m1GP6p0R@Y@nBji51 zn5_nld%`NS99nxCa{CI&BgFP{J3W7WUo5Uae5r-?u&6k<)tR`8F`6yzhI`^hd9N1rEMqBa5x)9sCq6Ut}6~Smsq9Tekjkc|<~0NpFyEZ{?sA&^SJwnbkQIs*C2i zY=Mt)zHv}A)KR3;&c(pMz{>_N(=>6*%E`7Ful~KzuE3CFJ?-v{b!!id#$be-7;UGj zOD$J8$@2_~Qyg50sf3sZyiWJ2U)a6;@L>)tj`Bhur$vN=jH?md{mG(~e4qAOc2#X$ zP0+;#WYpA|OEZk7^DS>csUoz3y6@GzLCc)^2GV^m6zQ|ngX*PjFvumjC2ZX zY6Cn{QR2=vs5Q2U=u;ljEk)dn5YtGJYWq}A-(F^u3UTuArZD2zLCzvA%uRj3C;5$# zmz+hh;9_VJnm)(WAYd#$j@dg!_tH!oEGK-jweTU?kr9!&*ym_TZ4FJ@vL&t)focD> zyAZ?$IbKqg@*|=au*%~@n-LB?3=J1h(F_f1=3z9TmO#*1<6m9#=d&2G+h6A&5~3A$ z$U=r89b7HYMH~a=W_l`N8 zytcJ8fht1loYp3)poy*iVM!7``YjPN;)xfyc3l449WTGbsEJfdoS)> zVc^eb*dMS2Ov`XS9RCW%%<={0;!Z27LMDevWIVyi(&}8XDTyZ23IT})Jb9o|Sd zp%nF~Q%V+rUM%c->Q!95l5j(N>70wFSfG*vn?A@wTJ!gtI*}N>tbAV;lApWp;oGC? z#B)9j3x_Hh6y^%ixcp-_a;o0h-venxR5C9vQ-`wxGDytQSib8)tOG_KJ zo-FJ=Mo^jD+2i&pLn6#?_Yaj+^5n0nili%ASrz8z=fmM}2N~n&%*+f0Z{)~>$Xic2 zISExFH7{hLPykNjaGHoLwgSi8aJqc>Qu{H)H(Wl(nkkPaZ^g9Y#q3oZ4f9z)iC2}8S?j@`;sIV6hiH2O5g5wNw+ z8zi(4UAV?hP9NoH6p&Ri|07$3rGwv6v+u^lW>zrdn;V%wSW>99HyACReG8dg?n_io z6d3L6gZ^VUzBENWs@1?em97|Eyp6#pbzF2M+4lNsML^25>D*~af5lC zfiU>eyJu_)yI=^coO(AZ)gH7~LpDaKrr-?$-F+f5lKqIOl2do5B-qhsx6oB|_rRno z_5RsKBl%2|vv0fJ71t?#+IXyG@3M4z!o>@v*aXi zkx&!w7SvJccd+)s)P>h%cx^9g_Cy;#>~%4}>vw`TI*+Yk8Gp&dp&Rq4`a zp`t%hCGN8}0i^g3qg1oB{+V<~G>A{@!pd^CS=UQ8k!7|-8ZdxG?NsA`GA?hDO#VdEvIoG1e(ue_4U?d?ua8N^;nGU$+sPrjsfv z%JVdsFVf(6u2Ts?2`ovk5oty3Y|3HyUA$V8xKi@HD8GzsUpF9%IB~JIwy=nF^>agO z3s?a&sE4!7+WO{GuGXSAUPt>$Vfv}`scG8tpt}kjPe7TGzU1B76&p#^mE_gHMvOxb zvJF(-siQca7xZzlo;7}XC?H>IVDHwHA9!4Q`f<$R_H})ChwtoPB8Ln6Jb@jf0G>_;13NZ#{^u=qS1x@IrpG$8Suj3d!Ih0Qa3d1Z=0~VDx%Ar>(wcFR z%;P}07IZUoIR(+{XE2^{cHFN0`aK^=&%mMi;7`$Vt8Jfs;Z%|RX zQuT^feGOV%x-B*G5$v$B%~kHhGr(P=`~(BfX_^Ikou;!I)XUtxaM-sd2qch@e2YET2{4 zjqx# z6;a=&N4r+t6a)u!YS+@|#Ygd9J@AxKohZju-yI23gwJ8SjIk8T-!N`KA&ex}M8h9z z`gO}~ntrof!Z`+m0%ox1MmA!nU1_N3DeR<_i-e^@&o*`9 z<0M33CGhB4HGnk$s61N8XegII*n0H%lFt;Yo@*0s%XRmlw+N;khUT`E($r$!8Yv{S z5^(R;H(u@7WS!pzu9AeW0#@R0SMjmW0!(ilSj9#}S31uACU?|y*Q782>IasE;(g-A zzk-9>NNpj4N|}Q+_k;y~j!ltwlmOsgpSi^nSjC`&&(6$;!iD`H@Yu(I*=ljlN?19AwMT@F9^=w(o4= ztG92z%%)F`rO8*Bg;>nszv+akx!RzQy6w-)FCftkb5&4ikF2abuuByLmsOQfiRUQ* z;~UJOwYz&}6q7D@o&cnG$q>pMQWW=)<$Z4DP@K=+lAEwGf|RuKIfKC~;W^yWCr!BY z=<>a9T;Jb3LEYW%Uccdl5@&l=c7@!fcR0?~-k#6OSc)z0jJbM%uZWWQW@$DwuTji< z-Lq9hnCZT5Sfl;s$g#iQK1^zM9of11x1I_w&)sBjrBE^1zOU;5J|<&h<1NT%6_!8D zLwW=bcZ_?1ra;owhDY*~Mvllp0}Zg1-b86=p?u|u4@)+!cW%9V|Nd{p2|FDP{BB^nx2}?&Z~m4Q@_UtXy%}t zqk`A#jy4vO5}*ghI1$tv?lBB~N#d(27AYKEH(o^FJ%}}$ySbYs`-;DyD-54)>8e{# z^)VyvPqp3Bvf6ms$5U1K#ToSE?&0+f0h}27J?-PCXK;5Gtg?kgZS-j0-Q=t_w#h7w z59$~rWo(L0`lE*Dt|edZM+k~d>)zuEnurQ|-Z?tTK}AZF599US=KVvxfgw%U`y_)% zq3x-`-Fq8ypQkJMR$CqxqaZKJvRH zaUw0dJqS@K7Ii+>5rp$v&Y$2h9&eni>ZRd`)#buQ_9L{rnZYe-c}Y&p&B6-;e1L7% z;FF?N{YR;Mbf{iX@bBG%PeR40wzefeQrNhmJ|LKC*vP8>I{=I`*7(o7h!`U4WtS2T zeYy<^4}oIh#U1SL2hkZ*KNQrs`~>Z~z1-*^rDeH^Vk?vz>kE$@mNQfgw~Lg{>yX{z zGH6lTnlolG$7fa_imLz!sp}oT@C{BDzmHbse$6-^Zp%-zp)sVS9wez0f?22WyP8Bs zSedifCQcQ9%j+8PFABo&X^~;dDZ5_MH<$)MKk-GeezwDP&DdL{dVSHa?#QLVMWW=VAJbZ8 z#`T>RVsA4y3*C@nk_6UHx)4(H(ynJ4A~msnY;THXZ)@5S$~a<3O~L5VXz>Bo7F#eOx==C1dKnELp8`b@G5p+-=!VQf0kHLG#qv+C+$MaW0^|nZMwDT%v-b z-bCs(j|z_Nc8hk;X>#|&C%+oD>V|!0OU^$`E;~o#pY2H=+3sq9iKcRDoo<~ezeH-H zs9AHI3(%p7yR%gq}{&<2NOd~4Q^-Vrq z(@3^`H10<&)$<-S?aZ|v@x$_PTFfygwdU5i$8%u@Yr?FV46O*w1X!lXU-0s8jn?>z zExlN*MVwx|uekK*SHgQFkqGeEU45k|L$t~++%n`W3LN~330Q2_?nAB5=AVUxF^5}Y z@>@1$sA2?wKimuSZ3{;FuE`QkiY_olP@S5efzJaO+!PcL)EOGMrsm%NZ{k87UBJNs>IFMHB)uEp4@ft3~2DsJ|zb4IqSt{LmoowI)dJiO@_ zw?QN2$z7w`*X_BH9vU&ro9#aRVYh_628UX78-l(Ei@P0^rME{!#5)s))Nz5^sI72= zj3IY1J9ej)#znstFBzqt@;993@vmP`httiwT?u8Q!{d|%{LREX@m)8L)rb!$t@%8T*%-Pqy+bnY!2YP+nA zjB902_vuz|D1s7|ZWVTiv)%Qa?Q{k_cf43vlCCXmud_R68eY`=QgnH9c&Fbl_TKuR zT>`Y@<$Y_rAN__2T)O>hX}KShuV~43PUYo&C)0g&;cWJ_?iFxO0LIf>7Fr;6HlYiq z9)hMOH>|vr<>trD-ECf#Eu{0o)7FdpUX!^oMmSw!S5Nmz_l*>vcld0qebPrTT)LdW$^8AKgxK@-oq+N_iW@75 ziDwS{^=td9l&EhAv(9B;j)+d}ulYS(3D#h(l^Lpc6aq0Zq{R}V($ig_aOC8C#=^s> zl<>Qb*soWoRx7l@vSLE-R2iU@{MdNz*yvcY-4$M{Gu?#(Y&+x?| zui6SVv^&_h3=wIZspj1niV~z#y3PO9QZ4=%Jpn8{D=Aal{ zbv57s?eIsiLoT)G!;Acpilogl`-Yd6mOb%z$E2k@o9Z;W-5*KP`zpV89_g20SZsj+@G*erL$Cf@x4Pv|sQKyu(jq~2& zjrug8)GoE!D(Vv@QF!>Xv&RuUblh+^OP#ie-SS!=N*~(1uaka<8_I`fN-^0ST>re( zrsW|zxvp-LqLMPKDQUM4OkyN+H!pOJC7Z3KMt&S^3CE51mBi8FyW` zFV3GvpK@}$5z3&v%cfmict8WYvDZ~&xtwlfW{GuCNz;IDYP|~eq!A%ymFVmQ`k#$d zxPJUere_WqvXYSC%&QGE=Bg?yOH*DH9B%q<0J02XY3gB{+rlb-axI|EZb;3$I;JT- zOCRwCkmtJWPeIGXMe`T%e*lu%P-HBlyWGLj*T$pcRMl%UNn%y_CX$1gbi)I3H%zrs zh;?09mGwIb9DP{us_;fJNF0#R)V6l;tdoNKQR@h@J1wtkEAT1{XyP~#L|#%5fm$4i z54(@;2~K|7jNAP!vcOl*Y6z~n3sled>PWGw-HTlTj^ZK&M@nhVnt z+`1nlzy;BeZ8>w1OTTA@E^FL04+~ji^V!FNu}fW1P*iJq7?H!t>2lwVc29MRE?N&s z6;zqCs415&Q=;InSI@(N7v}Kvym8-6>kK{Z^OM^1nOL04FwbRqArzRe+ER>)qYU8e zKUg+BH2!7~1})k&wKeU?1sWyPmv=WV+lf@q=RhaHqzlyb;9u~?{OJEL$NaMVS`AI& zzsBGzQ?A+Mc1NE@fZ%8D)nl6IJn+$A_c`?j$vOW=FW~X?RFde0IQ(;L&xbc11Ya%pj~WeUw!55^(T2Jf7|1cB?e>==Y>@a&j`vgGE$S z^x3nuU-Oyv_P3Kot2i3*j;b?B{EL%!9_vQe?SK$5kt*BiT6Ny*NIHc~y&C(06U0q- zXPcQ^1%>uo+2*_uF@e(+-X17@*po_0j^I8;PYbk}Xc6tG>4t^Kal`P@AOL3+WbR>3 z(8Ooet3xsL(1P5MjjS|+@}DHELX+3Ha}C6I&I79N0x@8pxP{j=-py8&goOlJ&to{O z^B{S|9zl@tM_TtGMo$b<&o*Lt*suen#OjTgLXG-sHILs|_3No$pW2H^Ant>-2hQ3>+yK>c*( zY1l;zovTiyF;X?0bgLUc9$4 zH9a|=R95mj9u#B>(?dF6{GDlrhjOm@ZI0ra_I`{U&tY!^i|B;M5$y5^%*~?Af8M9` zFgjWd0Pz+#{uDg59CTnf+zc38q)kPB2+*GIHq$47U^|`{ABT1;DJw=TD!1zPV*ggo z^~u!`a%azic0!c4M-!22_Rp6pKKAQ*5NkEK7r}=jDXUDAP1w4JZP=d7L z6Oh)nHhnBtifwgBD3l=ZC}@}+&V1u)JP9QtqU3+|s=MpiHqbAN@nqzNghU%ADYY5g z0`~gK0atyO3uC^UMgyB+!u$ehD{}?Z2_7SawC=1&&=Z`QqYDQ>C-|!>eAO#x9Q$Ae z95}Rs9;ogi!ldfm-98tBZ%Ao;fUN-4Y?a};ej_5~77e`|D&~h24=X!EK~8z9Z+=Zv zrt5o$EZAz<{=g=o-Y426xgmkMS`_ruiSF!Js^$90ZEKMwYYTdw9@{UQngXx=uibfo z76b0hC}26B*w2So-qjPKzE^Py=at84lRMXa2}kF=`QusYNi(b+Rk*SsYyvb*3FDbBhu` zCElL#m;+GZ=!98MVQB8Ku*aW;`VeGBZk?~M?;}PA2GNuLUTePaqnx3OBV2LN702{dcH0x6BB5=j4f3 zAPY>^X(&xU`5!u-oI{9b`T0?~Z72;1rqLYZ8x-3qwr8AqV_%!kHVGsjVWTgs^^w&Y zc5@M#Ojd=~S<9`9(!u1y)wrLfM$#nYJSAcIEDhN(HM=jV|2V*CH4u)CEsg>6_^CEJ zSue~lj$g02g;Sl>{TL#{gEIqkpezxw#H3Y!D>Zi7Bn_KwK%oST;|K~mW!s-#>d}{k zStdOZb*FBy=ahFi6LO^H^umb z5ksMhiu7ne_kt>4tZbG86ZO(Y0Nt|mS5JgoJI;)w&P9A|AlDG-0`+Wg;A9tZ$Dbwi zou)iN4>b3?bWc$8t`sMWeJr`W>BRC-M&-w-XgH;S^~SHF{dBs{q2b|cmTCs|R?#pt z5z*(3eU{cvMbCLGa^;NB5Ut+-+szHsRna+L5g{Q&d4F^cVJ(3j7_JE~b3BB&K>%RM zUHf~13^Ei%+ZC&Ims_qJSaip|lLFaB)VvO|N25OgW{<^uGUpAzbSZsLBCd1bFY)u~ z%I8IGu?gt}h-JOVGYy#$udA}bSgqlcT5%wZDAHb*nXVNCQ0@az@Fy;|=Wx0ZS~dj* z@{+15jh&xe9wvd!fRusP?aujnGSo6M)i-hMX{aQY&PevydYUMjZL-#6ogb`qZYxhm zd!fC?tWwmi6;T^R2CmKq$-bG?9M(70t!=A?+NVa&4jnBUMQUEQ5MOs?Wo6t3+Z15o z-kNDN5AtgUlcA(0&mdT0e8LTlL#;~lVrh7EEC9K-G0-zHw+;T#YWfe#3=ypi7 z_>FxFIC?|k!u)S~Mhh=xK{B6iB{5y5;7xIxPlA^=md{_JJ9&lZRl}Fh3D*P%hKqDW z$7i-p$*zFt(RC2|wlR_pwR=0qvHZcN)MdI4a!Z7As^etZXF5#laF1a;6%;mN(Fl>mikP`t}PRdNPRRTC-q52I~iq zjK-Lo)pe&4geD0H2;AARb*Z8DFMRFP$2=da1|Br5z0HtR=1zxPm;fNm0gZ+n2G)6X4KbKz{i+o5lu6R*u&?G^b` zzuw_|l7vx5d&$a8nFP({qa_4v#d4fLw@(;E`alkwY^U{-8K$2M)lCD|Bi`jH>oE9e z``c5JTb>o(h%aGHiFF$53t;GJ9|#(PkdaEi>%^e*T#3Wo_i)+ za|t?CTN9NYKJXamJuHY>bNIlEqf%6a6*t9cte#PC?$Xi-a(DYkd<~O0U2e^{EkH+) zSWZE3%$AmLaP4RRd8fbXE4&A+C?I{8p_S+D;II@&tgaMIu(?VFwrs`5*M%cs9C$0F(N{YyvT{|dXE)qVn~hh*hiB%wx$aS&IajMm~ah)&(On$ z=bLO4rVPI8?{#?LA7TsSx06-D`Vcp}Chk?`?KU1tzNK%wK=bH@&+6Jy&pz9q;D|rg zO;S@Q3Q1@4X_&xxCRVa!N`}H`43Dma4d1= ze1(^P18d0#F_h8Id5}242Vx`%E0$c!RTUNWR4(m9EoI=T@WA5is0ZhNXd~jV*%s%Y zDpv2BIp0_UqA8{;%`U=Ud5y+S*Ez+VwNP#d+Ng6$V#*IcfRL~|9rxN_B$a`*qyF%y zi&{uF6uzu1w(DSQ?9?8JJg2PHOC(i@MUX>`Un*K{hU$4V9WO|+1P26+uI=$4LF(hl z@yUB_f@667yHA4~BpzPKd zOaCFK_()aYOuiM+G~Ro@Tq8+d%Wlc5&`PDDhsmCKA>Zc{&$`Rap16ngvnU0i!=G(MsK4yPEczjxf;Y-9$wfY{`* zOc2?T{RY^1+;z0>Q&ZyRmotb($0uEEhGl8MnsaX6A4r(ifuBbckaBIAX?%c!w=O}6494Pz&}fgbU|Z=Bk12^f^JoD5hW!4@#{`@FjS3nNJ8jwE z->C4!?jM3I&PZ1n{rv%P-h8dZZBU-Y?5GicL+YnKcucK|FpWy9MR+)EjWpSku+%^B zMkr;v+)HpFx@BW(Yb#jd!zFbd(Rwqb$xdzzj{C`dhsNqvzqeF@^VH|9pLPCbXZGIH zBbmHx7>&5M<*6bEThd`|ri@AxzXv0GCTV7m17P)oJI~zo!BV~N_RZ{j5o|t!pm$E) z&_ty5luJUKu+pNSBjUktj@3f%kRER*EqwZ@IhfND+GXT#D8(R>Kd%VK6FYyxJYAM< zN#HoG6wz~#pV)k}0YDnU6O*Vs(8)W;uM`099Qu%;E=qhLo!whZ(9RasE@VW)$ zE|Fc%X@$EE$3X?}uapGn)GKNUVBPUK>PyWx7ZyAT2~h^zaL>Pn?tyhZjhPWSiRXeT zql>Io?X|f+}W|LsuHlVadr3(a;kf-PBoiTAf3+LVPFWG zxhRoKK;0l}a_<{%O`Z{yA2gHT3^!W5Y+nUytc(sM1vWk6w(vdSk&6%X2l^ zGnd~>%vS6-fGe%&jmH(r?UvZP0h7JVHAkgS+Ri5EOi_p@e7-g8=DqOP&V_!7rj?%X zqo98`Kj-5oCizxzt!So;6KrqbmRFXcu?z{{uaPmYJxcJxF*ltQ!WWsVc8gI zQP;JcZzsK~= z^adA#Lkj!bf!Yq{1LC_WMAipAeTz8zWk{^veesYo3_a>KuA~j|gE$fI=ep8zAF|M6|ib;VOEBxyS+n}BaO25f_o>MdE-KWsLDUl#m2xdE>1TLfB-X+ z#}568TR>bx&zP%$x9cN2vZ#gZlUTpDV|FX=MPXXp3 z65!efh9Wl1|9|uXz_0vV z{qN|1*`#)H3CtqNm(Md2`eX8^62J>_1tp(8MJ?1lLzkXF0QNn3*h%mGDkuy#wXv?=NN;Zp8FdX7s(=V1E`(8@Tgdr*@1C6 zW-<=gvRT15|9JQ?b>V{%r^Jm*SJV|)H1Jh{Il@*UM<2Aj8u$L9bRcBE5X#_U%=0Sum97sv;WRw{oh>*|1&M}e+|0W22*q;6LsX(`ta~yU-ip( zdnZ<49!{OQY-MY5Nn2D&NPl&CHsDJmit))&4VC{Mskd=$Fef=ds{g z6$|O>W|aSaic9i8k)dPDZnqTkg5LmY7FV}flHN(%h*Bo-hJ2wb)C~8 z$o&AfA5WBuoV=5W-fOGCu-qs^H9#6tO08R1`mpkWYYU#l7!#fCi%&y=Oc*{t)nRM| zEQ#XL+{)6Y+$jjAz^q@-9)@5=Xqdw%G4geKuWrjefYqH3gSzcOlqwhyECAZ=6ohV~ zRKcdOk?`ss z8miZ~a~=wj)Y6Va!`1a91A)a|0Wu1Z6&xJwBex$2(E|pJ%bTBY8|@k#Q16)(&=F%eyG5WB zJ+R(#zC4H@c79%`0wP2f^EKq?Ek$S#=&B_+o&stJ*dt?Buvaqm_ z=3eUnF7PE7A9i~Rd-G!!{i3YA99y@HU7hFw$K&$mOv44b%te?-=68>OT(nE-F@Vje zCzIl~%jPC_fHzZaJn$7siippiQUE96Ht|j!LhgBnAz9Hx&Q$(U))IiovtSjwHCcs9 zP-~9JTc3>KV|NNw6n%kELqFh$#&*A+_g_f1O^cEFhbw^;b>641Za;#ZKCkvOkb``k zoG!J^kGtnRHy%Ijuf*!Q9-~uC|6iYkY~9sr^d$|?i}RJH2?01d&*mPMCJ2nVS3k%! zQ6fR9Jk>lSAiz1;SGl8PJc>A(-EgJbv@ix*FF)t2cc|;UH~w~iHM(VO4f9Ywzh!7c zU#rQyP4^;$`ZPud7xSO@QQg*lZN3)oNNOu&za06k0XkcRNZ`K}*}{g-xa1J-yF2ar zHRE}he-^*m`T}@$0Isj+Wc9T;wo|ILwoXKJd4`&|@J*)nn?5lHemzs2v#XyfYlRgA zXe6AQ$Hr=vS4g3bWIyMCr~3^Y33k0;C+zj7&sRaJ|`;u$hZC%>_QH z?7M5TZfL55pOzMwNNT4HORk-pUlz+Vs{_$F>oF1U%1oTJ^+3?jWMv)_P@vzyU4i!} zHkZJ+kCnl<3#vAq(6KOVMbR!hTa1<%ZTR7!siR;Fh6R2~ibn$DsMaHW90bXyQo4UdGi>o$YbFYw7<3q?fF3U`IDKc7seyE1}p&$67P6GB?MbU5-RSmG%^0(>uBOT_qSTmst)1H$g)40#wk#aB7V08vIJO^JJ~R5=(G}+-LJ@AFRcVhq zN}tYcJj=_^@18)MMo(3l0F*Q$JSKvL!&I9d_f`~{vn zs7$aRHpP4}%RQQOUaWPg zs0PgUKgI2T%jNbj6QwqM9v$v`GqajFOM3IbpBV($0Krvx^Qy@T@13 zbsTQ~+FTK4Gmd?W>}_65A5f;^agjwgY1lvqx~$Z=N^H?F&K4RThwc7%Qj+isSsx?U z{sYlq!^o>yc{*90B@Tq-Z)+Yq9PK;C7B!yi<;A8xaT8kF%y*2vt9N-l+-Udt6FF7N zPq78!h8W~Pn_Py0i>>;;jq5kI8qNH@?U6IYN{2jS#`e-_SPG^F*QQEB634@0hMulo z41ZKKQ$Nt%tVB@q-`}WqSWG$7!6|8yUoN(I7S}&ya>wq0V`pe-ME!PDXX4G!=hz$d z;6VlJd1fM9u?_CtkQekCKzJCjYyvusHxDhC7>435Rx1M>e%3Ry2ygw=;cF+M78R{G zka4pVE0~X@Q;lMlH;&{_wZv>VI#|Ack>(8Q+nn!eXuOQJhx+bElnzzHa@49IqF}n(-rfARc^^0byR{ZP zhfOdidH|3^vN|X&9<3*?ObG2x-wZBfZ=IAwr7hifxUOLStazfr441B|A`^N{=7_JS z8aT9beCL1fTEw7znG09r;kOf&1kY0Y03iQTKyQPL-r&`-Mme<<^K9YutnqFYIkrK} zpgsps%~3sYmzx;Ue>|Q2kp=M^Am?a_g_$~+7`mPaW=(?hH{bT?z+$T13uAA!afwlYL9jd+`yTg^sZ{uzER%@T0az0VH?yngJf~&Nk6Z^i;G* z$Ic*G?LT+Y#f%|Eh(`Bx>pjsFqSx4z^AUis1xkSWBLWGCrjv&~d_OI7StNb&NSnG& z)x`B4Jgq~_XsPM_f6X4PYPP@5Ujhn*RD}(_ggudD$fmYhgEm2>Ji#_wGnFD9mQ>|5 z-4FPrZh(P{Osc8)omFYB?sV(<C|Ce-bm`6@atiVS2) z={nbQ#RT>;am&}$c4Mxq2F8@X3T(XEGSjcY+Qq|+{P%1yJY}xVzh@kTo&3Zw1;XbL zAk|>jW9_4xhE3<)uzMWu;}DM=_%Gv)9Eu2W`0)C6Ah|G+1IJ#{)z9z=Bi9Ir>t_9( z>02O!ZhGtnm)dnrrenp?kCwpaHd|q{J=qW7v1cV=gVNGJ(l2dq%X^*#?loo;dRQXTr%wK9(L;FO-)P5Jmm5=}7 z%_g{g4?n+ggtJ@LQ>+O(cJAA$J4h2pU4 zHyacEA`JXcmcM~cZBBs8NtG<~dtC_o7pn?D;z6B((z&T$kDBJb>WCP_l}IhQ4z-c- zuGMPE*tWrDB~r*U7KRnNE>PUuiLo<#!AzjDt4`i#1*ZmC*L;^jblz{v-=S7wa1CuO znl@AA@_T&?=q=^)2Gd-=wCSt|2Ti!UJ#Io>o|SOr8qsf=%IFx7d|6sP{ow8BB#5Vp zoTNzvlAHeX|BJczj%sS_qDQaSYeR*rph#B{kS@K0N|Rm_q$|Dm4gp04QHu25J0bLh znxN8q4V?t(1VRlxB=30d_x)ZOH-W)f3xuf@XGa&G1tS`pskt4OLWU+VinU9;8{XH%Joxmui z*;4O(AnL9_&^r6k#>b41U-LV_(xV^W2ZWjBDU3x+k;l6mw3t&2;H;Ep&IY$h7L5WT zzo!3jphO9#8q~JVu%~PH)`c{cf^wiuxodW7V;(hZ4E3=yBpX@vL&t(Aa0HPu|9V$L zbBZp&Gqj8rdu|pT*r$sw(@~_jXpvdE8z1VpEX?Yte;n~8N^gjqJpG&W3rrB%@|tkC zo^^lD>JX*@q&&UPOs0M>c~)AGa#~u}c7))6F(|=Z7SVo$OC4 zhHGW?ZHz4K^a)T@7k@*T#k`fwrj;sf`K(MORj}U#$c!KWL~6 zJm4+vAO86>@8rfo+|of9>d|&e%Lb65;WEKrYu1KK0k!lw59n&9?T>q$4ZqVdMb1+f z)&d(%+B}9RDI;8u@t7#ZPwzd)!ZcsPPX4}#C*(fop@G~BFQ#aeZU1| zPmUjG6_Y*Rf_zXd?7T91dGRe!!~O|9)qjiF6RPVms^tK$2)xC6UA08|D&rLxrp5JO zu+2lC3{WNSM}>wT;_W<&HLBVR#`}K70=~5d+Jd8X6=sDs?SFNQtEg%=`>h0AcpyM( z?)99;z^(yRE`o^4yDYQSYBm4l-zQXl3x5J$##8%ICkkQ)#j|+knw*<&@7!Nq1*mlt zE^J#y37Syos=;ZJ2){}6w+}}^F0AUW`=wdK@J2Iz;n*hh6x?JzK<Vti|PuKC{2xlr*Eb7qU$+|3sf9UIJH!P z+oP_L(bFAMi*tEOPGmYq9Q}i6JbRY*F5e4XVF-%;!g5;z5FO=u`W#Pq0UaxoNlDTk z1fw@%vRb}_k%5gYKP0Br00KSdslF1@5T6@e z6IbSadhA;SYlQ*AKaN#`O-6ipXE5OTGbfODeD4w>02TC2g)4TA_0`QX_j;BWr`Z1) z^k8n672nPTB5iI1n390b zzi8t_e!y5paRy1}dsnw**Hd@8h~ee^H@!nD9g`}3#F%1+mtbvh?TI8g6|Wtah@9*d zTzI86R8w8^ufQ?c#?m=9E~{{^NKs0NGJBG1<$Ladkzq<+r$b%9{D+at`Q=6BPCAxk z%h}wML?jjP@7<#~o&mLAJOG*wOxMQ5BI4>&5@zdg&x>?jXlkF)&pTJ9?Qz*ylLKdS zoq!Iz6dfQXyYa8(1w|ZUlxfa--^;I293l_u|G7@uq%B?s_4ksF5%SX&cf?qJBe3tt zpBYqEN`9F1|CD~dr&P^-j-92nZdS4mVWIu~seo0u!v7fqAzSrJ;o#!0TCYYiOS3o} z;cU#7-@b09x%@Zo_=gfq@tc3^{UpIRj)jq%PNp4W>i$)xOys zg}B~TUEMLO9)6PslYRRr7wdbLjT%ly0iDq@%8&Ccoy%c>WG>KJcAal|e~bJ09T+-~ zB&SsRD&F)wXfD-5L}y%&iu{5fX-vFDKI_tj56|`Y&kuBr<-f!~4B=M0d8P0PpR5XL z3wq|y>HCBblw!c<8z^P`8!UdxdSPsY6zlZ)HQ4XFT0lq9ELP*BJs&ik#dFPgjPrw2 z-qK-m{x*JZ)naQ3?mC{Lf~#t5^amJ1tjf3OMGLY$STZF&eQMkH|exFQT-|ZLuTmc!;#tPn{Ry? z-5A9gEyil@{KHr6)!t#3uyydVeUqimua4BWiI99pl`F)d5aU%sqa{u6H`Qdd*Q)1a@?Q#$Lz z1UMO&EdQElc*_g;VW z0VMPzUq^{$f`6*5k_}ifh60v`u7_7A{bX>z5gwBr@g>?5x?s|(nG>n2`dl;}VKIogbM4Zqp1E8ZR;;4UEiOXNk>Z=QAt4Pw=>|LCmo%O*F|I}_> zD9MwJ4t31coXyo(LSEDCo~SVCLE5(&Q67m2CAW2HnxpOl=y2@5O%%YnFf3!qz6Ch$ znl%K$L%-_bb8whMnL?A|F}#Euu^0Ucm>O20OA{`)PHO(4R+DH2S<#F42Lscg=MkW7 zh90BtK+~_E0V2fiLFssjSye>OKw=`q6`MfvgFj!cb>+)wUEW-}Sf-94Y@-z^+A5V=PLJy3l{LksB#w4wE%$|} zSB@JKvc>&Z(_Mp;QvbSgHIhXvFL06)C*^{zK0jv(k@H@43P->I*_pD0%cDNm%jmvU zx?w^KTQ81mPtcC&3n=p0Y4s7_Bdu^p<6;Yk{>;)!<}tTsHJYBaV1&56k~;bxt4GUk z4w72<7S{9uGWctI%gMSg`a_(Bm&0tiPZb%}_UP@e$wK1Hp`E>lSh=yD?yb|UN!RK8 zK@J&K*5z>t`*t6D>|D=?>|#Lwk!g5=${f;DP31Yb0{iH;CVcgmrLC2R{Q#h|za`<8 zH(WRa-?59r1)N5>BXG<<{8Vb&S=mn6w4*GiHf9GdV5;?6)0OGM`|7 zFVvTlFEf4CF!}J%GGF|#2^8%-gEB9k>?@J)0-8+^FeSlm;i4LXf`Zmp3ik(Mb_o6q zj@+gEQr{;suhrIi_EIs++{;dxgF2=U;?qo{KF0z1uDIr0f7RO&Z8w(p{`WDJ@QGQ2$1EYFu+RDeE?|P`4HpZUqXkRkZQxC za58%YwcK3V0^+Y5L+u?LLC5HDsu+g70pC`Vz#2bi!sq4O@a&`s>};jh-r-)r{zfvo z?>V1o(_^8S<~w8pbxf~R@cu@ zQJT2P(qACXHE9-33IElXxoT|GX9j(Skw+C#v^=IEK!XN^RieKN$Xx!b9SCR@&b9qy z&2U72+pmI7GLWV2JjH3SNioh<);4{-XOj66psYkkd3sef-^pHIx2wzwczD~=9^d!< z^sB*>Or6P>tAA57g#6O=yO4aZ8qp{z)xnXG5XBh?%iSdwx?Q{0d|v}sR;!_=MkN9H z(FM(u1g8;nU$4*I%C~T(A-U5u-7jnUaS7MR#WX9N+U)17Pl!^@+J00#=<9-2pC?r~ zr}Z1R@U1Gf^gmlGgz6z@M8-klzQ4YVSFUfctGw?wK|~}RZ|;R0ZC~bdIEYy~`B8nW z|Kd4g!!af_a2$y%pP3k$K=_ovC$=46Of%Nu`C!-tvVQUVrxNiW1K-#JT}iD=e{(+m z5~4pEjKTC7@FlD5Kj|{fa?O%C14T<56B1lpqz24N z!ftO>;ENOlulHm<*%(GaRX!N4cuXohzIVd~H&MnOI50l`8f9$Vb+7i0Fu>NbpRB2_ zsE96)33~PC2l*A={bBS-ECl9k^;^wd8iGvQU=dG$q}&}iSTMv00Ky3w32FIUZ3UEY zhr_V%IkRnHl-~jdI9N2G>)JmiG(B^IM~2Al5qfFw)q)KT4dt&MJ5SW<%uWo1%gE(q zAOWxLvUbKD)P+Mcf_LQ?CO0YzRisNiPSe;(rE4Xh!Ac`~-jR?Mrmb+o5PH9}b<#QR ztfNLQn8$q3j6MC?2A-+;2$}`|np_!4e~{+$y3DIDiMX%|_U=M}ydCUuQnzuF8&lrz zhBiE9nti$U@x5G55l>kcwAaj%M)U0FzYR-yxR7&_W5ypDi@83$q^x99G&J2e)AO=z zN1>~|@7;S;u#?HpDB~X z7#RZ;O4>&7z2510QyEiW3N9r@{Dyd0gQV$E>vWFh;nazOVI5-ML_7B8y{4un1J}Pg zJK4J_+Bu$BTiLmcEaDdZNTX;#hoV7Js<+B#g-zwg?HK7`u1ZstVSLq}0v)U>8frqq z!o#O)vrodFCZ9*;fKVBDHgY;3lkHOfa^E~)c>8G$jnTc{HB-^N%1WTpc{HYG_ANCr zv6a1s8^+6PH8}H5J`BEOoo;ND*1^H9@?8h%P$=IOw2Z8Qv+)V^;vq-5#wvfR8ETWf zx#NW4cDnraH10KFroHJ88M=GqU&*WcZL zADd|K8$2E|N~FN04F-g4Y1Gz_Bme$pZIy$Y(+(Ni^8%#`TTQ7Q`S|SJ*aH~O&a-tv z8y&H_m8D?#{`(JRw0vgK5z(_v-t{Qd>KoMC4*{C@n3Q7tVV1={_?fz`>HOMHjjI`U zYnu2q$iZ0kk}N=B-I|9&)md4QTBn_z?GR#*!AxPszQiU?5#7#C2uvx;VTPjB4x8(3 zSaPrlJ=_m87zS@LawWukRXc&~eaTr^Fs=`^Qr8{53}yJwA{iDGepX^qnkNm2t@!K{ z6icHWDqjO)(W9CV57){Xd8J&g&dCN5BjYXoBRKliqu4gOvsHc=x%l`XDl|uEH|C&3 zIdSRXqZ@H@8HtH7(`%K~wrn%kZ&4(^g}eMZrEMH;IBAD9Q>tVd;%1AAl1`UkyaVBc z%>cZTw)U%(n4OgX_& z-=cBvXCILZui>K~tHrh^5Kq@v`E;7;ZkkQTH#^sEYvv!su&tGWI{-5y+}XgxR%pHE zNyg4pg+H5Zc(_X;pssXo^Sa&Vn@B9xhw?I}b)eIxm%=N|{-nW1ME;Xe6hv38Q zmukB(%~yW7QMCy&xOw5XFnNm9mmx@{+neZ8J-z5rfR=gnHXXx#=IN1q=TCFnq8c9Q zoyLZSF0*wW_%PTtX(T3)Ts!T#R%wnbZ~#qyk{g_@_S_kqzZQIEKa%W8OWd8`TFcr@ zODdA!H$w%Pxn?ePQgYiCOJwufHsAE=er>fDsbzkM1EJ1WYeOwXU1nrgG7SgPRLk^? zsgL~v?j878-$N+S{O$Hiu%2Ay3#15j7mj7Z9B2`a15Xc+CwIP4?|3^l-Yb(ZLwVGK z4|``QSr@6}_O+I1ALkla=>zW*6MUQUMzUnp4_2(4h@hls>mVZ&buseR3v=_K>FS_2 zZm*wf+XZh%hVv1WzB>b0I^NXu(@qEX>Db!+sxoPpt6_}dTk{2PdBZ7=1a&R`L|aB^ zy&|**K!dkGF_T{lIQD(+PWiJi{_l2O&|Rt-rm{R$cVEuxndq0+uAS|jeJuHjg^dan zBh~IReP%Vao_Ur_3=@#;An#TG(6j0sQJifL;9oWVT*g1sKI?*qKa1R{akAWKJkQ_KKVYR7h_giNhk(;v`}pz- zt#a?u>cDr+`KvB7H3tKeU46O1*_iqBqm9^TL#vj$^Y#mf)FH66~WMFa&SBfn^9 z4ZYG(JC^m_d*SH=#roEC%PC-70BHDaE{qSd?=WOP)B3xgEXO6(mh z9y9?AKl46Zu=0&RJ;w0ei0DZ*G_)N$U&FF0<)!jAQ`%Risi><9iU@!=abO?!Ta-}^ z9x@NPxuy7I-RF}ax`q6~AV3tt3bL)x(VZZ5#LR0xh%kdWd@6GHc25}BtD!Qi{wy^D zToMFb%;SpqXS?P?zB)BcHT*RlAQ)9)WB^tljpC}%?9$ANw^GG;5XU9tFGB(Y0RxL> z_t{H5B36#qBOdX1=oN>%uxY3vdIy!Y&U%wZ=0sP<{aLVmX8{2|2yka-m@;xm<<8x^ z@m7DPf_{iH9mkg|s6M(BwM+wS>W)^Kb)VM(VMR~}|31_Q4l9?ATfY&H!a{eDjI!}O zi)iuL6?PD&xECrR|d7s$;uJ^~6A_L>DIGfPm9-lX`j0!#=-bga)Wl=Ee4&G%9F7 z6e4pmF|diHxW(y#&}{{bOk*X0sza51aRdSAbw?nWNl)p;O4z(7mzVX`Zq`OSr>nqT zOJNEvPA@h#;AIAUf_$7Fx#%)+wR^`qQE7ipz(DBj?L+b6GDUTm`0CcdE z=n~z7Z4EP)kqnJdmA@_l|11SE`q}F}t^N-&s`IF>QGgw>! zGZj7f_mN(>+<5=0N~uH_aeJOtR8V%U|1i>Q;4m4l2I72yH$rnng}faZO$5ROkjxc@ zV6)7WwR%sOWh9Y7F3Vln1A(rqYenqr@$Av~5oEp*j1;ill|4wZS-~9!1Ozb6&!P@i zpQJVf&l?@IUMxZ@LtZyJWa@JV9IX?7^!r)`jj`XBJNG*y?t}yex1Th@9a|1299bZe z2B4#s^CL%ojSNAzZgDlWuXJNrX&<7#05vzVs$-Zl5wYhnDECBgq%o+c!tOdn4EKmJ zP&ckbHZ?YGT%hM8`_%_bLp=gjx;Z$Iw{Hx7qvKc z8>Dj+*%}QioY0^1X0fI$TqLWD=~GQHQE`0=8lo}$`}EHMf|q<^U~A&zE;%o5`}_Li zNt0vT(=4xZUHk9CTFRccmR+$u$axE_s6Vc#@D_D{iDa}%uH}DhRb5-l>mfTK3Ra9J zhj0*eP{OO_l5qjX1S!y z8EEJGU^vbXPFliT|4k47+KmMTk@7VX#|0-c>OaDtFIZ3X9Kb1emaNAgu z^n9<6`6zyJvxV5iHen5sCam)$K}F$vpd)#>joL6dVo=coK6 zog><6c}Hq*iQgiOZ=@Rjum7BW)uo(E=!;=$*V8nQdwNqLSYX4k1teVxFRgy-p&Is2 z$7ENlrA5~kVnUo7C=-9(G;I_xB;$SA@}?!QcIKc1l)s(;_`JKY!m z!TkUC-!&kz{hyB5^S%En11$cRYu@o+#nCVSWv2gMywFCes+P82#=lLlg;MT;cljAn zssH(PL_vY0O!Y?cZy^NW=l}7C{l9kC7Y2Vun+F>=`S)ppHX1!S_5n0gYQy1wqgz?~J^!HwDhs~2h|x&*{r7A5y(6oB-}@16&SSJ+Tt=9qypef?Q9h?) z1aktu9o)A4M0F57pXm{Fwkk?3@hmDma7s!)`*?t=Y?QTm=eVPfS#tVL<&xv1D_BN= zLDWUc>tsD&JvBbw^H8Y@r!va3X!Gs&^ZVXv0MBhV3({O8jeLX`QlKR`U=Z_QL z-RPEa|1L*9=y^tqr$ESZ;S@u5rfEiO{9(?G-srySOWEgq5gKaRGE#prKD}~I$ru;8 z)OE!idDN@M8g|>LIUpi(4&j`PlLO=*Y}ejX1G;&MzxoU=2fuDXg{Ajh#Itb}Zp#rL zTxoyeyd$CX)_e?9cQTCTR{nEB;InL}$x@sfBW6CUiiGqvzH@dxb z&72hcYd`ATJT7)$*GfMFH^L|xq(Atp8SFS*WgpJz+6q{91d)aJ_HMYU`&XLfBPa*q zO-BzJ0*5Z)Pcoa2M4U%@4sr}g32jO!VY~GQcsC!Pqb>ecM>N7X5f_CCu(k20QV`LshN;ya1*Abs``Zn;aN|5^yn>>(b~K352wB0IJm=j|3zkLZC<`* z-4ak&T)n+MR%#iDA5%><0o*PsO+0!QWBr+A9<>QR3mWg6iSWf^_}St(t&^j-2#K0>DZFr`YY)f;+-$Srsq)fyrKvYLY+V|~_ziKR@t1i9cfqtDlwqm41`E)}sc##NTehvLn*`t20)fs-;!v#E;}b^(oVz^!;2g zxOdMrzD8Gn$SC!?_?8oF%ZZ1U)L5$B`AEyUAQM(7&1@}O95hH^5k8x`vGd~c85nBh zKN7bxbjSn;4tWoc9paYq|>0YtFyzrX)F2Lw-mV*xT9F5S)Xy? zw0SW19VHJ{)kgTz(kUZx6U+L3yrpgoMY03Rk}lTebU z^WEL+0{P(^>+9;-dwbz5Lpm!H_8jVIk85kHtLX)thC9?qLot5tLuOk3r9l$sQR%*~ z>K?UmAhpi3nycQ9A=(uNu4vNopJqw~4t8{?goT(B*V+v*C{GU6)r`c{L(F;hD-E;e z;q>}HA)4pWf&Mk`-@h2z&b(QnWn;72P0Oy@aIn!O#K^==(>NH%bL#;iRDo(AgMbx~;>MX%Gg@~u``;tgbE-p#+Iq8>TV|5NU(G#*4OGB+ziGqScU9G}KV5j!=y_vf* z-Is<~`6D4K#7}(ymKpoXVtBZEV}q{D!Z+Ipwmn{*3Co13KfFAUE;2G$OwQ4%iC}M$ zE&^+aok~X{)Bo1UB#k7S^X|I2N;$4J*Ib0(VhG&*W*}f)R|}7w?$xzw?SAfE@6b^m z^4u*&{)!a7N`YY_Y4upk($ZPN-Am3>Z|NAzXo`mCEi?5^?%-yT19g*1N*(QZ=^PaY zKGPe!t@lrU7+Yc(RuGO=yYuzXA;j&RG8Yk80ad(#?NXW*{ z%A_zyG27$VaryguFY;+k)wx>&oGN^m*+4GE4Ad>F?FDQtV8<#|GqNZk$i+S$kmihTvmnD-}&jzJUrJMy1HNIJPWoo5Aw#J@d9mt zr{byYm#EDHjm^zNa8i|hdMZ!NLJp9FnUYQ>^(F=e21(pKEYojEPSxZwQ~Ed9=y*UzPh?U2?(GRt2MoSn;Y>U;E1`#cIO9y z;599nUcYTOQ8&`knB9F6+ZO(c>xqE!o^`@B39&xJm1ugfMt%tN=95;7I<#2TgvKY6 zKhe{#_l>C;xS}gT1F)J}8yP9M5!~H8CYpB)fP(}Dsj1_goq4bj5&kkD`NnOQ-Xz6GKx7REZ}4nHSUqlo`oP8NG<#IzQp)yr=)5icq$%xWKn$SNr*)p(`yNJ#}QfBrG!CBHS-q|{l0#_7r0 zu53&Rw8590ot@95C*^Bmc6X(inPz$)q{7%2#JA9`u9d82T~y5bfalz@3JV8^a)h*1 zVognr(k+&P%-Zqnu1IE-ZDsKgojKk{KunBbc&9N`F`lCJ$@P$m*GL<*F4Zw&CbpG` z5SAvIHNV@QmlSiII9Ydhr4tnpn7Ci7>+&I1$TsNa6~;=WEPnTw<=t(4Xi*O>WXLqZ zRBs+;M6#BOj=tx!Bk^z+J2kc5SGh_Xw9~$~1k6tWP}R}ah+(?RXBFzsm1~S_GZ52e zzixR^r{{09O-zEEoPvPSJS(fZ@^_AGw9|3f!6Z;C3i+fyU~|Br&egwN@V=JE4NpBw zkcvsG|M|EC8JM{5b5l(lu6u3&dJ@PoPm<*-s_LHkx}tM54$h8Sb#&thbzZ+~x_2R& z@v+GS??d`2{gC#JSpz$Lx9@U%Ii;pBhx>zBk7Q{rDwT^nTyID|7D`B_p8@*{@`~XW zlrx#q;GT9x#0fl^Z#M zn`8u7--QBfASiR{(aXe8p-S)}rYbwTm8-rjIs~;6uuT?Q>qi=d*H>33oi&mZ9Y3wI zt9BAG;}CmbFEc435JcVa<|a=4G6+bOEv zpj4ML1o89d&lc@PR>Me9*;gva$-m!hu!NjuHGd`%)P(#f zW`XP%TCoWgX~n)#lu~L|mE$#e5)-P|>!)&`!i7=5?Roap)D+;v9jfT&_SDtFB92TL z@in)y8ebMG#?OYl3VfvXqLr8E@)PPhK-!n`PvEva{#o)?$kxs%Q(BddrMG?R8T@1J z2hR=%;C+^Dv zWKH)ngBUCP2dk!Q&cr0;gH=S!tG?b>1F9?yzD5cTbQ!43GtL0$=D`IkF~D8N<``IeS; zAo0rD5aVzBrmmsFbnbG+`KlZ`>}H~=NUpq_4xZA)R3v%rq9j8UIgJt>UtYYykjJ}DvdHOZcER)66qoncVR1ra)T!ajJx2(OD<|)b70>mc?jkE2loqMY%kAN(wL180|>lhc4CNf7E$n+J^oryMVed}0UaS^1M%`2XI z)Eg_tuH}S9c%SYcU76+7@5h^r&7Mad+54SjzI*o&?=~-)2F((6##A7YpQftZ)kq8s zjq4UFc?1aTSv!uDdZ|8yD?(BKf6siJ7^j?xsZ@RgD*$m zyPG{#0r|!(WY)w4cxIuk4Qcx3&2AZbbGFXK#m;W3+@3Y!#wz3>f3mQh}9>>%gYHmmh3%utqXXn7aaN!iLp@(XsVlEKSxEOlSZbd z?11ctWKiL-R*T!Ytydg!0tAGaXx@E$gN@~$K!!E&^92m#8jwLe$$HvA;xFlwVzGT7d6{kkHA1J})PHi?4H(&5YuQYqviSDem z%AV{W4>Ti~ldw)(_9jCITrM)hSh=WkaQAeY zpKsyKy8SVpT23Ng!gH>z6!cPQf3s_q>8m(7o!oi5UK%uHH0|(XFCI)H5WEPf2A1k+PeXqfNp&4+$ExNBJK46Y=fgkdKIX5(^4Mz66``&aBvCFRwz(be7}Tb zsS|Y7@;`)676ZUc5TPXZ$B#!f!$eV42}C z%KqVTT2u`iJYL02Xk|XNTAc1@7BcA#z!w86V_HIb?_6%WwK-{MU@)dyV*2&!{wqxtApy~G zl!N`-vx$$n02lObc?sIzumD=y7Va8_Myxh?t$=yO43K#hZU4X>cCxW)XKK{VX)&ZE z-_59?o$(kQ`#m_1=^7%#sm#`-uWY|P?ecx|>6Vk;`&8lV7hH{jigxrP$6gK>9#O&B z8K;oW(rk4?q**aLjW2!N9`LG4O3|eQcNO^q+<1l5;TM3Ka!RILC`HE7=+NZ z2)~UlJ0ZjYEV=BjOS)G~iL=Ku2&xrEs2Wr7*x5Zn9+BOh`^31oxJBKb({tA~0@E{# zt($EjL^C~JY@L4>ka{R8DXB2|ZD+&;pL*K|D1}uHJpyfCNOfujcusC#?&;|fg!*Cv zG~yCsY&6D>{ORu8k=R4mNBc1)f2m)liD!?zIBDC!U7M7yt*ymo9{mc_FmC(#*y@Y~ z3FJF*iAEGhNcmcUU0Gy3)NMdOXc}S7&OX}SJazTj89y&L-dHkg-kcm0z8uT5g!h&6 z!zU)M4xTnFWQe(7?$gur`777+G0X9!K%GN>jWlZQ;=e(YwN{3lhld63cD2uq2Fy8BQ+&X|^nB7kQ`cYAMM-KR)zvTZL+_)uM0F z;7px$GYwc-SvfEK6x$#BRN?SVc95uV4-0o&+fGA+_ywsiXnMRxtlo(TJ(=e?$lf1} z{16`9Bres%-V|_n2H+`dlixn|3fE+Y5KoUCgJo-0lljgy0uJYa5~N@pn46*Qib<7N zV#1RGwCO%=VTVvBXD1PQOXl>Mtv|kcojQDPRGSPx!j}ZZ1)J0a+huFU_)%N`K7lFr zKkbYnA7Rb|i6>8~`5T*7+QwQl{3G^I@#~WH2DYk*r?h#%y*N+hR1o-O!$7lwPEoHd*97Y<>gyo zIEA<{NNfFxEv`g6Hv{oC>SgE$`GT{|kgxHffH)wdvPrW^MeAPS0#YADw1@blt?geQ`c2n)sVb91huNIE(?!Gmy@sgsY(95Yg3PTB1R zG@e3|3%?%+>C@G%^i`W*MC;l~X6r@byCo7Kck4gBlLhll19BAa)~0ZkdtgGuP=|>W zqmPZnzxa`LEyi^4w5g>2h>`M!D2RHK2(rQO1a_WsOxua7nRpunO=RDi>M;#;(P# zrN#OGbnO3lS5v@qmGMSuPyN`omE4%!Qt(~4f^B=3%@cj$8Zm?P5)ze)VXFJa#zq>U zz$sDZju%Mss$^5r)vHCgu2TZLwRKm4ZQ)wU)X7ub?Q81R&N-qWInA%I~J*i`iOHD1g4Pr}XHFuvjl#sCAq|yNPvw;EZ;Pt>>uT6yNvK)v=F0KN; zMsDvM6*-eIl~eaw(D*dr-c8l0&;6hEWGdSGBX{c(^ffL3>eSLVJiDK&{0vfh@@gp`y>1%Rc+?+<5G z42a~VOYe#0V@6vB87&IM>ywpYboeCDm0WGimWRE5|9+29y@oH3Ip}NcA6lKn?f;12@RX1j2XzIFwk8gNVUdQ`(0UrNzC50k zEv}~g`-R_2y5Dt6m37^tr}Y$hT&LjbAIW>_RH+`Jr6NQ7x<>H6$D-03Bbys#X|$nj z7IC`zempUUm$)UpzG`WRhn#!l#tLxm6@TC>VcuH5oxN2v(nL}*agR?)Ddz$u6J1GC zf}f3FHh)76?oTL17j+{Mu$J7|5hYIm>#(wU( zI@4OhPqft4f#%JSL4}##1y-7h*|5f|QT36k{;pc`#eHIx+lslPl^+`CjSW8jP`0e< zu6OxFY_FMKd)ajWKP+`eN1!Z(^VIq4SN?=vhLx}o z2e&|Td$?C$bR`t84YM!2dPVujF;MHu3&huY5tD)TzWJ&0C)r8Z6PW-?b*kExxbX9;Jb5W{{eM3^!x2(!R&bD@|1E zo5Fk|!IK^G@@dk=x&V^+IdRjs)Z1G~s5Xp%Hz^#ILuJRu>9q`5pXvz=gC=Qw9kV8SU$c8DP5SR1uD$$;YrQyi6-kV3TS>I?T zA>G2iV$5~{Xu2_wP-0knHk|umd4qI0i$pRgZaGcEMeyEvPDhiSvK5F)aeSZsUo4>2 z6ndA1i$)7~`W5U#^ux8Eo6j_zYXg%}z0*ce=021>$1K9m%C4l6D-f|Qy9GJQOD$DV zb(l%XMVe-Cb$91;Kzz64fRyw;L-w-<-{h;_a}+4WK`kCrIGZprn$gQJB|X|771KB! ze`XydZOyjud5Fio$z61Pu`E8F-Iv~%thU0t#nBi;@#V&}pUrV(lxI@(DOl?`g7J{u zs&Hx@D0KPARo{f(i@`4`Z_rO7Ecd80Zc;ie6r{Ad(AcB*YGF{&;|0G}BYlUbROqEI zS}Gvj#@5y-Q+(6%ANM~*k40x$|-%{y9H=?D(P9u zckLg|6nhBo9*iCW7PXxEg_DxL8vx7pOUGY~y^mkC%?w<-dVDDEM<|bicPzFdP)B)1 zs~bz3v|ds{Gs`+$Oz6r!QRYM0$)Zyv;n#~wmuc^*>d^TDfy;NPXN6#GD|yp|B~{Z24P@6S1fpM-OB3W^(*%gZ?d zJOBd|7_MKRs%R|RlJLY}cV=L8uBfNV7Hj1+g<}u`-?TuNo%Xt``ivpdtWLB#;TH=f z&*AMRI|sW_WJNfl%xxXFm&hTi#NOGESSdKYbVV6T(NT$r3f}mO(Id`bNr;Eejq~&p$*thD*h0hQSb3Hedd^tFjsrJ zrE*Oz-4Jrspay<`@`kLP3VDQjkymauV@b~)Xs}xt4?F6QLke9{ zkNap~rTa2@=8(e|_vou}@K#kW3bRlIvkC=qAVo7JQqy8b;4U9-Fg4*;nai@NDE)F_ z;kV|Zqew2t6E}an;Pdg>@!DAe59XePwloos>5yjzb%?|pva>Y^k-=Z4{Qn~`V&HG-dr3JB0Tp1Id&ej~;tV11nSLo-$m zX(LsM!Q_=4%*3xXjl(bn9+*8}CGe3lgHJtFr!f#oPi0ySNv`0P@!;ZJOmZ-|m<}YB4J~oQ-WoWMGYd{B*GynlvrvKD%m;@p3sX&vNZn3%sD@ zmuq~)#}*tk8_NLk4Ic4LgO#rGykA3Gwg$rDAY4oH(E9#t&?}SnbVs>pjcA6Z^~4QR zbC^yp?&RLsRZ(0cVG0X4*hABhoAu`*a&&GJqumxx&=;AGh%uD)kM|w4FE6+a4T5UN z)Lf1#(s=K?`D~1|wo&KJ%^RUsNuk^Et+_U@73#Q@3!PUNz#WEzN7uqX8hCq;qUi5Y zACHZC1OBSnkXphiDH96v8Lgw-m8PSk=Afnu7;F>wRaRE&a-Q4v=N6yq`GNASN8IC{ zqBUtx^(0Bj#1t1~;Qcw-SKbX`_UlU>JZ6TFOBGFX&b2r3U%f~DCjg?&^cA^o$RSf&Mm1G5HE zr!DjLMedzMZi()<*{P)=T?h>;$3D7erlw5TuLf*|x@!nI57)7&u}!ZZ@0MOv_MU9S zi#R_U=vPk@E%zVUY5I$iF4a&J{6DpKom`nnWBHC# zW1yUa*43M$0UL&M??1Gm&kivFbF2rKbC*)*Jke9My1FYI&0kw41q!Nh#3eU;`XrR~ zOzwP>!(*RXg6f$^2dUI=mQ~g}_hsl_p7QlzuOZCj?#3v@wApv!_!a2Zx&&ajAnLi;o&1L(eCu#9Obi%Xk|VFy%s@#~#%6RZ14j5qa4<&HX%yOW zjvgjWAAh?Ld}k-ejbJy>!J4fz=mv21SVvnj$QmsBKmg;-k(QJ&7U+4Ezj_vv!OK>T z*c@pd`AeYlS~9^%x#Jf%8RbYS|8}GE>S5kxQg!w4Fy?Ogxc=ZzJZ7riuTA>;#^~38 zpF`8Vouujnz2mBWQXKp3ViuRlEx(iIob2puXa+Z%OL|Y!;`?6{RbTu;&gLJpvroFh zYgABy1n2UO>l;p1u6zrSOP^moyb;9H7*UG*@pirp{jIXk%l;i!e}ai}QCk zw8k!@fSwl6%ul28D_)5mXe&?d@5?0Kb^O zR!~LI%2MYTh-NodGyL3rrC*b8xJp|1kaeY)mkqHV?KD#vXY`L*x1(SH$O zk;O|sdQ;x-#+a6x%tyyK9(+vh!11imupr+YMZ&0Dop)smhCZoHP^=yxSowND+fXN? zVV{ax|DnWeB9pdR&9kN-1=~t@IYmX}#Q3)#R@{?P%1Gm-nk7t>OkBH#zc*5H#Cd}HRVHDkB!68n)EIDyPLc5J;(73 z)Gae86b*-kwkVxNvAS1_OcI-4gK=%`P9K( zC%*10qqEyY|AC

v_4ET8PF}A%;f|V&!}=)7M5f$;%&;T_+|!^USt)!(y_l*Fd{H z=}U&zC@rQVzzh|T#4daORm9Q6QG_?0Sql_PlN$V{o0&~mi0ur!1bCl0sA{(Gq@Ti@ z9S{yQ3%=|;1sc@DH-LkM6r5$EaRSlUlqgCeT2GU8qD$m^y>(j#vi(6g_5s41>ynh?qz+j%dJVkQjbGSqW=W!8pRB6V)gs*P;4I z!B;6{L;=`#g%Q#`&=3^O#a~}oSg_LnfJ@wErjBYsg_yB>a&1r3_gJmiOGlqZRr0Ew zpgqK5wJ{^0-^Un6{k1|gSS;H3p)S)EOvy{zlNi}Hm z@E!pj+rU1{@BRREZFquv$sCy>dBW`Jwnn}NK`i81qLkNb#ur7AgXCVt|(uI_4J z(X5rJC+nxHd%p3ZNmhG0JSxeW6+jhqDJL<%oR~CqgjLm4#ao!{SSX>>Q&R=PRF;?h zu;wn?K@ufKZpn#IOa+G=Iw-heVgidoHvaH3+q+(K-c_vSl2MYK8?u6<&D1nHV+VfF z_k~x2rpldG)^K>V-@aje_L-BAlDu-t=1&7z>Q4GcU00wj9koF0HOv;F2e|%^wP!nEI zc&R`lkjT~(5e&-F4uj1ml&kqL7#*{SvS|7QD8T&D%#CWKw3lbf2h{}2t&!3&noplY z)qi}Sxb3)B&5IB6`MfA>-5E7q?RDD89{fFUtc$@iQAs_0XD`y)hClG}hTce9*WI9s z{`-*shtA3E?!7c;26d^Q`6E4QqB=5yDCugTd=DSjHZDPQv-+jQ{~T#ZLY1nrVC?Yh+Q(hAoe#cFWfbfh%kVCz@)6j&6BYld1tUa5zZv&Q_&rd<*BMO6j*=!@|G{OD2r<{L`3ZJt# z&n!%r5^`5H^sl$ByFU*1M~u7ZyQlL`x2VR=I%MjIrTaj)kb*x04=)i=kb zUt8GinL8n20SA>yd{`lVCmCtsdlga}53OEgMQ8#2ymX_%#0t=YU7g+XUXjW7`LrJy zgwipI*Bjk8KZk$#a2S2KoQztR?8~Y?u0;fcRSbEJwvd=P@^;s<2Dj^>V5phK@R7kqt~rSQ`4HE8i3QeIK#&n!!v) zLH3NWk^bTYeFE#9Yz`?)&{StCEt~4mP`Kd|MTZ{E!ZPhUr;p_(Qf-EzF zvQUz(ac(Bu+l39+p+NlPXNi_8w7{u(r&DbSU-r`OBar%V+a2)$F+4S`D^I)4R0aNL z6c~4OlJ$spEbaXiaBP+q9Eqya?3<+>Q@%se24P;*_=O=H=D8PATc5(56oW;TF z0qIvkwZz_~Euz!Cwb|3sXMP+-3RpgDsYBl;YP5hD;}NuXOcN15qU}N@P*6E{SP>~DPgiI^6cZNT*R{3$oO+@Fnh;|{|T6gl@Uop zUe~n=fL|7`&m!($L*-el>~0AK2BYpk$&V}F44D^oO7?v`Z1o+TxbW$Pa_IvAA4 zuPilJOit?ixGWV)VUPS*Bv378@qYDAxu^EV0Y8<(m<%$VpFs2Dv%JQnN4RRf0%FHm{(_84sMN6O^p^!SA(~Pnre|ZZxX*{#m6V7ee2#E$}sRw4qGp-l&sm(>9n|f zGO~SwUq)U6N&8{y^`a%BGsPLQG@a$am}As2NMJA8-pU4FQ_E$FhIJgm6Pxq} zSNMW!c>d_CE1fL+uIIuiRk$<*gMyi)tBo^{)&KUl$PvZJp-pq^Zm zgA%xp2sSLY)U44y^@7NI916Sl06tkQ3dtewc*qG~bl8nc`RE$fWY9eoo31@K)7qx7We29aR8xjFmhmK0 zf<_v;s|Gv7*0#B%k2Ea?*MMDX@U7YDux4@1+>6W^)RvSp3_Oxa9K18u!v#k89~Ax8G^@oG z-qVewEKjl9HIw?x^tds9`8`e!nIB^Ks<*@M@Eh^)@W4$_=Fp2@1caXG<3tWWZU-mN zz3HA0-K&@K?D}Dcof0N5)ho4}Pgl4&K_HsS0jpg)bR=(;=9t-L#<@-N-2+FE2{mtF zvq(!StE}`BhhcZzvdd)li^ZBok5gsLw6b!2h=M+HV(C*jJZzeF2H{atXQKEZUfld} zAA|rCcB~Y7do{Lmp3lcS>cP66gVolzBU4lp-#9otg={fsY2D$<(pkDJq7uVmARFb} zW+5XnT4K=F)#b4I_2QKG=AN{B8ECe7+FR-|F*8%;)$%`m;yAS0WhJa?PrD*?vN^F|ttVXzf$rtUuKeUU zCAmUIHTGS&%_yhr4B(82FCuv=y(e?BcXmM4=U;p7m%{*lSQuZdoNqJ{$i8}7n3H`4U_G#%wyd;ORo+Ag#VS}bzt{DI5xI{%L|c~ z1L>H$pB8Rnq64Y^e07JbW(2sCA+rviMpzK&Fi$DL?W0$$Now)D?Zl^mF`-> zr=*d{71qdyug zB(nnJbo89jtoY8MR%cva2(U+!SPOmOYG5wmJlhfoHv*2wS~>d(f|==VaFw0aao-7s z6!>(egkM=ezc9rq!8xLjkb$|9(Ho^9W%NL*Zwmfwf$dulU~qfQeOXx@MX9g@6<4XF zuM#V@ot%>Dy%u_M++b1`z~8(yc*%u9W6|s?fZ>A z#ZK(vWSr#0MIIj(O`H_ewOP1iUY1-#bo-D_a5B?9Mr=6nJC2mb_A&!rOPRRT^plzF za{E6Zwx!kEzdDVYpo3x2s`#6+h~hMfKOX&>96s1ev|D5Jdl3?D%1e*sZ)n);Sn^l_giD#9J?3S+GkldRWs9DRYisInEx`C%|lYt;Mr zmU_hQqf5Msh8lhF|naANqLoj8{X2M<{q{OT3mgCZ^oir(Gb522peMO9bi;BQE%b z<^IFsqOcCGS4*?PPv#4hoCpFM)Dp#0^=_67g0q{?1WZ0>`Q7aN_(4bHSj@;#?($tx z<@NJD=-j({+a`^7a*s4Fqh}ZH1lC9FZH&L5R~8^jBTmUl!%@9mpEy+wfMKzj>o83k zA~uSqZGpZ_QzIhNYWEU@ow6U19RV!!hvf{lKit!jSZXl~lmcp{cIg?xm@-R3#1eb0 zUPm2dkJ#5n)dDiad?%*@X%|NROesai8H-u)ItheYKe>~AFYDJMs0TBl^3S_>l}VhMW`K5VIwr}I&(sX*sTty>=%zT=!p{J)*kr3ren2Ojhu|2xBVaj3pwR}19RT4~c z3;}*EEn{&T>4zq&YkNw&Pe~{tUViocNQ&fsj@tFQuH|p0H{%8+eB^cLucxM^E2?y2 zVyns#29zhdKa=N!9D%x&JE7PdgA+;G-T4xH)Uma za)h6>I@~jQ;+QGm{s?j!a4zIjznK!!V!%vlJ}p_U?3irTX#Ef2YL{hQEx8kssf6YW zdwnHcnT3yF5Pl_GbrfktW_#w%{p6wla$*%2;i~VsNuC{hnplHqnz~>!`(1D5syW(d zz1l%wrJpZTPyYET~l3OS1Tkg0c(r^=&u|dK4ZW z4#-|9LV%Ptf6vR6r>U@zyWC8IB<=mOMxce?(b19LunZk^R_M*W*8<=6yYiIjiE~w zhoEEy<>ah7>#i?(G}TB2Q|90QY}*HA$*j~+&iqRwM!vnkUo;E(e4SIc;nvNy&3XO2 z&r`q|t-wJ?M+fM%4vh($cl!FZI%QMSIfZ3q@u|(fv9j8F{k;`9I+BG>JAZYAhADA! za@{pnlO;UQ`%EOUKjoy-%#cXFL}Wh-0EYKupG^Yuh-S<87Ya6ETE{l_u`=^s&ifrL zGCl;sivw4-_2SKt4NoC66;dP34&ChP=L4IwtTH~c$XmRap}uCNNISkc`DJehAoG45 zdCjO++ahzHmTrBdtZpaPi@li`e>nZzQ=?hB9)g*!pS}_sMOlV5XDClNpbb1fLc)wZ zybNbuF5P9J$5*T3b6fyF1!7vvtW!Hn-f#IMIbY31 z!VO>714H+`ksNI~8_8pL&HA1=m2ke5ET9Ejx{n$Nl{N+qOJ1BEY>; z2N9p3;2M5H+(|udkJU{{eC;@&`V;yr_8jG>J(E6?E?H8FQq#xuD(V%XXJm{Q>l!vE zim`Zw^!Ez6IX~tq{p1B}Y9B*pt91Zj`L8pN)E*9JU~gl8SM2AiVMyQ1;i-Q=9O511 zM=DUbB@(vHm@2k$I~v#N2_UgcUjo;25IP4 zYi^~u>%VkAo|f()-weEpCM`za?`E*Ek-AnR%TTd}OeHxb$}334W`B*hvv=iQ^NkY| zK+m=BqfLGdv6cxaT&?}4uG$|kpj9g&eheTG)833-IxjA*b4e5J8=o@?y45%-opp6* zgqFH9w7r1b5`D5vFd4^}FQuH>cahe+3Lssx-|`9q6krZM}x72fq7wvi7-TI zQyth8?j7Ajmu1fdy9#f4^7O$2zm*RQz%DP)cJme_ej)G>u(Spd`TF*_X3MO~U7*c2 zYxH6h6UL!Rib!dvKU^6rPwktgAA-as;&Ft_^i~dSk*^tZWNzHJZ3;KML$KwolQ4`~ zB79VMSSx%CqD#ctB1)P{vQsZHCrISHnvsV+kvV#WK^3WKw=@W&Bw-pBReReN7SfHX z^n>5BWBXA}JNV!assK@q)BN)#+UM74_WqbpoIdPG;%}11Ytn<#j;i(Hwr^GkJ}~-d zM>y~k+$q=}KjtWDn5)c;GaK&|z4g_$M#{Ff2!&c1j5ltqdL10`_8^C><~sR% zO}DBQ`|SonvSsm-7Ofwv7tB5%TI)Jld?tUgO>;H;U1Nrxjh$_GoN^GzB3{k6)`jT~ zYyIim?ih%P`)6aO>39|Ci@qu#BXPkVOYr4-aH(qWNiSnx&&SgutyI7t^eXE=ip#z) zayLt%S)V~>sBCzdkqF}};`tB;lk@z=+ozqVvz`RQ?unnfTOES|9_x#gZ#Fij=@~I> zL+Ua?zlXfauN~ytx)ewmB1xJ>u#~i7$9ci2N=Pg%{gOG7ts#IQ0px;_6STEKfSS|x z+V(a1PkIB5Z3@`mu0tiGrQ$89*a$_9-&W}JX4+t6T4P?n^NL+pR2LC8jDC@#MQu3{ zlUWb_Z9m13dhcF+$&j(7=%{4#ZEs@Iooy&}X8D3~TYF({9NKO7MacJ(;dEv$7;GXv zUG>pBo(Gh1uk%a*rtsDU%yj#fL2qFAwRm^&Vts>`)Vx8??t7y(a+-VwgXjZNu{AH_ zn(j@Dz}C_#^`wv6V9SEZk_45-k>1VCG-isahnhzm0Xg>r7}|PW{MJZLL~sOmo=XF2 zV12R*^SdOh!&`m7&#+FV;shi8r=rqHD1NlUSiyem4$l*QZoRF>RG)*W&@Gw(@IR32s~PMM+P$ai3#< zWAl%_-b+=@n)(xXZfPl{hnEdpJ8$V3LA3jOW4 zOJ_CmnJ}=#PmRab8{N29bN_7ElYC_5s_WnLvou|-kwF1YV|GwN#!9%CEcg+P6<_8@ zJqsW#(ovtkmi1f0Or&RIWIyvz{oHTaP?A@wx9SXREvo^Lc8Q*k=Y|ZPg=xrhH2l7 zOLzUx;y!Q7*=g)i7xn8N===d!eTiS#A4{9%r^L&;j1!@VW)x9+_ghHc4`wLOT&>H*A-Q>mzc3 zlaj%#eVMXjm1sMFKRf!!2p_Z?FR!VgnJ*{o_TxK;igwoC+Q4_0Ad&veZm6|}#wOP6 z;xU$etNsn8Ihxs!U4HAVeG|S|Y^@YV9!XFUfe3T}{{59cizAP@e?rHKWX?rO z$l?_KG^)@Y$V+WpgkynGSO7kyZsg)PTjJJ|%|?ydlh$kks9x_#?sP^xonj1RF2*=s z$*rfp5~Qdx!oWNGu;!dAQhb$&x?wHh1f|XngGcs0vU7Dwp=kx7CO{$qTfemyQsw50 zpH`El)>j9g+#A$0n|2W`Ui$Zk1yfTvyG>fDj9ab{M{_2S`-LWo9_KJ z(k>!cZ)E2so=N{Na-`4X*uM4nQb)OsRxJlk;Vjvh|a zBSMK5zoporMmtpPh&l^H86Z>cvOQbZlO^rF189XZB=)3K1zNi@LBVXoMPooluqv@n zOr?C4X*aB_%$anVAIfv$hDPx^Le9^ay=9nvcM(ZqGd)HXgx7+6I(k1hOyCJNEF26d zAyq$U{ar$7{gVM4eC@RtehVa4BkF=Jn_WH7-K`PFL485fu@27{?ru|1n{!F(5{1jx zvm#~m#)c7-d)8su8S>DD#p2k<)*}bmXLe;~6Wd`(_u;1vCZ%LH5mSS5x%sir+UA=#MHW{e`Pl;l?TFxoyY<$&y-`sD9%g2%r6V=eH0PchjSwG55+JY=1Kvlquo&0R` z&&bHp^-`}KCu2-UuOd$NL-t7m#R5IT_0xT`so~(Mos&kyq!G`PGw%NFSeAr&!;Sd_Ly1UhEMPwM<#_2 zZce-2Hb_@Cpv$oezm4>)->bVNYpOx<&@=Roi7u|HYgdbtDeyw50mp#zjFl_lwcK9L zj(cSgbyr*G-I;H6-j>q7;`7TJY}T-{vjd%UhzXO4XX%X9@Q|eZmUzLaJd}fJ#(igV zMxNrk8tWe;UOMg=Q3fA9uO-QVHpy0$5A9M?CflBA4n_8_0g3)M_C(v1h^~m4G@gB9 z7z-QB$QoX!CT=xsZo8b)=EUcm-lS?OIiE=SRvhkzyNs|4K3}<};Q#8ZB|{~0I~X?| z`Lngvvv)r|;fnjv^UIXUN+@UagbyJ=Z-7V&8C ztagFRYPZST>F}fcB7dnJLT!@yS0?Hmmy~$9zj(?+&>f3srUIr_$NP};uT0heF#|DH zS?e@$UzG0<2Vuy#$5>!*L$fubBO`bJsYZ+p^RSnmtZIw|xW1|WbJ9g>Cv0Ux*2y`t zDW$4v_dO+WIfz&u8Cerqv!CM8BIQl~rM_VK1vLdNii?X24Mk=Wwr85`y9i$|&7NnT z-qm%q=9CKB5f}AMuai~}AK-!?yEa)5S;@|bC+BQ#KGB3oK=6j>mq;pVS^~bN zB_K3x_WQ(Wp9C+jUt34HZZ0Uu3&TFDbLDe>)<;-)VICi8)+n zru?IV--H~fA9vupj{lzJ+_t!AL$H9D);e%Qty=TDbAnIe&}e*A8hhyHPg#e%x);w4 z{*tc)k3Xl?1jozg9+syA-8FnA;r*{0laZ{8Uq~nAb0rm8fJz)!sqisYq@UzES$T*s zd%+$^jAY^I>VEIamQMMTE2PMm6qUst+W=)-@?Oo_JFMF?GpV;r0_fp(fyXrfwRm_1 zfdX9${Va3GD^Tstt5n>~{-eFTywXub7wUa2h|b4kimt`{kr>cgDX&Yi6_gy4+Gj9J z&rszS^)FhEtjbl8Dkk)&H|;F=@PLJ_dG!{*Y#6@jDwC4WNOdrk)ed-pd)Azp<;6fs z_P8sMwD9P-idn8Q>~Q|uS(sKWWabyXuf!@)qE7{%0-#l#l0B;gywk z6jJgm-E=?tm6m|KnZ(lntZkY3B}~81Qf>F;!&`9W87Dt{wP`ogSb?p*xi+%gzcqd+ zLF4*WALXxKlWp?t@85Y(7D<~hQFZK2?x1TqcM`wFLzNXDswf#B=@gtDEZD7JYiGZ) zmVAFnwfmLi5N-?!eSJ6|pEJ8Y*;E)}ZE9B-{$dbo?+)F-Nc6ZUH=U8wuKm17uX;Q4 zKCd8lcwlGn8huXsTPyvi^lBQKNvruqQny9i9vEn-dB@_Ny4#7+oH(e6()S9X0$KrU zwlm|pe4PB%i802s2@*&-?rXZMox|&QCiL2?8^y#Cy5l!p?18|}`(%q$*391$FWd%* ztksaoquO2pKMRn>G`ss-aAOsDZ91!;4jQYbC|d2s!3eRl*|{es5cg8OKP55Q6?_yM z>lr7*B`1!Z9;iH$HNY-D5X4+XCNUNAA$N{ss4G~QhZd!-5G(LVaa(L>2KO1I@>F4| zyQiDaT@4tW7Yfb}J-?iQ)6<~2dTzp%&z;76XHZ@JMTPPmeXh8)BEy!O3P;RKz*E`=e`q3myEbuBBLLIMk2HUezSy-=amo* z0H_u}OL!HYSX=q_B==KqS*G7Y&TybNFyq0gGLEv((a*QAu((U9{id0b49I-oQO9QA z!Tu0xVv6*>Q^IUSWTa22q1VyQgZS6x&$b0WbRj($E?;Nm6}Mt$-eBBDo%S_}8Ny9R zYgCM+lA#M&AFjZ!jI>PS{X;`&{)TVLx8#2M}qK zD98?BsxxMDtmD?DGxRE5xams>Xv|xbuc{T8oQ-9f^!Po$WLPdMQFdDDVXp~a1RBpw zV&1I<`>;%lOu1CznzsCTa&|Tzp$@xxrw7UgOx$Ps{N%#7+?-#ywMeLL9|jX;AT4Mh z%0@YwS%BAx17Bkf{mqxjxxD=xOII~zqUlqtiTRAo{lcjhw4UR+!Z^y+VQv^?Di zS)06OY?-|?Sz_QM4BCk!aL=kG@r}8=m-v!bFA_f5Eb(Y$2AtHwn}-|R6s~**kCf>Y z3H%6-CO_#C&gyFmx_yljWoKj8bKuYQtW`sccfNk_!0)^_3W^=VOibNwv?=_0N~g3q zrdI8n7tHphN4UY%!cZuq0pzY$!V+P8wqGP5%OUpQLGfG`Xb&zt8WFDfwVGj*)jX-` z#iHWy!W{`d?!Li34GSjm&}#a3Uw#bl{3^ShO z{s&titush6+A~0sxDW`$XcL^uqPPQAP>_+4aR*tpx;n1X{KrFS>5kS`GUXU^;*bzY zAE(5_&3(tuIuWKIp$@PfRgLaHc%Vazi;iYTd=xi1mC)O;fpBx0J4ri6Ll z+hb09l~fK_9b9e5t1z5(LV_k&AwLSYeJ*d26jK^*5{`H0kjfKqCRb~qfxahaa?PWO&m(HD zG*c3`LQS>F@?rsXi&Tc~`8G!IKMR#@O4;_)+g9^a#}ogwXI2TEsdis_BWICR5vkK1 zGBwR?lj*(lOS(?t;>*4a+7UJvV{cNE`IijC&kcf|&e4C}ujV7b z9DK9KLEP-+pA32KswTzVCdB})(y^phK#C)MzqbF&de3fVjK1MPlz7rz4k_8ve=xwd z+G(yTZ`2C^GD%;!z;+R<`4?sI!u$Urwy$OJC6-2MK13f59nJsO$1c1s0@xZ?ZmFpQ z|M3S~aGbIH4OzbMgQKgrC;ssZ12r{!Woc;Z1LV65(M=k^71Bo`Mu6w(;|YVp26QAz0e+^{;KxzV8Y|? zwWjY@i`7EPqAMyYisL?n*fyF<61A%1{-VwS+&MXVhhJQrjBCXjQ3qTakILUG@bJKf z{V!vCdU}8sgckOTX3rCe;RHteB^!+VKVSL?f*Tys*>IST0zl_t9Ef~ofB*EmEG4?S z=@-L#`G7=u;pJC=878?aB;X&*dy%J?J{S>`T@e%h-IUoS@$LR~gSE>nCDTy~%pMMo zo&T_Ze*|YpUm2CYlxevfX!tQn@RJ}5-9eBlChsH)-%{eO$|e=*F zYyUoLz#aJiaEE(Fcj3bMd9{hWByfF~|9T-)UAS_ObM>>G#gD(%TvbU+vEt>MkN*Y5 COi^I~ literal 0 HcmV?d00001 diff --git a/package-lock.json b/package-lock.json index 3376a2e..147c174 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "debug": "^4.4.3", "ink": "^5.2.1", "ink-spinner": "^5.0.0", + "ndjson": "^2.0.0", "oauth4webapi": "^3.0.0", "open": "^10.1.0", "react": "^18.3.1", @@ -30,6 +31,7 @@ "@playwright/test": "^1.49.0", "@release-it-plugins/lerna-changelog": "^8.0.1", "@types/debug": "^4.1.12", + "@types/ndjson": "^2.0.4", "@types/node": "^22.10.2", "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", @@ -5083,6 +5085,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/ndjson": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/ndjson/-/ndjson-2.0.4.tgz", + "integrity": "sha512-ajAl7AjhFstF6waORYNSS49GL5iBKisqJlgvXuprXFKCX9fto4ordlNU3+XMgkMddgeR0WoQQBmKUk0v0dJ4pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/through": "*" + } + }, "node_modules/@types/node": { "version": "22.19.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.3.tgz", @@ -5149,6 +5162,16 @@ "@types/node": "*" } }, + "node_modules/@types/through": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/@types/through/-/through-0.0.33.tgz", + "integrity": "sha512-HsJ+z3QuETzP3cswwtzt2vEIiHBk/dCcHGhbmG5X3ecnwFD/lPrMpliGXxSCg03L9AhrdwA4Oz/qfspkDW+xGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -10190,7 +10213,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "dev": true, "license": "ISC" }, "node_modules/json-with-bigint": { @@ -11732,7 +11754,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -11944,7 +11965,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ndjson/-/ndjson-2.0.0.tgz", "integrity": "sha512-nGl7LRGrzugTtaFcJMhLbpzJM6XdivmbkdlaGcrk/LXg2KL/YBC6z1g70xh0/al+oFuVFP8N8kiWRucmeEH/qQ==", - "dev": true, "license": "BSD-3-Clause", "dependencies": { "json-stringify-safe": "^5.0.1", @@ -13628,7 +13648,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -14114,7 +14133,6 @@ "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "devOptional": true, "funding": [ { "type": "github", @@ -14443,7 +14461,6 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", - "dev": true, "license": "ISC", "dependencies": { "readable-stream": "^3.0.0" @@ -14534,7 +14551,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -14874,7 +14890,6 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", - "dev": true, "license": "MIT", "dependencies": { "readable-stream": "3" @@ -16438,7 +16453,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/validate-npm-package-name": { diff --git a/package.json b/package.json index d5c89dc..5af7e9f 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,9 @@ "preview-reporter": "tsx scripts/preview-reporter.ts", "test": "vitest run", "test:all": "npm run build && npm run format:check && npm run lint && npm run typecheck && npm test", + "test:external-host": "vitest run --config vitest.external-host.config.mts", + "test:external-host:chat": "vitest run --config vitest.external-host.config.mts -t \"Claude Chat\"", + "test:external-host:cowork": "vitest run --config vitest.external-host.config.mts -t \"Claude Cowork\"", "test:playwright": "playwright test", "test:watch": "vitest", "typecheck": "tsc --noEmit" @@ -84,6 +87,7 @@ "debug": "^4.4.3", "ink": "^5.2.1", "ink-spinner": "^5.0.0", + "ndjson": "^2.0.0", "oauth4webapi": "^3.0.0", "open": "^10.1.0", "react": "^18.3.1", @@ -95,6 +99,7 @@ "@playwright/test": "^1.49.0", "@release-it-plugins/lerna-changelog": "^8.0.1", "@types/debug": "^4.1.12", + "@types/ndjson": "^2.0.4", "@types/node": "^22.10.2", "@types/react": "^18.3.12", "@types/react-dom": "^18.3.1", diff --git a/src/assertions/validators/toolCalls.test.ts b/src/assertions/validators/toolCalls.test.ts index 42da8b7..637c613 100644 --- a/src/assertions/validators/toolCalls.test.ts +++ b/src/assertions/validators/toolCalls.test.ts @@ -182,7 +182,7 @@ describe('validateToolCalls', () => { calls: [{ name: 'search' }], }); expect(v.pass).toBe(false); - expect(v.message).toContain('mcp_host'); + expect(v.message).toContain('host simulation response'); }); }); @@ -292,6 +292,6 @@ describe('validateToolCallCount', () => { it('returns error when response is not an MCPHostSimulationResult', () => { const v = validateToolCallCount('not a simulation result', { exact: 1 }); expect(v.pass).toBe(false); - expect(v.message).toContain('mcp_host'); + expect(v.message).toContain('host simulation response'); }); }); diff --git a/src/assertions/validators/toolCalls.ts b/src/assertions/validators/toolCalls.ts index ccb36fe..2a11b70 100644 --- a/src/assertions/validators/toolCalls.ts +++ b/src/assertions/validators/toolCalls.ts @@ -102,9 +102,9 @@ function findMatchingCall( } /** - * Validates tool calls made during an MCP host simulation. + * Validates tool calls made during a host simulation. * - * @param response - Must be an MCPHostSimulationResult (from mcp_host mode) + * @param response - Must be an MCPHostSimulationResult-compatible response * @param expectation - Expected tool call specification */ export function validateToolCalls( @@ -115,7 +115,7 @@ export function validateToolCalls( return { pass: false, message: - 'toolsTriggered expectation requires mcp_host mode — response must be an MCPHostSimulationResult', + 'toolsTriggered expectation requires a host simulation response with structured tool calls', }; } @@ -206,9 +206,9 @@ export function validateToolCalls( } /** - * Validates the number of tool calls made during an MCP host simulation. + * Validates the number of tool calls made during a host simulation. * - * @param response - Must be an MCPHostSimulationResult (from mcp_host mode) + * @param response - Must be an MCPHostSimulationResult-compatible response * @param options - Count constraints (min, max, exact) */ export function validateToolCallCount( @@ -219,7 +219,7 @@ export function validateToolCallCount( return { pass: false, message: - 'toolCallCount expectation requires mcp_host mode — response must be an MCPHostSimulationResult', + 'toolCallCount expectation requires a host simulation response with structured tool calls', }; } diff --git a/src/assertions/validators/validators.test.ts b/src/assertions/validators/validators.test.ts index d4f19cc..1e7b361 100644 --- a/src/assertions/validators/validators.test.ts +++ b/src/assertions/validators/validators.test.ts @@ -183,6 +183,18 @@ describe('validateText', () => { const result = validateText(response, 'result'); expect(result.pass).toBe(true); }); + + it('should prefer host simulation final response over metadata JSON', () => { + const response = { + response: 'final answer text', + externalHost: { + traceLimitations: ['metadata-only text'], + }, + }; + + expect(validateText(response, 'final answer').pass).toBe(true); + expect(validateText(response, 'metadata-only').pass).toBe(false); + }); }); }); diff --git a/src/evals/datasetTypes.test.ts b/src/evals/datasetTypes.test.ts index 94201d7..e1df9df 100644 --- a/src/evals/datasetTypes.test.ts +++ b/src/evals/datasetTypes.test.ts @@ -92,6 +92,68 @@ describe('datasetTypes', () => { expect(() => validateEvalCase(evalCase)).not.toThrow(); }); + it('should accept external_host eval case configuration', () => { + const evalCase = { + id: 'external-1', + mode: 'external_host' as const, + scenario: 'Reply with exactly hello', + externalHost: { + driver: { + provider: 'anthropic', + product: 'claude', + surface: 'cowork', + runtime: 'desktop-app', + platform: 'macos', + }, + name: 'Claude Cowork Desktop', + timeoutMs: 120000, + capabilities: { + control: [ + { uses: 'builtin:platform.macos' }, + { uses: 'builtin:anthropic.claude.coworkSurface' }, + ], + input: { + uses: 'builtin:desktop.macos.accessibilitySubmit', + with: { createNewConversation: false }, + }, + completion: { + uses: 'builtin:anthropic.claude.localAgentTrace', + provides: ['trace'], + }, + normalize: { + uses: 'builtin:anthropic.claude.localAgentNormalize', + }, + }, + correlation: { + strategy: 'prompt_marker', + includeInPrompt: false, + promptTemplate: 'trace: {{marker}}', + }, + options: { + appName: 'Claude', + newConversationShortcut: 'cmd+n', + }, + }, + }; + + const result = validateEvalCase(evalCase); + + expect(result).toEqual(evalCase); + }); + + it('should reject external_host configuration without a driver', () => { + const evalCase = { + id: 'external-1', + mode: 'external_host' as const, + scenario: 'Reply with exactly hello', + externalHost: { + name: 'Claude Cowork Desktop', + }, + }; + + expect(() => validateEvalCase(evalCase)).toThrow(ZodError); + }); + it('should accept eval case with complex args', () => { const evalCase = { id: 'test-1', diff --git a/src/evals/datasetTypes.ts b/src/evals/datasetTypes.ts index 0a7d7ba..01fcd7e 100644 --- a/src/evals/datasetTypes.ts +++ b/src/evals/datasetTypes.ts @@ -1,5 +1,7 @@ import { z } from 'zod'; import type { MCPHostConfig } from './mcpHost/mcpHostTypes.js'; +import type { ExternalHostConfig } from './externalHost/types.js'; +import { ExternalHostConfigSchema } from './externalHost/schema.js'; import type { SnapshotSanitizer } from '../assertions/validators/types.js'; import type { BuiltInRubric } from '../judge/judgeTypes.js'; @@ -16,13 +18,14 @@ export type { /** * Evaluation mode */ -export type EvalMode = 'direct' | 'mcp_host'; +export type EvalMode = 'direct' | 'mcp_host' | 'external_host'; /** * A single eval test case * * For 'direct' mode: toolName and args are required * For 'mcp_host' mode: scenario and mcpHostConfig are required + * For 'external_host' mode: scenario and externalHost are required */ export interface EvalCase { /** @@ -38,7 +41,8 @@ export interface EvalCase { /** * Evaluation mode * - 'direct': Direct API calls to MCP tools (default) - * - 'mcp_host': LLM-driven tool selection via natural language + * - 'mcp_host': SDK/CLI host simulation via natural language + * - 'external_host': Real external MCP host driven by configured capabilities * * @default 'direct' */ @@ -55,7 +59,7 @@ export interface EvalCase { args?: Record; /** - * Natural language scenario for LLM to execute (optional, required for 'mcp_host' mode) + * Natural language scenario for LLM to execute (required for 'mcp_host' and 'external_host' modes) * * @example "Get the weather for London and tell me if I need an umbrella" */ @@ -68,6 +72,11 @@ export interface EvalCase { */ mcpHostConfig?: MCPHostConfig; + /** + * External host configuration (required for 'external_host' mode) + */ + externalHost?: ExternalHostConfig; + /** * Additional metadata for this test case * @@ -241,8 +250,9 @@ export interface EvalExpectBlock { }; /** - * Asserts which tools the LLM called during a mcp_host simulation. - * Only meaningful for mcp_host mode — direct mode has no tool call trace. + * Asserts which tools the LLM called during a host simulation. + * Only meaningful for mcp_host or external_host runs with high-confidence + * structured tool evidence — direct mode has no tool call trace. */ toolsTriggered?: { /** Expected tool calls */ @@ -264,7 +274,8 @@ export interface EvalExpectBlock { }; /** - * Asserts the number of tool calls made during a mcp_host simulation. + * Asserts the number of tool calls made during a host simulation. + * External-host runs require high-confidence structured tool evidence. */ toolCallCount?: { /** Minimum number of tool calls */ @@ -447,11 +458,12 @@ const EvalExpectBlockSchema = z.object({ export const EvalCaseSchema = z.object({ id: z.string().min(1, 'id must not be empty'), description: z.string().optional(), - mode: z.enum(['direct', 'mcp_host']).optional(), + mode: z.enum(['direct', 'mcp_host', 'external_host']).optional(), toolName: z.string().min(1, 'toolName must not be empty').optional(), args: z.record(z.string(), z.unknown()).optional(), scenario: z.string().optional(), mcpHostConfig: MCPHostConfigSchema.optional(), + externalHost: ExternalHostConfigSchema.optional(), metadata: z.record(z.string(), z.unknown()).optional(), iterations: z.number().int().min(1).optional(), accuracyThreshold: z.number().min(0).max(1).optional(), diff --git a/src/evals/evalRunner.externalHost.test.ts b/src/evals/evalRunner.externalHost.test.ts new file mode 100644 index 0000000..7aaf10c --- /dev/null +++ b/src/evals/evalRunner.externalHost.test.ts @@ -0,0 +1,362 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { MCPFixtureApi } from '../mcp/fixtures/mcpFixture.js'; +import type { + ExternalHostRunResult, + ExternalHostSimulationResult, +} from './externalHost/types.js'; + +const TEST_CORRELATION = { + strategy: 'prompt_marker', + marker: 'MCP_SERVER_TESTER_TEST', + includedInPrompt: true, +} as const; + +vi.mock('./externalHost/runtime.js', () => ({ + runExternalHostScenario: vi.fn(async () => { + const result: ExternalHostSimulationResult = { + success: true, + response: 'external host trace acknowledged.', + toolCalls: [{ name: 'search', arguments: { query: 'planning' } }], + scenario: 'unused', + usage: { + inputTokens: 10, + outputTokens: 5, + totalCostUsd: 0.01, + durationMs: 1000, + }, + externalHost: { + driver: { + provider: 'anthropic', + product: 'claude', + surface: 'cowork', + runtime: 'desktop-app', + platform: 'macos', + }, + driverSlug: 'anthropic.claude.cowork.desktop-app.macos', + displayName: 'Claude Cowork Desktop', + hostName: 'Claude Cowork Desktop', + hostType: 'desktop', + capabilitiesUsed: [ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', + ], + traceSource: 'host-local-transcript', + traceConfidence: 'high', + traceLimitations: ['fixture limitation'], + artifacts: [ + { + kind: 'audit', + name: 'Claude audit log', + path: '/tmp/audit.jsonl', + }, + ], + session: { + id: 'local_123', + runMarker: 'MCP_SERVER_TESTER_TEST', + requestId: 'req_123', + }, + correlation: TEST_CORRELATION, + sources: { + finalAnswer: 'host-local-transcript', + toolCalls: 'host-local-transcript', + usage: 'host-local-transcript', + cost: 'host-local-transcript', + }, + evidence: { + finalAnswer: { + source: 'host-local-transcript', + confidence: 'high', + }, + toolCalls: { + source: 'host-local-transcript', + confidence: 'high', + }, + usage: { source: 'host-local-transcript', confidence: 'high' }, + cost: { source: 'host-local-transcript', confidence: 'high' }, + }, + }, + }; + return result; + }), +})); + +const { runEvalCase } = await import('./evalRunner.js'); +const { runExternalHostScenario } = await import('./externalHost/runtime.js'); + +function makeContext(): { mcp: MCPFixtureApi } { + return { + mcp: { + authType: 'none', + project: 'external-host-test', + } as MCPFixtureApi, + }; +} + +describe('runEvalCase external_host mode', () => { + it('runs an external host case through existing expectations and preserves trace metadata', async () => { + const result = await runEvalCase( + { + id: 'external-host-case', + mode: 'external_host', + scenario: 'Say hello and search.', + externalHost: { + driver: 'anthropic.claude.cowork.desktop-app.macos', + name: 'Claude Cowork Desktop', + }, + expect: { + containsText: 'trace acknowledged', + toolsTriggered: { + calls: [{ name: 'search', arguments: { query: 'planning' } }], + }, + }, + }, + makeContext() + ); + + expect(runExternalHostScenario).toHaveBeenCalledWith( + 'Say hello and search.', + { + driver: 'anthropic.claude.cowork.desktop-app.macos', + name: 'Claude Cowork Desktop', + }, + { caseId: 'external-host-case' } + ); + expect(result.pass).toBe(true); + expect(result.toolName).toBe('external_host'); + expect(result.hostUsage).toMatchObject({ totalCostUsd: 0.01 }); + expect(result.externalHost).toMatchObject({ + hostName: 'Claude Cowork Desktop', + traceSource: 'host-local-transcript', + traceConfidence: 'high', + session: { id: 'local_123', requestId: 'req_123' }, + }); + expect(result.request?.externalHost).toEqual({ + driver: 'anthropic.claude.cowork.desktop-app.macos', + driverSlug: 'anthropic.claude.cowork.desktop-app.macos', + name: 'Claude Cowork Desktop', + hostType: undefined, + variant: undefined, + timeoutMs: undefined, + usesBuiltInDefaults: true, + correlation: { + strategy: 'prompt_marker', + includeInPrompt: true, + }, + options: undefined, + capabilities: { + control: [ + { uses: 'builtin:platform.macos' }, + { uses: 'builtin:anthropic.claude.coworkSurface' }, + ], + input: [ + { + uses: 'builtin:desktop.macos.accessibilitySubmit', + with: { appName: 'Claude', createNewConversation: false }, + }, + ], + completion: [ + { + uses: 'builtin:anthropic.claude.localAgentTrace', + provides: ['trace'], + }, + ], + normalize: [{ uses: 'builtin:anthropic.claude.localAgentNormalize' }], + }, + }); + expect(result.mcpHostTrace?.calls).toEqual([ + { + name: 'search', + arguments: { query: 'planning' }, + status: 'expected', + }, + ]); + }); + + it('fails tool assertions as trace insufficiency when external host evidence is low confidence', async () => { + vi.mocked(runExternalHostScenario).mockResolvedValueOnce({ + success: true, + response: 'external host trace acknowledged.', + toolCalls: [{ name: 'search', arguments: { query: 'planning' } }], + externalHost: { + driver: { + provider: 'anthropic', + product: 'claude', + surface: 'chat', + runtime: 'desktop-app', + platform: 'macos', + }, + driverSlug: 'anthropic.claude.chat.desktop-app.macos', + displayName: 'Claude Chat Desktop', + hostName: 'Claude Chat Desktop', + hostType: 'desktop', + capabilitiesUsed: [ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', + ], + traceSource: 'accessibility', + traceConfidence: 'low', + artifacts: [], + session: { runMarker: 'MCP_SERVER_TESTER_TEST' }, + correlation: TEST_CORRELATION, + sources: { + finalAnswer: 'accessibility', + toolCalls: 'none', + usage: 'none', + cost: 'none', + }, + evidence: { + finalAnswer: { source: 'accessibility', confidence: 'low' }, + toolCalls: { source: 'none', confidence: 'unknown' }, + }, + }, + }); + + const result = await runEvalCase( + { + id: 'external-host-low-confidence', + mode: 'external_host', + scenario: 'Say hello and search.', + externalHost: { + driver: 'anthropic.claude.chat.desktop-app.macos', + }, + expect: { + containsText: 'trace acknowledged', + toolsTriggered: { + calls: [{ name: 'search' }], + }, + }, + }, + makeContext() + ); + + expect(result.pass).toBe(false); + expect(result.mcpHostTrace).toBeUndefined(); + expect(result.expectations.toolsTriggered?.details).toContain( + 'cannot support tool-call assertions' + ); + }); + + it('requires high-confidence structured evidence for tool assertions when per-field evidence is absent', async () => { + vi.mocked(runExternalHostScenario).mockResolvedValueOnce({ + success: true, + response: 'external host trace acknowledged.', + toolCalls: [{ name: 'search', arguments: { query: 'planning' } }], + externalHost: { + driver: { + provider: 'anthropic', + product: 'claude', + surface: 'cowork', + runtime: 'desktop-app', + platform: 'macos', + }, + driverSlug: 'anthropic.claude.cowork.desktop-app.macos', + displayName: 'Claude Cowork Desktop', + hostName: 'Claude Cowork Desktop', + hostType: 'desktop', + capabilitiesUsed: [ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', + ], + traceSource: 'host-local-transcript', + traceConfidence: 'medium', + artifacts: [], + session: { runMarker: 'MCP_SERVER_TESTER_TEST' }, + correlation: TEST_CORRELATION, + sources: { + finalAnswer: 'host-local-transcript', + toolCalls: 'host-local-transcript', + }, + }, + }); + + const result = await runEvalCase( + { + id: 'external-host-medium-confidence', + mode: 'external_host', + scenario: 'Say hello and search.', + externalHost: { + driver: 'anthropic.claude.cowork.desktop-app.macos', + }, + expect: { + toolsTriggered: { + calls: [{ name: 'search' }], + }, + }, + }, + makeContext() + ); + + expect(result.pass).toBe(false); + expect(result.mcpHostTrace).toBeUndefined(); + expect(result.expectations.toolsTriggered?.details).toContain( + 'cannot support tool-call assertions' + ); + }); + + it('counts external host driver failures as infrastructure failures across iterations', async () => { + const failure: ExternalHostRunResult = { + success: false, + error: 'Failed to submit prompt to Claude: automation permission denied', + toolCalls: [], + externalHost: { + driver: { + provider: 'anthropic', + product: 'claude', + surface: 'cowork', + runtime: 'desktop-app', + platform: 'macos', + }, + driverSlug: 'anthropic.claude.cowork.desktop-app.macos', + displayName: 'Claude Cowork Desktop', + hostName: 'Claude Cowork Desktop', + hostType: 'desktop', + capabilitiesUsed: [], + traceSource: 'none', + traceConfidence: 'unknown', + artifacts: [], + session: { runMarker: 'MCP_SERVER_TESTER_TEST' }, + correlation: TEST_CORRELATION, + failureKind: 'automation_permission_denied', + }, + }; + const deniedAgain: ExternalHostRunResult = { + ...failure, + error: 'Failed to submit prompt to Claude: still denied', + }; + vi.mocked(runExternalHostScenario) + .mockResolvedValueOnce(failure) + .mockResolvedValueOnce(deniedAgain); + + const result = await runEvalCase( + { + id: 'external-host-driver-failure', + mode: 'external_host', + scenario: 'Say hello.', + externalHost: { + driver: 'anthropic.claude.cowork.desktop-app.macos', + }, + iterations: 2, + expect: { + containsText: 'hello', + }, + }, + makeContext() + ); + + expect(result.pass).toBe(false); + expect(result.infrastructureErrorCount).toBe(2); + expect(result.infrastructureErrorRate).toBe(1); + expect(result.iterationResults?.every((r) => r.isInfrastructureError)).toBe( + true + ); + }); +}); diff --git a/src/evals/evalRunner.test.ts b/src/evals/evalRunner.test.ts index f2f7200..82d9f66 100644 --- a/src/evals/evalRunner.test.ts +++ b/src/evals/evalRunner.test.ts @@ -608,7 +608,9 @@ describe('toolsTriggered and toolCallCount expectations in eval runner', () => { const result = await runEvalCase(evalCase, createContext(mcp)); expect(result.expectations.toolsTriggered?.pass).toBe(false); - expect(result.expectations.toolsTriggered?.details).toContain('mcp_host'); + expect(result.expectations.toolsTriggered?.details).toContain( + 'host simulation response' + ); }); it('validates toolCallCount correctly from simulation result', async () => { diff --git a/src/evals/evalRunner.ts b/src/evals/evalRunner.ts index 5c943c5..235a712 100644 --- a/src/evals/evalRunner.ts +++ b/src/evals/evalRunner.ts @@ -5,6 +5,18 @@ import type { Tool } from '@modelcontextprotocol/sdk/types.js'; import type { ZodType } from 'zod'; import { simulateMCPHost } from './mcpHost/mcpHostSimulation.js'; import type { MCPHostSimulationResult } from './mcpHost/mcpHostTypes.js'; +import { runExternalHostScenario } from './externalHost/runtime.js'; +import type { + ExternalHostCapabilitiesConfig, + ExternalHostCorrelationConfig, + ExternalHostMetadata, + ExternalHostSimulationResult, +} from './externalHost/types.js'; +import { + driverToSlug, + normalizeHostDriver, +} from './externalHost/driverIdentity.js'; +import { getRegisteredExternalHostConfig } from './externalHost/hostRegistry.js'; import type { EvalExpectationResult, UsageMetrics } from '../types/index.js'; import type { EvalCaseResult, @@ -411,6 +423,33 @@ async function executeToolCall( throw new Error(simulationResult.error || 'MCP host simulation failed'); } + return { response: simulationResult }; + } else if (mode === 'external_host') { + if (!evalCase.scenario) { + throw new Error( + `Eval case ${evalCase.id}: scenario is required for external_host mode` + ); + } + + if (!evalCase.externalHost) { + throw new Error( + `Eval case ${evalCase.id}: externalHost is required for external_host mode` + ); + } + + const simulationResult = await runExternalHostScenario( + evalCase.scenario, + evalCase.externalHost, + { caseId: evalCase.id } + ); + + if (!simulationResult.success) { + return { + response: simulationResult, + error: simulationResult.error || 'External host simulation failed', + }; + } + return { response: simulationResult }; } else { // Direct mode - call tool directly @@ -670,11 +709,26 @@ function buildRequest( evalCase: EvalCase, toolOverrideVariantId?: string ): EvalCaseRequest { - const request: EvalCaseRequest = {}; + const request: EvalCaseRequest = { + mode: evalCase.mode ?? 'direct', + }; if (evalCase.description) request.description = evalCase.description; if (toolOverrideVariantId !== undefined) { request.toolOverrideVariantId = toolOverrideVariantId; } + if (evalCase.iterations !== undefined) + request.iterations = evalCase.iterations; + if (evalCase.accuracyThreshold !== undefined) { + request.accuracyThreshold = evalCase.accuracyThreshold; + } + if (evalCase.judgeReps !== undefined) request.judgeReps = evalCase.judgeReps; + if (evalCase.tags) request.tags = evalCase.tags; + if (evalCase.expect) { + request.expect = sanitizeReporterValue(evalCase.expect) as Record< + string, + unknown + >; + } if (evalCase.mode === 'mcp_host') { if (evalCase.scenario) request.scenario = evalCase.scenario; @@ -686,6 +740,45 @@ function buildRequest( }), }; } + } else if (evalCase.mode === 'external_host') { + if (evalCase.scenario) request.scenario = evalCase.scenario; + if (evalCase.externalHost) { + let driverSlug: string | undefined; + try { + driverSlug = driverToSlug( + normalizeHostDriver(evalCase.externalHost.driver) + ); + } catch { + driverSlug = undefined; + } + const registeredConfig = driverSlug + ? getRegisteredExternalHostConfig(driverSlug) + : undefined; + const effectiveOptions = mergeReporterOptions( + registeredConfig?.options, + evalCase.externalHost.options + ); + const effectiveCapabilities = mergeReporterCapabilities( + registeredConfig?.capabilities, + evalCase.externalHost.capabilities + ); + const effectiveCorrelation = mergeReporterCorrelation( + registeredConfig?.correlation, + evalCase.externalHost.correlation + ); + request.externalHost = { + driver: evalCase.externalHost.driver, + driverSlug, + name: evalCase.externalHost.name ?? registeredConfig?.name, + hostType: evalCase.externalHost.hostType, + variant: evalCase.externalHost.variant, + timeoutMs: evalCase.externalHost.timeoutMs, + usesBuiltInDefaults: registeredConfig !== undefined, + correlation: effectiveCorrelation, + options: sanitizeReporterRecord(effectiveOptions), + capabilities: serializeExternalHostCapabilities(effectiveCapabilities), + }; + } } else { if (evalCase.args) request.args = evalCase.args; } @@ -693,6 +786,130 @@ function buildRequest( return request; } +function mergeReporterOptions( + base: Record | undefined, + override: Record | undefined +): Record | undefined { + if (!base) { + return override; + } + if (!override) { + return base; + } + return { + ...base, + ...override, + }; +} + +function mergeReporterCapabilities( + base: ExternalHostCapabilitiesConfig | undefined, + override: ExternalHostCapabilitiesConfig | undefined +): ExternalHostCapabilitiesConfig | undefined { + if (!base) { + return override; + } + if (!override) { + return base; + } + return { + ...base, + ...override, + }; +} + +function mergeReporterCorrelation( + base: ExternalHostCorrelationConfig | undefined, + override: ExternalHostCorrelationConfig | undefined +): ExternalHostCorrelationConfig | undefined { + if (!base) { + return override; + } + if (!override) { + return base; + } + return { + ...base, + ...override, + }; +} + +function serializeExternalHostCapabilities( + capabilities: ExternalHostCapabilitiesConfig | undefined +): NonNullable['capabilities'] { + if (!capabilities || typeof capabilities !== 'object') { + return undefined; + } + + const serialized: NonNullable< + NonNullable['capabilities'] + > = {}; + + for (const [capability, bindingOrBindings] of Object.entries(capabilities)) { + const bindings = Array.isArray(bindingOrBindings) + ? bindingOrBindings + : [bindingOrBindings]; + serialized[capability] = bindings + .filter((binding): binding is NonNullable => + Boolean(binding) + ) + .map((binding) => ({ + uses: binding.uses, + ...(binding.provides !== undefined && { + provides: [...binding.provides], + }), + ...(binding.with !== undefined && { + with: sanitizeReporterRecord(binding.with), + }), + })); + } + + return serialized; +} + +function sanitizeReporterRecord( + value: Record | undefined +): Record | undefined { + if (!value) { + return undefined; + } + return sanitizeReporterValue(value) as Record; +} + +function sanitizeReporterValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => sanitizeReporterValue(item)); + } + + if (value && typeof value === 'object') { + const sanitized: Record = {}; + for (const [key, nestedValue] of Object.entries( + value as Record + )) { + sanitized[key] = isSecretLikeKey(key) + ? '[redacted]' + : sanitizeReporterValue(nestedValue); + } + return sanitized; + } + + if (typeof value === 'function') { + return '[function]'; + } + + if (typeof value === 'bigint') { + return value.toString(); + } + + return value; +} + +function isSecretLikeKey(key: string): boolean { + return /token|secret|password|credential|authorization|api[-_]?key/i.test( + key + ); +} + function isMCPHostSimulationResult( value: unknown ): value is MCPHostSimulationResult { @@ -705,6 +922,12 @@ function isMCPHostSimulationResult( ); } +function isExternalHostSimulationResult( + value: unknown +): value is ExternalHostSimulationResult { + return isMCPHostSimulationResult(value) && 'externalHost' in value; +} + /** * Runs a single iteration of an eval case (the atomic unit of work). * Extracted from runEvalCase to support multi-iteration accuracy loops. @@ -718,6 +941,10 @@ async function runSingleIteration( // Execute tool call const { response, error } = await executeToolCall(evalCase, context.mcp); + const externalHost = + isExternalHostSimulationResult(response) && response.externalHost + ? response.externalHost + : undefined; // Collect expectation results from expect block let expectationResults: EvalCaseResult['expectations'] = {}; @@ -741,10 +968,24 @@ async function runSingleIteration( toolPrecision = tp; toolRecall = tr; + if (evalCase.mode === 'external_host' && externalHost) { + applyExternalHostEvidenceGating( + evalCase.expect, + externalHost, + expectationResults + ); + if (expectationResults.toolsTriggered?.pass === false) { + toolPrecision = undefined; + toolRecall = undefined; + } + } + // Build mcpHostTrace when toolsTriggered expectation is present if ( evalCase.expect.toolsTriggered !== undefined && - isMCPHostSimulationResult(response) + isMCPHostSimulationResult(response) && + (evalCase.mode !== 'external_host' || + (externalHost !== undefined && hasStructuredToolEvidence(externalHost))) ) { const expectedNames = new Set( evalCase.expect.toolsTriggered.calls.map((c) => c.name) @@ -780,7 +1021,11 @@ async function runSingleIteration( id: evalCase.id, datasetName: options.datasetName ?? 'single-case', toolName: - evalCase.scenario != null ? 'mcp_host' : (evalCase.toolName ?? 'unknown'), + evalCase.mode === 'external_host' + ? 'external_host' + : evalCase.scenario != null + ? 'mcp_host' + : (evalCase.toolName ?? 'unknown'), source: 'eval', pass: didCasePass(error, expectationResults), request: buildRequest(evalCase, options.toolOverrideVariantId), @@ -795,9 +1040,60 @@ async function runSingleIteration( toolRecall, mcpHostTrace, hostUsage, + externalHost, }; } +function applyExternalHostEvidenceGating( + expectBlock: EvalExpectBlock, + externalHost: ExternalHostMetadata, + expectationResults: EvalCaseResult['expectations'] +): void { + const needsToolEvidence = + expectBlock.toolsTriggered !== undefined || + expectBlock.toolCallCount !== undefined; + + if (!needsToolEvidence || hasStructuredToolEvidence(externalHost)) { + return; + } + + const details = `External host trace source ${ + externalHost.sources?.toolCalls ?? externalHost.traceSource + } (${externalHost.traceConfidence} confidence) cannot support tool-call assertions. Use protocol traces or host-native structured traces for toolsTriggered/toolCallCount.`; + + if (expectBlock.toolsTriggered !== undefined) { + expectationResults.toolsTriggered = { pass: false, details }; + } + if (expectBlock.toolCallCount !== undefined) { + expectationResults.toolCallCount = { pass: false, details }; + } +} + +function hasStructuredToolEvidence( + externalHost: ExternalHostMetadata +): boolean { + const structuredSources = [ + 'mcp-proxy', + 'mcp-server-logs', + 'host-local-transcript', + 'host-native-export', + ]; + const evidence = externalHost.evidence?.toolCalls; + + if (evidence) { + return ( + evidence.confidence === 'high' && + structuredSources.includes(evidence.source) + ); + } + + const source = externalHost.sources?.toolCalls ?? externalHost.traceSource; + return ( + externalHost.traceConfidence === 'high' && + structuredSources.includes(source) + ); +} + /** * Returns true when the error message appears to be caused by network or * infrastructure issues (connection resets, timeouts, rate limits, etc.) @@ -830,6 +1126,12 @@ function isInfrastructureError(err: unknown): boolean { msg.includes('429') || msg.includes('503') || msg.includes('network') || + msg.includes('automation permission') || + msg.includes('automation/accessibility') || + msg.includes('no matching claude session') || + msg.includes('timed out waiting for claude session') || + msg.includes('failed to submit prompt to claude') || + msg.includes('failed to submit prompt to desktop host') || // Prompt/context overflow — LLM couldn't run, not a tool discoverability failure msg.includes('prompt is too long') || msg.includes('context length exceeded') || @@ -842,6 +1144,12 @@ function isInfrastructureError(err: unknown): boolean { ); } +function isExternalHostInfrastructureFailure( + externalHost: ExternalHostMetadata | undefined +): boolean { + return externalHost?.failureKind !== undefined; +} + /** * Runs a single eval case and returns the result. * When `evalCase.iterations > 1`, runs the case N times and returns accuracy. @@ -884,7 +1192,8 @@ export async function runEvalCase( // Check whether the tool call itself failed due to infrastructure (the // error is surfaced as result.error since executeToolCall swallows throws) const infraError = - result.error != null && isInfrastructureError(result.error); + isExternalHostInfrastructureFailure(result.externalHost) || + (result.error != null && isInfrastructureError(result.error)); iterationResults.push({ pass: result.pass, durationMs: result.durationMs, @@ -892,6 +1201,7 @@ export async function runEvalCase( isInfrastructureError: infraError, mcpHostTrace: result.mcpHostTrace, hostUsage: result.hostUsage, + externalHost: result.externalHost, }); } catch (err) { // runSingleIteration should not throw, but guard defensively @@ -920,7 +1230,11 @@ export async function runEvalCase( id: evalCase.id, datasetName: options.datasetName ?? 'single-case', toolName: - evalCase.scenario != null ? 'mcp_host' : (evalCase.toolName ?? 'unknown'), + evalCase.mode === 'external_host' + ? 'external_host' + : evalCase.scenario != null + ? 'mcp_host' + : (evalCase.toolName ?? 'unknown'), source: 'eval', pass: false, error: iterationResults[0]?.error, @@ -1080,7 +1394,7 @@ export async function runEvalDataset( // Preflight cost warning: estimate the number of LLM judge API calls this run will make const estimatedJudgeCalls = casesToRun.reduce((sum, c) => { const effectiveIterations = - c.mode === 'mcp_host' + c.mode === 'mcp_host' || c.mode === 'external_host' ? (c.iterations ?? defaultLlmIterations ?? 1) : (c.iterations ?? 1); if (c.expect?.passesJudge == null) return sum; @@ -1102,10 +1416,10 @@ export async function runEvalDataset( // Build task factories for all cases const tasks = casesToRun.map((evalCase) => async () => { - // Apply defaultLlmIterations to mcp_host cases that don't specify iterations. + // Apply defaultLlmIterations to host-driven cases that don't specify iterations. // Direct mode cases are deterministic — they always stay at 1 iteration. const withIterations = - evalCase.mode === 'mcp_host' && + (evalCase.mode === 'mcp_host' || evalCase.mode === 'external_host') && evalCase.iterations === undefined && defaultLlmIterations !== undefined ? { ...evalCase, iterations: defaultLlmIterations } @@ -1116,11 +1430,11 @@ export async function runEvalDataset( // Single-iteration mcp_host runs (the default) are a valid smoke-test pattern // and are not warned about — the warning is scoped to cases that have // explicitly chosen a multi-iteration count that is too small to be reliable. - if (evalCase.mode === 'mcp_host') { + if (evalCase.mode === 'mcp_host' || evalCase.mode === 'external_host') { const effectiveIterations = withIterations.iterations ?? 1; if (effectiveIterations > 1 && effectiveIterations < 10) { console.warn( - `[mcp-server-tester] Eval case "${evalCase.id}": running ${effectiveIterations} iterations in mcp_host mode ` + + `[mcp-server-tester] Eval case "${evalCase.id}": running ${effectiveIterations} iterations in ${evalCase.mode} mode ` + `may not be statistically reliable. Consider using 10+ iterations for accuracy measurements you can trust.` ); } diff --git a/src/evals/externalHost/builtinCapabilities.ts b/src/evals/externalHost/builtinCapabilities.ts new file mode 100644 index 0000000..ec3e55c --- /dev/null +++ b/src/evals/externalHost/builtinCapabilities.ts @@ -0,0 +1,22 @@ +import { ANTHROPIC_CLAUDE_CAPABILITIES } from './builtins/anthropicClaude.js'; +import { MACOS_DESKTOP_CAPABILITIES } from './builtins/macosDesktop.js'; +import type { ExternalHostCapabilityImplementation } from './types.js'; + +const BUILTIN_CAPABILITIES = new Map< + string, + ExternalHostCapabilityImplementation +>( + [...MACOS_DESKTOP_CAPABILITIES, ...ANTHROPIC_CLAUDE_CAPABILITIES].map( + (implementation) => [implementation.id, implementation] + ) +); + +export function listBuiltinExternalHostCapabilities(): ExternalHostCapabilityImplementation[] { + return Array.from(BUILTIN_CAPABILITIES.values()); +} + +export function resolveBuiltinExternalHostCapability( + uses: string +): ExternalHostCapabilityImplementation | undefined { + return BUILTIN_CAPABILITIES.get(uses); +} diff --git a/src/evals/externalHost/builtins/anthropicClaude.integration.test.ts b/src/evals/externalHost/builtins/anthropicClaude.integration.test.ts new file mode 100644 index 0000000..4687173 --- /dev/null +++ b/src/evals/externalHost/builtins/anthropicClaude.integration.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest'; +import { runExternalHostScenario } from '../runtime.js'; + +describe('Claude external host integrations', () => { + it('drives Claude Chat Desktop and captures low-confidence visible response evidence', async () => { + const result = await runExternalHostScenario( + 'Please reply with exactly: external host integration acknowledged.', + { + driver: 'anthropic.claude.chat.desktop-app.macos', + name: 'Claude Chat Desktop', + timeoutMs: 30_000, + }, + { caseId: 'claude-chat-desktop-integration' } + ); + + if (!result.success) { + throw new Error( + `${result.externalHost.failureKind ?? 'unknown'}: ${result.error}` + ); + } + + expect(result.response?.toLowerCase()).toContain( + 'external host integration acknowledged' + ); + expect(result.externalHost.driverSlug).toBe( + 'anthropic.claude.chat.desktop-app.macos' + ); + expect(result.externalHost.traceSource).toBe('accessibility'); + expect(result.externalHost.traceConfidence).toBe('low'); + expect(result.externalHost.artifacts.length).toBeGreaterThan(0); + expect(result.externalHost.session.runMarker).toContain( + 'MCP_SERVER_TESTER_' + ); + }, 150_000); + + it('drives the active Claude Cowork Desktop surface and captures high-confidence local-agent trace evidence', async () => { + const result = await runExternalHostScenario( + 'Please reply with exactly: external host integration acknowledged.', + { + driver: 'anthropic.claude.cowork.desktop-app.macos', + name: 'Claude Cowork Desktop', + timeoutMs: 60_000, + options: { + newConversationShortcut: 'none', + }, + }, + { caseId: 'claude-cowork-desktop-integration' } + ); + + if (!result.success) { + throw new Error( + `${result.externalHost.failureKind ?? 'unknown'}: ${result.error}` + ); + } + + expect(result.response?.toLowerCase()).toContain( + 'external host integration acknowledged' + ); + expect(result.externalHost.driverSlug).toBe( + 'anthropic.claude.cowork.desktop-app.macos' + ); + expect(result.externalHost.traceSource).toBe('host-local-transcript'); + expect(result.externalHost.traceConfidence).toBe('high'); + expect(result.externalHost.artifacts.length).toBeGreaterThan(0); + expect(result.externalHost.session.id).toBeDefined(); + expect(result.externalHost.session.runMarker).toContain( + 'MCP_SERVER_TESTER_' + ); + }, 150_000); +}); diff --git a/src/evals/externalHost/builtins/anthropicClaude.test.ts b/src/evals/externalHost/builtins/anthropicClaude.test.ts new file mode 100644 index 0000000..1a9b5d2 --- /dev/null +++ b/src/evals/externalHost/builtins/anthropicClaude.test.ts @@ -0,0 +1,835 @@ +import { mkdtemp, mkdir, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + buildClaudeTraceMetadata, + findMatchingClaudeSessions, + extractAccessibilityResponse, + getClaudeDataDir, + looksLikeClaudeChatSurface, + parseClaudeTrace, + snapshotClaudeSessions, + waitForClaudeTrace, + type SessionCandidate, +} from './anthropicClaude.js'; + +const COWORK_DRIVER = { + provider: 'anthropic', + product: 'claude', + surface: 'cowork', + runtime: 'desktop-app', + platform: 'macos', +} as const; + +async function writeJsonl(path: string, events: unknown[]): Promise { + await writeFile( + path, + events.map((event) => JSON.stringify(event)).join('\n'), + 'utf-8' + ); +} + +describe('anthropicClaude trace parsing', () => { + it('parses final answer, usage, tool calls, and artifacts from local Claude files', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-trace-')); + const sessionId = 'local_test'; + const cliSessionId = 'cli-session'; + const sessionDir = join(root, sessionId); + const transcriptDir = join( + sessionDir, + '.claude', + 'projects', + '-sessions-test' + ); + await mkdir(transcriptDir, { recursive: true }); + + const metadataPath = join(root, `${sessionId}.json`); + await writeFile( + metadataPath, + JSON.stringify({ + sessionId, + cliSessionId, + initialMessage: 'marker MCP_SERVER_TESTER_TEST', + cwd: '/sessions/test', + createdAt: '2026-05-09T00:00:00.000Z', + }), + 'utf-8' + ); + + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { + type: 'result', + result: 'trace spike acknowledged.', + requestId: 'req_123', + duration_ms: 1234, + duration_api_ms: 1000, + total_cost_usd: 0.01, + usage: { + input_tokens: 10, + output_tokens: 5, + cache_read_input_tokens: 2, + }, + timestamp: '2026-05-09T00:00:02.000Z', + }, + ]); + + await writeJsonl(join(transcriptDir, `${cliSessionId}.jsonl`), [ + { + type: 'assistant', + message: { + content: [ + { + type: 'tool_use', + id: 'toolu_1', + name: 'mcp__server__search', + input: { query: 'planning' }, + }, + ], + }, + }, + ]); + + const candidate: SessionCandidate = { + id: sessionId, + metadataPath, + sessionDir, + statMtimeMs: Date.now(), + metadata: { + sessionId, + cliSessionId, + initialMessage: 'marker MCP_SERVER_TESTER_TEST', + cwd: '/sessions/test', + }, + }; + + const trace = await parseClaudeTrace(candidate); + + expect(trace.finalAnswer).toBe('trace spike acknowledged.'); + expect(trace.requestId).toBe('req_123'); + expect(trace.usage).toMatchObject({ + inputTokens: 10, + outputTokens: 5, + totalCostUsd: 0.01, + durationMs: 1234, + durationApiMs: 1000, + cacheReadInputTokens: 2, + }); + expect(trace.toolCalls).toEqual([ + { + id: 'toolu_1', + name: 'search', + arguments: { query: 'planning' }, + }, + ]); + expect(trace.transcriptPath).toContain(`${cliSessionId}.jsonl`); + expect(trace.isComplete).toBe(true); + expect(trace.auditParsed).toBe(true); + expect(trace.transcriptParsed).toBe(true); + expect(trace.usageAvailable).toBe(true); + expect(trace.costAvailable).toBe(true); + }); + + it('does not treat assistant text without a result event as a completed run', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-pending-')); + const sessionId = 'local_pending'; + const sessionDir = join(root, sessionId); + await mkdir(sessionDir, { recursive: true }); + const metadataPath = join(root, `${sessionId}.json`); + await writeFile( + metadataPath, + JSON.stringify({ + sessionId, + initialMessage: 'marker MCP_SERVER_TESTER_PENDING', + createdAt: new Date().toISOString(), + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { + type: 'assistant', + message: { + content: [{ type: 'text', text: 'partial assistant response' }], + }, + }, + ]); + + const candidate: SessionCandidate = { + id: sessionId, + metadataPath, + sessionDir, + statMtimeMs: Date.now(), + metadata: { + sessionId, + initialMessage: 'marker MCP_SERVER_TESTER_PENDING', + }, + }; + + const trace = await parseClaudeTrace(candidate); + + expect(trace.finalAnswer).toBe('partial assistant response'); + expect(trace.isComplete).toBe(false); + }); + + it('continues parsing valid JSONL events when one line is malformed', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-jsonl-')); + const sessionId = 'local_jsonl'; + const sessionDir = join(root, sessionId); + await mkdir(sessionDir, { recursive: true }); + const metadataPath = join(root, `${sessionId}.json`); + await writeFile( + metadataPath, + JSON.stringify({ + sessionId, + initialMessage: 'marker MCP_SERVER_TESTER_JSONL', + }), + 'utf-8' + ); + await writeFile( + join(sessionDir, 'audit.jsonl'), + [ + JSON.stringify({ type: 'assistant', result: 'ignored' }), + '{not-json', + JSON.stringify({ type: 'result', result: 'final answer' }), + ].join('\n'), + 'utf-8' + ); + + const candidate: SessionCandidate = { + id: sessionId, + metadataPath, + sessionDir, + statMtimeMs: Date.now(), + metadata: { + sessionId, + initialMessage: 'marker MCP_SERVER_TESTER_JSONL', + }, + }; + + const trace = await parseClaudeTrace(candidate); + + expect(trace.finalAnswer).toBe('final answer'); + expect(trace.isComplete).toBe(true); + expect(trace.parseWarnings.join('\n')).toContain( + 'discarded 1 malformed JSONL line' + ); + }); + + it('only marks evidence fields high confidence when the parsed trace supports them', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-evidence-')); + const sessionId = 'local_evidence'; + const sessionDir = join(root, sessionId); + await mkdir(sessionDir, { recursive: true }); + const metadataPath = join(root, `${sessionId}.json`); + await writeFile( + metadataPath, + JSON.stringify({ + sessionId, + initialMessage: 'marker MCP_SERVER_TESTER_EVIDENCE', + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { + type: 'result', + result: 'final answer', + total_cost_usd: 0.01, + usage: { input_tokens: 1, output_tokens: 2 }, + }, + ]); + + const trace = await parseClaudeTrace({ + id: sessionId, + metadataPath, + sessionDir, + statMtimeMs: Date.now(), + metadata: { + sessionId, + initialMessage: 'marker MCP_SERVER_TESTER_EVIDENCE', + }, + }); + const metadata = buildClaudeTraceMetadata({ + config: { + driver: COWORK_DRIVER, + name: 'Claude Cowork Desktop', + }, + context: { + runId: 'run', + caseId: 'case', + scenario: 'scenario', + submittedScenario: 'scenario', + marker: 'MCP_SERVER_TESTER_EVIDENCE', + correlation: { + strategy: 'prompt_marker', + marker: 'MCP_SERVER_TESTER_EVIDENCE', + includedInPrompt: true, + }, + timeoutMs: 1000, + startedAtMs: Date.now(), + }, + driver: COWORK_DRIVER, + displayName: 'Claude Cowork Desktop', + artifacts: [], + trace, + limitations: [], + }); + + expect(metadata.evidence?.finalAnswer).toEqual({ + source: 'host-local-transcript', + confidence: 'high', + }); + expect(metadata.evidence?.toolCalls).toEqual({ + source: 'none', + confidence: 'unknown', + }); + expect(metadata.evidence?.usage).toEqual({ + source: 'host-local-transcript', + confidence: 'high', + }); + expect(metadata.evidence?.cost).toEqual({ + source: 'host-local-transcript', + confidence: 'high', + }); + expect(metadata.traceConfidence).toBe('high'); + expect(metadata.traceLimitations?.join('\n')).toContain( + 'Tool-call evidence is unavailable' + ); + }); + + it('allows capability-local Claude data directory options to override driver-wide options', () => { + expect( + getClaudeDataDir( + { + driver: COWORK_DRIVER, + options: { dataDir: '/global/claude' }, + }, + { with: { dataDir: '/capability/claude' } } + ) + ).toBe('/capability/claude'); + }); + + it('matches sessions by marker instead of timing alone', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-match-')); + const sessionDir = join(root, 'local_match'); + await mkdir(sessionDir, { recursive: true }); + const metadataPath = join(root, 'local_match.json'); + + await writeFile( + metadataPath, + JSON.stringify({ + sessionId: 'local_match', + initialMessage: 'hello MCP_SERVER_TESTER_MATCH', + createdAt: new Date().toISOString(), + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { type: 'result', result: 'done' }, + ]); + + const matches = await findMatchingClaudeSessions({ + dataDir: root, + marker: 'MCP_SERVER_TESTER_MATCH', + snapshot: new Map(), + startedAtMs: Date.now() - 1000, + }); + + expect(matches).toHaveLength(1); + expect(matches[0]?.finalAnswer).toBe('done'); + }); + + it('handles numeric Claude metadata timestamps when checking recency', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-numeric-time-')); + const sessionDir = join(root, 'local_numeric_time'); + await mkdir(sessionDir, { recursive: true }); + await writeFile( + join(root, 'local_numeric_time.json'), + JSON.stringify({ + sessionId: 'local_numeric_time', + initialMessage: 'hello MCP_SERVER_TESTER_NUMERIC_TIME', + createdAt: Date.now(), + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { type: 'result', result: 'numeric timestamp done' }, + ]); + + const snapshot = await snapshotClaudeSessions(root); + const matches = await findMatchingClaudeSessions({ + dataDir: root, + marker: 'MCP_SERVER_TESTER_NUMERIC_TIME', + snapshot, + startedAtMs: Date.now() - 1000, + }); + + expect(matches).toHaveLength(1); + expect(matches[0]?.finalAnswer).toBe('numeric timestamp done'); + }); + + it('snapshots existing sessions so old unchanged files are ignored', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-snapshot-')); + await mkdir(join(root, 'local_old'), { recursive: true }); + await writeFile( + join(root, 'local_old.json'), + JSON.stringify({ + sessionId: 'local_old', + initialMessage: 'MCP_SERVER_TESTER_OLD', + }), + 'utf-8' + ); + + const snapshot = await snapshotClaudeSessions(root); + const matches = await findMatchingClaudeSessions({ + dataDir: root, + marker: 'MCP_SERVER_TESTER_OLD', + snapshot, + startedAtMs: Date.now(), + }); + + expect(matches).toEqual([]); + }); + + it('detects reused sessions when audit files change after the snapshot', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-reuse-')); + const sessionDir = join(root, 'local_reuse'); + await mkdir(sessionDir, { recursive: true }); + await writeFile( + join(root, 'local_reuse.json'), + JSON.stringify({ + sessionId: 'local_reuse', + initialMessage: 'MCP_SERVER_TESTER_REUSE', + }), + 'utf-8' + ); + + const snapshot = await snapshotClaudeSessions(root); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { type: 'result', result: 'reuse done' }, + ]); + + const matches = await findMatchingClaudeSessions({ + dataDir: root, + marker: 'MCP_SERVER_TESTER_REUSE', + snapshot, + startedAtMs: Date.now(), + }); + + expect(matches).toHaveLength(1); + expect(matches[0]?.finalAnswer).toBe('reuse done'); + }); + + it('does not use a pre-marker result as completion for a reused session', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-reuse-marker-')); + const sessionId = 'local_reuse_marker'; + const sessionDir = join(root, sessionId); + await mkdir(sessionDir, { recursive: true }); + const metadataPath = join(root, `${sessionId}.json`); + await writeFile( + metadataPath, + JSON.stringify({ + sessionId, + initialMessage: 'old run', + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { type: 'result', result: 'old completed answer' }, + { + type: 'assistant', + message: { + content: [ + { + type: 'text', + text: 'MCP_SERVER_TESTER_REUSED_MARKER partial new response', + }, + ], + }, + }, + ]); + + const trace = await parseClaudeTrace( + { + id: sessionId, + metadataPath, + sessionDir, + statMtimeMs: Date.now(), + metadata: { + sessionId, + initialMessage: 'old run', + }, + }, + 'MCP_SERVER_TESTER_REUSED_MARKER' + ); + + expect(trace.finalAnswer).toBe( + 'MCP_SERVER_TESTER_REUSED_MARKER partial new response' + ); + expect(trace.isComplete).toBe(false); + }); + + it('does not use a pre-marker result when metadata contains the marker but the audit is still pending', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-metadata-marker-')); + const sessionId = 'local_metadata_marker'; + const sessionDir = join(root, sessionId); + await mkdir(sessionDir, { recursive: true }); + const metadataPath = join(root, `${sessionId}.json`); + await writeFile( + metadataPath, + JSON.stringify({ + sessionId, + initialMessage: 'MCP_SERVER_TESTER_METADATA_MARKER prompt', + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { type: 'result', result: 'old completed answer' }, + { + type: 'assistant', + message: { + content: [ + { + type: 'text', + text: 'MCP_SERVER_TESTER_METADATA_MARKER partial new response', + }, + ], + }, + }, + ]); + + const trace = await parseClaudeTrace( + { + id: sessionId, + metadataPath, + sessionDir, + statMtimeMs: Date.now(), + metadata: { + sessionId, + initialMessage: 'MCP_SERVER_TESTER_METADATA_MARKER prompt', + }, + }, + 'MCP_SERVER_TESTER_METADATA_MARKER' + ); + + expect(trace.finalAnswer).toBe( + 'MCP_SERVER_TESTER_METADATA_MARKER partial new response' + ); + expect(trace.isComplete).toBe(false); + }); + + it('does not combine a transcript marker with a pre-marker audit result', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-cross-source-marker-')); + const sessionId = 'local_cross_source_marker'; + const cliSessionId = 'cli-cross-source'; + const sessionDir = join(root, sessionId); + const transcriptDir = join(sessionDir, '.claude', 'projects', '-project'); + await mkdir(transcriptDir, { recursive: true }); + const metadataPath = join(root, `${sessionId}.json`); + await writeFile( + metadataPath, + JSON.stringify({ + sessionId, + cliSessionId, + initialMessage: 'old run', + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { type: 'result', result: 'old completed answer' }, + ]); + await writeJsonl(join(transcriptDir, `${cliSessionId}.jsonl`), [ + { + type: 'assistant', + message: { + content: [ + { + type: 'text', + text: 'MCP_SERVER_TESTER_CROSS_SOURCE partial new response', + }, + ], + }, + }, + ]); + + const trace = await parseClaudeTrace( + { + id: sessionId, + metadataPath, + sessionDir, + statMtimeMs: Date.now(), + metadata: { + sessionId, + cliSessionId, + initialMessage: 'old run', + }, + }, + 'MCP_SERVER_TESTER_CROSS_SOURCE' + ); + + expect(trace.finalAnswer).toBe( + 'MCP_SERVER_TESTER_CROSS_SOURCE partial new response' + ); + expect(trace.isComplete).toBe(false); + }); + + it('normalizes MCP tool names when server names contain underscores', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-tool-name-')); + const sessionId = 'local_tool_name'; + const cliSessionId = 'cli-session'; + const sessionDir = join(root, sessionId); + const transcriptDir = join(sessionDir, '.claude', 'projects', '-project'); + await mkdir(transcriptDir, { recursive: true }); + const metadataPath = join(root, `${sessionId}.json`); + await writeFile( + metadataPath, + JSON.stringify({ + sessionId, + cliSessionId, + initialMessage: 'marker MCP_SERVER_TESTER_TOOL', + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { type: 'result', result: 'done' }, + ]); + await writeJsonl(join(transcriptDir, `${cliSessionId}.jsonl`), [ + { + type: 'assistant', + message: { + content: [ + { + type: 'tool_use', + id: 'toolu_1', + name: 'mcp__my_server__search', + input: { query: 'planning' }, + }, + ], + }, + }, + ]); + + const trace = await parseClaudeTrace({ + id: sessionId, + metadataPath, + sessionDir, + statMtimeMs: Date.now(), + metadata: { + sessionId, + cliSessionId, + initialMessage: 'marker MCP_SERVER_TESTER_TOOL', + }, + }); + + expect(trace.toolCalls[0]?.name).toBe('search'); + }); + + it('waits for a terminal result event before returning a matched trace', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-wait-result-')); + const sessionDir = join(root, 'local_wait'); + await mkdir(sessionDir, { recursive: true }); + await writeFile( + join(root, 'local_wait.json'), + JSON.stringify({ + sessionId: 'local_wait', + initialMessage: 'MCP_SERVER_TESTER_WAIT', + createdAt: new Date().toISOString(), + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { + type: 'assistant', + message: { content: [{ type: 'text', text: 'partial' }] }, + }, + ]); + + const tracePromise = waitForClaudeTrace({ + dataDir: root, + marker: 'MCP_SERVER_TESTER_WAIT', + correlation: { + strategy: 'prompt_marker', + marker: 'MCP_SERVER_TESTER_WAIT', + includedInPrompt: true, + }, + snapshot: new Map(), + timeoutMs: 2_500, + startedAtMs: Date.now() - 1000, + }); + + await new Promise((resolve) => setTimeout(resolve, 900)); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { + type: 'assistant', + message: { content: [{ type: 'text', text: 'partial' }] }, + }, + { type: 'result', result: 'complete' }, + ]); + + await expect(tracePromise).resolves.toMatchObject({ + finalAnswer: 'complete', + isComplete: true, + }); + }); + + it('waits briefly for an expected embedded transcript after the result event', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-wait-transcript-')); + const sessionId = 'local_wait_transcript'; + const cliSessionId = 'cli-session'; + const sessionDir = join(root, sessionId); + const transcriptDir = join(sessionDir, '.claude', 'projects', '-project'); + await mkdir(transcriptDir, { recursive: true }); + await writeFile( + join(root, `${sessionId}.json`), + JSON.stringify({ + sessionId, + cliSessionId, + initialMessage: 'MCP_SERVER_TESTER_TRANSCRIPT', + createdAt: new Date().toISOString(), + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { type: 'result', result: 'complete' }, + ]); + + const tracePromise = waitForClaudeTrace({ + dataDir: root, + marker: 'MCP_SERVER_TESTER_TRANSCRIPT', + correlation: { + strategy: 'prompt_marker', + marker: 'MCP_SERVER_TESTER_TRANSCRIPT', + includedInPrompt: true, + }, + snapshot: new Map(), + timeoutMs: 3_500, + startedAtMs: Date.now() - 1000, + }); + + await new Promise((resolve) => setTimeout(resolve, 900)); + await writeJsonl(join(transcriptDir, `${cliSessionId}.jsonl`), [ + { + type: 'assistant', + message: { + content: [ + { + type: 'tool_use', + id: 'toolu_1', + name: 'mcp__server__search', + input: { query: 'planning' }, + }, + ], + }, + }, + ]); + + await expect(tracePromise).resolves.toMatchObject({ + finalAnswer: 'complete', + transcriptParsed: true, + toolCalls: [{ name: 'search', arguments: { query: 'planning' } }], + }); + }); + + it('discovers nested Claude local-agent session metadata', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-nested-')); + const nested = join(root, 'workspace', 'project'); + await mkdir(join(nested, 'local_nested'), { recursive: true }); + await writeFile( + join(nested, 'local_nested.json'), + JSON.stringify({ + sessionId: 'local_nested', + initialMessage: 'MCP_SERVER_TESTER_NESTED', + createdAt: new Date().toISOString(), + }), + 'utf-8' + ); + await writeJsonl(join(nested, 'local_nested', 'audit.jsonl'), [ + { type: 'result', result: 'nested done' }, + ]); + + const matches = await findMatchingClaudeSessions({ + dataDir: root, + marker: 'MCP_SERVER_TESTER_NESTED', + snapshot: new Map(), + startedAtMs: Date.now() - 1000, + }); + + expect(matches).toHaveLength(1); + expect(matches[0]?.finalAnswer).toBe('nested done'); + }); + + it('can match a single fresh Claude local-agent session without a prompt marker', async () => { + const root = await mkdtemp(join(tmpdir(), 'claude-no-marker-')); + const sessionDir = join(root, 'local_no_marker'); + await mkdir(sessionDir, { recursive: true }); + await writeFile( + join(root, 'local_no_marker.json'), + JSON.stringify({ + sessionId: 'local_no_marker', + initialMessage: 'plain prompt without marker', + createdAt: new Date().toISOString(), + }), + 'utf-8' + ); + await writeJsonl(join(sessionDir, 'audit.jsonl'), [ + { type: 'result', result: 'plain prompt done' }, + ]); + + const matches = await findMatchingClaudeSessions({ + dataDir: root, + marker: 'MCP_SERVER_TESTER_NOT_IN_PROMPT', + correlation: { + strategy: 'none', + marker: 'MCP_SERVER_TESTER_NOT_IN_PROMPT', + includedInPrompt: false, + }, + snapshot: new Map(), + startedAtMs: Date.now() - 1000, + }); + + expect(matches).toHaveLength(1); + expect(matches[0]?.finalAnswer).toBe('plain prompt done'); + }); + + it('extracts final answer from accessibility fallback text', () => { + expect( + extractAccessibilityResponse( + [ + 'You said: Please reply with exactly: external host integration acknowledged.', + '[eval-run-marker:MCP_SERVER_TESTER_TEST]', + 'Claude responded: external host integration acknowledged.', + 'Write a message...', + ].join('\n') + ) + ).toBe('external host integration acknowledged.'); + }); + + it('extracts final answer from comma-separated accessibility fallback text', () => { + expect( + extractAccessibilityResponse( + 'You said: prompt [eval-run-marker:MCP_SERVER_TESTER_TEST], Claude responded: external host integration acknowledged., Write a message...' + ) + ).toBe('external host integration acknowledged.'); + }); + + it('recognizes the regular Claude Chat surface from visible controls', () => { + expect( + looksLikeClaudeChatSurface( + [ + 'New chat', + 'Projects', + 'Artifacts', + 'Ask your org', + 'Write a message...', + ].join('\n') + ) + ).toBe(true); + }); + + it('does not classify a local-agent surface from generic composer text alone', () => { + expect( + looksLikeClaudeChatSurface( + ['Claude Code', 'Session', 'Write a message...'].join('\n') + ) + ).toBe(false); + }); +}); diff --git a/src/evals/externalHost/builtins/anthropicClaude.ts b/src/evals/externalHost/builtins/anthropicClaude.ts new file mode 100644 index 0000000..786cbf8 --- /dev/null +++ b/src/evals/externalHost/builtins/anthropicClaude.ts @@ -0,0 +1,1331 @@ +import { randomUUID } from 'node:crypto'; +import { readdir, readFile, stat } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { Readable } from 'node:stream'; +import { parse as parseNdjson } from 'ndjson'; +import type { LLMToolCall } from '../../mcpHost/mcpHostTypes.js'; +import type { + ExternalHostConfig, + ExternalHostCapabilityContext, + ExternalHostCapabilityImplementation, + ExternalHostFailureKind, + ExternalHostMetadata, + ExternalHostRunResult, + HostArtifact, + HostCapability, + HostDriverId, + HostRunContext, +} from '../types.js'; +import type { UsageMetrics } from '../../../types/index.js'; +import { driverToSlug, hostTypeFromDriver } from '../driverIdentity.js'; +import { + readMacosAccessibilityText, + readMacosFrontWindowContents, +} from './macosDesktop.js'; + +const DEFAULT_APP_NAME = 'Claude'; +const POLL_INTERVAL_MS = 750; +const TRACE_SETTLE_AFTER_COMPLETE_MS = 1_500; +const CLAUDE_DESKTOP_MACOS_CAPABILITIES = [ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', +] as const; + +export interface ClaudeSessionMetadata { + sessionId?: string; + cliSessionId?: string; + createdAt?: string | number; + lastActivityAt?: string | number; + cwd?: string; + model?: string; + title?: string; + initialMessage?: string; +} + +export interface SessionCandidate { + id: string; + metadataPath: string; + sessionDir: string; + statMtimeMs: number; + metadata: ClaudeSessionMetadata; +} + +interface SnapshotEntry { + mtimeMs: number; +} + +export type ClaudeSessionSnapshot = Map; + +export interface ClaudeTrace { + candidate: SessionCandidate; + auditPath?: string; + transcriptPath?: string; + finalAnswer?: string; + toolCalls: LLMToolCall[]; + usage?: UsageMetrics; + requestId?: string; + completedAt?: string; + llmDurationMs?: number; + terminalReason?: string; + isError?: boolean; + isComplete: boolean; + auditParsed: boolean; + transcriptParsed: boolean; + usageAvailable: boolean; + costAvailable: boolean; + parseWarnings: string[]; + rawText: string; +} + +interface ClaudeAuditEvent { + type?: string; + result?: unknown; + is_error?: boolean; + duration_ms?: number; + duration_api_ms?: number; + total_cost_usd?: number; + requestId?: string; + request_id?: string; + usage?: Record; + message?: { + content?: Array<{ + type?: string; + id?: string; + name?: string; + input?: Record; + text?: string; + }>; + }; + timestamp?: string; + terminal_reason?: string; +} + +export const ANTHROPIC_CLAUDE_CAPABILITIES: ExternalHostCapabilityImplementation[] = + [ + { + id: 'builtin:anthropic.claude.coworkSurface', + capabilities: ['control'], + run: rejectClaudeChatSurfaceCapability, + }, + { + id: 'builtin:anthropic.claude.accessibilityTrace', + capabilities: ['completion', 'trace', 'normalize'], + run: captureClaudeChatAccessibilityResultCapability, + }, + { + id: 'builtin:anthropic.claude.localAgentTrace', + capabilities: ['completion', 'trace'], + setup: snapshotClaudeSessionsCapability, + run: captureClaudeCoworkAgentTraceCapability, + }, + { + id: 'builtin:anthropic.claude.localAgentNormalize', + capabilities: ['normalize'], + run: normalizeClaudeCoworkAgentTraceCapability, + }, + ]; + +async function rejectClaudeChatSurfaceCapability({ + config, + run, + binding, + state, +}: ExternalHostCapabilityContext): Promise { + const appName = + runStringOption(config, binding, 'appName') ?? DEFAULT_APP_NAME; + const chatSurfaceReason = await detectClaudeChatSurface(appName); + if (!chatSurfaceReason) { + return; + } + + return failureResult({ + config, + context: run, + driver: state.driver, + displayName: state.displayName, + capabilitiesUsed: state.capabilitiesUsed, + failureKind: 'submission_failed', + error: `${state.displayName} surface is not active: ${chatSurfaceReason}`, + artifacts: [], + limitations: [ + 'Cowork is a distinct Claude Desktop surface; this driver will not submit Cowork evals through the regular Claude Chat composer.', + 'Open or focus an active Cowork/local-agent session before running this driver, or add a deterministic Cowork launch step.', + ], + }); +} + +async function snapshotClaudeSessionsCapability({ + config, + run, + binding, + state, +}: ExternalHostCapabilityContext): Promise { + const dataDir = getClaudeDataDir(config, binding); + state.data.claudeDataDir = dataDir; + + try { + state.data.claudeSessionSnapshot = await snapshotClaudeSessions(dataDir); + } catch (err) { + return failureResult({ + config, + context: run, + driver: state.driver, + displayName: state.displayName, + capabilitiesUsed: state.capabilitiesUsed, + failureKind: 'parse_failure', + error: `Failed to snapshot Claude session directory: ${formatError(err)}`, + artifacts: [], + limitations: [`Claude data directory: ${dataDir}`], + }); + } +} + +async function captureClaudeChatAccessibilityResultCapability({ + config, + run, + binding, + state, +}: ExternalHostCapabilityContext): Promise { + try { + return await waitForAccessibilityTrace({ + config, + context: run, + driver: state.driver, + displayName: state.displayName, + capabilitiesUsed: state.capabilitiesUsed, + timeoutMs: run.timeoutMs, + appName: runStringOption(config, binding, 'appName'), + }); + } catch (err) { + const message = formatError(err); + return failureResult({ + config, + context: run, + driver: state.driver, + displayName: state.displayName, + capabilitiesUsed: state.capabilitiesUsed, + failureKind: classifyTraceFailure(message), + error: message, + artifacts: [], + limitations: [ + 'Claude Chat Desktop currently uses Accessibility as the fallback trace source; IndexedDB parsing has not been stabilized.', + ], + }); + } +} + +async function captureClaudeCoworkAgentTraceCapability({ + config, + run, + binding, + state, +}: ExternalHostCapabilityContext): Promise { + const dataDir = + typeof state.data.claudeDataDir === 'string' + ? state.data.claudeDataDir + : getClaudeDataDir(config, binding); + const snapshot = state.data.claudeSessionSnapshot as + | ClaudeSessionSnapshot + | undefined; + + if (!snapshot) { + return failureResult({ + config, + context: run, + driver: state.driver, + displayName: state.displayName, + capabilitiesUsed: state.capabilitiesUsed, + failureKind: 'parse_failure', + error: 'Claude Cowork trace step requires a session snapshot.', + artifacts: [], + limitations: [`Claude data directory: ${dataDir}`], + }); + } + + try { + state.data.claudeTrace = await waitForClaudeTrace({ + dataDir, + marker: run.marker, + correlation: run.correlation, + snapshot, + timeoutMs: run.timeoutMs, + startedAtMs: run.startedAtMs, + }); + } catch (err) { + const message = formatError(err); + return failureResult({ + config, + context: run, + driver: state.driver, + displayName: state.displayName, + capabilitiesUsed: state.capabilitiesUsed, + failureKind: classifyTraceFailure(message), + error: message, + artifacts: [], + limitations: [`Claude data directory: ${dataDir}`], + }); + } +} + +async function normalizeClaudeCoworkAgentTraceCapability({ + config, + run, + state, +}: ExternalHostCapabilityContext): Promise { + const trace = state.data.claudeTrace as ClaudeTrace | undefined; + if (!trace) { + return failureResult({ + config, + context: run, + driver: state.driver, + displayName: state.displayName, + capabilitiesUsed: state.capabilitiesUsed, + failureKind: 'parse_failure', + error: 'Claude Cowork trace normalization requires a parsed trace.', + artifacts: [], + limitations: [], + }); + } + + const artifacts = buildArtifacts(trace); + const metadata = buildClaudeTraceMetadata({ + config, + context: run, + driver: state.driver, + displayName: state.displayName, + capabilitiesUsed: state.capabilitiesUsed, + artifacts, + trace, + limitations: trace.parseWarnings, + }); + + if (trace.isError) { + return { + success: false, + toolCalls: trace.toolCalls, + error: + trace.finalAnswer ?? + `Claude host run failed${trace.terminalReason ? `: ${trace.terminalReason}` : ''}`, + externalHost: { + ...metadata, + failureKind: 'host_run_failed', + }, + }; + } + + if (trace.finalAnswer === undefined) { + return { + success: false, + toolCalls: trace.toolCalls, + error: 'Claude trace completed but did not include a final answer.', + externalHost: { + ...metadata, + failureKind: 'parse_failure', + }, + }; + } + + return { + success: true, + toolCalls: trace.toolCalls, + response: trace.finalAnswer, + conversationHistory: trace.finalAnswer + ? [{ role: 'assistant', content: trace.finalAnswer }] + : undefined, + usage: trace.usage, + llmDurationMs: trace.llmDurationMs, + externalHost: metadata, + }; +} + +function stringOption( + options: Record | undefined, + key: string +): string | undefined { + const value = options?.[key]; + return typeof value === 'string' ? value : undefined; +} + +function configStringOption( + config: ExternalHostConfig, + key: string +): string | undefined { + const value = config.options?.[key]; + return typeof value === 'string' ? value : undefined; +} + +function runStringOption( + config: ExternalHostConfig, + binding: { with?: Record }, + key: string +): string | undefined { + return stringOption(binding.with, key) ?? configStringOption(config, key); +} + +export function getClaudeDataDir( + config: ExternalHostConfig, + binding?: { with?: Record } +): string { + const configuredDataDir = binding + ? runStringOption(config, binding, 'dataDir') + : configStringOption(config, 'dataDir'); + + return ( + configuredDataDir ?? + join( + homedir(), + 'Library', + 'Application Support', + 'Claude', + 'local-agent-mode-sessions' + ) + ); +} + +export async function snapshotClaudeSessions( + dataDir: string +): Promise { + const snapshot = new Map(); + const sessions = await listSessionCandidates(dataDir); + for (const session of sessions) { + snapshot.set(session.metadataPath, { mtimeMs: session.statMtimeMs }); + } + return snapshot; +} + +export async function waitForClaudeTrace(options: { + dataDir: string; + marker: string; + correlation: HostRunContext['correlation']; + snapshot: ClaudeSessionSnapshot; + timeoutMs: number; + startedAtMs: number; +}): Promise { + const deadline = Date.now() + options.timeoutMs; + let lastPending: ClaudeTrace | undefined; + let completeTraceFirstSeenAtMs: number | undefined; + + while (Date.now() < deadline) { + const matches = await findMatchingClaudeSessions(options); + + if (matches.length > 1) { + throw new Error( + `Ambiguous Claude sessions for ${describeCorrelation(options)}: ${matches + .map((m) => m.candidate.id) + .join(', ')}` + ); + } + + if (matches.length === 1) { + const trace = matches[0]!; + if (isTraceReady(trace, completeTraceFirstSeenAtMs)) { + return trace; + } + if (trace.isComplete && completeTraceFirstSeenAtMs === undefined) { + completeTraceFirstSeenAtMs = Date.now(); + } + lastPending = trace; + } + + await delay(POLL_INTERVAL_MS); + } + + if (lastPending) { + throw new Error( + `Timed out waiting for Claude session ${lastPending.candidate.id} to complete` + ); + } + + throw new Error( + `No matching Claude session found for ${describeCorrelation(options)}` + ); +} + +function isTraceReady( + trace: ClaudeTrace, + completeTraceFirstSeenAtMs: number | undefined +): boolean { + if (!trace.isComplete) { + return false; + } + + if (!trace.candidate.metadata.cliSessionId || trace.transcriptParsed) { + return true; + } + + return ( + completeTraceFirstSeenAtMs !== undefined && + Date.now() - completeTraceFirstSeenAtMs >= TRACE_SETTLE_AFTER_COMPLETE_MS + ); +} + +export async function findMatchingClaudeSessions(options: { + dataDir: string; + marker: string; + correlation?: HostRunContext['correlation']; + snapshot: ClaudeSessionSnapshot; + startedAtMs: number; +}): Promise { + const sessions = await listSessionCandidates(options.dataDir); + const traces: ClaudeTrace[] = []; + + for (const session of sessions) { + const previous = options.snapshot.get(session.metadataPath); + const isNewOrUpdated = + previous === undefined || session.statMtimeMs > previous.mtimeMs; + const createdAtMs = metadataTimestampMs(session.metadata.createdAt); + const isRecent = + !Number.isNaN(createdAtMs) && createdAtMs >= options.startedAtMs - 5_000; + + if (!isNewOrUpdated && !isRecent) { + continue; + } + + const trace = await parseClaudeTrace( + session, + options.correlation?.includedInPrompt === false + ? undefined + : options.marker + ); + if ( + sessionMatchesCorrelation({ + session, + trace, + marker: options.marker, + correlation: options.correlation, + isNewOrUpdated, + isRecent, + }) + ) { + traces.push(trace); + } + } + + return traces; +} + +function describeCorrelation(options: { + marker: string; + correlation?: HostRunContext['correlation']; +}): string { + if (options.correlation?.includedInPrompt) { + return `marker ${options.marker}`; + } + return `${options.correlation?.strategy ?? 'none'} correlation near the run start`; +} + +async function readAccessibilityFallback( + config: ExternalHostConfig, + context: HostRunContext, + driver: HostDriverId, + displayName: string, + capabilitiesUsed: readonly HostCapability[], + options: { appName?: string } = {} +): Promise { + let visibleText: string; + try { + visibleText = await readMacosAccessibilityText( + options.appName ?? + configStringOption(config, 'appName') ?? + DEFAULT_APP_NAME + ); + } catch { + return undefined; + } + + if (!visibleText.includes(context.marker)) { + return undefined; + } + + const response = extractAccessibilityResponse(visibleText); + if (!response) { + return undefined; + } + + return { + success: true, + toolCalls: [], + response, + conversationHistory: [{ role: 'assistant', content: response }], + externalHost: { + ...buildHostIdentityMetadata(config, driver, displayName), + hostVariant: config.variant, + capabilitiesUsed: [...capabilitiesUsed], + traceSource: 'accessibility', + traceConfidence: 'low', + traceLimitations: [ + 'Claude did not produce a matching local-agent transcript; final answer was captured from the visible Accessibility tree.', + 'Tool calls, token usage, cost, and hidden context are unavailable from this fallback source.', + ], + artifacts: [ + { + kind: 'trace', + name: 'Claude visible accessibility text', + contentType: 'text/plain', + summary: visibleText.slice(0, 1000), + }, + ], + session: { + runMarker: context.marker, + }, + correlation: context.correlation, + sources: { + finalAnswer: 'accessibility', + toolCalls: 'none', + usage: 'none', + cost: 'none', + }, + evidence: { + finalAnswer: { source: 'accessibility', confidence: 'low' }, + toolCalls: { source: 'none', confidence: 'unknown' }, + usage: { source: 'none', confidence: 'unknown' }, + cost: { source: 'none', confidence: 'unknown' }, + }, + }, + }; +} + +async function detectClaudeChatSurface( + appName: string +): Promise { + let surfaceText: string; + try { + surfaceText = await readMacosFrontWindowContents(appName); + } catch (err) { + return `could not verify active Claude surface via Accessibility: ${formatError(err)}`; + } + + if (looksLikeClaudeChatSurface(surfaceText)) { + return 'visible controls match the regular Claude Chat surface'; + } + + return undefined; +} + +export function looksLikeClaudeChatSurface(visibleText: string): boolean { + const chatSignals = [ + 'New chat', + 'Projects', + 'Artifacts', + 'Ask your org', + 'Write a message', + ]; + const signalCount = chatSignals.filter((signal) => + visibleText.includes(signal) + ).length; + return signalCount >= 3; +} + +async function waitForAccessibilityTrace(options: { + config: ExternalHostConfig; + context: HostRunContext; + driver: HostDriverId; + displayName: string; + capabilitiesUsed: readonly HostCapability[]; + timeoutMs: number; + appName?: string; +}): Promise { + const deadline = Date.now() + options.timeoutMs; + + while (Date.now() < deadline) { + const fallback = await readAccessibilityFallback( + options.config, + options.context, + options.driver, + options.displayName, + options.capabilitiesUsed, + { appName: options.appName } + ); + if (fallback) { + return fallback; + } + await delay(POLL_INTERVAL_MS); + } + + throw new Error( + `Timed out waiting for Claude Chat Desktop visible response for marker ${options.context.marker}` + ); +} + +export async function parseClaudeTrace( + candidate: SessionCandidate, + marker?: string +): Promise { + const parseWarnings: string[] = []; + const auditPath = join(candidate.sessionDir, 'audit.jsonl'); + const transcriptPath = candidate.metadata.cliSessionId + ? await findFile( + candidate.sessionDir, + `${candidate.metadata.cliSessionId}.jsonl` + ) + : undefined; + + let auditEvents: ClaudeAuditEvent[] = []; + let transcriptEvents: ClaudeAuditEvent[] = []; + let rawAudit = ''; + let rawTranscript = ''; + let auditParsed = false; + let transcriptParsed = false; + + try { + rawAudit = await readFile(auditPath, 'utf-8'); + const parsed = await parseNdjsonContent( + rawAudit, + 'Claude audit log' + ); + auditEvents = parsed.events; + auditParsed = parsed.events.length > 0; + parseWarnings.push(...parsed.warnings); + } catch (err) { + parseWarnings.push(`Could not read Claude audit log: ${formatError(err)}`); + } + + if (transcriptPath) { + try { + rawTranscript = await readFile(transcriptPath, 'utf-8'); + const parsed = await parseNdjsonContent( + rawTranscript, + 'Claude transcript' + ); + transcriptEvents = parsed.events; + transcriptParsed = parsed.ok; + parseWarnings.push(...parsed.warnings); + } catch (err) { + parseWarnings.push( + `Could not read Claude transcript: ${formatError(err)}` + ); + } + } else if (candidate.metadata.cliSessionId) { + parseWarnings.push( + `Could not locate transcript for cliSessionId ${candidate.metadata.cliSessionId}.` + ); + } + + const auditEventsForRun = selectEventsForMarker( + candidate.metadata, + auditEvents, + marker + ); + const transcriptEventsForRun = selectEventsForMarker( + candidate.metadata, + transcriptEvents, + marker + ); + const combinedEventsForRun = [ + ...auditEventsForRun, + ...transcriptEventsForRun, + ]; + const resultEvent = + findLastResultEvent(auditEventsForRun) ?? + findLastResultEvent(transcriptEventsForRun); + const finalAnswer = + typeof resultEvent?.result === 'string' + ? resultEvent.result + : extractAssistantText(combinedEventsForRun); + const usage = resultEvent ? extractUsage(resultEvent) : undefined; + const toolCalls = extractToolCalls( + transcriptEventsForRun.length > 0 + ? transcriptEventsForRun + : combinedEventsForRun + ); + + return { + candidate, + auditPath, + transcriptPath, + finalAnswer, + toolCalls, + usage, + requestId: resultEvent?.requestId ?? resultEvent?.request_id, + completedAt: resultEvent?.timestamp, + llmDurationMs: resultEvent?.duration_api_ms ?? resultEvent?.duration_ms, + terminalReason: resultEvent?.terminal_reason, + isError: resultEvent?.is_error === true, + isComplete: resultEvent !== undefined, + auditParsed, + transcriptParsed, + usageAvailable: usage !== undefined, + costAvailable: typeof resultEvent?.total_cost_usd === 'number', + parseWarnings, + rawText: `${rawAudit}\n${rawTranscript}`, + }; +} + +function selectEventsForMarker( + metadata: ClaudeSessionMetadata, + events: ClaudeAuditEvent[], + marker?: string +): ClaudeAuditEvent[] { + if (!marker) { + return events; + } + + const markerIndex = events.findIndex((event) => + JSON.stringify(event).includes(marker) + ); + if (markerIndex < 0) { + return metadata.initialMessage?.includes(marker) ? events : []; + } + + return events.slice(markerIndex); +} + +export function buildClaudeTraceMetadata(options: { + config: ExternalHostConfig; + context: HostRunContext; + driver: HostDriverId; + displayName: string; + capabilitiesUsed?: readonly HostCapability[]; + artifacts: HostArtifact[]; + trace: ClaudeTrace; + limitations: string[]; +}): ExternalHostMetadata { + const correlationLimitations = options.context.correlation.includedInPrompt + ? [] + : [ + 'Trace was matched by recently updated host artifacts because no prompt marker was included.', + ]; + const limitations = buildTraceLimitations(options.trace, [ + ...options.limitations, + ...correlationLimitations, + ]); + const traceConfidence = getTraceConfidence( + options.trace, + options.context.correlation + ); + const finalAnswerEvidence = buildEvidence( + options.trace.isComplete && options.trace.finalAnswer !== undefined, + traceConfidence + ); + const toolCallsEvidence = buildEvidence( + options.trace.transcriptParsed, + traceConfidence + ); + const usageEvidence = buildEvidence( + options.trace.usageAvailable, + traceConfidence + ); + const costEvidence = buildEvidence( + options.trace.costAvailable, + traceConfidence + ); + + return { + ...buildHostIdentityMetadata( + options.config, + options.driver, + options.displayName + ), + hostVariant: options.config.variant, + capabilitiesUsed: [ + ...(options.capabilitiesUsed ?? CLAUDE_DESKTOP_MACOS_CAPABILITIES), + ], + traceSource: 'host-local-transcript', + traceConfidence, + traceLimitations: limitations.length > 0 ? limitations : undefined, + artifacts: options.artifacts, + session: { + id: + options.trace.candidate.metadata.sessionId ?? + options.trace.candidate.id, + runMarker: options.context.marker, + requestId: options.trace.requestId, + cliSessionId: options.trace.candidate.metadata.cliSessionId, + cwd: options.trace.candidate.metadata.cwd, + startedAt: metadataTimestampString( + options.trace.candidate.metadata.createdAt + ), + completedAt: options.trace.completedAt, + }, + correlation: options.context.correlation, + sources: { + finalAnswer: finalAnswerEvidence.source, + toolCalls: toolCallsEvidence.source, + usage: usageEvidence.source, + cost: costEvidence.source, + }, + evidence: { + finalAnswer: finalAnswerEvidence, + toolCalls: toolCallsEvidence, + usage: usageEvidence, + cost: costEvidence, + }, + }; +} + +function buildEvidence( + available: boolean, + confidence: ExternalHostMetadata['traceConfidence'] +) { + return available + ? ({ source: 'host-local-transcript', confidence } as const) + : ({ source: 'none', confidence: 'unknown' } as const); +} + +function getTraceConfidence( + trace: ClaudeTrace, + correlation: HostRunContext['correlation'] +): ExternalHostMetadata['traceConfidence'] { + if (!trace.isComplete || !trace.auditParsed) { + return 'unknown'; + } + if ( + trace.parseWarnings.some((warning) => + warning.startsWith('Claude audit log discarded') + ) + ) { + return 'medium'; + } + return correlation.includedInPrompt ? 'high' : 'medium'; +} + +function buildTraceLimitations( + trace: ClaudeTrace, + limitations: string[] +): string[] { + const output = [...limitations]; + + if (!trace.transcriptParsed) { + output.push( + 'Tool-call evidence is unavailable because a complete structured Claude transcript was not found or could not be parsed.' + ); + } + + if (!trace.usageAvailable) { + output.push('Usage evidence is unavailable from the parsed Claude trace.'); + } + + if (!trace.costAvailable) { + output.push('Cost evidence is unavailable from the parsed Claude trace.'); + } + + return Array.from(new Set(output)); +} + +function failureResult(options: { + config: ExternalHostConfig; + context: HostRunContext; + driver: HostDriverId; + displayName: string; + capabilitiesUsed?: readonly HostCapability[]; + failureKind: ExternalHostFailureKind; + error: string; + artifacts: HostArtifact[]; + limitations: string[]; +}): ExternalHostRunResult { + return { + success: false, + toolCalls: [], + error: options.error, + externalHost: { + ...buildHostIdentityMetadata( + options.config, + options.driver, + options.displayName + ), + hostVariant: options.config.variant, + capabilitiesUsed: [...(options.capabilitiesUsed ?? [])], + traceSource: 'none', + traceConfidence: 'unknown', + traceLimitations: options.limitations, + artifacts: options.artifacts, + session: { runMarker: options.context.marker }, + correlation: options.context.correlation, + failureKind: options.failureKind, + }, + }; +} + +function buildHostIdentityMetadata( + config: ExternalHostConfig, + driver: HostDriverId, + displayName: string +): Pick< + ExternalHostMetadata, + 'driver' | 'driverSlug' | 'displayName' | 'hostName' | 'hostType' +> { + return { + driver, + driverSlug: driverToSlug(driver), + displayName, + hostName: displayName, + hostType: config.hostType ?? hostTypeFromDriver(driver), + }; +} + +function buildArtifacts(trace: ClaudeTrace): HostArtifact[] { + const artifacts: HostArtifact[] = [ + { + kind: 'metadata', + name: 'Claude session metadata', + path: trace.candidate.metadataPath, + contentType: 'application/json', + }, + ]; + + if (trace.auditPath) { + artifacts.push({ + kind: 'audit', + name: 'Claude audit log', + path: trace.auditPath, + contentType: 'application/x-ndjson', + }); + } + + if (trace.transcriptPath) { + artifacts.push({ + kind: 'transcript', + name: 'Claude transcript', + path: trace.transcriptPath, + contentType: 'application/x-ndjson', + }); + } + + return artifacts; +} + +export function extractAccessibilityResponse( + visibleText: string +): string | undefined { + const lines = visibleText + .split('\n') + .map((line) => line.trim()) + .filter(Boolean); + const responseLine = [...lines] + .reverse() + .find((line) => line.startsWith('Claude responded: ')); + if (responseLine) { + return responseLine.slice('Claude responded: '.length).trim(); + } + + const inlineResponseMatch = /Claude responded:\s*([^,\n]+)/.exec(visibleText); + if (inlineResponseMatch?.[1]) { + return inlineResponseMatch[1].trim(); + } + + const markerIndex = lines.findIndex((line) => + line.includes('[eval-run-marker:') + ); + if (markerIndex >= 0) { + return lines + .slice(markerIndex + 1) + .find( + (line) => + !line.startsWith('Write a message') && + !line.includes('Claude is AI and can make mistakes') + ); + } + + return undefined; +} + +async function listSessionCandidates( + dataDir: string +): Promise { + const metadataPaths = await findClaudeMetadataFiles(dataDir); + const candidates: SessionCandidate[] = []; + + for (const metadataPath of metadataPaths) { + try { + const metadata = JSON.parse( + await readFile(metadataPath, 'utf-8') + ) as ClaudeSessionMetadata; + const metadataStat = await stat(metadataPath); + const id = basename(metadataPath, '.json'); + const sessionDir = join(dirname(metadataPath), id); + const statMtimeMs = await getSessionObservedMtime({ + sessionDir, + cliSessionId: metadata.cliSessionId, + metadataMtimeMs: metadataStat.mtimeMs, + }); + candidates.push({ + id, + metadataPath, + sessionDir, + statMtimeMs, + metadata, + }); + } catch { + continue; + } + } + + return candidates; +} + +async function getSessionObservedMtime(options: { + sessionDir: string; + cliSessionId?: string; + metadataMtimeMs: number; +}): Promise { + const observed = [ + options.metadataMtimeMs, + await getFileMtime(join(options.sessionDir, 'audit.jsonl')), + await getFileMtime(options.sessionDir), + ]; + + if (options.cliSessionId) { + const transcriptPath = await findFile( + options.sessionDir, + `${options.cliSessionId}.jsonl` + ); + if (transcriptPath) { + observed.push(await getFileMtime(transcriptPath)); + } + } + + return Math.max( + ...observed.filter((mtime): mtime is number => mtime !== undefined) + ); +} + +async function getFileMtime(path: string): Promise { + try { + return (await stat(path)).mtimeMs; + } catch { + return undefined; + } +} + +async function findClaudeMetadataFiles(root: string): Promise { + const stack = [root]; + const matches: string[] = []; + + while (stack.length > 0) { + const current = stack.pop()!; + let entries; + try { + entries = await readdir(current, { withFileTypes: true }); + } catch { + continue; + } + + for (const entry of entries) { + const path = join(current, entry.name); + if (entry.isFile() && /^local_.+\.json$/.test(entry.name)) { + matches.push(path); + } else if (entry.isDirectory()) { + stack.push(path); + } + } + } + + return matches; +} + +function sessionMatchesMarker( + session: SessionCandidate, + trace: ClaudeTrace, + marker: string +): boolean { + if (session.metadata.initialMessage?.includes(marker)) { + return true; + } + if (trace.finalAnswer?.includes(marker)) { + return true; + } + return trace.rawText.includes(marker); +} + +function sessionMatchesCorrelation(options: { + session: SessionCandidate; + trace: ClaudeTrace; + marker: string; + correlation?: HostRunContext['correlation']; + isNewOrUpdated: boolean; + isRecent: boolean; +}): boolean { + if (options.correlation?.includedInPrompt !== false) { + return sessionMatchesMarker(options.session, options.trace, options.marker); + } + + return options.isNewOrUpdated || options.isRecent; +} + +async function parseNdjsonContent( + content: string, + sourceName: string +): Promise<{ events: T[]; ok: boolean; warnings: string[] }> { + const events: T[] = []; + const parser = parseNdjson({ strict: false }); + + await new Promise((resolve, reject) => { + parser.on('data', (event: T) => events.push(event)); + parser.on('error', reject); + parser.on('end', resolve); + Readable.from([content]).pipe(parser); + }); + + const nonEmptyLineCount = content + .split('\n') + .filter((line) => line.trim().length > 0).length; + const discardedLineCount = nonEmptyLineCount - events.length; + const warnings = + discardedLineCount > 0 + ? [ + `${sourceName} discarded ${discardedLineCount} malformed JSONL line${ + discardedLineCount === 1 ? '' : 's' + } using ndjson strict=false parsing.`, + ] + : []; + + return { events, ok: warnings.length === 0, warnings }; +} + +function findLastResultEvent( + events: ClaudeAuditEvent[] +): ClaudeAuditEvent | undefined { + return [...events] + .reverse() + .find((event) => event.type === 'result' || event.result !== undefined); +} + +async function findFile( + root: string, + filename: string +): Promise { + const stack = [root]; + + while (stack.length > 0) { + const current = stack.pop()!; + let entries; + try { + entries = await readdir(current, { withFileTypes: true }); + } catch { + continue; + } + + for (const entry of entries) { + const path = join(current, entry.name); + if (entry.isFile() && entry.name === filename) { + return path; + } + if (entry.isDirectory()) { + stack.push(path); + } + } + } + + return undefined; +} + +function extractAssistantText(events: ClaudeAuditEvent[]): string | undefined { + const parts: string[] = []; + + for (const event of events) { + for (const block of event.message?.content ?? []) { + if (block.type === 'text' && block.text) { + parts.push(block.text); + } + } + } + + return parts.length > 0 ? parts.join('') : undefined; +} + +function extractToolCalls(events: ClaudeAuditEvent[]): LLMToolCall[] { + const toolCalls: LLMToolCall[] = []; + + for (const event of events) { + for (const block of event.message?.content ?? []) { + if (block.type !== 'tool_use' || !block.name) { + continue; + } + const mcpMatch = /^mcp__(.+)__(.+)$/.exec(block.name); + toolCalls.push({ + name: mcpMatch ? mcpMatch[2]! : block.name, + arguments: block.input ?? {}, + id: block.id, + }); + } + } + + return toolCalls; +} + +function extractUsage(event: ClaudeAuditEvent): UsageMetrics | undefined { + const usage = event.usage; + const inputTokens = + getNumber(usage, 'input_tokens') ?? getNumber(usage, 'inputTokens'); + const outputTokens = + getNumber(usage, 'output_tokens') ?? getNumber(usage, 'outputTokens'); + + if ( + inputTokens === undefined && + outputTokens === undefined && + event.total_cost_usd === undefined && + event.duration_ms === undefined + ) { + return undefined; + } + + return { + inputTokens: inputTokens ?? 0, + outputTokens: outputTokens ?? 0, + totalCostUsd: event.total_cost_usd ?? 0, + durationMs: event.duration_ms ?? 0, + durationApiMs: event.duration_api_ms, + cacheReadInputTokens: + getNumber(usage, 'cache_read_input_tokens') ?? + getNumber(usage, 'cacheReadInputTokens'), + cacheCreationInputTokens: + getNumber(usage, 'cache_creation_input_tokens') ?? + getNumber(usage, 'cacheCreationInputTokens'), + }; +} + +function getNumber( + object: Record | undefined, + key: string +): number | undefined { + const value = object?.[key]; + return typeof value === 'number' ? value : undefined; +} + +function metadataTimestampMs(value: string | number | undefined): number { + if (typeof value === 'number') { + return value; + } + if (typeof value === 'string') { + const parsed = Date.parse(value); + return Number.isNaN(parsed) ? Number.NaN : parsed; + } + return Number.NaN; +} + +function metadataTimestampString( + value: string | number | undefined +): string | undefined { + if (typeof value === 'string') { + return value; + } + if (typeof value === 'number') { + return new Date(value).toISOString(); + } + return undefined; +} + +function classifyTraceFailure(message: string): ExternalHostFailureKind { + const lower = message.toLowerCase(); + if (lower.includes('ambiguous')) return 'ambiguous_matching_sessions'; + if (lower.includes('timed out')) return 'timeout'; + if (lower.includes('no matching')) return 'no_matching_session'; + if (lower.includes('parse')) return 'parse_failure'; + return 'unknown'; +} + +function formatError(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function createExternalHostRunId(caseId: string): string { + return `${caseId}-${randomUUID()}`; +} diff --git a/src/evals/externalHost/builtins/macosDesktop.test.ts b/src/evals/externalHost/builtins/macosDesktop.test.ts new file mode 100644 index 0000000..4e2ce29 --- /dev/null +++ b/src/evals/externalHost/builtins/macosDesktop.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest'; +import { + buildMacosDesktopSubmitScript, + MACOS_DESKTOP_CAPABILITIES, +} from './macosDesktop.js'; + +describe('macOS desktop built-in capabilities', () => { + it('declares reusable platform and accessibility submit capabilities', () => { + expect( + MACOS_DESKTOP_CAPABILITIES.map((capability) => ({ + id: capability.id, + capabilities: capability.capabilities, + })) + ).toEqual([ + { + id: 'builtin:platform.macos', + capabilities: ['control'], + }, + { + id: 'builtin:desktop.macos.accessibilitySubmit', + capabilities: ['control', 'input'], + }, + ]); + }); + + it('builds a submit script that prefers direct accessibility actions over global keystrokes', () => { + const script = buildMacosDesktopSubmitScript('hello marker', { + appName: 'Example', + createNewConversation: false, + settleDelayMs: 500, + }); + + expect(script).toContain('set value of textAreaElement to "hello marker"'); + expect(script).toContain('perform action "AXPress" of submitButtonElement'); + expect(script).toContain('keystroke "v" using command down'); + expect(script).toContain('key code 36'); + }); +}); diff --git a/src/evals/externalHost/builtins/macosDesktop.ts b/src/evals/externalHost/builtins/macosDesktop.ts new file mode 100644 index 0000000..a743a2a --- /dev/null +++ b/src/evals/externalHost/builtins/macosDesktop.ts @@ -0,0 +1,441 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import type { + ExternalHostCapabilityContext, + ExternalHostCapabilityImplementation, + ExternalHostFailureKind, + ExternalHostRunResult, +} from '../types.js'; +import { driverToSlug, hostTypeFromDriver } from '../driverIdentity.js'; + +const execFileAsync = promisify(execFile); +const DEFAULT_SETTLE_DELAY_MS = 500; +const DEFAULT_SUBMIT_BUTTON_NAMES = ['Send', 'Submit']; + +export const MACOS_DESKTOP_CAPABILITIES: ExternalHostCapabilityImplementation[] = + [ + { + id: 'builtin:platform.macos', + capabilities: ['control'], + run: requireMacosCapability, + }, + { + id: 'builtin:desktop.macos.accessibilitySubmit', + capabilities: ['control', 'input'], + run: submitPromptCapability, + }, + ]; + +export async function runAppleScript(script: string): Promise { + const result = await execFileAsync('osascript', ['-e', script]); + return result.stdout; +} + +export function writeMacosClipboard(value: string): Promise { + return new Promise((resolve, reject) => { + const child = execFile('pbcopy', (error) => { + if (error) { + reject(new Error(error.message)); + return; + } + resolve(); + }); + child.stdin?.end(value); + }); +} + +export async function readMacosAccessibilityText( + appName: string +): Promise { + const script = ` +on collectText(theElement) + set output to {} + try + tell application "System Events" to set elementRole to role of theElement + tell application "System Events" to set elementValue to value of theElement + if (elementRole is "AXStaticText" or elementRole is "AXTextArea") and elementValue is not missing value then set end of output to (elementValue as text) + end try + try + tell application "System Events" to set uiChildren to UI elements of theElement + repeat with childElement in uiChildren + set output to output & my collectText(childElement) + end repeat + end try + return output +end collectText + +tell application "System Events" to tell process ${JSON.stringify(appName)} + set textItems to my collectText(front window) +end tell +set AppleScript's text item delimiters to linefeed +return textItems as text +`; + return runAppleScript(script); +} + +export async function readMacosFrontWindowContents( + appName: string +): Promise { + const script = `tell application "System Events" to tell process ${JSON.stringify( + appName + )} to get entire contents of front window`; + return runAppleScript(script); +} + +async function requireMacosCapability({ + config, + run, + binding, + state, +}: ExternalHostCapabilityContext): Promise { + if (process.platform === 'darwin') { + return; + } + + return desktopFailureResult({ + config, + context: run, + state, + failureKind: 'unsupported_host', + error: + stringOption(binding.with, 'error') ?? + `${state.displayName} currently requires macOS automation support.`, + limitations: [ + stringOption(binding.with, 'limitation') ?? + 'Windows UI Automation support has not been added yet.', + ], + }); +} + +async function submitPromptCapability({ + config, + run, + binding, + state, +}: ExternalHostCapabilityContext): Promise { + try { + const appName = + runStringOption(config, binding, 'appName') ?? state.displayName; + await submitPromptToMacosDesktopApp(run.submittedScenario, { + appName, + createNewConversation: shouldCreateNewConversation( + binding.with?.createNewConversation, + config + ), + settleDelayMs: runNumberOption(config, binding, 'settleDelayMs'), + submitButtonNames: stringArrayOption(binding.with, 'submitButtonNames'), + }); + } catch (err) { + const message = formatError(err); + return desktopFailureResult({ + config, + context: run, + state, + failureKind: classifyDesktopSubmissionFailure(message), + error: `Failed to submit prompt to desktop host: ${message}`, + limitations: [ + 'The desktop host app must be installed, signed in, and allowed in macOS Automation/Accessibility settings.', + ], + }); + } +} + +export async function submitPromptToMacosDesktopApp( + prompt: string, + options: { + appName: string; + createNewConversation: boolean; + settleDelayMs?: number; + submitButtonNames?: string[]; + } +): Promise { + const settleDelayMs = options.settleDelayMs ?? DEFAULT_SETTLE_DELAY_MS; + const script = buildMacosDesktopSubmitScript(prompt, { + ...options, + settleDelayMs, + }); + await writeMacosClipboard(prompt); + await runAppleScript(script); +} + +export function buildMacosDesktopSubmitScript( + prompt: string, + options: { + appName: string; + createNewConversation: boolean; + settleDelayMs: number; + submitButtonNames?: string[]; + } +): string { + const settleDelayMs = options.settleDelayMs; + const promptLiteral = JSON.stringify(prompt); + const verificationNeedle = prompt.includes('[eval-run-marker:') + ? '[eval-run-marker:' + : prompt.trim().slice(0, 120); + const verificationNeedleLiteral = JSON.stringify(verificationNeedle); + const submitButtonNamesLiteral = appleScriptListLiteral( + options.submitButtonNames?.length + ? options.submitButtonNames + : DEFAULT_SUBMIT_BUTTON_NAMES + ); + + const newConversation = options.createNewConversation + ? `keystroke "n" using command down + delay ${Math.max(settleDelayMs, 1500) / 1000}` + : ''; + + return ` +on findTextArea(theElement) + try + tell application "System Events" to if role of theElement is "AXTextArea" then return theElement + end try + try + tell application "System Events" to set uiChildren to UI elements of theElement + repeat with childElement in uiChildren + set foundElement to my findTextArea(childElement) + if foundElement is not equal to missing value then return foundElement + end repeat + end try + return missing value +end findTextArea + +on normalizeText(valueToNormalize) + try + return my lowercaseText(valueToNormalize as text) + on error + return "" + end try +end normalizeText + +on lowercaseText(inputText) + set upperChars to "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + set lowerChars to "abcdefghijklmnopqrstuvwxyz" + set outputText to "" + repeat with currentChar in characters of inputText + set currentCharText to currentChar as text + set charIndex to offset of currentCharText in upperChars + if charIndex is greater than 0 then + set outputText to outputText & character charIndex of lowerChars + else + set outputText to outputText & currentCharText + end if + end repeat + return outputText +end lowercaseText + +on elementLabel(theElement) + set labels to {} + try + tell application "System Events" to if name of theElement is not missing value then set end of labels to name of theElement + end try + try + tell application "System Events" to if description of theElement is not missing value then set end of labels to description of theElement + end try + try + tell application "System Events" to if value of theElement is not missing value then set end of labels to value of theElement + end try + set AppleScript's text item delimiters to " " + return my normalizeText(labels as text) +end elementLabel + +on labelMatches(theElement, buttonNames) + set labelText to my elementLabel(theElement) + repeat with buttonName in buttonNames + if labelText contains my normalizeText(buttonName) then return true + end repeat + return false +end labelMatches + +on findSubmitButton(theElement, buttonNames) + try + tell application "System Events" to if role of theElement is "AXButton" and my labelMatches(theElement, buttonNames) then return theElement + end try + try + tell application "System Events" to set uiChildren to UI elements of theElement + repeat with childElement in uiChildren + set foundElement to my findSubmitButton(childElement, buttonNames) + if foundElement is not equal to missing value then return foundElement + end repeat + end try + return missing value +end findSubmitButton + +tell application ${JSON.stringify(options.appName)} to activate +delay ${settleDelayMs / 1000} +tell application "System Events" + ${newConversation} + tell process ${JSON.stringify(options.appName)} + set frontmost to true + set textAreaElement to my findTextArea(front window) + if textAreaElement is equal to missing value then error "No composer text area found" + click textAreaElement + try + set focused of textAreaElement to true + end try + try + set value of textAreaElement to ${promptLiteral} + end try + end tell + delay 0.1 + tell process ${JSON.stringify(options.appName)} + set textAreaElement to my findTextArea(front window) + if textAreaElement is equal to missing value then error "No composer text area found before submit" + if value of textAreaElement does not contain ${verificationNeedleLiteral} then + set frontmost to true + click textAreaElement + try + set focused of textAreaElement to true + end try + keystroke "v" using command down + delay 0.5 + end if + if value of textAreaElement does not contain ${verificationNeedleLiteral} then error "Composer did not receive pasted eval prompt" + set submitButtonElement to my findSubmitButton(front window, ${submitButtonNamesLiteral}) + if submitButtonElement is not equal to missing value then + perform action "AXPress" of submitButtonElement + else + set frontmost to true + click textAreaElement + try + set focused of textAreaElement to true + end try + key code 36 + end if + end tell +end tell +`; +} + +function shouldCreateNewConversation( + option: unknown, + config: { options?: Record } +): boolean { + if (option === 'unless-disabled') { + return configStringOption(config, 'newConversationShortcut') !== 'none'; + } + return option === true; +} + +function desktopFailureResult({ + config, + context, + state, + failureKind, + error, + limitations, +}: { + config: ExternalHostCapabilityContext['config']; + context: ExternalHostCapabilityContext['run']; + state: ExternalHostCapabilityContext['state']; + failureKind: ExternalHostFailureKind; + error: string; + limitations: string[]; +}): ExternalHostRunResult { + return { + success: false, + toolCalls: [], + error, + externalHost: { + driver: state.driver, + driverSlug: driverToSlug(state.driver), + displayName: state.displayName, + hostName: state.displayName, + hostType: config.hostType ?? hostTypeFromDriver(state.driver), + hostVariant: config.variant, + capabilitiesUsed: state.capabilitiesUsed, + traceSource: 'none', + traceConfidence: 'unknown', + traceLimitations: limitations, + artifacts: [], + session: { runMarker: context.marker }, + correlation: context.correlation, + failureKind, + }, + }; +} + +function runStringOption( + config: { options?: Record }, + binding: { with?: Record }, + key: string +): string | undefined { + return stringOption(binding.with, key) ?? configStringOption(config, key); +} + +function runNumberOption( + config: { options?: Record }, + binding: { with?: Record }, + key: string +): number | undefined { + const value = binding.with?.[key]; + return typeof value === 'number' ? value : configNumberOption(config, key); +} + +function configStringOption( + config: { options?: Record }, + key: string +): string | undefined { + return stringOption(config.options, key); +} + +function configNumberOption( + config: { options?: Record }, + key: string +): number | undefined { + const value = config.options?.[key]; + return typeof value === 'number' ? value : undefined; +} + +function stringOption( + options: Record | undefined, + key: string +): string | undefined { + const value = options?.[key]; + return typeof value === 'string' ? value : undefined; +} + +function stringArrayOption( + options: Record | undefined, + key: string +): string[] | undefined { + const value = options?.[key]; + if (!Array.isArray(value)) { + return undefined; + } + const strings = value.filter( + (item): item is string => typeof item === 'string' + ); + return strings.length > 0 ? strings : undefined; +} + +function appleScriptListLiteral(values: string[]): string { + return `{${values.map((value) => JSON.stringify(value)).join(', ')}}`; +} + +function classifyDesktopSubmissionFailure( + message: string +): ExternalHostFailureKind { + const lower = message.toLowerCase(); + if ( + lower.includes('not authorized') || + lower.includes('not permitted') || + lower.includes('assistive access') || + lower.includes('accessibility') || + lower.includes('automation') + ) { + return 'automation_permission_denied'; + } + if ( + lower.includes('can’t get application') || + lower.includes("can't get application") || + lower.includes('application isn’t running') || + lower.includes("application isn't running") + ) { + return 'app_unavailable'; + } + return 'submission_failed'; +} + +function formatError(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/src/evals/externalHost/capabilities.test.ts b/src/evals/externalHost/capabilities.test.ts new file mode 100644 index 0000000..a5692bd --- /dev/null +++ b/src/evals/externalHost/capabilities.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; +import { validateHostCapabilities } from './capabilities.js'; + +describe('validateHostCapabilities', () => { + it('passes when all required external host capabilities are present', () => { + expect( + validateHostCapabilities([ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', + ]) + ).toEqual([]); + }); + + it('reports missing required capabilities', () => { + expect(validateHostCapabilities(['control', 'input'])).toEqual([ + 'completion', + 'trace', + 'normalize', + ]); + }); +}); diff --git a/src/evals/externalHost/capabilities.ts b/src/evals/externalHost/capabilities.ts new file mode 100644 index 0000000..6877e36 --- /dev/null +++ b/src/evals/externalHost/capabilities.ts @@ -0,0 +1,18 @@ +import type { HostCapability } from './types.js'; + +export const REQUIRED_HOST_CAPABILITIES: HostCapability[] = [ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', +]; + +export function validateHostCapabilities( + capabilities: readonly HostCapability[] +): HostCapability[] { + const provided = new Set(capabilities); + return REQUIRED_HOST_CAPABILITIES.filter( + (capability) => !provided.has(capability) + ); +} diff --git a/src/evals/externalHost/capabilityRuntime.test.ts b/src/evals/externalHost/capabilityRuntime.test.ts new file mode 100644 index 0000000..76a639b --- /dev/null +++ b/src/evals/externalHost/capabilityRuntime.test.ts @@ -0,0 +1,155 @@ +import { describe, expect, it } from 'vitest'; +import { + loadExternalHostConfig, + loadExternalHostRunner, + registerExternalHostCapability, +} from './capabilityRuntime.js'; + +const TEST_DRIVER = { + provider: 'test', + product: 'host', + surface: 'chat', + runtime: 'desktop-app', + platform: 'macos', +} as const; + +const TEST_CORRELATION = { + strategy: 'prompt_marker', + marker: 'MCP_SERVER_TESTER_CAPABILITY', + includedInPrompt: true, +} as const; + +describe('external host capability runtime', () => { + it('composes a runner from config-declared capability bindings', async () => { + const calls: string[] = []; + + registerExternalHostCapability({ + id: 'test.capability.success', + capabilities: ['control', 'input', 'completion', 'trace', 'normalize'], + async setup({ state }) { + calls.push('setup'); + state.data.setupSeen = true; + }, + async run({ run, state }) { + calls.push('run'); + expect(state.driverSlug).toBe('test.host.chat.desktop-app.macos'); + expect(state.data.setupSeen).toBe(true); + return { + success: true, + response: 'composed result', + toolCalls: [], + externalHost: { + driver: state.driver, + driverSlug: state.driverSlug, + displayName: state.displayName, + hostName: state.displayName, + hostType: 'custom', + capabilitiesUsed: state.capabilitiesUsed, + traceSource: 'manual-import', + traceConfidence: 'high', + artifacts: [], + session: { runMarker: run.marker }, + correlation: run.correlation, + }, + }; + }, + }); + + const runner = await loadExternalHostRunner({ + driver: TEST_DRIVER, + capabilities: { + control: { + uses: 'test.capability.success', + provides: ['input', 'completion', 'trace', 'normalize'], + }, + }, + }); + + const result = await runner.run({ + runId: 'run', + caseId: 'case', + scenario: 'scenario', + submittedScenario: 'scenario', + marker: 'MCP_SERVER_TESTER_CAPABILITY', + correlation: TEST_CORRELATION, + timeoutMs: 1000, + startedAtMs: Date.now(), + }); + + expect(calls).toEqual(['setup', 'run']); + expect(result).toMatchObject({ + success: true, + response: 'composed result', + externalHost: { + driverSlug: 'test.host.chat.desktop-app.macos', + capabilitiesUsed: [ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', + ], + }, + }); + }); + + it('treats binding provides as additional capabilities', async () => { + registerExternalHostCapability({ + id: 'test.capability.extraControl', + capabilities: ['control'], + }); + registerExternalHostCapability({ + id: 'test.capability.inputTrace', + capabilities: ['input', 'trace'], + }); + + const loaded = await loadExternalHostConfig({ + driver: TEST_DRIVER, + capabilities: { + control: { uses: 'test.capability.extraControl' }, + input: { + uses: 'test.capability.inputTrace', + provides: ['completion', 'normalize'], + }, + }, + }); + + expect(loaded.capabilitiesUsed).toEqual([ + 'control', + 'input', + 'trace', + 'completion', + 'normalize', + ]); + }); + + it('fails config loading when required capabilities are missing', async () => { + registerExternalHostCapability({ + id: 'test.capability.controlOnly', + capabilities: ['control'], + }); + + await expect( + loadExternalHostConfig({ + driver: TEST_DRIVER, + capabilities: { + control: { uses: 'test.capability.controlOnly' }, + }, + }) + ).rejects.toThrow('missing capabilities'); + }); + + it('fails config loading for unavailable capability implementations', async () => { + await expect( + loadExternalHostConfig({ + driver: TEST_DRIVER, + capabilities: { + control: { + uses: 'missing.capability', + provides: ['input', 'completion', 'trace', 'normalize'], + }, + }, + }) + ).rejects.toThrow('not available'); + }); +}); diff --git a/src/evals/externalHost/capabilityRuntime.ts b/src/evals/externalHost/capabilityRuntime.ts new file mode 100644 index 0000000..152044f --- /dev/null +++ b/src/evals/externalHost/capabilityRuntime.ts @@ -0,0 +1,336 @@ +import { + REQUIRED_HOST_CAPABILITIES, + validateHostCapabilities, +} from './capabilities.js'; +import { + getRegisteredExternalHostConfig, + getRegisteredExternalHostDisplayName, +} from './hostRegistry.js'; +import { + listBuiltinExternalHostCapabilities, + resolveBuiltinExternalHostCapability, +} from './builtinCapabilities.js'; +import { + driverToSlug, + hostTypeFromDriver, + normalizeHostDriver, +} from './driverIdentity.js'; +import type { + ExternalHostCapabilityBinding, + ExternalHostCapabilityContext, + ExternalHostCapabilityImplementation, + ExternalHostCapabilitiesConfig, + ExternalHostConfig, + ExternalHostRunResult, + ExternalHostRunState, + ExternalHostRunner, + HostCapability, + HostDriverId, + HostRunContext, +} from './types.js'; + +const CAPABILITIES = new Map(); + +export interface LoadedExternalHostCapability { + capability: HostCapability; + binding: ExternalHostCapabilityBinding; + implementation: ExternalHostCapabilityImplementation; +} + +export interface LoadedExternalHostConfig { + config: ExternalHostConfig; + driver: HostDriverId; + driverSlug: string; + displayName: string; + loadedCapabilities: LoadedExternalHostCapability[]; + capabilitiesUsed: HostCapability[]; +} + +export function registerExternalHostCapability( + implementation: ExternalHostCapabilityImplementation +): void { + CAPABILITIES.set(implementation.id, implementation); +} + +export function listExternalHostCapabilities(): ExternalHostCapabilityImplementation[] { + return Array.from( + new Map( + [...listBuiltinExternalHostCapabilities(), ...CAPABILITIES.values()].map( + (implementation) => [implementation.id, implementation] + ) + ).values() + ); +} + +export async function resolveExternalHostCapability( + uses: string +): Promise { + const registered = CAPABILITIES.get(uses); + if (registered) { + return registered; + } + + const configuredBuiltin = resolveBuiltinExternalHostCapability(uses); + if (configuredBuiltin) { + return configuredBuiltin; + } + + if (uses.startsWith('module:')) { + return loadModuleCapability(uses); + } + + return undefined; +} + +export async function loadExternalHostRunner( + config: ExternalHostConfig +): Promise { + const loaded = await loadExternalHostConfig(config); + + return createExternalHostRunner(loaded); +} + +export function createExternalHostRunner( + loaded: LoadedExternalHostConfig +): ExternalHostRunner { + return { + async run(context: HostRunContext): Promise { + return runLoadedExternalHost(loaded, context); + }, + }; +} + +export async function loadExternalHostConfig( + config: ExternalHostConfig +): Promise { + const driver = normalizeHostDriver(config.driver); + const driverSlug = driverToSlug(driver); + const registeredConfig = getRegisteredExternalHostConfig(driverSlug); + const effectiveConfig = mergeExternalHostConfig(config, registeredConfig); + const capabilitiesConfig = effectiveConfig.capabilities; + + if (!capabilitiesConfig) { + throw new Error( + `External host ${driverSlug} does not declare capabilities and has no built-in defaults.` + ); + } + + const loadedCapabilities: LoadedExternalHostCapability[] = []; + const providedCapabilities = new Set(); + + for (const capability of REQUIRED_HOST_CAPABILITIES) { + const bindings = normalizeCapabilityBindings( + capabilitiesConfig[capability] + ); + for (const binding of bindings) { + const implementation = await resolveExternalHostCapability(binding.uses); + if (!implementation) { + throw new Error( + `External host capability implementation is not available: ${binding.uses}` + ); + } + + loadedCapabilities.push({ + capability, + binding, + implementation, + }); + providedCapabilities.add(capability); + for (const provided of [ + ...implementation.capabilities, + ...(binding.provides ?? []), + ]) { + providedCapabilities.add(provided); + } + } + } + + const capabilitiesUsed = Array.from(providedCapabilities); + const missingCapabilities = validateHostCapabilities(capabilitiesUsed); + if (missingCapabilities.length > 0) { + throw new Error( + `External host ${driverSlug} is missing capabilities: ${missingCapabilities.join(', ')}` + ); + } + + return { + config: effectiveConfig, + driver, + driverSlug, + displayName: + effectiveConfig.name ?? + getRegisteredExternalHostDisplayName(driverSlug) ?? + driverSlug, + loadedCapabilities, + capabilitiesUsed, + }; +} + +async function runLoadedExternalHost( + loaded: LoadedExternalHostConfig, + context: HostRunContext +): Promise { + const state: ExternalHostRunState = { + driver: loaded.driver, + driverSlug: loaded.driverSlug, + displayName: loaded.displayName, + capabilitiesUsed: loaded.capabilitiesUsed, + data: {}, + }; + + for (const loadedCapability of loaded.loadedCapabilities) { + const result = await loadedCapability.implementation.setup?.( + capabilityContext(loaded, context, state, loadedCapability) + ); + if (result) { + return result; + } + if (state.result) { + return state.result; + } + } + + for (const loadedCapability of loaded.loadedCapabilities) { + const result = await loadedCapability.implementation.run?.( + capabilityContext(loaded, context, state, loadedCapability) + ); + if (result) { + return result; + } + if (state.result) { + return state.result; + } + } + + return runtimeFailure( + loaded, + context, + `External host ${loaded.driverSlug} completed without producing a result.` + ); +} + +function capabilityContext( + loaded: LoadedExternalHostConfig, + run: HostRunContext, + state: ExternalHostRunState, + loadedCapability: LoadedExternalHostCapability +): ExternalHostCapabilityContext { + return { + config: loaded.config, + run, + capability: loadedCapability.capability, + binding: loadedCapability.binding, + state, + }; +} + +function mergeExternalHostConfig( + config: ExternalHostConfig, + builtin: Partial | undefined +): ExternalHostConfig { + if (!builtin) { + return config; + } + + return { + ...builtin, + ...config, + capabilities: mergeCapabilities(builtin.capabilities, config.capabilities), + correlation: { + ...builtin.correlation, + ...config.correlation, + }, + options: { + ...builtin.options, + ...config.options, + }, + }; +} + +function mergeCapabilities( + base: ExternalHostCapabilitiesConfig | undefined, + override: ExternalHostCapabilitiesConfig | undefined +): ExternalHostCapabilitiesConfig | undefined { + if (!base) { + return override; + } + if (!override) { + return base; + } + return { + ...base, + ...override, + }; +} + +function normalizeCapabilityBindings( + binding: + | ExternalHostCapabilityBinding + | ExternalHostCapabilityBinding[] + | undefined +): ExternalHostCapabilityBinding[] { + if (!binding) { + return []; + } + return Array.isArray(binding) ? binding : [binding]; +} + +async function loadModuleCapability( + uses: string +): Promise { + const target = uses.slice('module:'.length); + const [specifier, exportName = 'default'] = target.split('#'); + if (!specifier) { + throw new Error(`Invalid external host module capability id: ${uses}`); + } + + const module = (await import(specifier)) as Record; + const implementation = module[exportName]; + if (!isExternalHostCapabilityImplementation(implementation)) { + throw new Error( + `External host module capability ${uses} did not export a valid implementation.` + ); + } + return implementation; +} + +function isExternalHostCapabilityImplementation( + value: unknown +): value is ExternalHostCapabilityImplementation { + return ( + typeof value === 'object' && + value !== null && + typeof (value as ExternalHostCapabilityImplementation).id === 'string' && + Array.isArray((value as ExternalHostCapabilityImplementation).capabilities) + ); +} + +function runtimeFailure( + loaded: LoadedExternalHostConfig, + context: HostRunContext, + error: string +): ExternalHostRunResult { + return { + success: false, + toolCalls: [], + error, + externalHost: { + driver: loaded.driver, + driverSlug: loaded.driverSlug, + displayName: loaded.displayName, + hostName: loaded.displayName, + hostType: loaded.config.hostType ?? hostTypeFromDriver(loaded.driver), + hostVariant: loaded.config.variant, + capabilitiesUsed: loaded.capabilitiesUsed, + traceSource: 'none', + traceConfidence: 'unknown', + traceLimitations: [ + 'The external host capability runner did not produce a result.', + ], + artifacts: [], + session: { runMarker: context.marker }, + correlation: context.correlation, + failureKind: 'unsupported_host', + }, + }; +} diff --git a/src/evals/externalHost/driverIdentity.ts b/src/evals/externalHost/driverIdentity.ts new file mode 100644 index 0000000..4ca7d6f --- /dev/null +++ b/src/evals/externalHost/driverIdentity.ts @@ -0,0 +1,77 @@ +import type { + ExternalHostType, + HostDriverConfig, + HostDriverId, +} from './types.js'; + +export const CLAUDE_CHAT_DESKTOP_MACOS_DRIVER: HostDriverId = { + provider: 'anthropic', + product: 'claude', + surface: 'chat', + runtime: 'desktop-app', + platform: 'macos', +}; + +export const CLAUDE_COWORK_DESKTOP_MACOS_DRIVER: HostDriverId = { + provider: 'anthropic', + product: 'claude', + surface: 'cowork', + runtime: 'desktop-app', + platform: 'macos', +}; + +export const CLAUDE_CODE_CLI_MACOS_DRIVER: HostDriverId = { + provider: 'anthropic', + product: 'claude', + surface: 'code', + runtime: 'cli', + platform: 'macos', +}; + +export function driverToSlug(driver: HostDriverId): string { + return [ + driver.provider, + driver.product, + driver.surface, + driver.runtime, + driver.platform, + driver.channel, + ] + .filter((part): part is string => Boolean(part)) + .join('.'); +} + +export function parseDriverSlug(slug: string): HostDriverId { + const [provider, product, surface, runtime, platform, ...rest] = + slug.split('.'); + + if (!provider || !product || !surface || !runtime) { + throw new Error( + `External host driver slug must include provider.product.surface.runtime: ${slug}` + ); + } + + return { + provider, + product, + surface, + runtime, + ...(platform ? { platform } : {}), + ...(rest.length > 0 ? { channel: rest.join('.') } : {}), + }; +} + +export function normalizeHostDriver(driver: HostDriverConfig): HostDriverId { + if (typeof driver === 'string') { + return parseDriverSlug(driver); + } + + return driver; +} + +export function hostTypeFromDriver(driver: HostDriverId): ExternalHostType { + if (driver.runtime === 'cli' || driver.runtime === 'tui') return 'cli'; + if (driver.runtime === 'browser') return 'browser'; + if (driver.runtime === 'desktop-app') return 'desktop'; + return 'custom'; +} diff --git a/src/evals/externalHost/hostRegistry.test.ts b/src/evals/externalHost/hostRegistry.test.ts new file mode 100644 index 0000000..8f498fd --- /dev/null +++ b/src/evals/externalHost/hostRegistry.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest'; +import { + CLAUDE_COWORK_DESKTOP_MACOS_DRIVER, + driverToSlug, + getRegisteredExternalHostConfig, + loadExternalHostConfig, + listRegisteredExternalHostSlugs, + normalizeHostDriver, + parseDriverSlug, +} from './index.js'; + +describe('external host driver identity and built-in defaults', () => { + it('round-trips structured driver ids to slugs', () => { + const slug = driverToSlug(CLAUDE_COWORK_DESKTOP_MACOS_DRIVER); + + expect(slug).toBe('anthropic.claude.cowork.desktop-app.macos'); + expect(parseDriverSlug(slug)).toEqual(CLAUDE_COWORK_DESKTOP_MACOS_DRIVER); + }); + + it('normalizes driver slug strings to structured ids', () => { + expect( + normalizeHostDriver('anthropic.claude.cowork.desktop-app.macos') + ).toEqual(CLAUDE_COWORK_DESKTOP_MACOS_DRIVER); + }); + + it('declares Claude Cowork as capability bindings, not a concrete runner', () => { + const config = getRegisteredExternalHostConfig( + 'anthropic.claude.cowork.desktop-app.macos' + ); + + expect(config?.name).toBe('Claude Cowork Desktop'); + expect(config?.correlation).toEqual({ + strategy: 'prompt_marker', + includeInPrompt: true, + }); + expect(config?.capabilities).toMatchObject({ + control: [ + { uses: 'builtin:platform.macos' }, + { uses: 'builtin:anthropic.claude.coworkSurface' }, + ], + input: { uses: 'builtin:desktop.macos.accessibilitySubmit' }, + completion: { + uses: 'builtin:anthropic.claude.localAgentTrace', + provides: ['trace'], + }, + normalize: { + uses: 'builtin:anthropic.claude.localAgentNormalize', + }, + }); + }); + + it('loads Claude Cowork defaults into concrete capability providers at runtime', async () => { + const loaded = await loadExternalHostConfig({ + driver: 'anthropic.claude.cowork.desktop-app.macos', + }); + + expect(loaded.displayName).toBe('Claude Cowork Desktop'); + expect(loaded.capabilitiesUsed).toEqual([ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', + ]); + expect( + loaded.loadedCapabilities.map((capability) => capability.binding.uses) + ).toEqual([ + 'builtin:platform.macos', + 'builtin:anthropic.claude.coworkSurface', + 'builtin:desktop.macos.accessibilitySubmit', + 'builtin:anthropic.claude.localAgentTrace', + 'builtin:anthropic.claude.localAgentNormalize', + ]); + }); + + it('returns no built-in defaults for syntactically valid unsupported drivers', () => { + expect( + getRegisteredExternalHostConfig('openai.chatgpt.chat.browser.web') + ).toBeUndefined(); + }); + + it('lists registered external hosts by structured driver slug', () => { + expect(listRegisteredExternalHostSlugs()).toEqual([ + 'anthropic.claude.chat.desktop-app.macos', + 'anthropic.claude.cowork.desktop-app.macos', + ]); + }); +}); diff --git a/src/evals/externalHost/hostRegistry.ts b/src/evals/externalHost/hostRegistry.ts new file mode 100644 index 0000000..9a7a5ad --- /dev/null +++ b/src/evals/externalHost/hostRegistry.ts @@ -0,0 +1,85 @@ +import type { ExternalHostConfig } from './types.js'; +import { + CLAUDE_CHAT_DESKTOP_MACOS_DRIVER, + CLAUDE_COWORK_DESKTOP_MACOS_DRIVER, + driverToSlug, +} from './driverIdentity.js'; + +const EXTERNAL_HOST_REGISTRY: Record< + string, + Partial & { name: string; description: string } +> = { + [driverToSlug(CLAUDE_CHAT_DESKTOP_MACOS_DRIVER)]: { + driver: CLAUDE_CHAT_DESKTOP_MACOS_DRIVER, + name: 'Claude Chat Desktop', + description: + 'Drives the regular Claude Desktop chat surface on macOS and captures low-confidence visible response evidence via Accessibility.', + correlation: { + strategy: 'prompt_marker', + includeInPrompt: true, + }, + capabilities: { + control: { uses: 'builtin:platform.macos' }, + input: { + uses: 'builtin:desktop.macos.accessibilitySubmit', + with: { + appName: 'Claude', + createNewConversation: 'unless-disabled', + }, + }, + completion: { + uses: 'builtin:anthropic.claude.accessibilityTrace', + provides: ['trace', 'normalize'], + }, + }, + }, + [driverToSlug(CLAUDE_COWORK_DESKTOP_MACOS_DRIVER)]: { + driver: CLAUDE_COWORK_DESKTOP_MACOS_DRIVER, + name: 'Claude Cowork Desktop', + description: + 'Drives the Claude Desktop Cowork surface on macOS and captures high-confidence local-agent trace evidence.', + correlation: { + strategy: 'prompt_marker', + includeInPrompt: true, + }, + capabilities: { + control: [ + { uses: 'builtin:platform.macos' }, + { uses: 'builtin:anthropic.claude.coworkSurface' }, + ], + input: { + uses: 'builtin:desktop.macos.accessibilitySubmit', + with: { appName: 'Claude', createNewConversation: false }, + }, + completion: { + uses: 'builtin:anthropic.claude.localAgentTrace', + provides: ['trace'], + }, + normalize: { + uses: 'builtin:anthropic.claude.localAgentNormalize', + }, + }, + }, +}; + +export function getRegisteredExternalHostConfig( + driverSlug: string +): Partial | undefined { + return EXTERNAL_HOST_REGISTRY[driverSlug]; +} + +export function getRegisteredExternalHostDisplayName( + driverSlug: string +): string | undefined { + return EXTERNAL_HOST_REGISTRY[driverSlug]?.name; +} + +export function getRegisteredExternalHostDescription( + driverSlug: string +): string | undefined { + return EXTERNAL_HOST_REGISTRY[driverSlug]?.description; +} + +export function listRegisteredExternalHostSlugs(): string[] { + return Object.keys(EXTERNAL_HOST_REGISTRY); +} diff --git a/src/evals/externalHost/index.ts b/src/evals/externalHost/index.ts new file mode 100644 index 0000000..31d3e1a --- /dev/null +++ b/src/evals/externalHost/index.ts @@ -0,0 +1,69 @@ +export { runExternalHostScenario } from './runtime.js'; +export { + REQUIRED_HOST_CAPABILITIES, + validateHostCapabilities, +} from './capabilities.js'; +export { + getRegisteredExternalHostConfig, + getRegisteredExternalHostDescription, + getRegisteredExternalHostDisplayName, + listRegisteredExternalHostSlugs, +} from './hostRegistry.js'; +export { + listBuiltinExternalHostCapabilities, + resolveBuiltinExternalHostCapability, +} from './builtinCapabilities.js'; +export { + listExternalHostCapabilities, + loadExternalHostConfig, + loadExternalHostRunner, + registerExternalHostCapability, + resolveExternalHostCapability, +} from './capabilityRuntime.js'; +export type { + LoadedExternalHostCapability, + LoadedExternalHostConfig, +} from './capabilityRuntime.js'; +export { + CLAUDE_CHAT_DESKTOP_MACOS_DRIVER, + CLAUDE_CODE_CLI_MACOS_DRIVER, + CLAUDE_COWORK_DESKTOP_MACOS_DRIVER, + driverToSlug, + hostTypeFromDriver, + normalizeHostDriver, + parseDriverSlug, +} from './driverIdentity.js'; +export { + ExternalHostCapabilityBindingSchema, + ExternalHostConfigSchema, + ExternalHostCorrelationSchema, + getExternalHostConfigJsonSchema, + getExternalHostReference, + HostCapabilitySchema, + HostDriverIdSchema, + listExternalHostDriverReferences, +} from './schema.js'; +export type { ExternalHostDriverReference } from './schema.js'; +export type { + EvidenceSource, + ExternalHostCapabilityBinding, + ExternalHostCapabilityContext, + ExternalHostCapabilityImplementation, + ExternalHostCapabilitiesConfig, + ExternalHostConfig, + ExternalHostFailureKind, + ExternalHostMetadata, + ExternalHostRunState, + ExternalHostRunResult, + ExternalHostRunner, + ExternalHostSession, + ExternalHostSimulationResult, + ExternalHostType, + HostArtifact, + HostCapability, + HostDriverConfig, + HostDriverId, + HostRunContext, + ObservationConfidence, + TraceSource, +} from './types.js'; diff --git a/src/evals/externalHost/runtime.test.ts b/src/evals/externalHost/runtime.test.ts new file mode 100644 index 0000000..c936463 --- /dev/null +++ b/src/evals/externalHost/runtime.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest'; +import { formatSubmittedScenario, runExternalHostScenario } from './runtime.js'; + +describe('external host runtime', () => { + it('adds an evaluator marker with an instruction not to mention it', () => { + const submitted = formatSubmittedScenario( + 'Reply with exactly: acknowledged.', + 'MCP_SERVER_TESTER_run_123' + ); + + expect(submitted).toContain('Reply with exactly: acknowledged.'); + expect(submitted).toContain('[eval-run-marker:MCP_SERVER_TESTER_run_123]'); + expect(submitted).toContain('do not mention this marker'); + }); + + it('leaves the submitted scenario unchanged when prompt correlation is disabled', () => { + const submitted = formatSubmittedScenario( + 'Reply with exactly: acknowledged.', + 'MCP_SERVER_TESTER_run_123', + { strategy: 'none' } + ); + + expect(submitted).toBe('Reply with exactly: acknowledged.'); + }); + + it('supports prompt marker correlation without including it in the prompt', () => { + const submitted = formatSubmittedScenario( + 'Reply with exactly: acknowledged.', + 'MCP_SERVER_TESTER_run_123', + { strategy: 'prompt_marker', includeInPrompt: false } + ); + + expect(submitted).toBe('Reply with exactly: acknowledged.'); + }); + + it('infers host type for unsupported driver failures', async () => { + const result = await runExternalHostScenario( + 'hello', + { driver: 'openai.chatgpt.chat.browser.web' }, + { runId: 'unsupported-browser' } + ); + + expect(result).toMatchObject({ + success: false, + externalHost: { + driverSlug: 'openai.chatgpt.chat.browser.web', + hostType: 'browser', + failureKind: 'unsupported_host', + correlation: { + strategy: 'none', + includedInPrompt: false, + }, + }, + }); + }); +}); diff --git a/src/evals/externalHost/runtime.ts b/src/evals/externalHost/runtime.ts new file mode 100644 index 0000000..2b9ace7 --- /dev/null +++ b/src/evals/externalHost/runtime.ts @@ -0,0 +1,139 @@ +import { randomUUID } from 'node:crypto'; +import type { + ExternalHostCorrelationConfig, + ExternalHostCorrelationMetadata, + ExternalHostConfig, + ExternalHostRunResult, + HostRunContext, +} from './types.js'; +import { + driverToSlug, + hostTypeFromDriver, + normalizeHostDriver, +} from './driverIdentity.js'; +import { + createExternalHostRunner, + loadExternalHostConfig, +} from './capabilityRuntime.js'; + +const DEFAULT_TIMEOUT_MS = 120_000; +const DEFAULT_PROMPT_MARKER_TEMPLATE = + 'Trace marker for MCP Server Tester; do not mention this marker in your response: [eval-run-marker:{{marker}}]'; + +export function formatSubmittedScenario( + scenario: string, + marker: string, + correlation: ExternalHostCorrelationConfig = { + strategy: 'prompt_marker', + includeInPrompt: true, + } +): string { + const metadata = normalizeCorrelation(correlation, marker); + if (!metadata.includedInPrompt) { + return scenario; + } + + const template = correlation.promptTemplate ?? DEFAULT_PROMPT_MARKER_TEMPLATE; + return `${scenario}\n\n${template.replaceAll('{{marker}}', marker)}`; +} + +export async function runExternalHostScenario( + scenario: string, + config: ExternalHostConfig, + options: { caseId?: string; runId?: string } = {} +): Promise { + const runId = options.runId ?? `external-host-${randomUUID()}`; + const marker = `MCP_SERVER_TESTER_${runId}`; + + let loaded; + try { + loaded = await loadExternalHostConfig(config); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return unsupportedHostResult(config, marker, message); + } + + const timeoutMs = loaded.config.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const correlation = normalizeCorrelation(loaded.config.correlation, marker); + const submittedScenario = formatSubmittedScenario( + scenario, + marker, + loaded.config.correlation + ); + + const context: HostRunContext = { + runId, + caseId: options.caseId ?? 'unknown', + scenario, + submittedScenario, + marker, + correlation, + timeoutMs, + startedAtMs: Date.now(), + }; + + const runner = createExternalHostRunner(loaded); + + return runner.run(context); +} + +function normalizeCorrelation( + correlation: ExternalHostCorrelationConfig | undefined, + marker: string +): ExternalHostCorrelationMetadata { + const strategy = correlation?.strategy ?? 'none'; + const includedInPrompt = + strategy === 'prompt_marker' + ? (correlation?.includeInPrompt ?? true) + : false; + + return { + strategy, + marker, + includedInPrompt, + }; +} + +function unsupportedHostResult( + config: ExternalHostConfig, + marker: string, + error: string +): ExternalHostRunResult { + const driver = (() => { + try { + return normalizeHostDriver(config.driver); + } catch { + return { + provider: 'unknown', + product: 'unknown', + surface: 'unknown', + runtime: 'unknown', + }; + } + })(); + const driverSlug = driverToSlug(driver); + + return { + success: false as const, + toolCalls: [], + error, + externalHost: { + driver, + driverSlug, + displayName: config.name ?? driverSlug, + hostName: config.name ?? driverSlug, + hostType: config.hostType ?? hostTypeFromDriver(driver), + hostVariant: config.variant, + capabilitiesUsed: [], + traceSource: 'none', + traceConfidence: 'unknown', + traceLimitations: [ + 'The external host capability configuration could not be loaded.', + ], + artifacts: [], + session: { runMarker: marker }, + correlation: normalizeCorrelation(config.correlation, marker), + failureKind: 'unsupported_host', + }, + }; +} diff --git a/src/evals/externalHost/schema.test.ts b/src/evals/externalHost/schema.test.ts new file mode 100644 index 0000000..25431f0 --- /dev/null +++ b/src/evals/externalHost/schema.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest'; +import { + ExternalHostConfigSchema, + getExternalHostConfigJsonSchema, + getExternalHostReference, + listExternalHostDriverReferences, +} from './schema.js'; + +describe('external host schema and reference', () => { + it('validates minimal built-in external host config', () => { + const parsed = ExternalHostConfigSchema.parse({ + driver: 'anthropic.claude.cowork.desktop-app.macos', + timeoutMs: 60_000, + }); + + expect(parsed).toEqual({ + driver: 'anthropic.claude.cowork.desktop-app.macos', + timeoutMs: 60_000, + }); + }); + + it('exposes known driver slugs in the JSON schema for editor autocomplete', () => { + const schema = getExternalHostConfigJsonSchema(); + const driver = (schema.properties as Record) + .driver as Record; + const choices = driver.anyOf as Array>; + + expect(choices[0]).toMatchObject({ + type: 'string', + enum: [ + 'anthropic.claude.chat.desktop-app.macos', + 'anthropic.claude.cowork.desktop-app.macos', + ], + }); + }); + + it('lists built-in drivers with examples and internal capability defaults', () => { + const references = listExternalHostDriverReferences(); + const cowork = references.find( + (reference) => + reference.slug === 'anthropic.claude.cowork.desktop-app.macos' + ); + + expect(cowork).toMatchObject({ + name: 'Claude Cowork Desktop', + builtIn: true, + example: { + mode: 'external_host', + externalHost: { + driver: 'anthropic.claude.cowork.desktop-app.macos', + }, + }, + }); + expect(cowork?.capabilities?.input).toMatchObject({ + uses: 'builtin:desktop.macos.accessibilitySubmit', + with: { appName: 'Claude' }, + }); + }); + + it('bundles schema and driver references for agents and docs generators', () => { + const reference = getExternalHostReference(); + + expect(reference).toMatchObject({ + schema: { title: 'MCP Server Tester ExternalHostConfig' }, + drivers: expect.any(Array), + }); + }); +}); diff --git a/src/evals/externalHost/schema.ts b/src/evals/externalHost/schema.ts new file mode 100644 index 0000000..9c13e27 --- /dev/null +++ b/src/evals/externalHost/schema.ts @@ -0,0 +1,281 @@ +import { z } from 'zod'; +import { + getRegisteredExternalHostConfig, + getRegisteredExternalHostDescription, + listRegisteredExternalHostSlugs, +} from './hostRegistry.js'; +import { driverToSlug, normalizeHostDriver } from './driverIdentity.js'; +import type { + ExternalHostCapabilitiesConfig, + ExternalHostConfig, + HostDriverId, +} from './types.js'; + +export const HostDriverIdSchema = z.object({ + provider: z.string().min(1), + product: z.string().min(1), + surface: z.string().min(1), + runtime: z.string().min(1), + platform: z.string().optional(), + channel: z.string().optional(), +}); + +export const HostCapabilitySchema = z.enum([ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', +]); + +export const ExternalHostCapabilityBindingSchema = z.object({ + uses: z.string().min(1), + with: z.record(z.string(), z.unknown()).optional(), + provides: z.array(HostCapabilitySchema).optional(), +}); + +export const ExternalHostCorrelationSchema = z.object({ + strategy: z + .enum(['prompt_marker', 'host_session_metadata', 'none']) + .optional(), + includeInPrompt: z.boolean().optional(), + promptTemplate: z.string().optional(), +}); + +export const ExternalHostConfigSchema = z.object({ + driver: z.union([HostDriverIdSchema, z.string().min(1)]), + name: z.string().optional(), + hostType: z.enum(['cli', 'browser', 'desktop', 'custom']).optional(), + variant: z.string().optional(), + timeoutMs: z.number().int().positive().optional(), + capabilities: z + .partialRecord( + HostCapabilitySchema, + z.union([ + ExternalHostCapabilityBindingSchema, + z.array(ExternalHostCapabilityBindingSchema), + ]) + ) + .optional(), + correlation: ExternalHostCorrelationSchema.optional(), + options: z.record(z.string(), z.unknown()).optional(), +}); + +export interface ExternalHostDriverReference { + slug: string; + driver: HostDriverId; + name: string; + description?: string; + builtIn: true; + defaultConfig: ExternalHostConfig; + capabilities?: ExternalHostCapabilitiesConfig; + example: { + mode: 'external_host'; + scenario: string; + externalHost: Pick; + expect: { containsText: string }; + }; +} + +export function listExternalHostDriverReferences(): ExternalHostDriverReference[] { + return listRegisteredExternalHostSlugs().map((slug) => { + const config = getRegisteredExternalHostConfig(slug); + const driver = normalizeHostDriver(slug); + const name = config?.name ?? slug; + + return { + slug, + driver, + name, + description: getRegisteredExternalHostDescription(slug), + builtIn: true, + defaultConfig: { + driver, + ...(config ?? {}), + }, + capabilities: config?.capabilities, + example: { + mode: 'external_host', + scenario: 'Ask the host to complete the task you want to evaluate.', + externalHost: { + driver: slug, + timeoutMs: config?.timeoutMs ?? 60_000, + }, + expect: { + containsText: 'expected text', + }, + }, + }; + }); +} + +export function getExternalHostConfigJsonSchema(): Record { + const driverSlugs = listRegisteredExternalHostSlugs(); + + return { + $schema: 'https://json-schema.org/draft/2020-12/schema', + $id: 'https://github.com/gleanwork/mcp-server-tester/schemas/external-host-config.schema.json', + title: 'MCP Server Tester ExternalHostConfig', + description: + 'Configuration for running an MCP eval through an external host driver.', + type: 'object', + additionalProperties: false, + required: ['driver'], + properties: { + driver: { + description: + 'Canonical built-in driver slug, custom driver slug, or structured driver identity.', + anyOf: [ + { + type: 'string', + enum: driverSlugs, + description: + 'Known built-in driver slug. Prefer this form for normal eval datasets.', + }, + { + type: 'string', + minLength: 1, + description: + 'Custom driver slug. Use when registering project-local capabilities.', + }, + hostDriverIdJsonSchema(), + ], + }, + name: { + type: 'string', + description: 'Optional display name shown in reports.', + }, + hostType: { + type: 'string', + enum: ['cli', 'browser', 'desktop', 'custom'], + description: 'Host category shown in reports.', + }, + variant: { + type: 'string', + description: 'Optional variant label for matrix-style runs.', + }, + timeoutMs: { + type: 'integer', + minimum: 1, + description: 'End-to-end timeout for the host run in milliseconds.', + }, + correlation: externalHostCorrelationJsonSchema(), + options: { + type: 'object', + additionalProperties: true, + description: + 'Driver-wide options interpreted by the selected driver or capability bindings.', + }, + capabilities: externalHostCapabilitiesJsonSchema(), + }, + examples: listExternalHostDriverReferences().map((reference) => ({ + driver: reference.slug, + timeoutMs: reference.example.externalHost.timeoutMs, + })), + }; +} + +export function getExternalHostReference(): Record { + return { + schema: getExternalHostConfigJsonSchema(), + drivers: listExternalHostDriverReferences(), + }; +} + +function hostDriverIdJsonSchema(): Record { + return { + type: 'object', + additionalProperties: false, + required: ['provider', 'product', 'surface', 'runtime'], + properties: { + provider: { type: 'string', minLength: 1 }, + product: { type: 'string', minLength: 1 }, + surface: { type: 'string', minLength: 1 }, + runtime: { type: 'string', minLength: 1 }, + platform: { type: 'string' }, + channel: { type: 'string' }, + }, + }; +} + +function externalHostCorrelationJsonSchema(): Record { + return { + type: 'object', + additionalProperties: false, + description: + 'How a submitted host run is correlated with host-native trace evidence.', + properties: { + strategy: { + type: 'string', + enum: ['prompt_marker', 'host_session_metadata', 'none'], + }, + includeInPrompt: { + type: 'boolean', + description: + 'Whether to include the generated run marker in the host-visible prompt.', + }, + promptTemplate: { + type: 'string', + description: 'Prompt suffix template. Supports {{marker}}.', + }, + }, + }; +} + +function externalHostCapabilitiesJsonSchema(): Record { + return { + type: 'object', + additionalProperties: false, + description: + 'Advanced escape hatch for overriding the capability recipe. Most users should choose a built-in driver instead.', + properties: Object.fromEntries( + HostCapabilitySchema.options.map((capability) => [ + capability, + { + anyOf: [ + externalHostCapabilityBindingJsonSchema(), + { + type: 'array', + items: externalHostCapabilityBindingJsonSchema(), + }, + ], + }, + ]) + ), + }; +} + +function externalHostCapabilityBindingJsonSchema(): Record { + return { + type: 'object', + additionalProperties: false, + required: ['uses'], + properties: { + uses: { + type: 'string', + minLength: 1, + description: + 'Capability implementation id. Built-ins use builtin:; custom integrations may use module:#.', + }, + with: { + type: 'object', + additionalProperties: true, + description: + 'Binding-local options interpreted by the selected capability implementation.', + }, + provides: { + type: 'array', + items: { + type: 'string', + enum: HostCapabilitySchema.options, + }, + }, + }, + }; +} + +export function externalHostDriverSlugForConfig( + config: ExternalHostConfig +): string { + return driverToSlug(normalizeHostDriver(config.driver)); +} diff --git a/src/evals/externalHost/types.ts b/src/evals/externalHost/types.ts new file mode 100644 index 0000000..650cc38 --- /dev/null +++ b/src/evals/externalHost/types.ts @@ -0,0 +1,274 @@ +import type { + LLMToolCall, + MCPHostSimulationResult, +} from '../mcpHost/mcpHostTypes.js'; +import type { UsageMetrics } from '../../types/index.js'; + +export type ExternalHostType = 'cli' | 'browser' | 'desktop' | 'custom'; + +export type HostCapability = + | 'control' + | 'input' + | 'completion' + | 'trace' + | 'normalize'; + +export type TraceSource = + | 'mcp-proxy' + | 'mcp-server-logs' + | 'host-local-transcript' + | 'host-native-export' + | 'browser-api' + | 'accessibility' + | 'dom' + | 'screenshot' + | 'stdout' + | 'manual-import' + | 'none'; + +export type ObservationConfidence = 'high' | 'medium' | 'low' | 'unknown'; + +export type ExternalHostCorrelationStrategy = + | 'prompt_marker' + | 'host_session_metadata' + | 'none'; + +export interface HostDriverId { + provider: string; + product: string; + surface: string; + runtime: string; + platform?: string; + channel?: string; +} + +export type HostDriverConfig = HostDriverId | string; + +export type ExternalHostFailureKind = + | 'app_unavailable' + | 'automation_permission_denied' + | 'submission_failed' + | 'no_matching_session' + | 'ambiguous_matching_sessions' + | 'timeout' + | 'parse_failure' + | 'host_run_failed' + | 'unsupported_host' + | 'unknown'; + +export interface HostArtifact { + kind: + | 'stdout' + | 'stderr' + | 'log' + | 'transcript' + | 'audit' + | 'metadata' + | 'screenshot' + | 'video' + | 'har' + | 'trace'; + name: string; + path?: string; + contentType?: string; + summary?: string; +} + +export interface ExternalHostSession { + id?: string; + runMarker: string; + requestId?: string; + cliSessionId?: string; + cwd?: string; + startedAt?: string; + completedAt?: string; +} + +export interface ExternalHostCorrelationConfig { + /** + * How this run should be correlated with host-native evidence. + * + * - prompt_marker: append a marker to the submitted prompt. + * - host_session_metadata: rely on host-native session metadata. + * - none: no host-visible marker is submitted. + */ + strategy?: ExternalHostCorrelationStrategy; + /** + * Whether the marker should be included in the host-visible prompt. + * Defaults to true only for prompt_marker. + */ + includeInPrompt?: boolean; + /** + * Optional prompt suffix template. Supports {{marker}}. + */ + promptTemplate?: string; +} + +export interface ExternalHostCorrelationMetadata { + strategy: ExternalHostCorrelationStrategy; + marker: string; + includedInPrompt: boolean; +} + +export interface ExternalHostMetadata { + driver: HostDriverId; + driverSlug: string; + displayName: string; + hostName: string; + hostType: ExternalHostType; + hostVariant?: string; + capabilitiesUsed: HostCapability[]; + traceSource: TraceSource; + traceConfidence: ObservationConfidence; + traceLimitations?: string[]; + artifacts: HostArtifact[]; + session: ExternalHostSession; + correlation: ExternalHostCorrelationMetadata; + failureKind?: ExternalHostFailureKind; + sources?: { + finalAnswer?: TraceSource; + toolCalls?: TraceSource; + usage?: TraceSource; + cost?: TraceSource; + }; + evidence?: { + finalAnswer?: EvidenceSource; + toolCalls?: EvidenceSource; + usage?: EvidenceSource; + cost?: EvidenceSource; + }; +} + +export interface ExternalHostConfig { + /** + * Canonical structured driver identity or derived slug. + * Example: `anthropic.claude.cowork.desktop-app.macos`. + */ + driver: HostDriverConfig; + /** + * Human-readable host name shown in reports. + */ + name?: string; + /** + * Host type shown in reports. + */ + hostType?: ExternalHostType; + /** + * Optional variant label for matrix-style runs. + */ + variant?: string; + /** + * End-to-end timeout for the host run. + */ + timeoutMs?: number; + /** + * Capability bindings used to compose this external host runner. + * If omitted, the runtime may provide a built-in default for known drivers. + */ + capabilities?: ExternalHostCapabilitiesConfig; + /** + * Run correlation strategy. Built-in drivers may provide defaults. + */ + correlation?: ExternalHostCorrelationConfig; + /** + * Driver-wide options available to capability implementations. + */ + options?: Record; +} + +export interface HostRunContext { + runId: string; + caseId: string; + scenario: string; + submittedScenario: string; + marker: string; + correlation: ExternalHostCorrelationMetadata; + timeoutMs: number; + startedAtMs: number; +} + +export interface ExternalHostSimulationResult extends MCPHostSimulationResult { + externalHost: ExternalHostMetadata; +} + +export interface ExternalHostRunSuccess { + success: true; + response?: string; + toolCalls: LLMToolCall[]; + conversationHistory?: MCPHostSimulationResult['conversationHistory']; + usage?: UsageMetrics; + llmDurationMs?: number; + mcpDurationMs?: number; + externalHost: ExternalHostMetadata; +} + +export interface ExternalHostRunFailure { + success: false; + error: string; + toolCalls: LLMToolCall[]; + externalHost: ExternalHostMetadata; +} + +export type ExternalHostRunResult = + | ExternalHostRunSuccess + | ExternalHostRunFailure; + +export type ExternalHostCapabilitiesConfig = Partial< + Record< + HostCapability, + ExternalHostCapabilityBinding | ExternalHostCapabilityBinding[] + > +>; + +export interface ExternalHostCapabilityBinding { + /** + * Implementation identifier. Built-ins use `builtin:`; callers may use + * `module:#` to load project-local integrations. + */ + uses: string; + /** + * Binding-local options interpreted only by the selected implementation. + */ + with?: Record; + /** + * Extra capabilities this binding should satisfy beyond its map key. + */ + provides?: HostCapability[]; +} + +export interface ExternalHostRunState { + driver: HostDriverId; + driverSlug: string; + displayName: string; + capabilitiesUsed: HostCapability[]; + data: Record; + result?: ExternalHostRunResult; +} + +export interface ExternalHostCapabilityContext { + config: ExternalHostConfig; + run: HostRunContext; + capability: HostCapability; + binding: ExternalHostCapabilityBinding; + state: ExternalHostRunState; +} + +export interface ExternalHostCapabilityImplementation { + id: string; + capabilities: HostCapability[]; + setup?( + context: ExternalHostCapabilityContext + ): Promise; + run?( + context: ExternalHostCapabilityContext + ): Promise; +} + +export interface ExternalHostRunner { + run(context: HostRunContext): Promise; +} + +export interface EvidenceSource { + source: TraceSource; + confidence: ObservationConfidence; +} diff --git a/src/index.ts b/src/index.ts index b5b91d6..ddf422b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -243,6 +243,40 @@ export { getMissingDependencyMessage, } from './evals/mcpHost/index.js'; +// External Host Evals (experimental) +export type { + EvidenceSource, + ExternalHostCapabilityBinding, + ExternalHostCapabilityContext, + ExternalHostCapabilityImplementation, + ExternalHostCapabilitiesConfig, + ExternalHostConfig, + ExternalHostDriverReference, + ExternalHostFailureKind, + ExternalHostMetadata, + ExternalHostRunResult, + ExternalHostSession, + ExternalHostSimulationResult, + ExternalHostType, + HostArtifact, + HostCapability, + HostDriverConfig, + HostDriverId, + HostRunContext, + ObservationConfidence, + TraceSource, +} from './evals/externalHost/index.js'; +export { + driverToSlug, + normalizeHostDriver, + parseDriverSlug, + getExternalHostConfigJsonSchema, + getExternalHostReference, + listExternalHostDriverReferences, + registerExternalHostCapability, + runExternalHostScenario, +} from './evals/externalHost/index.js'; + // Judge export { createJudge } from './judge/judgeClient.js'; export { diff --git a/src/mcp/response.ts b/src/mcp/response.ts index 1189250..5e5130b 100644 --- a/src/mcp/response.ts +++ b/src/mcp/response.ts @@ -190,6 +190,11 @@ export function extractText(response: unknown): string { return r.text; } + // Host simulation results expose the final answer as `response`. + if (typeof r.response === 'string') { + return r.response; + } + // Fallback to JSON return JSON.stringify(r); } diff --git a/src/reporters/mcpReporter.test.ts b/src/reporters/mcpReporter.test.ts index 2cc490b..1bbeb75 100644 --- a/src/reporters/mcpReporter.test.ts +++ b/src/reporters/mcpReporter.test.ts @@ -382,6 +382,77 @@ describe('MCPReporter.buildRunData()', () => { }); }); + describe('external host metadata', () => { + it('preserves external host trace metadata in run data', () => { + setResults(reporter, [ + makeResult({ + pass: true, + toolName: 'external_host', + externalHost: { + driver: { + provider: 'anthropic', + product: 'claude', + surface: 'cowork', + runtime: 'desktop-app', + platform: 'macos', + }, + driverSlug: 'anthropic.claude.cowork.desktop-app.macos', + displayName: 'Claude Cowork Desktop', + hostName: 'Claude Cowork Desktop', + hostType: 'desktop', + capabilitiesUsed: [ + 'control', + 'input', + 'completion', + 'trace', + 'normalize', + ], + traceSource: 'host-local-transcript', + traceConfidence: 'high', + traceLimitations: ['fixture limitation'], + artifacts: [ + { + kind: 'audit', + name: 'Claude audit log', + path: '/tmp/audit.jsonl', + }, + ], + session: { + id: 'local_123', + runMarker: 'MCP_SERVER_TESTER_TEST', + requestId: 'req_123', + }, + correlation: { + strategy: 'prompt_marker', + marker: 'MCP_SERVER_TESTER_TEST', + includedInPrompt: true, + }, + evidence: { + finalAnswer: { + source: 'host-local-transcript', + confidence: 'high', + }, + toolCalls: { + source: 'host-local-transcript', + confidence: 'high', + }, + }, + }, + }), + ]); + + const data = callBuildRunData(reporter, 100); + + expect(data.results[0]?.externalHost).toMatchObject({ + driverSlug: 'anthropic.claude.cowork.desktop-app.macos', + hostName: 'Claude Cowork Desktop', + traceSource: 'host-local-transcript', + traceConfidence: 'high', + session: { id: 'local_123', requestId: 'req_123' }, + }); + }); + }); + describe('conformanceChecks and serverCapabilities', () => { it('returns undefined conformanceChecks when none are recorded', () => { setResults(reporter, [makeResult({ pass: true })]); diff --git a/src/reporters/ui-src/components/Results/DetailModal.tsx b/src/reporters/ui-src/components/Results/DetailModal.tsx index d8dbcdf..bbe02ac 100644 --- a/src/reporters/ui-src/components/Results/DetailModal.tsx +++ b/src/reporters/ui-src/components/Results/DetailModal.tsx @@ -19,6 +19,213 @@ function formatResponsePreview(response: unknown): string { return JSON.stringify(response, null, 2) ?? ''; } +function getExternalHostEvidenceRows( + externalHost: NonNullable +) { + const labels = { + finalAnswer: 'Final answer', + toolCalls: 'Tool calls', + usage: 'Usage', + cost: 'Cost', + } as const; + const keys = Object.keys(labels) as Array; + + return keys + .map((key) => { + const evidence = externalHost.evidence?.[key]; + const source = evidence?.source ?? externalHost.sources?.[key]; + const confidence = evidence?.confidence; + + if (!source && !confidence) { + return undefined; + } + + return { + key, + label: labels[key], + source: source ?? 'unknown', + confidence: confidence ?? externalHost.traceConfidence, + }; + }) + .filter((row): row is NonNullable => row !== undefined); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function responseRecord(result: EvalCaseResult): Record { + return isRecord(result.response) ? result.response : {}; +} + +function resultToolCalls( + result: EvalCaseResult +): Array<{ id?: string; name: string; arguments: Record }> { + const toolCalls = responseRecord(result).toolCalls; + if (!Array.isArray(toolCalls)) { + return []; + } + + return toolCalls.filter( + ( + call + ): call is { + id?: string; + name: string; + arguments: Record; + } => + isRecord(call) && + typeof call.name === 'string' && + isRecord(call.arguments) + ); +} + +function finalAnswer(result: EvalCaseResult): string | undefined { + const response = responseRecord(result).response; + return typeof response === 'string' ? response : undefined; +} + +function usageForResult( + result: EvalCaseResult +): Record | undefined { + const responseUsage = responseRecord(result).usage; + return ( + (result.hostUsage as unknown as Record | undefined) ?? + (isRecord(responseUsage) ? responseUsage : undefined) + ); +} + +function numberField( + value: Record | undefined, + key: string +): number | undefined { + const nested = value?.[key]; + return typeof nested === 'number' ? nested : undefined; +} + +function formatNumber(value: number | undefined): string { + return value === undefined ? 'unknown' : value.toLocaleString(); +} + +function formatCost(value: number | undefined): string { + if (value === undefined) { + return 'unknown'; + } + return `$${value.toFixed(value === 0 ? 2 : 4)}`; +} + +function formatMs(value: number | undefined): string { + if (value === undefined) { + return 'unknown'; + } + return value >= 1000 + ? `${(value / 1000).toFixed(1)}s` + : `${value.toFixed(0)}ms`; +} + +function jsonPreview(value: unknown): string { + return JSON.stringify(value, null, 2) ?? ''; +} + +function InfoField({ + label, + value, +}: { + label: string; + value: React.ReactNode; +}) { + return ( +

+

+ {label} +

+
{value}
+
+ ); +} + +function JsonBlock({ value }: { value: unknown }) { + return ( + 
+      {jsonPreview(value)}
+
+ ); +} + +function expectationEntries(result: EvalCaseResult) { + return Object.entries(result.expectations ?? {}).filter( + (entry): entry is [string, NonNullable<(typeof entry)[1]>] => + entry[1] !== undefined + ); +} + +function failedExpectationEntries(result: EvalCaseResult) { + return expectationEntries(result).filter(([, expectation]) => { + return !expectation.pass; + }); +} + +function getVerdictSummary(result: EvalCaseResult): { + category: string; + reason: string; +} { + const failedAssertions = failedExpectationEntries(result).map( + ([type]) => type + ); + + if (result.pass) { + return { + category: 'Pass', + reason: 'All configured assertions passed.', + }; + } + + if (result.externalHost?.failureKind) { + return { + category: 'Host or automation failure', + reason: `The driver failed before producing trustworthy eval evidence: ${result.externalHost.failureKind}.`, + }; + } + + if (result.error) { + const firstLine = stripAnsiCodes(result.error).split('\n')[0]; + return { + category: 'Execution failure', + reason: firstLine, + }; + } + + if (failedAssertions.length > 0) { + return { + category: 'Assertion failure', + reason: `${failedAssertions.length} configured assertion${failedAssertions.length === 1 ? '' : 's'} failed: ${failedAssertions.join(', ')}.`, + }; + } + + return { + category: 'Failure', + reason: + 'The run failed without a specific assertion or host-driver error in the report.', + }; +} + +function evidenceSummary( + externalHost: NonNullable | undefined, + key: 'finalAnswer' | 'toolCalls' | 'usage' | 'cost' +): string { + if (!externalHost) { + return 'not reported'; + } + const evidence = externalHost.evidence?.[key]; + const source = evidence?.source ?? externalHost.sources?.[key]; + const confidence = evidence?.confidence ?? externalHost.traceConfidence; + + if (!source) { + return 'not reported'; + } + return `${source} · ${confidence}`; +} + interface DetailModalProps { result: EvalCaseResult | null; onClose: () => void; @@ -81,12 +288,24 @@ export function DetailModal({ result, onClose }: DetailModalProps) { const responseText = formatResponsePreview(result.response); const isLargeResponse = responseText.length > 500; - const hasAssertions = Object.keys(result.expectations ?? {}).length > 0; + const expectationRows = expectationEntries(result); + const failedExpectationRows = failedExpectationEntries(result); + const hasAssertions = expectationRows.length > 0; const hasIterations = result.iterationResults && result.iterationResults.length > 0; const iterations = result.iterationResults!; const displayRate = result.assertionPassRate; const infraErrorRate = result.infrastructureErrorRate; + const externalHostEvidenceRows = result.externalHost + ? getExternalHostEvidenceRows(result.externalHost) + : []; + const hostToolCalls = resultToolCalls(result); + const hostUsage = usageForResult(result); + const answer = finalAnswer(result); + const llmDurationMs = numberField(responseRecord(result), 'llmDurationMs'); + const mcpDurationMs = numberField(responseRecord(result), 'mcpDurationMs'); + const externalHostConfig = result.request?.externalHost; + const verdict = getVerdictSummary(result); return ( <> @@ -229,6 +448,24 @@ export function DetailModal({ result, onClose }: DetailModalProps) { {result.project} )} + {result.externalHost && ( + <> + + {result.externalHost.hostName} + + + {result.externalHost.traceConfidence} trace + + + )} {result.durationMs.toFixed(0)}ms @@ -244,18 +481,155 @@ export function DetailModal({ result, onClose }: DetailModalProps) { )} - {/* Request — show what was sent */} + +
+
+
+ + {verdict.category} + + {failedExpectationRows.length > 0 && ( + + {failedExpectationRows.length} failed assertion + {failedExpectationRows.length === 1 ? '' : 's'} + + )} +
+

+ {verdict.reason} +

+
+ + {result.externalHost && ( +
+ + + {result.externalHost.displayName} + +

+ {result.externalHost.driverSlug} +

+ + } + /> + + + + + +
+ )} +
+ + + {/* Setup and configuration — show what the eval was configured to run */} {result.request && (result.request.args || result.request.scenario || - result.request.description) && ( - -
+ result.request.externalHost || + result.request.description || + result.request.expect) && ( + +
{result.request.description && (

{result.request.description}

)} + +
+ + {result.request.mode ?? result.toolName} + + } + /> + + {result.datasetName} + + } + /> + + {result.request.accuracyThreshold !== undefined && ( + + )} + {result.request.judgeReps !== undefined && ( + + )} + {result.request.tags && + result.request.tags.length > 0 && ( + + {result.request.tags.map((tag) => ( + + {tag} + + ))} +
+ } + /> + )} +
+ {result.request.scenario && (

@@ -266,6 +640,7 @@ export function DetailModal({ result, onClose }: DetailModalProps) {

)} + {result.request.mcpHostConfig && (
@@ -278,14 +653,210 @@ export function DetailModal({ result, onClose }: DetailModalProps) { )}
)} + + {externalHostConfig && ( +
+

+ External Host Driver +

+
+ + {externalHostConfig.driverSlug ?? + (typeof externalHostConfig.driver === 'string' + ? externalHostConfig.driver + : 'external host')} + + } + /> + + {typeof externalHostConfig.driver === 'object' && ( + <> + + + {externalHostConfig.driver.platform && ( + + )} + {externalHostConfig.driver.channel && ( + + )} + + )} + {externalHostConfig.hostType && ( + + )} + {externalHostConfig.variant && ( + + )} + {externalHostConfig.timeoutMs !== undefined && ( + + )} + {externalHostConfig.usesBuiltInDefaults !== + undefined && ( + + )} + {externalHostConfig.correlation && ( + + + {externalHostConfig.correlation.strategy ?? + 'none'} + + {externalHostConfig.correlation + .includeInPrompt !== undefined && ( +

+ prompt marker:{' '} + {externalHostConfig.correlation + .includeInPrompt + ? 'included' + : 'not included'} +

+ )} + {externalHostConfig.correlation + .promptTemplate && ( +

+ template:{' '} + { + externalHostConfig.correlation + .promptTemplate + } +

+ )} +
+ } + /> + )} +
+ + {externalHostConfig.capabilities && + Object.keys(externalHostConfig.capabilities).length > + 0 && ( +
+

+ Capability Bindings +

+
+ + + + + + + + + + + {Object.entries( + externalHostConfig.capabilities + ).flatMap(([capability, bindings]) => + bindings.map((binding, index) => ( + + + + + + + )) + )} + +
+ Capability + + Implementation + + Provides + + Options +
+ {capability} + + {binding.uses} + + {binding.provides?.join(', ') ?? + '-'} + + {binding.with ? ( + 
+                                                {jsonPreview(binding.with)}
+
+ ) : ( + '-' + )} +
+
+
+ )} + + {externalHostConfig.options && + Object.keys(externalHostConfig.options).length > + 0 && ( +
+

+ Driver Options +

+ +
+ )} +
+ )} + + {result.request.expect && ( +
+

+ Configured Expectations +

+ +
+ )} + {result.request.args && (

Arguments

- 
-                          {JSON.stringify(result.request.args, null, 2)}
-
+
)} @@ -311,45 +882,334 @@ export function DetailModal({ result, onClose }: DetailModalProps) { defaultOpen={true} badge={ - { - Object.values(result.expectations).filter((e) => e?.pass) - .length - } - /{Object.values(result.expectations).filter(Boolean).length}{' '} - passed + {expectationRows.filter(([, e]) => e.pass).length}/ + {expectationRows.length} passed } >
- {Object.entries(result.expectations) - .filter(([_, exp]) => exp !== undefined) - .map(([type, exp]) => ( -
-
+ {expectationRows.map(([type, exp]) => ( +
+
+ + {exp.pass ? '✓' : '✗'} {type} + +
+ {exp.details && ( + 
+                          {stripAnsiCodes(exp.details)}
+
+ )} +
+ ))} +
+ + )} + + {result.externalHost && ( + +
+ {answer && ( +
+

+ Final Answer +

+

+ {answer} +

+
+ )} + +
+
+
+ Tool Calls +
+
+ {hostToolCalls.length} +
+
+
+
+ Input Tokens +
+
+ {formatNumber(numberField(hostUsage, 'inputTokens'))} +
+
+
+
+ Output Tokens +
+
+ {formatNumber(numberField(hostUsage, 'outputTokens'))} +
+
+
+
Cost
+
+ {formatCost(numberField(hostUsage, 'totalCostUsd'))} +
+
+
+ + {hostToolCalls.length > 0 && ( +
+

+ Observed Tool Calls +

+
+ {hostToolCalls.map((call, i) => ( +
+
+ {call.name} + {call.id && ( + + {call.id} + + )} +
+ +
+ ))} +
+
+ )} + + {hostUsage && ( +
+

+ Usage & Durations +

+
+ + + + + + + {numberField(hostUsage, 'cacheReadInputTokens') !== + undefined && ( + + )} + {numberField(hostUsage, 'cacheCreationInputTokens') !== + undefined && ( + + )} +
+
+ )} + +
+ +

+ {result.externalHost.displayName} + {result.externalHost.hostVariant + ? ` / ${result.externalHost.hostVariant}` + : ''} +

+

+ {result.externalHost.driverSlug} +

+ + } + /> + + + {result.externalHost.session.id ?? 'unknown'} + + } + /> + + {result.externalHost.session.requestId ?? 'unknown'} + + } + /> + + {result.externalHost.session.runMarker} + + } + /> + + + {result.externalHost.correlation.strategy} + +

+ prompt marker{' '} + {result.externalHost.correlation.includedInPrompt + ? 'included' + : 'not included'} +

+ + } + /> + {result.externalHost.session.cliSessionId && ( + + {result.externalHost.session.cliSessionId} + + } + /> + )} +
+ +
+

+ Capabilities +

+
+ {result.externalHost.capabilitiesUsed.map( + (capability) => ( - {exp.pass ? '✓' : '✗'} {type} + {capability} -
- {exp.details && ( - 
-                            {stripAnsiCodes(exp.details)}
-
- )} + ) + )} +
+
+ + {externalHostEvidenceRows.length > 0 && ( +
+

+ Evidence Sources +

+
+ {externalHostEvidenceRows.map((row) => ( +
+
{row.label}
+
+ {row.source} · {row.confidence} +
+
+ ))}
- ))} +
+ )} + + {result.externalHost.failureKind && ( +
+ Host failure: {result.externalHost.failureKind} +
+ )} + + {result.externalHost.traceLimitations && + result.externalHost.traceLimitations.length > 0 && ( +
+

+ Limitations +

+
    + {result.externalHost.traceLimitations.map( + (limitation, i) => ( +
  • {limitation}
    • + ) + )} +
+
+ )} + + {result.externalHost.artifacts.length > 0 && ( +
+

+ Artifacts +

+
+ {result.externalHost.artifacts.map((artifact, i) => ( +
+
{artifact.name}
+
+ {artifact.kind} + {artifact.contentType + ? ` · ${artifact.contentType}` + : ''} +
+ {artifact.path && ( + 
+                                {artifact.path}
+
+ )} +
+ ))} +
+
+ )}
)} @@ -446,6 +1306,11 @@ export function DetailModal({ result, onClose }: DetailModalProps) { Tools called )} + {iterations.some((r) => r.externalHost) && ( + + Host trace + + )} Error @@ -522,6 +1387,24 @@ export function DetailModal({ result, onClose }: DetailModalProps) { )} )} + {iterations.some((r) => r.externalHost) && ( + + {iter.externalHost ? ( + + {iter.externalHost.driverSlug ?? + iter.externalHost.hostName}{' '} + · {iter.externalHost.traceConfidence} + + ) : ( + + — + + )} + + )} {iter.error ? stripAnsiCodes(iter.error) : '—'} diff --git a/src/reporters/ui-src/components/Results/ResultsTable.tsx b/src/reporters/ui-src/components/Results/ResultsTable.tsx index f9885d2..171381a 100644 --- a/src/reporters/ui-src/components/Results/ResultsTable.tsx +++ b/src/reporters/ui-src/components/Results/ResultsTable.tsx @@ -26,6 +26,65 @@ function formatMs(ms: number): string { return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${ms.toFixed(0)}ms`; } +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function toolCallCount(result: EvalCaseResult): number { + const response = isRecord(result.response) ? result.response : undefined; + const toolCalls = response?.toolCalls; + return Array.isArray(toolCalls) ? toolCalls.length : 0; +} + +function usageRecord( + result: EvalCaseResult +): Record | undefined { + if (result.hostUsage) { + return result.hostUsage as unknown as Record; + } + const response = isRecord(result.response) ? result.response : undefined; + return isRecord(response?.usage) ? response.usage : undefined; +} + +function numberField( + value: Record | undefined, + key: string +): number | undefined { + const nested = value?.[key]; + return typeof nested === 'number' ? nested : undefined; +} + +function formatCost(value: number): string { + return `$${value.toFixed(value === 0 ? 2 : 4)}`; +} + +function failedExpectationTypes(result: EvalCaseResult): string[] { + return Object.entries(result.expectations ?? {}) + .filter(([, expectation]) => expectation !== undefined && !expectation.pass) + .map(([type]) => type); +} + +function failureLabel(result: EvalCaseResult): string | undefined { + if (result.pass) { + return undefined; + } + + if (result.externalHost?.failureKind) { + return `host: ${result.externalHost.failureKind}`; + } + + if (result.error) { + return 'execution error'; + } + + const failedAssertions = failedExpectationTypes(result); + if (failedAssertions.length > 0) { + return `assertion: ${failedAssertions.join(', ')}`; + } + + return 'failed'; +} + interface ResultRowProps { result: EvalCaseResult; onSelectResult?: (result: EvalCaseResult) => void; @@ -45,6 +104,13 @@ function ResultRow({ const iterDots = result.iterationResults ?? []; const cappedDots = iterDots.slice(0, 10); const hasMore = iterDots.length > 10; + const observedToolCallCount = toolCallCount(result); + const usage = usageRecord(result); + const inputTokens = numberField(usage, 'inputTokens') ?? 0; + const outputTokens = numberField(usage, 'outputTokens') ?? 0; + const totalTokens = inputTokens + outputTokens; + const totalCostUsd = numberField(usage, 'totalCostUsd'); + const rowFailureLabel = failureLabel(result); const ariaLabel = `${result.toolName ? result.toolName + ': ' : ''}${result.id}, ${result.pass ? 'passed' : 'failed'}`; @@ -90,6 +156,14 @@ function ResultRow({ ▲ fixed )} + {rowFailureLabel && ( + + {rowFailureLabel} + + )} {result.assertionPassRate !== undefined && ( )} + {result.externalHost && ( + <> + + {result.externalHost.driver.provider}/ + {result.externalHost.driver.product} + + + {result.externalHost.driver.surface} ·{' '} + {result.externalHost.driver.runtime} + {result.externalHost.driver.platform + ? ` · ${result.externalHost.driver.platform}` + : ''} + + + {result.externalHost.traceConfidence} trace + + + {observedToolCallCount} tool + {observedToolCallCount === 1 ? '' : 's'} + + {totalTokens > 0 && ( + + {totalTokens.toLocaleString()} tokens + + )} + {totalCostUsd !== undefined && ( + + {formatCost(totalCostUsd)} + + )} + + )} + {isEval ? ( {result.id} - {result.toolName && result.toolName !== 'mcp_host' ? ( + {result.toolName && + result.toolName !== 'mcp_host' && + result.toolName !== 'external_host' ? ( {result.toolName} @@ -186,6 +319,10 @@ function ResultRow({ mcp_host + ) : result.toolName === 'external_host' ? ( + + external_host + ) : null} {showProjectBadge && result.project && ( diff --git a/src/types/reporter.ts b/src/types/reporter.ts index 66cf00a..fdd1f43 100644 --- a/src/types/reporter.ts +++ b/src/types/reporter.ts @@ -14,6 +14,17 @@ import type { ExpectationBreakdown, UsageMetrics, } from './index.js'; +import type { + ExternalHostCorrelationConfig, + ExternalHostMetadata, + HostDriverId, +} from '../evals/externalHost/types.js'; + +export interface SerializedExternalHostCapabilityBinding { + uses: string; + provides?: string[]; + with?: Record; +} /** * Configuration options for MCP Eval Reporter @@ -194,6 +205,8 @@ export interface IterationResult { }; /** Token usage from mcp_host LLM simulation in this iteration */ hostUsage?: UsageMetrics; + /** External host metadata for this iteration */ + externalHost?: ExternalHostMetadata; } /** @@ -201,11 +214,29 @@ export interface IterationResult { * Preserves what was sent so results are self-contained for debugging. */ export interface EvalCaseRequest { + /** Eval execution mode */ + mode?: string; + /** Human-readable description of the case */ description?: string; /** Runtime tool override variant identifier, when one was used */ toolOverrideVariantId?: string; + /** Number of iterations configured for this case */ + iterations?: number; + + /** Accuracy threshold configured for this case */ + accuracyThreshold?: number; + + /** Judge repetitions configured for this case */ + judgeReps?: number; + + /** Tags from the source eval case */ + tags?: string[]; + + /** Configured expectation block, sanitized for reporter output */ + expect?: Record; + // Direct mode fields /** Tool arguments (direct mode) */ args?: Record; @@ -218,6 +249,19 @@ export interface EvalCaseRequest { provider?: string; model?: string; }; + /** External host configuration summary (external_host mode) */ + externalHost?: { + driver: HostDriverId | string; + driverSlug?: string; + name?: string; + hostType?: string; + variant?: string; + timeoutMs?: number; + usesBuiltInDefaults?: boolean; + correlation?: ExternalHostCorrelationConfig; + options?: Record; + capabilities?: Record; + }; } /** @@ -377,6 +421,12 @@ export interface EvalCaseResult { * Summed across all iterations. Only populated for mcp_host mode cases. */ hostUsage?: UsageMetrics; + + /** + * External host trace and evidence metadata. + * Populated for external_host mode cases. + */ + externalHost?: ExternalHostMetadata; } /** diff --git a/vitest.config.mts b/vitest.config.mts index 63e3f76..8bddbb9 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -5,7 +5,12 @@ export default defineConfig({ globals: true, environment: 'node', include: ['src/**/*.test.ts', 'tests/**/*.test.ts'], - exclude: ['node_modules', 'dist', 'tests/**/*.spec.ts'], + exclude: [ + 'node_modules', + 'dist', + 'tests/**/*.spec.ts', + 'src/**/*.integration.test.ts', + ], coverage: { provider: 'v8', reporter: ['text', 'json', 'html'], diff --git a/vitest.external-host.config.mts b/vitest.external-host.config.mts new file mode 100644 index 0000000..438d7de --- /dev/null +++ b/vitest.external-host.config.mts @@ -0,0 +1,10 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + include: ['src/**/*.integration.test.ts'], + testTimeout: 150_000, + }, +}); From c3be568f73a360be725df4a5b89fce02f67df09c Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Tue, 26 May 2026 17:19:18 -0700 Subject: [PATCH 2/7] fix(external-host): make Cowork driver work end-to-end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several gaps prevented the anthropic.claude.cowork.desktop-app.macos driver from running reliably: - runAppleScript hit the default 1MB stdout buffer when the Claude AX tree was fully loaded, surfacing as opaque "stdout maxBuffer length exceeded" errors. Bump to 64MB and add a 30s per-script timeout so individual osascript invocations cannot hang the eval. - The reject-only coworkSurface capability required the user to be manually on the Cowork tab before a run, breaking CI use. Add activateCoworkSurface, which sends Cmd+2 to switch surfaces deterministically. Idempotent — no-op if already on Cowork. - Add wakeAccessibility capability that clicks the front window to force Chromium to populate its accessibility tree. Without this, recursive AX walks return empty on Electron apps until something interacts with the window. - Replace recursive findTextArea/findSubmitButton AppleScript helpers with coordinate-based clicks plus paste-and-Return. The composer role varies across Claude versions (AXTextArea, AXTextField, contenteditable); coordinate clicks work regardless. The Cowork driver registry now composes: control: [platform.macos, activateCoworkSurface, wakeAccessibility] input: accessibilitySubmit (coordinate-click + paste + Return) Manually verified on macOS: a two-case dataset (echo + Glean search tool call) passes end-to-end with traceConfidence=high and traceSource=host-local-transcript on both cases. No Cowork pre-flight required. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/evals/evalRunner.externalHost.test.ts | 9 +- .../externalHost/builtins/anthropicClaude.ts | 58 +++++ .../builtins/macosDesktop.test.ts | 22 +- .../externalHost/builtins/macosDesktop.ts | 219 ++++++++---------- src/evals/externalHost/hostRegistry.test.ts | 12 +- src/evals/externalHost/hostRegistry.ts | 9 +- 6 files changed, 193 insertions(+), 136 deletions(-) diff --git a/src/evals/evalRunner.externalHost.test.ts b/src/evals/evalRunner.externalHost.test.ts index 7aaf10c..3283d96 100644 --- a/src/evals/evalRunner.externalHost.test.ts +++ b/src/evals/evalRunner.externalHost.test.ts @@ -149,7 +149,14 @@ describe('runEvalCase external_host mode', () => { capabilities: { control: [ { uses: 'builtin:platform.macos' }, - { uses: 'builtin:anthropic.claude.coworkSurface' }, + { + uses: 'builtin:anthropic.claude.activateCoworkSurface', + with: { appName: 'Claude' }, + }, + { + uses: 'builtin:desktop.macos.wakeAccessibility', + with: { appName: 'Claude' }, + }, ], input: [ { diff --git a/src/evals/externalHost/builtins/anthropicClaude.ts b/src/evals/externalHost/builtins/anthropicClaude.ts index 786cbf8..2f852bc 100644 --- a/src/evals/externalHost/builtins/anthropicClaude.ts +++ b/src/evals/externalHost/builtins/anthropicClaude.ts @@ -22,6 +22,7 @@ import { driverToSlug, hostTypeFromDriver } from '../driverIdentity.js'; import { readMacosAccessibilityText, readMacosFrontWindowContents, + runAppleScript, } from './macosDesktop.js'; const DEFAULT_APP_NAME = 'Claude'; @@ -111,6 +112,11 @@ export const ANTHROPIC_CLAUDE_CAPABILITIES: ExternalHostCapabilityImplementation capabilities: ['control'], run: rejectClaudeChatSurfaceCapability, }, + { + id: 'builtin:anthropic.claude.activateCoworkSurface', + capabilities: ['control'], + run: activateCoworkSurfaceCapability, + }, { id: 'builtin:anthropic.claude.accessibilityTrace', capabilities: ['completion', 'trace', 'normalize'], @@ -129,6 +135,53 @@ export const ANTHROPIC_CLAUDE_CAPABILITIES: ExternalHostCapabilityImplementation }, ]; +/** + * Deterministically switches the Claude desktop app to the Cowork surface via + * Cmd+2 (the app's built-in shortcut for the Cowork sidebar tab). Idempotent — + * sending Cmd+2 while already on Cowork is a no-op. Replaces the older + * rejectClaudeChatSurface capability for use cases that need automatic surface + * activation (e.g. CI runs). + */ +async function activateCoworkSurfaceCapability({ + config, + run, + binding, + state, +}: ExternalHostCapabilityContext): Promise { + const appName = + runStringOption(config, binding, 'appName') ?? DEFAULT_APP_NAME; + const settleDelayMs = 700; + const script = ` +tell application ${JSON.stringify(appName)} to activate +delay 0.4 +tell application "System Events" + tell process ${JSON.stringify(appName)} + set frontmost to true + keystroke "2" using command down + end tell +end tell +delay ${settleDelayMs / 1000} +return "ok" +`; + try { + await runAppleScript(script, { timeoutMs: 8_000 }); + } catch (err) { + return failureResult({ + config, + context: run, + driver: state.driver, + displayName: state.displayName, + capabilitiesUsed: state.capabilitiesUsed, + failureKind: 'submission_failed', + error: `Failed to activate Cowork surface via Cmd+2: ${formatError(err)}`, + artifacts: [], + limitations: [ + 'Cowork surface activation depends on Cmd+2 being bound to the Cowork sidebar tab in the user-installed Claude app version.', + ], + }); + } +} + async function rejectClaudeChatSurfaceCapability({ config, run, @@ -594,6 +647,11 @@ async function detectClaudeChatSurface( ): Promise { let surfaceText: string; try { + // `entire contents of front window` is a single IPC batch transfer; it can + // be multi-MB on a fully-loaded Electron window (handled by the maxBuffer + // bump in runAppleScript). The recursive AppleScript alternative does one + // IPC round-trip per element and hits the per-script timeout on large + // trees. surfaceText = await readMacosFrontWindowContents(appName); } catch (err) { return `could not verify active Claude surface via Accessibility: ${formatError(err)}`; diff --git a/src/evals/externalHost/builtins/macosDesktop.test.ts b/src/evals/externalHost/builtins/macosDesktop.test.ts index 4e2ce29..f7939b2 100644 --- a/src/evals/externalHost/builtins/macosDesktop.test.ts +++ b/src/evals/externalHost/builtins/macosDesktop.test.ts @@ -5,7 +5,7 @@ import { } from './macosDesktop.js'; describe('macOS desktop built-in capabilities', () => { - it('declares reusable platform and accessibility submit capabilities', () => { + it('declares reusable platform, accessibility submit, and AX wake capabilities', () => { expect( MACOS_DESKTOP_CAPABILITIES.map((capability) => ({ id: capability.id, @@ -20,19 +20,33 @@ describe('macOS desktop built-in capabilities', () => { id: 'builtin:desktop.macos.accessibilitySubmit', capabilities: ['control', 'input'], }, + { + id: 'builtin:desktop.macos.wakeAccessibility', + capabilities: ['control'], + }, ]); }); - it('builds a submit script that prefers direct accessibility actions over global keystrokes', () => { + it('builds a submit script that focuses the composer via coordinate click then pastes and submits', () => { const script = buildMacosDesktopSubmitScript('hello marker', { appName: 'Example', createNewConversation: false, settleDelayMs: 500, }); - expect(script).toContain('set value of textAreaElement to "hello marker"'); - expect(script).toContain('perform action "AXPress" of submitButtonElement'); + expect(script).toContain('tell application "Example" to activate'); + expect(script).toContain('click at {centerX as integer, composerY as integer}'); expect(script).toContain('keystroke "v" using command down'); expect(script).toContain('key code 36'); }); + + it('emits Cmd+N when createNewConversation is enabled', () => { + const script = buildMacosDesktopSubmitScript('hello marker', { + appName: 'Example', + createNewConversation: true, + settleDelayMs: 500, + }); + + expect(script).toContain('keystroke "n" using command down'); + }); }); diff --git a/src/evals/externalHost/builtins/macosDesktop.ts b/src/evals/externalHost/builtins/macosDesktop.ts index a743a2a..30c1ca7 100644 --- a/src/evals/externalHost/builtins/macosDesktop.ts +++ b/src/evals/externalHost/builtins/macosDesktop.ts @@ -10,7 +10,8 @@ import { driverToSlug, hostTypeFromDriver } from '../driverIdentity.js'; const execFileAsync = promisify(execFile); const DEFAULT_SETTLE_DELAY_MS = 500; -const DEFAULT_SUBMIT_BUTTON_NAMES = ['Send', 'Submit']; +const DEFAULT_APPLESCRIPT_TIMEOUT_MS = 30_000; +const DEFAULT_APPLESCRIPT_MAX_BUFFER = 64 * 1024 * 1024; export const MACOS_DESKTOP_CAPABILITIES: ExternalHostCapabilityImplementation[] = [ @@ -24,10 +25,22 @@ export const MACOS_DESKTOP_CAPABILITIES: ExternalHostCapabilityImplementation[] capabilities: ['control', 'input'], run: submitPromptCapability, }, + { + id: 'builtin:desktop.macos.wakeAccessibility', + capabilities: ['control'], + run: wakeAccessibilityCapability, + }, ]; -export async function runAppleScript(script: string): Promise { - const result = await execFileAsync('osascript', ['-e', script]); +export async function runAppleScript( + script: string, + options: { timeoutMs?: number; maxBuffer?: number } = {} +): Promise { + const result = await execFileAsync('osascript', ['-e', script], { + maxBuffer: options.maxBuffer ?? DEFAULT_APPLESCRIPT_MAX_BUFFER, + timeout: options.timeoutMs ?? DEFAULT_APPLESCRIPT_TIMEOUT_MS, + killSignal: 'SIGKILL', + }); return result.stdout; } @@ -82,6 +95,63 @@ export async function readMacosFrontWindowContents( return runAppleScript(script); } +/** + * Forces a Chromium-based app (Electron) to populate its accessibility tree by + * activating the app and simulating a click in the lower-center of the front + * window — the area where chat composers typically live. Without this, the AX + * tree exposes only window chrome (close/minimize buttons) and downstream + * findTextArea/findSubmitButton calls fail with "no composer found". + */ +export async function wakeMacosAccessibility( + appName: string, + options: { settleDelayMs?: number } = {} +): Promise { + const settleDelayMs = options.settleDelayMs ?? 800; + const script = ` +tell application ${JSON.stringify(appName)} to activate +delay 0.3 +tell application "System Events" + tell process ${JSON.stringify(appName)} + set frontmost to true + set winPos to position of front window + set winSize to size of front window + set centerX to (item 1 of winPos) + (item 1 of winSize) / 2 + set composerY to (item 2 of winPos) + (item 2 of winSize) - 90 + click at {centerX as integer, composerY as integer} + end tell +end tell +delay ${settleDelayMs / 1000} +return "ok" +`; + await runAppleScript(script, { timeoutMs: 10_000 }); +} + +async function wakeAccessibilityCapability({ + config, + run, + binding, + state, +}: ExternalHostCapabilityContext): Promise { + try { + const appName = + runStringOption(config, binding, 'appName') ?? state.displayName; + await wakeMacosAccessibility(appName, { + settleDelayMs: runNumberOption(config, binding, 'settleDelayMs'), + }); + } catch (err) { + return desktopFailureResult({ + config, + context: run, + state, + failureKind: classifyDesktopSubmissionFailure(formatError(err)), + error: `Failed to wake macOS accessibility tree: ${formatError(err)}`, + limitations: [ + 'Chromium/Electron apps require a real mouse interaction before the macOS accessibility tree is populated.', + ], + }); + } +} + async function requireMacosCapability({ config, run, @@ -159,7 +229,7 @@ export async function submitPromptToMacosDesktopApp( } export function buildMacosDesktopSubmitScript( - prompt: string, + _prompt: string, options: { appName: string; createNewConversation: boolean; @@ -168,16 +238,6 @@ export function buildMacosDesktopSubmitScript( } ): string { const settleDelayMs = options.settleDelayMs; - const promptLiteral = JSON.stringify(prompt); - const verificationNeedle = prompt.includes('[eval-run-marker:') - ? '[eval-run-marker:' - : prompt.trim().slice(0, 120); - const verificationNeedleLiteral = JSON.stringify(verificationNeedle); - const submitButtonNamesLiteral = appleScriptListLiteral( - options.submitButtonNames?.length - ? options.submitButtonNames - : DEFAULT_SUBMIT_BUTTON_NAMES - ); const newConversation = options.createNewConversation ? `keystroke "n" using command down @@ -185,123 +245,30 @@ export function buildMacosDesktopSubmitScript( : ''; return ` -on findTextArea(theElement) - try - tell application "System Events" to if role of theElement is "AXTextArea" then return theElement - end try - try - tell application "System Events" to set uiChildren to UI elements of theElement - repeat with childElement in uiChildren - set foundElement to my findTextArea(childElement) - if foundElement is not equal to missing value then return foundElement - end repeat - end try - return missing value -end findTextArea - -on normalizeText(valueToNormalize) - try - return my lowercaseText(valueToNormalize as text) - on error - return "" - end try -end normalizeText - -on lowercaseText(inputText) - set upperChars to "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - set lowerChars to "abcdefghijklmnopqrstuvwxyz" - set outputText to "" - repeat with currentChar in characters of inputText - set currentCharText to currentChar as text - set charIndex to offset of currentCharText in upperChars - if charIndex is greater than 0 then - set outputText to outputText & character charIndex of lowerChars - else - set outputText to outputText & currentCharText - end if - end repeat - return outputText -end lowercaseText - -on elementLabel(theElement) - set labels to {} - try - tell application "System Events" to if name of theElement is not missing value then set end of labels to name of theElement - end try - try - tell application "System Events" to if description of theElement is not missing value then set end of labels to description of theElement - end try - try - tell application "System Events" to if value of theElement is not missing value then set end of labels to value of theElement - end try - set AppleScript's text item delimiters to " " - return my normalizeText(labels as text) -end elementLabel - -on labelMatches(theElement, buttonNames) - set labelText to my elementLabel(theElement) - repeat with buttonName in buttonNames - if labelText contains my normalizeText(buttonName) then return true - end repeat - return false -end labelMatches - -on findSubmitButton(theElement, buttonNames) - try - tell application "System Events" to if role of theElement is "AXButton" and my labelMatches(theElement, buttonNames) then return theElement - end try - try - tell application "System Events" to set uiChildren to UI elements of theElement - repeat with childElement in uiChildren - set foundElement to my findSubmitButton(childElement, buttonNames) - if foundElement is not equal to missing value then return foundElement - end repeat - end try - return missing value -end findSubmitButton - tell application ${JSON.stringify(options.appName)} to activate delay ${settleDelayMs / 1000} tell application "System Events" ${newConversation} tell process ${JSON.stringify(options.appName)} set frontmost to true - set textAreaElement to my findTextArea(front window) - if textAreaElement is equal to missing value then error "No composer text area found" - click textAreaElement - try - set focused of textAreaElement to true - end try - try - set value of textAreaElement to ${promptLiteral} - end try - end tell - delay 0.1 - tell process ${JSON.stringify(options.appName)} - set textAreaElement to my findTextArea(front window) - if textAreaElement is equal to missing value then error "No composer text area found before submit" - if value of textAreaElement does not contain ${verificationNeedleLiteral} then - set frontmost to true - click textAreaElement - try - set focused of textAreaElement to true - end try - keystroke "v" using command down - delay 0.5 - end if - if value of textAreaElement does not contain ${verificationNeedleLiteral} then error "Composer did not receive pasted eval prompt" - set submitButtonElement to my findSubmitButton(front window, ${submitButtonNamesLiteral}) - if submitButtonElement is not equal to missing value then - perform action "AXPress" of submitButtonElement - else - set frontmost to true - click textAreaElement - try - set focused of textAreaElement to true - end try - key code 36 - end if + -- Click the lower-center of the front window where chat composers live. + -- This focuses the composer AND wakes the Chromium AX tree as a side + -- effect. Using a coordinate-based click avoids fragile recursive + -- searches for AXTextArea — Cowork's composer may use a different role + -- (AXTextField, AXTextInput) depending on Electron/Claude version. + set winPos to position of front window + set winSize to size of front window + set centerX to (item 1 of winPos) + (item 1 of winSize) / 2 + set composerY to (item 2 of winPos) + (item 2 of winSize) - 90 + click at {centerX as integer, composerY as integer} + delay 0.6 end tell + -- Paste the prompt from clipboard. The caller has already written the + -- prompt to the macOS clipboard via writeMacosClipboard. + keystroke "v" using command down + delay 0.4 + -- Submit via Return. + key code 36 end tell `; } @@ -408,10 +375,6 @@ function stringArrayOption( return strings.length > 0 ? strings : undefined; } -function appleScriptListLiteral(values: string[]): string { - return `{${values.map((value) => JSON.stringify(value)).join(', ')}}`; -} - function classifyDesktopSubmissionFailure( message: string ): ExternalHostFailureKind { diff --git a/src/evals/externalHost/hostRegistry.test.ts b/src/evals/externalHost/hostRegistry.test.ts index 8f498fd..52c84b1 100644 --- a/src/evals/externalHost/hostRegistry.test.ts +++ b/src/evals/externalHost/hostRegistry.test.ts @@ -36,7 +36,14 @@ describe('external host driver identity and built-in defaults', () => { expect(config?.capabilities).toMatchObject({ control: [ { uses: 'builtin:platform.macos' }, - { uses: 'builtin:anthropic.claude.coworkSurface' }, + { + uses: 'builtin:anthropic.claude.activateCoworkSurface', + with: { appName: 'Claude' }, + }, + { + uses: 'builtin:desktop.macos.wakeAccessibility', + with: { appName: 'Claude' }, + }, ], input: { uses: 'builtin:desktop.macos.accessibilitySubmit' }, completion: { @@ -66,7 +73,8 @@ describe('external host driver identity and built-in defaults', () => { loaded.loadedCapabilities.map((capability) => capability.binding.uses) ).toEqual([ 'builtin:platform.macos', - 'builtin:anthropic.claude.coworkSurface', + 'builtin:anthropic.claude.activateCoworkSurface', + 'builtin:desktop.macos.wakeAccessibility', 'builtin:desktop.macos.accessibilitySubmit', 'builtin:anthropic.claude.localAgentTrace', 'builtin:anthropic.claude.localAgentNormalize', diff --git a/src/evals/externalHost/hostRegistry.ts b/src/evals/externalHost/hostRegistry.ts index 9a7a5ad..95bfe4c 100644 --- a/src/evals/externalHost/hostRegistry.ts +++ b/src/evals/externalHost/hostRegistry.ts @@ -45,7 +45,14 @@ const EXTERNAL_HOST_REGISTRY: Record< capabilities: { control: [ { uses: 'builtin:platform.macos' }, - { uses: 'builtin:anthropic.claude.coworkSurface' }, + { + uses: 'builtin:anthropic.claude.activateCoworkSurface', + with: { appName: 'Claude' }, + }, + { + uses: 'builtin:desktop.macos.wakeAccessibility', + with: { appName: 'Claude' }, + }, ], input: { uses: 'builtin:desktop.macos.accessibilitySubmit', From dba37ad1e4f7167ec6eead62432cbf4af9846975 Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Wed, 27 May 2026 12:26:04 -0700 Subject: [PATCH 3/7] chore: drop unreferenced reporter screenshot The PNG at docs/img/external-host-cowork-reporter.png was only referenced from the PR body, not from any docs or code in the repo. Inlined screenshots in PR descriptions are better hosted via GitHub's drag-and-drop upload (user-images.githubusercontent.com) than committed to the source tree, where they bloat git history without serving documentation. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/img/external-host-cowork-reporter.png | Bin 71836 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/img/external-host-cowork-reporter.png diff --git a/docs/img/external-host-cowork-reporter.png b/docs/img/external-host-cowork-reporter.png deleted file mode 100644 index fb279b13b06bfaa4ec9fce6f1c097f4eab26cd03..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 71836 zcmdRWWmH>j*CzF+1qT=Z_(oJR$LR@tt|x#6n9N=3GNms?xaX?g1buy z5CU^b-*0Aq&7WD{nl=17D>>)n-22{g?d#fyAVqme!UxYE;NjsBN_`Yl#>2bw6A$m! z`ad^;Pjblyv+(f##gh{Ipz4;gJ$uJ%bd{>>U{o(3k2@>7vbLe2Ve58h3b=nq*swnl z`^@jH3GkUr)hI6O%7ENu%j)UtN|2E4u5-yuG(#P2JSD~}THq~Wa8Moi0pRaP zV*a0>!o&Nt{Lj1VgU>@W^VbJ>65*e3T^~IYy!r6@;M4K{o0I4qF52@kxpK`O+>4ML zX(=f|mkSdQ5093l=b@{z>vh}P+g~$-6PY+UIhU8o+|{KeV92z8cqvFpOJ*J=zB%4@ zhwB@7RV|X9&YXp4X=y!>Ccbg+&x+=A)(pHx`g4p>U7)!ut7rV?yR9{IxbCqr36Aej z^NR)TvH;bID$i!V>ff!6DzmkZ#v86)n-g6H4Dj*RVHLbC{D1r)s0v%YqoCk}9LZHL zItP0dj9)pOy9V@xBr?5yYq`H4$gEnRRbuZcS+l94@6&LaAeuV@Ki=f$h(4F&tM$B7 zwKY?cS4zJv?N;JLV=7*M^1|qK%&!dRKva1|3tGExnbRD>>@;OW@=oAF8_`7bf=Lyo zm76Vqp%UQX;NY8EwjQ_TvGKxh=nd6zI$IwpLH?GcP)k?-)P%e^2_#e2&o$&@SXo)8T{qa>zcuLGY0Ha=8U5<-p6lT3 zOor(m;dj|d6<%8#>2j6x@mMV3T$8=pl&l)uvi07d4R00{vcG6bwVtj4pB-B74W=E% z^A+Ufb3Gr3cobf!RdW`CKnZ(%>SJaXVqF;#x-3tq!d~=I31JtpAbs!M{-9Y--!l&c z$}e2rL!N@$^)jmIR+ny_gId5bcpNmFo$x5nvMg;1XL`Mf5OA2UdK8+Bp&FyKj zxXe23Qn1Cw8)B#ztTmvtL^W3q9-o;M*5~zI#uzl~SD7t|K1pMAQu6b=Y5LGW?&{ZM z&qfs}x9RZQGeGJ7{h*Ne;z6CWE~t4=MSgzJ;ira@8(oXpFv_&kLNTrKJ(3^!e~gA_ z%1pm1fn;hDy^H1+xPWP>Nt&9PQt){-njX=E78i|-mriU9W^*+(+P?XdtfoqrjO*y= zz)vq!{O%qXKT^cSk~%&=@x)LlJCu+`uBSs%=P`5cRrZ&`RdhSm#y&5c&?}4x$|WT+OPe^4J525 zuRCv@u1}p^yFBCc4$0b}K_S^XduC%JQ%^>3PF$1-sq_9Xl;6FR$aeFCuXBY(MWig+ zX|CHCVVv1BA+P1r9u@PauY>Cx4A!0M?ZXA=A`?PGL%qIPkAcqtk87wdE+p1fx%Rgh zqPJzB0yQ*trH0p+RadpYSssn zT`srP=;@|JN2?slGutwJ@T)Y3x}9hLt{^pO?V|s2bW+TcNqg%7Z#i69T!C(lEBFINf!0i{q z^jF76w%Kp5#uth8JPBtjO#}I)1t-%1m*bHB`ortmP6dtr@omKd2mLa{%pS5n=WpE^ z-MRyadmOUI}?NWoLIlkK((BA*gy4#v6%ec?aQr zNB8!cisQOlVH*JvPMPzxxrmh5pXrX_ErDS##y2aW@>j*iHD98l@{Ikg#v&vjO;U%f zrKfYr?0;h4SfNpowX|m>+Q9qZjKRQTfic@6Ki|^8=g85?sgN4?2=3`pq}bxTT3E-y zlHLhDA`K_!oOx4SXu*|~*(Cy*`L5DCC@AbF`l{xUF{MtW$qLB8=QY)GfZX(@cI`#( zo-jGTRd*NE8+O;BCPfO!`^v)u`~4#$@K1Mw%9m+00#j2diV6!4!(zXg&Rw>}x3v!R z%O-P!H9HWXx#hA2W!^Lj@?9e7~T##-ebtZz+LHkeJwk zp0PGIX5F~1i1RIqCGtfDYF@d%=V^6yXLU!UW~}KUzHNU} z#e4Wa**`@{HMP5ZS+#}CMkCDR$aiL{_M#EZp7%!5GFH3P7E@Wqm4)Th2tn)ELc#4tu0( zvma7TH4H_-$UZ`bGZA*qZYU~YLbB2EH&(0L?zWeEqN2t00s=E>o@UCZ6r05->@2#C zul5L55SjLw9*5=^=gbH#P0iQ-J-xjiCuifnsb{&jUk9`X5N`I~eA>*n5_q9jqTf(f z`yex*|6;{@Wp^9p_);d?)XYl#V$$eO)b!5uW;+tJ-MBgcx za3Ucpj>Sk3n3OECN#+XblbQ;eRdm3*S`L?Qr8hU`CT=r>a$7w0sRW|W)qv&fL<^Xn?P2sZuwWDoW41 zvkNEJTx2RVp98>$aD3_MXr7_GEk*f<(RgzaL|?_Z3lQwhKv0aMQnW8_b}`U)S2*rY zzF;DaPl0JKEb-+-K0ndnP)P$c5im)$9TxAwNl^KGgooY>6@$2E4BO`-LLAOt0UtvDen7-Wvh2+xJbV$L5;lE<+b zsFdbxCoNIdB~tc~6F#5lnuZOJ?KIHMM!W-$QQ^~~`dr~7eJW|YV^yL?Mu)v>Tcm7S zkqHTvM#ys_;WO=%A}>yZX8YuJTWN^w>4+<|XP20a40m8Y_fell#Tz%v?&`bP=KT^T zY&JU3z{jpxD=YGv>>0mD9<^)X;#%gPU37c%?%hPgJAUuu4Ny)_4tfiji0<;@)xH>A z3V+SJX1eB*J(Q6}>XdgV$89qzW=T9fJ-stg=PC)=OXj%Pl;H)6=^>@}c3q?~10faP zd`O9JRFvf&TjnAFGXZ6}m?3$Tj>Lvi7+|lZzGXY#Xje!?#DH_of6_G%j-#fkMZDVLS zRH8sc2ZH>mP2Aw&pQm?re;3CB2&;tis!+M3Mk_i10JB&}l#zl0CXN-t&Z8_1kIksD6_?vL4XQ~ZI%Yc#VSW-uQSL#<}YO3IFWbSh2`Sy}4)Z40+ z%9kxB{xQ2$eVO^(7XX}^Z=r!~dqT4$dCBKE81cp17k3b{3lA)sF6ExXeS1J>-&sy< zjfAVC*DgBz(@YLdRoJb#p3`Cap|(vIimZiX&wf|h`_Z#HqCS$a82lP42KTvuwR_*s zp=O`+6k?Y-dG)p-lhqZUeo^25&bNO;&R*cl`MAKF8+WVc=vA?#{zH6_6Z!{KAhIHqNeJ&LK#%w6-U#aWw&8TAbUI zaJ;sh&q65k0xx{q>k7?qXl{4U5mjeuimK}N9Owp6-{@!2rsszceL9AL$$G=3F#vy0 zPH6U>Y)rx(QO)mzoSV;2;`JIZ{wV@9zIx8q_sGuK;=WlcLFBU_fnP+s`aTz?PN@X7 zryMHv9(J{NR$C$)y&Kd4n9P$>&*O2iWNEo^h+*$zH!RquEYjhLZIt~3Bkd6VskqEa z#2l1IuV)A3)a$6wZTN|>c z38NC`c03W_?PT$X`W`}#%t9f&$dtL}_|BaAN*9-E+vc|`8w`}O3(n-#Ug#YJLP(H$ z?$D+;HTF%v^bc}Q*_&bX!}2yWKNFno9UOEUjMBv33=gY1cn0cn`ARHF@a8&q^EI7K z9R(zbvzFQ*;QV9VH=9pZRW08nB_>t@#r$Dta`K#1n^*hq8|;n8CwU8g(heE#^F1#+ zKy9t9+4osC0vdk=heeu)H?uP`G1CX|55$lpE_{Bc{O}ioL)qhsaCwG}&gEPxa>2`>JNWV7=~~&TWfXF=C&%4kxXpj?sDa9t z^SFh&-g%?c<4kxw>!Y^)JA|{k71ayLaDOfMiTo>_hxfAEm2a;j74V!1`MD{D3@hDR$EXib*2!X%honFIy(cx;J4Ut< zt6P#o4)r-6bVSHTjYjZWCd-HD2W1cP8^s#&VjX9eQuqJPi6SJBK_V zpiCMiPZ8nDUORGe(YAIEV-pdWMJ^5>PyBrp1?hgD9h`PHOgPk}$Lu^mO=UJ(Ft-{z zKBrS|Q)Tk+S%Dz-V$13-wf(PQ|B4&{>-zb5+iVkBPOmi#N^|@pEDT)`y?4AfE!re@ zmuS6cIqgN6=`>qaYYr>_D3{gg?@Lz!^A(TNlFdU<+3%Ka+Z$?^O+_2MS63xlV}f| zuINP^&#;r|$95V7P|G>Il*{>scd@weJDWSUz0!;Z8Lt9Y=w=hncder!h0~=g`Q`@s zda%`1=MaQJ?L^u1dnu66`8+Eb-A3oknarpT%x1R98o=sKJ2OpceK|ELz9XH4Ca%KH ze5yP%TP3n_kI|^+ovG@=xM{!Q4<@RWe2UDNZ;8{+E4@fdD&O>nHDHx)xG*hyL;w!p zqY9P-)SzBzBP~Db>fCc5g6ZX6%+ABGfs7W*g=~J1qtnD$4;{v_CT;hG^0s#`_*vb(+bw#s9i1x@J43!w2_6xl{8O2~ zQ5>q7*1lZd+VfE>(MjZ9BCl!=Pn+fEwnD8e=W{x)vZWl(JaLa-fM3CrO0ZOaA`PSB z;1K*Gv3a(8elUlL7@%wPcHHK?u`7)1&4LRDYvvcnrl{3it`w;E z0$TjY?CCaB)o854FKX%z6ovIDbG*JgPSDcriW{RL<*Mzvo;k81lz4LDh>%xqSbLL2 zE_nVU?~x)HXhLH5Sv$iiE{9=z0OU^N^2koab4C%si?d$m&1{lG{l{C~#^^*KLq4 zJ~h>Q{pt~lDuns*7su?IaeO+BF>R2Umqv=aD80G1yAcxCymKre%{b| zzI*6jZbBso{u`0zlNOJUr*PiQ)@LuE?uSZ$BH{gHzUGjq;4d#Kz_Jb|dM$#+@t#pl zvB>76M1`qimymxjjd0E7>HWRwWAbh!QjL1(7m#!}}9H|z%CDCVPt6ww0h)8t^2AM5K0MOcF zVR40mAJf^6icp99`Qd)vfD^1I#Yc8uW)X^ceX`YT$0(+Lnog2#INHPxY0+lWny{12TGDn;-WC1Q70ma4;$$ z^qhbO$jnUL?Ub7ib4=*ol*}8S((bV7PLh%5>VBNA#P7P3%I_ARM71k zR_sLX+%^Cy7BN;{38$c-I1KY%+=vgOL|7Ull5r|1$h(`?r>=Wb)EK=Pohpl|EPL;* zX4?W?0%m6Bs;*fLeXma8S}nli)M0{l`*|gZ0bY>p*!pDdCY<56N(5>? zEtJoCKHQ#kRfPX8zM8kkx)#ASYShymPtXU-M@e5RV!v!6FP{Ff;QPAe`A>TW@PdU} zPyNv6W7AERbzH&m#hNrbTPI*09*e1-w%LKsPKi3quHmnPp30i!s$C(kV`BiE6{n`| z%r6kJ>orL{+-`nzc0ib1Ep@9l7>bBPhs@3}y{@smn0WNqu0OR|46eQi^k(>D`DD)& z8`R~bKma{T<>?@e3OH`U1tJ?9Zr{7Jx@F=7ge%!@hVlkBqEUU4 zm7CKoNka_Kx{;vyh{ySbE-IsiQf&Z4Vdmvk)f2@CZ18fJoY5`(hx$XdOLjE@@&Lds zt|K#LTFPRXHM|zf0!`jCr3g5$9x0$yc0-h#LZtPFg`h`ROaq3}Vxmyx(+&xURq{OkByq9$Zwb) zK&T3WrQWUS`_kU%S~Iw`12_e;^#p#Mw?Ie67Pz82>bQN{Qy)6jXS2m+mAiVaZrM@M z=4KId^3ZU~#Cv)UDBIHrF$%nrsN?tWK8cKQ+?3<#>s`)e=v~-XTB0Np^lZxG%asod zdv>G=aOJx?&B2&`f`@dNSNKv?W5sh(v9ZWZ9R}V(_KnWZ)Jt$UvrN5GSw^6PWdFcG zSJ?9cl!DX6#Dch*HNd%KoHa`&ykgx1h_fD=y@{akACci=bt($79&SB{+=hk=K!(~H z+cNkNl*L3P@2x^{xrH#8a@r)P^w}BeO7jFF-&AT;2f+g#-X98-m_5`pxz!{L_OP-V zoJ-En-vF9R!pvQVLAvAJX?B_8m-qWzg)dKV3Y>t7Lnj;Q05B<*d(G6h5DICP;4X;! z7SM+FbTQr9ywCE1+uG?AD_;*ZSNy&UQ3Q=Zr{b;)pk|i?xv?FJavD6-HX`g^b7;k1qN2u9YD%}&tfl8Wu9(#0}Fzo zds+9F(%c=sW|2GWHv?&jccZh@)HMZ?H>xXIoli^Ap}IAkKf8FRc?GU0Qf)Kv>ob6- z_p!D@b;$uvXe+g_?P)HfiO-F*zTE_L1`}{_l}EDX!qQTeA==;;7USUR8XY4!bv2xT z*!XI4HWQ`mNBj0mmeHmiwwm<_QrVE=cwZ``%;J*Z>^2Uj5&Kzjn6O97WXA=fMNO3# zV(kLOyAD(;n;*1>T64@se&3J|(#VV~#THlH_f`jB-Ct8q|ILWafd_0gKY z<>Z{LU$gL#P>-(;MP0acR%$SOe0796))M6BHmG+AOolnm6|>5e3uMbY)M&=-b^$tL z6iLl<_HJsh$c7C9QvHYiUJ&q1uxe?m5g7ALu~T9jMWo z2+I|i!N))b-PH=$mHOR5Q69JNK|cHQ{TCN-hpEHCu9RK#bzMMu&9z8h>$>s<-?@7wFYq z(8yXz;Y@8V?U&Y>?BwM$Ts&pa_h6j;N$`0UZe=A+lmTp0DC7nHy3As5QC|9i)tA>I zH6h01-SW%`5s{sjpF?rtpt@;Mq|X&AI#pA9<*&^RCS@WM@rIUOKwe!HcSv|2f+<2oKgN zH|xqt4NT_whl3MgDP(SDMT`E3=zj2UKbgZ)v*|p(SXbEdz^bV=lYyOJXQXAQMSVEJ zjL>g%v;17K#2dfMTdKM>wRp0Be5@S$>61EDWfiAZpFV^5EJ{E&ifJej7IHXl zPxHP=zZlH;VAW=qbPFka+YaKm#aKEQ0c`OnW94g~v@1i(DcFWO`liV8Ce2?YG zndb0kUCE6=x9y=atV{9ER2}?!P~;b`+^{P>htO(jdUl(*HS?>ZiBn3tM(_7-a`zI= zp8nBoUMFda>dH5^%It5JWe1C>DR#hARMWQS(+XVq21k7cB&Ili#b<(feeFu{qqvDF z&E%w{kNxRQvL74Wx!d;e$B_A2=u8kLb{!p*7Ul7@cHnzmqLw((xL?=csk>)yvb2N! z>P0Met|P#{Ztg~4)HHI_yFrTJ+Ljw+TGN;pTT{hB=~+>^$w4)q7&X#xfv(<%X^B`6=-<&+;_MgCv?{akZ${Qa_W z0|Qsi%vzS%vAo?1mo1~EW6Ghkk=vzO=VpP2ri+C;DI7-0~nJW2w%+6F$DZ9V@rBi`ZUm2I%_=57JF~o{mnj zVq@?1|Cqa)sdaB~z&d%*(c!9vNfX6#HLi_QYukHD5LOCP!q)&d*zt;FBDJ-&M)OuV zfhvM|Am1zKxVgQurDP8zIJF;N+Xv+b_48Tyg2fLN$_eSqBo7W zST{t!)M%?~Zou1c_;>JbaWDK}e?Kj_%-w786qv{0Y%J6tP^e?);>-T6FP^!GR`7;C zT{iaVvt;f6k74}E1v;A?+9>aNCUcRK^fQAZNZ%1ktqp5Mn zxT0y(gp27!ne1|cj{%3Xdwrv=qm@6j@W-2vFNyZNE*ts{SW_z8mP#BoU~LT#zcn5l zlzbkD98e&D4R^LLFVP(WBhsTcE2S7MB^A<27FCo2 zouXUc%E%I8p0bYZIqi=577-{V8NFwyTmDBDFV4D4zZlz;2>c~AqM)| z7nEdVPu5f>MH%l+RYQDEX8e&`JO%9WEPAnVY5L6X_Gj~<-k7C+26{mO^z1l*G`Bz& zHcj_4GmJcr$@H0D+YMdJY4g}}cpbM#=MFwavjIj}X=z%;X=^gpi2P^7&9;|i{-dn4 zoV*llS&Q1BE=D)>d!$>s538@EW5rS~yL5W{i}6fbt|9{!BA3kY*Y!jat&(a6cJp;pjz9mPx^rfITc0Urkwh=?v5a^0E9#OekA`Ui`9AYaWUN znnscAY(3|MJ4yLW^ZFM%rwYvz=}!BTMI_$`B%neWX^Pz;l>`A=ittAz6!fGnG`i&6 zE>BTT+RGcR!)4oOg}z(WT2MtE_;AzO$cO_N?N|z|7P{Kb>Y}8;UMzRfHGU5XLGZ(9 zy5xb0tH93C_0jZ!aRXWYN>^iX#48%|(B0z44~oLb|BkEeMssZWwRKJnw~I+3y8AWK zfD~?ROqW?$#IiM7=yGxFA6E4IV?VWj%NMm`$=Tml&J9H0#JW~1r;%GKKoqKDxAS&2 zCYSekoaKBC?s@7Y{)S~F_tVw}KvRpHQJ{JUR_U~w9o~B5IeW5qE}LI(vjz3!qhbZJ zRy&x->E_>Xasn*(!4G%qtXBWhe|m&s%75o-j9+@&x=j3|bw4vGNnU#j>@w%Emz!Cf#N*(* zRk!*F`pe6?ly`Smx3?|WiF&vJptrudm^TLE#^v&|wuNqvzVq4qDxWLt%u^_Z`{`wy zH3+zXQfh3Y#@-;Sti!F__a)f^#J}nP6My51NuM!)<1Oeh?cqO%7`0vzczspZkLMXe z65%fjNgk3-favI)v4e1c%E?nlmPjq#RX%MWVMmj_IbSb(A}=3CxH-=Fj+vBn&97m+ z`Dm;UAd=-g4#ltL&C<;gr0AwqGgo4TnU&@1937*pg%ZzBGRVeB4t|pqBj>A*5LoQ3 zTKAdN&S&4cs{!KV6!bW3Y=5C3ETAae$Xq9bH&p8Kek+H!2(xWNCrBe&9wo9iSNUN8 z>={l0WFq9o=J+83G&T;*66o+0ujsdE_0&9i6)UmV*4Nc( zz2`2Wz=rkpC0&|2-jz&eQa`qst40c(cx*oUGD;~ZTsf~oZ5z)ol1l{H*#(<% zXTMr#&T@vWs!mQ^02(?$U@)~=v*E)uFdf&3>kRR4^7l`SO1?Qu4N1SoNI)Gbj~=3i zPxJFn`tmz&tm;X!>D4v4^j#XP73FX0iK7k66}Zm2;+C=k%iNXABM>_gsGks&aZ!h#rQv=&b>^jnb`jx zROdd`SDKDnW7n-V-}`_*e`;-QEh!lYXyE}V0u?T}RYcQi(|jQbtCr=~HfD*v0vK`Q zcRnv%rfB)_MI2r-IlDt)W?JWXU9KrvOx^DW!czxEhzJuJY>o_=YpkmHn3+Rw20Gu8EbJuPPFmb7M*^CvxBrxghK_1I zl-wQyAr8*Z9N)^!G!_(`CDe-}5g!h%HGw-|btz;A>csx2Ab57oF#!Ht(b6_^kv`wq zp*`WDuU1cK`VTPr=bfGPh88cGq}0jNCr>(BI@_nBoUUiw?F=5-{S*UW5icvhjLrVQwT{`}fK{;7W4Z)$U;vYo}ld<~Cl>u*CczfA>KO!N#?~fxA@>fjNGD`L)MiZ z-Y(VGc?FNsCWjt?-#H{CX7VoTIXT(BP5x3Y`FH>2s2M#j{I*QM;A!pXs3PMCyh9-W z_^*q8$Q{|JsN^eBkB^AjSo{PwH?}k~%7{rDCU|F}-jknLUJDs?`Sb1upn4b@Evgi< z3{cns75VG~jI~PWoixLIM6Ecq$WBu8`VvKv$298| z6@BA$On+}-`u)G5Wpm6HkQ&Z1+blQv)zUgR)nER}f156spYYMSj9$!-USQ6+sI`wpTC3ql-z*m?txuSO!@13;8!Er$kw02i6QG zR4r0D@5dv*4An8HUkJ{CcaU)RuDDWdp~odDGWATid`^f784M;8vVYONb5PaMK zP--WeqZtz1wrwdZJ0^qRBqTTY+iUFsuY(E8`7S_XPau2uV~A11vMM4ig64=AtV`&R za!K3#wehWVx*!gbz5I_HWRA+}TcmkwGCR^FE1;sp&wuAE%UXubHrW{>CoAD)s#7Cf z&71n8$$nnaotOTn=X5YS)?sfy$k9+EGv+FzLoa8x95m@jTJl`vc5Egbj0hW$l?x$w z6cwcc+TnWT9HqOPP5$xKZj<#_B@3C-kRRn6 zQ(p-5)UmyULlD?qyES} zCmkfknCwac(PdW*5cU9P!1V@s`TbHT^u6{1PuopM83JMB#uJ&D@XP*ZsqzfB z{7H6sy@p3-P3`tl6mxZ|@vLz1()!^E9uqC@?b*|fJQ z1tKSGd+-D9KTy3TDPiLyg%}~Pw>Tx83d_303 ztm+s^vDpN;owu8q6TtBvh&Fg?^h#--Q1QZTaK*2C30SO70nh_o6zFU83@a;rDXc2-|kFt zl$T1_dx#ce6$1!xd{^GYKVnodN7B+#YS+b9oSC#Hh&3%b+gm(2h519q@RGAT$M^4} z&zxVrsaZk*nfdkl-r@JkiN=-jOCPO~$#KPX>op)?u=|^-w)!a!`?>4Kr%IQzQyRUN zVM1`Mqx%+gy0N@*t4lqHgnbuBFPE&;tz*WlRCW*;2K6>ROt2SiVAm|;-qs%Erh$QW zz^FH;(I6^Y)!fc)aq=u$L7JoR4)V0*HzrfePk^-YB$r((J&Rm#n&her&R$ue8h5 zU`Y5;>uDNA5>CKe98f0?p)LR;@>~sW>&qv;!!BxS2+0}QD6gm>qQp-rnF%XQJ43|O}(yjO%e2>IX1b-rr-VGFE3OoNzQPO z-@Ou|y#~kzKKg0lVcXeZR%376HtJKQih@t{ddlriJo!|1Tj%{6^_;%J;^K4V~Fz?MzPiyy! zI;W9y@ICU;hp#K&l-8G|7RlAPoRRjxGO)4MN-f+Mmp)yu(IU_O_~Cn{A2jp-xa60k z&ke~E9@Q($JsCB0e!0Gfm+=iLmZNWMYL)&lIVs^T-x*~Hzb`AR*(}m~wlP`Cu$+FU zL?#(u-XbDPDzzMh8Dt!QI+u3W&Aj9j)K5bu_dUOxF|riXnvj^JZf9SwH8OB^k!{WK zQWC8Jo+5`Pu%rj*Ls{tnQ9ARU^BMS3J5+)>U-D|Vh^2p^i)f1tykWZKio`Dv`YB1? zBXl@8IXE`;#tRU$07a7KmGMA-KcH^{uTWS<#H9g%=w$a;Yg6XF+~!?1jmw5%&hlb0 zv3sfw(VXv!^}@9pOnn2@35&%$i)Tu6CA(5AdRhio50QD5$FfrCBt~Fsazk`PQ8ZL$QXC<0z+PB3M1Z3B3 zZEi;-W}jIO_A*8Xs9|6!2{GqWlFfPCI$Q;GM3x!=@#ZtIit}!$E?m{wK;}vSHIh}Y z=BVb(THnj7R{7f;X`!`hO=m^;!zh83&~P7ar?rCB7d&HQW9o5x@Z(YeW}?sdhU2sZ z4lb3=ewNc*Fi&2A!wpbVWo4zs*vnd(=@dt{a~Es$j;#zsGcMHTSL7`|3L$&>A^|hS zc(S8$n`zJf(b3Un=@q40{G@8@dOf(vM}mL|cYrmp&Ta%()ay-g#T^3LyJSbZH+n0h zzYwSfmpeM6;h{3uuNvWrbB2A+%#ic9)sc%NcqJN-Zbg+4telYWG*@~f?bsyl!*E_^ zTJ>y4ZZl`#@$i#j2~Pe~h(k&fO2X)D!I8Dzo3*w|NvljXJ+(37704F^2mi{HD?L&wD4 zDk*eLH0>osrACX0iFFoFHSVPw&cd1M)zC$>W#8pmi_<{RjRpB{6x)UGj$~)saKIYq1FlueovFT7uuovu(-`4eW*Q`n+%>V%a%56-e^>Z$lz zm$)IbceVb>+}PHf)82{7^49o+!owB~$CT9VD(%@*!`avj^#3k}%7|r<@L2zQ+&824 zNU01V_lyH#QL|^pHrs?rkbwPxBOY^cR|OTMZPQ#uu$epLTO#t#jGJjJ{&$~a6)v*6k{z)7TkL@Vb)+bDwVp$=UB=P7W{_GJB5Ps^F~Px}&qo2S@cEubM?ZC?{}N zpsM9uPzrLbAOc9^no4^>sYm;b_I-u)9gL8gF5Oed`~~*e(%N5RXSWCXr58|<5+rsP zpl3H9Uh_hq?>)N_Xdf+?77??!=)?fmyUuR=P^I7-fkVeF2L=EG@*hq^F?hFn`Y)OE zndASOFpAawWoVTFtn=?3f8M5*h$m&TjEt#di3te_$%z(~ zDJ|FKA5U5=`ib4g3`IV3`xtFyBU|IhgsAju(1jQC9^hK8FRS8w2h5pmgZlx#=C4Nl z*{GvE_uWK-OoC$5;Cw{(TW;>EifS&<0R7V6i#sCLf6b_tRaO6ao|_NrMbK zlpL2X)$^NoQb#KxS!>ng?M%>LE&X1qd@f>mlS><*cmm*6iCN^UtGxMNl%=IP=OeT> z(C5i-C3Dna@!5#6_^Ivdr~h2W!L8KaFWWCCXH(QaYLWR=VNOvRG_UBu`H*{6F`w=) z&*WB`uNAN{Co3x}{dQJST5w)5|81a1{a*uI=FmUSrUcKV@RD8AeRywNj`99p_x~M_ z_kZ%ISj=b95`@rW2R43weuJ}~ig<4AA_sl|o0|_BDDA2`NGI{RTWNRJ$&CM%g%u$& z@t>FKYMMhsWw;s}|0Ov%G3PXWNyqKo5ipy-L3Xx@NT|`4V5|WqMBT?)ojX)Dn{l6{ zwIh!j9aueYs?+ zuRBP*pBId{&r91Kc7OnplS3cy7%qgpSb=dDoV~75fQ`V<4t`jc$VV&yeTxQUA~h;W zsMLJAk$9B_<+eR~hX}OxL2&wxD!LaGNN4T3JLTIt)>$vvl@rh!ogN=wG)GJd9nFJ; zF5vi*5)x!_o8eh8l?pxyI9;KFL}h{k7RdM7U*WZR`wSZ?I>W{4?bA z8^DFPJFH!{1)ZeTt7WD;Q>VH9@{I4bUa!cg0!=`PIm?;zF4KMP(Z^&F@yf5I#12LR zB*hFG9rV0`;`Y4PwtY>v{xR7TOjicF?cBu+>wgfHo|S`Yv{)%2)&Y_)Kl_ip$@EEt zqk2{jZp$cc_MQSIALaStQXAC6;uoOnjJ8hHw$G6ilFh(&bP%D@3u?(H(hDKc0lX8F z0!RBl*>;}yX%|k_x&rl8#lT0kHKa-hBXcKIo%#MpRAS&aeIakQZCpH+`&myU_8~M( zLH2s;7w^v+KDQ0GUifzfQ1Ux31WxwRO+^F}bjvVufjF6crhp07tIKgdbeYFdlb5kV zfIdVsSHg}^SkiOa(3^kVWUTH{?-%2AQEa7aX-yHi?rd)P{~+!?qncd5c3+n+JHoPo zbd}zv_n@HkfKsI^ReA^ME)|g8LhrqV&>?iBLugV%=q+>zhTg)tS?j;wan9LiydU-$ zdmp|yhCmXYazFE)^SXXlr*FxRdR1z1w?^>5P1)w8vFR5V6;NdYde)INjhCQP1L|2& zN=1qYE7+KSG&D41VEbUosBEi9vuM!HW>)oTw6*vRS?$mGcrA($0bz!|QOrD$z10Z` z|L1C9hY=W9)je62;`ug*X_tKtPgUI9md_r$vG^jB?C{^4ZHkoEOO_s$lNJq#3Wx~I zwD!HBuBt4pDzA+ByJC;1n`|Z-qQIdnD<2`2k(W<&ZG}^R+Gj5~!7|J24z!UWg35R_ zN8E!X8P$c7SCr>xSBE^P)+G+}#06>fD-kDa^M9JGPx-f3j%pEKGgey&UZJHtIaa=F z{Qib~q|H<$(4>L=jYv}VU_org_kCs8}Q|EqMn0uDnxw9w-^Q}!O%l#WzdFS_Oq@Cfo+ z4i^?>enZG@dueyv+d7!;&7M4@8UEyqNEkUNT1s&(;IyVNAP!5yZy$m-2el4=aN~V0 zU$9{W9gjpHUCm+4QGwX+^-*RzebVwdXxT?UHxfmfS2zz0lFgj=cT5z#;<(rt6uhIC z1@7KNS6YnMP8QM4n@9F1|IP|b+Ue_l3_nCMCF>Utud{P-;G@k<&FPO`@UoqKhZVAi z#0LxXNeFO=507=Q+6Gt=VVfTA#2vLZ1@!fI0FtB$raUy4C3OW&0|x|V}Ze7PzOs_5p4kQ6<9?t%qcnh z@#O%9hLS!RSFVWVDJp_d(&a4h zt%f{Ds1GuK9$dAXTAn*RFiI8558X5l!R;lBt(k z&m#XlGyls_E)-<$R?yggswmg=;A^4Nx#iM|3gfQQng71@6pLfFB2b+5I9f=%G`}-o zm$Rh!G{ZBl!eYt7lFz15RQGqr}72Ozw{oSaHE0F87n<*e6H;%sLz zf=(v-A|qMU#ruWxL{V}bOqAbwy`p|n47YfS^Le}7HO=+-%3O@bcF#C1$_)n8aZPE# z19QxN((rMuS#`p~_Iyf0g8Q*UoEWEGW4+z{RfV2O2rWH?ldu|*>2l!G8Ae^{TB7;( z2@tqCPnw#US@H=>!0shZfQsvnD^tEku@$sjvNezLPq)oGPe$4CA$ql z>SHe>_b7@O=rx?3!{uMtDzu#u8J@rqHkq(=o{#hUPDOH0{QUfIR#-%Md$P_JQ-%di z8`AsF1}eXPlM@X5jql;|HMkemnXmdx-X<@-5!Trj@bJ-6ng#!?s#16uzg4RH~*0q@w!8?gYm^m%h9!=?8%Ws ziLS0{w`aX*1E1k^FPZLFGw!=vH||k*5iomPjYkeGnGa3WWV__Z_t%(GyX`bI#yV+z zhUzp^e^R`n^9xJhH4B~JqVaOdP!o088<$$7B`(miL>K9ax@^KC4`;l5FheFhZ+TxF zRcg-uvZ(YS9GjKraAbwD%gM_Zh}00(6DAT7I+X~Gz9tW33a{x|P7D(hLr=xG8^ppM z2b;g1wVs_@I&F@-ogPs89X2?v;e%B5Eif6Cg(GD0_cVdG?C9^ zKN_9!JTq(Q8wmu4OyRezO7B0#LDi9wDu(DHeGe>F)1=>dV=R?#EnvFNb%k2gb*4Uo zPP(UKbF}=>pVk}fg~8oRa1mEJ+%;bL8&P;SG;-DF=;rNwatxfrxa(!xy@p3A4Fz6X z$fXq9HfUUmkdIDnx`T?vLB(`RXL($kA;+a#l8R3_FRd1|BfSAtZ(dG?A5mN=F12LumGT&SwN zAuE?_u;zNb+N7IcYLP)?Zb^3$(umh__#AAbCbPjFg6~}3qm(;2>=a!(Z*uM_WzUyz z`+^9oJ5K6bi}I1xJ*k2VYm$RvXz?S}ADz?T0>?uPJ;rJ8Y=TaFS-5htvpMw3O4Q*R zc!#rUe+`5-NBT=}%Z#4Lb$5FlY)6h&pOl&O#3YM4-!8mgZH0N@R^zfIA}46P*#V^l zyGJQ0sXCW6!ZM}pJu0D>&q|D%E4Gt7__U%xGm)Vt}{$KY%M!C928{v;Q8_?pUAn8b4Z&KWG8*G%rGeNS}X zM(-N$#uy@Bvq@h~%_){s?+`cHL2Y(QGUX zyj~|{kv2pWd^>fxrs!Co{^ZlC@_{BHY&(aJAX6Jt{lWT%hK7bE8WDkb7BrFVhwS38S&+g1MOBEFrHFzBMcXfp`)<*7n3>v!@$wxa-tjmPa z@G1&<9=(vNefKMZPFT|?m|RqQ=pOkkA|k8V#$;pW&QNN>5x0~0T)gx6V4*qhJ}jJ+ zBcZ3f*I;6o=U0W6s#|DiV2l})S?*zOHpO zDQ{CuR9Z6u<2bHI?pr-Myw|c>7_>0;?)%#;YkrXre|;7^`z6I= zhaWMwwA@G$cmGs5VLgdd=+1#AVrvYV8s!7>bv8>o8E16s@6#C{44B*nKU7zQ1hmFvX0O0rXjpKW4fyvru$2+*vjU3R^k#*N>D`&hQ-FcJ})78kg{bfT9_Q- zfX+HU!;%T3z2XuQ1JvrFhV>hj`fJ#V6NCk*^ zF89WVQj6v>fQzD6FxovisP+Zk%EUUmvDeiVkn{-Rq^YD{`&+yj5m1GP6p0R@Y@nBji51 zn5_nld%`NS99nxCa{CI&BgFP{J3W7WUo5Uae5r-?u&6k<)tR`8F`6yzhI`^hd9N1rEMqBa5x)9sCq6Ut}6~Smsq9Tekjkc|<~0NpFyEZ{?sA&^SJwnbkQIs*C2i zY=Mt)zHv}A)KR3;&c(pMz{>_N(=>6*%E`7Ful~KzuE3CFJ?-v{b!!id#$be-7;UGj zOD$J8$@2_~Qyg50sf3sZyiWJ2U)a6;@L>)tj`Bhur$vN=jH?md{mG(~e4qAOc2#X$ zP0+;#WYpA|OEZk7^DS>csUoz3y6@GzLCc)^2GV^m6zQ|ngX*PjFvumjC2ZX zY6Cn{QR2=vs5Q2U=u;ljEk)dn5YtGJYWq}A-(F^u3UTuArZD2zLCzvA%uRj3C;5$# zmz+hh;9_VJnm)(WAYd#$j@dg!_tH!oEGK-jweTU?kr9!&*ym_TZ4FJ@vL&t)focD> zyAZ?$IbKqg@*|=au*%~@n-LB?3=J1h(F_f1=3z9TmO#*1<6m9#=d&2G+h6A&5~3A$ z$U=r89b7HYMH~a=W_l`N8 zytcJ8fht1loYp3)poy*iVM!7``YjPN;)xfyc3l449WTGbsEJfdoS)> zVc^eb*dMS2Ov`XS9RCW%%<={0;!Z27LMDevWIVyi(&}8XDTyZ23IT})Jb9o|Sd zp%nF~Q%V+rUM%c->Q!95l5j(N>70wFSfG*vn?A@wTJ!gtI*}N>tbAV;lApWp;oGC? z#B)9j3x_Hh6y^%ixcp-_a;o0h-venxR5C9vQ-`wxGDytQSib8)tOG_KJ zo-FJ=Mo^jD+2i&pLn6#?_Yaj+^5n0nili%ASrz8z=fmM}2N~n&%*+f0Z{)~>$Xic2 zISExFH7{hLPykNjaGHoLwgSi8aJqc>Qu{H)H(Wl(nkkPaZ^g9Y#q3oZ4f9z)iC2}8S?j@`;sIV6hiH2O5g5wNw+ z8zi(4UAV?hP9NoH6p&Ri|07$3rGwv6v+u^lW>zrdn;V%wSW>99HyACReG8dg?n_io z6d3L6gZ^VUzBENWs@1?em97|Eyp6#pbzF2M+4lNsML^25>D*~af5lC zfiU>eyJu_)yI=^coO(AZ)gH7~LpDaKrr-?$-F+f5lKqIOl2do5B-qhsx6oB|_rRno z_5RsKBl%2|vv0fJ71t?#+IXyG@3M4z!o>@v*aXi zkx&!w7SvJccd+)s)P>h%cx^9g_Cy;#>~%4}>vw`TI*+Yk8Gp&dp&Rq4`a zp`t%hCGN8}0i^g3qg1oB{+V<~G>A{@!pd^CS=UQ8k!7|-8ZdxG?NsA`GA?hDO#VdEvIoG1e(ue_4U?d?ua8N^;nGU$+sPrjsfv z%JVdsFVf(6u2Ts?2`ovk5oty3Y|3HyUA$V8xKi@HD8GzsUpF9%IB~JIwy=nF^>agO z3s?a&sE4!7+WO{GuGXSAUPt>$Vfv}`scG8tpt}kjPe7TGzU1B76&p#^mE_gHMvOxb zvJF(-siQca7xZzlo;7}XC?H>IVDHwHA9!4Q`f<$R_H})ChwtoPB8Ln6Jb@jf0G>_;13NZ#{^u=qS1x@IrpG$8Suj3d!Ih0Qa3d1Z=0~VDx%Ar>(wcFR z%;P}07IZUoIR(+{XE2^{cHFN0`aK^=&%mMi;7`$Vt8Jfs;Z%|RX zQuT^feGOV%x-B*G5$v$B%~kHhGr(P=`~(BfX_^Ikou;!I)XUtxaM-sd2qch@e2YET2{4 zjqx# z6;a=&N4r+t6a)u!YS+@|#Ygd9J@AxKohZju-yI23gwJ8SjIk8T-!N`KA&ex}M8h9z z`gO}~ntrof!Z`+m0%ox1MmA!nU1_N3DeR<_i-e^@&o*`9 z<0M33CGhB4HGnk$s61N8XegII*n0H%lFt;Yo@*0s%XRmlw+N;khUT`E($r$!8Yv{S z5^(R;H(u@7WS!pzu9AeW0#@R0SMjmW0!(ilSj9#}S31uACU?|y*Q782>IasE;(g-A zzk-9>NNpj4N|}Q+_k;y~j!ltwlmOsgpSi^nSjC`&&(6$;!iD`H@Yu(I*=ljlN?19AwMT@F9^=w(o4= ztG92z%%)F`rO8*Bg;>nszv+akx!RzQy6w-)FCftkb5&4ikF2abuuByLmsOQfiRUQ* z;~UJOwYz&}6q7D@o&cnG$q>pMQWW=)<$Z4DP@K=+lAEwGf|RuKIfKC~;W^yWCr!BY z=<>a9T;Jb3LEYW%Uccdl5@&l=c7@!fcR0?~-k#6OSc)z0jJbM%uZWWQW@$DwuTji< z-Lq9hnCZT5Sfl;s$g#iQK1^zM9of11x1I_w&)sBjrBE^1zOU;5J|<&h<1NT%6_!8D zLwW=bcZ_?1ra;owhDY*~Mvllp0}Zg1-b86=p?u|u4@)+!cW%9V|Nd{p2|FDP{BB^nx2}?&Z~m4Q@_UtXy%}t zqk`A#jy4vO5}*ghI1$tv?lBB~N#d(27AYKEH(o^FJ%}}$ySbYs`-;DyD-54)>8e{# z^)VyvPqp3Bvf6ms$5U1K#ToSE?&0+f0h}27J?-PCXK;5Gtg?kgZS-j0-Q=t_w#h7w z59$~rWo(L0`lE*Dt|edZM+k~d>)zuEnurQ|-Z?tTK}AZF599US=KVvxfgw%U`y_)% zq3x-`-Fq8ypQkJMR$CqxqaZKJvRH zaUw0dJqS@K7Ii+>5rp$v&Y$2h9&eni>ZRd`)#buQ_9L{rnZYe-c}Y&p&B6-;e1L7% z;FF?N{YR;Mbf{iX@bBG%PeR40wzefeQrNhmJ|LKC*vP8>I{=I`*7(o7h!`U4WtS2T zeYy<^4}oIh#U1SL2hkZ*KNQrs`~>Z~z1-*^rDeH^Vk?vz>kE$@mNQfgw~Lg{>yX{z zGH6lTnlolG$7fa_imLz!sp}oT@C{BDzmHbse$6-^Zp%-zp)sVS9wez0f?22WyP8Bs zSedifCQcQ9%j+8PFABo&X^~;dDZ5_MH<$)MKk-GeezwDP&DdL{dVSHa?#QLVMWW=VAJbZ8 z#`T>RVsA4y3*C@nk_6UHx)4(H(ynJ4A~msnY;THXZ)@5S$~a<3O~L5VXz>Bo7F#eOx==C1dKnELp8`b@G5p+-=!VQf0kHLG#qv+C+$MaW0^|nZMwDT%v-b z-bCs(j|z_Nc8hk;X>#|&C%+oD>V|!0OU^$`E;~o#pY2H=+3sq9iKcRDoo<~ezeH-H zs9AHI3(%p7yR%gq}{&<2NOd~4Q^-Vrq z(@3^`H10<&)$<-S?aZ|v@x$_PTFfygwdU5i$8%u@Yr?FV46O*w1X!lXU-0s8jn?>z zExlN*MVwx|uekK*SHgQFkqGeEU45k|L$t~++%n`W3LN~330Q2_?nAB5=AVUxF^5}Y z@>@1$sA2?wKimuSZ3{;FuE`QkiY_olP@S5efzJaO+!PcL)EOGMrsm%NZ{k87UBJNs>IFMHB)uEp4@ft3~2DsJ|zb4IqSt{LmoowI)dJiO@_ zw?QN2$z7w`*X_BH9vU&ro9#aRVYh_628UX78-l(Ei@P0^rME{!#5)s))Nz5^sI72= zj3IY1J9ej)#znstFBzqt@;993@vmP`httiwT?u8Q!{d|%{LREX@m)8L)rb!$t@%8T*%-Pqy+bnY!2YP+nA zjB902_vuz|D1s7|ZWVTiv)%Qa?Q{k_cf43vlCCXmud_R68eY`=QgnH9c&Fbl_TKuR zT>`Y@<$Y_rAN__2T)O>hX}KShuV~43PUYo&C)0g&;cWJ_?iFxO0LIf>7Fr;6HlYiq z9)hMOH>|vr<>trD-ECf#Eu{0o)7FdpUX!^oMmSw!S5Nmz_l*>vcld0qebPrTT)LdW$^8AKgxK@-oq+N_iW@75 ziDwS{^=td9l&EhAv(9B;j)+d}ulYS(3D#h(l^Lpc6aq0Zq{R}V($ig_aOC8C#=^s> zl<>Qb*soWoRx7l@vSLE-R2iU@{MdNz*yvcY-4$M{Gu?#(Y&+x?| zui6SVv^&_h3=wIZspj1niV~z#y3PO9QZ4=%Jpn8{D=Aal{ zbv57s?eIsiLoT)G!;Acpilogl`-Yd6mOb%z$E2k@o9Z;W-5*KP`zpV89_g20SZsj+@G*erL$Cf@x4Pv|sQKyu(jq~2& zjrug8)GoE!D(Vv@QF!>Xv&RuUblh+^OP#ie-SS!=N*~(1uaka<8_I`fN-^0ST>re( zrsW|zxvp-LqLMPKDQUM4OkyN+H!pOJC7Z3KMt&S^3CE51mBi8FyW` zFV3GvpK@}$5z3&v%cfmict8WYvDZ~&xtwlfW{GuCNz;IDYP|~eq!A%ymFVmQ`k#$d zxPJUere_WqvXYSC%&QGE=Bg?yOH*DH9B%q<0J02XY3gB{+rlb-axI|EZb;3$I;JT- zOCRwCkmtJWPeIGXMe`T%e*lu%P-HBlyWGLj*T$pcRMl%UNn%y_CX$1gbi)I3H%zrs zh;?09mGwIb9DP{us_;fJNF0#R)V6l;tdoNKQR@h@J1wtkEAT1{XyP~#L|#%5fm$4i z54(@;2~K|7jNAP!vcOl*Y6z~n3sled>PWGw-HTlTj^ZK&M@nhVnt z+`1nlzy;BeZ8>w1OTTA@E^FL04+~ji^V!FNu}fW1P*iJq7?H!t>2lwVc29MRE?N&s z6;zqCs415&Q=;InSI@(N7v}Kvym8-6>kK{Z^OM^1nOL04FwbRqArzRe+ER>)qYU8e zKUg+BH2!7~1})k&wKeU?1sWyPmv=WV+lf@q=RhaHqzlyb;9u~?{OJEL$NaMVS`AI& zzsBGzQ?A+Mc1NE@fZ%8D)nl6IJn+$A_c`?j$vOW=FW~X?RFde0IQ(;L&xbc11Ya%pj~WeUw!55^(T2Jf7|1cB?e>==Y>@a&j`vgGE$S z^x3nuU-Oyv_P3Kot2i3*j;b?B{EL%!9_vQe?SK$5kt*BiT6Ny*NIHc~y&C(06U0q- zXPcQ^1%>uo+2*_uF@e(+-X17@*po_0j^I8;PYbk}Xc6tG>4t^Kal`P@AOL3+WbR>3 z(8Ooet3xsL(1P5MjjS|+@}DHELX+3Ha}C6I&I79N0x@8pxP{j=-py8&goOlJ&to{O z^B{S|9zl@tM_TtGMo$b<&o*Lt*suen#OjTgLXG-sHILs|_3No$pW2H^Ant>-2hQ3>+yK>c*( zY1l;zovTiyF;X?0bgLUc9$4 zH9a|=R95mj9u#B>(?dF6{GDlrhjOm@ZI0ra_I`{U&tY!^i|B;M5$y5^%*~?Af8M9` zFgjWd0Pz+#{uDg59CTnf+zc38q)kPB2+*GIHq$47U^|`{ABT1;DJw=TD!1zPV*ggo z^~u!`a%azic0!c4M-!22_Rp6pKKAQ*5NkEK7r}=jDXUDAP1w4JZP=d7L z6Oh)nHhnBtifwgBD3l=ZC}@}+&V1u)JP9QtqU3+|s=MpiHqbAN@nqzNghU%ADYY5g z0`~gK0atyO3uC^UMgyB+!u$ehD{}?Z2_7SawC=1&&=Z`QqYDQ>C-|!>eAO#x9Q$Ae z95}Rs9;ogi!ldfm-98tBZ%Ao;fUN-4Y?a};ej_5~77e`|D&~h24=X!EK~8z9Z+=Zv zrt5o$EZAz<{=g=o-Y426xgmkMS`_ruiSF!Js^$90ZEKMwYYTdw9@{UQngXx=uibfo z76b0hC}26B*w2So-qjPKzE^Py=at84lRMXa2}kF=`QusYNi(b+Rk*SsYyvb*3FDbBhu` zCElL#m;+GZ=!98MVQB8Ku*aW;`VeGBZk?~M?;}PA2GNuLUTePaqnx3OBV2LN702{dcH0x6BB5=j4f3 zAPY>^X(&xU`5!u-oI{9b`T0?~Z72;1rqLYZ8x-3qwr8AqV_%!kHVGsjVWTgs^^w&Y zc5@M#Ojd=~S<9`9(!u1y)wrLfM$#nYJSAcIEDhN(HM=jV|2V*CH4u)CEsg>6_^CEJ zSue~lj$g02g;Sl>{TL#{gEIqkpezxw#H3Y!D>Zi7Bn_KwK%oST;|K~mW!s-#>d}{k zStdOZb*FBy=ahFi6LO^H^umb z5ksMhiu7ne_kt>4tZbG86ZO(Y0Nt|mS5JgoJI;)w&P9A|AlDG-0`+Wg;A9tZ$Dbwi zou)iN4>b3?bWc$8t`sMWeJr`W>BRC-M&-w-XgH;S^~SHF{dBs{q2b|cmTCs|R?#pt z5z*(3eU{cvMbCLGa^;NB5Ut+-+szHsRna+L5g{Q&d4F^cVJ(3j7_JE~b3BB&K>%RM zUHf~13^Ei%+ZC&Ims_qJSaip|lLFaB)VvO|N25OgW{<^uGUpAzbSZsLBCd1bFY)u~ z%I8IGu?gt}h-JOVGYy#$udA}bSgqlcT5%wZDAHb*nXVNCQ0@az@Fy;|=Wx0ZS~dj* z@{+15jh&xe9wvd!fRusP?aujnGSo6M)i-hMX{aQY&PevydYUMjZL-#6ogb`qZYxhm zd!fC?tWwmi6;T^R2CmKq$-bG?9M(70t!=A?+NVa&4jnBUMQUEQ5MOs?Wo6t3+Z15o z-kNDN5AtgUlcA(0&mdT0e8LTlL#;~lVrh7EEC9K-G0-zHw+;T#YWfe#3=ypi7 z_>FxFIC?|k!u)S~Mhh=xK{B6iB{5y5;7xIxPlA^=md{_JJ9&lZRl}Fh3D*P%hKqDW z$7i-p$*zFt(RC2|wlR_pwR=0qvHZcN)MdI4a!Z7As^etZXF5#laF1a;6%;mN(Fl>mikP`t}PRdNPRRTC-q52I~iq zjK-Lo)pe&4geD0H2;AARb*Z8DFMRFP$2=da1|Br5z0HtR=1zxPm;fNm0gZ+n2G)6X4KbKz{i+o5lu6R*u&?G^b` zzuw_|l7vx5d&$a8nFP({qa_4v#d4fLw@(;E`alkwY^U{-8K$2M)lCD|Bi`jH>oE9e z``c5JTb>o(h%aGHiFF$53t;GJ9|#(PkdaEi>%^e*T#3Wo_i)+ za|t?CTN9NYKJXamJuHY>bNIlEqf%6a6*t9cte#PC?$Xi-a(DYkd<~O0U2e^{EkH+) zSWZE3%$AmLaP4RRd8fbXE4&A+C?I{8p_S+D;II@&tgaMIu(?VFwrs`5*M%cs9C$0F(N{YyvT{|dXE)qVn~hh*hiB%wx$aS&IajMm~ah)&(On$ z=bLO4rVPI8?{#?LA7TsSx06-D`Vcp}Chk?`?KU1tzNK%wK=bH@&+6Jy&pz9q;D|rg zO;S@Q3Q1@4X_&xxCRVa!N`}H`43Dma4d1= ze1(^P18d0#F_h8Id5}242Vx`%E0$c!RTUNWR4(m9EoI=T@WA5is0ZhNXd~jV*%s%Y zDpv2BIp0_UqA8{;%`U=Ud5y+S*Ez+VwNP#d+Ng6$V#*IcfRL~|9rxN_B$a`*qyF%y zi&{uF6uzu1w(DSQ?9?8JJg2PHOC(i@MUX>`Un*K{hU$4V9WO|+1P26+uI=$4LF(hl z@yUB_f@667yHA4~BpzPKd zOaCFK_()aYOuiM+G~Ro@Tq8+d%Wlc5&`PDDhsmCKA>Zc{&$`Rap16ngvnU0i!=G(MsK4yPEczjxf;Y-9$wfY{`* zOc2?T{RY^1+;z0>Q&ZyRmotb($0uEEhGl8MnsaX6A4r(ifuBbckaBIAX?%c!w=O}6494Pz&}fgbU|Z=Bk12^f^JoD5hW!4@#{`@FjS3nNJ8jwE z->C4!?jM3I&PZ1n{rv%P-h8dZZBU-Y?5GicL+YnKcucK|FpWy9MR+)EjWpSku+%^B zMkr;v+)HpFx@BW(Yb#jd!zFbd(Rwqb$xdzzj{C`dhsNqvzqeF@^VH|9pLPCbXZGIH zBbmHx7>&5M<*6bEThd`|ri@AxzXv0GCTV7m17P)oJI~zo!BV~N_RZ{j5o|t!pm$E) z&_ty5luJUKu+pNSBjUktj@3f%kRER*EqwZ@IhfND+GXT#D8(R>Kd%VK6FYyxJYAM< zN#HoG6wz~#pV)k}0YDnU6O*Vs(8)W;uM`099Qu%;E=qhLo!whZ(9RasE@VW)$ zE|Fc%X@$EE$3X?}uapGn)GKNUVBPUK>PyWx7ZyAT2~h^zaL>Pn?tyhZjhPWSiRXeT zql>Io?X|f+}W|LsuHlVadr3(a;kf-PBoiTAf3+LVPFWG zxhRoKK;0l}a_<{%O`Z{yA2gHT3^!W5Y+nUytc(sM1vWk6w(vdSk&6%X2l^ zGnd~>%vS6-fGe%&jmH(r?UvZP0h7JVHAkgS+Ri5EOi_p@e7-g8=DqOP&V_!7rj?%X zqo98`Kj-5oCizxzt!So;6KrqbmRFXcu?z{{uaPmYJxcJxF*ltQ!WWsVc8gI zQP;JcZzsK~= z^adA#Lkj!bf!Yq{1LC_WMAipAeTz8zWk{^veesYo3_a>KuA~j|gE$fI=ep8zAF|M6|ib;VOEBxyS+n}BaO25f_o>MdE-KWsLDUl#m2xdE>1TLfB-X+ z#}568TR>bx&zP%$x9cN2vZ#gZlUTpDV|FX=MPXXp3 z65!efh9Wl1|9|uXz_0vV z{qN|1*`#)H3CtqNm(Md2`eX8^62J>_1tp(8MJ?1lLzkXF0QNn3*h%mGDkuy#wXv?=NN;Zp8FdX7s(=V1E`(8@Tgdr*@1C6 zW-<=gvRT15|9JQ?b>V{%r^Jm*SJV|)H1Jh{Il@*UM<2Aj8u$L9bRcBE5X#_U%=0Sum97sv;WRw{oh>*|1&M}e+|0W22*q;6LsX(`ta~yU-ip( zdnZ<49!{OQY-MY5Nn2D&NPl&CHsDJmit))&4VC{Mskd=$Fef=ds{g z6$|O>W|aSaic9i8k)dPDZnqTkg5LmY7FV}flHN(%h*Bo-hJ2wb)C~8 z$o&AfA5WBuoV=5W-fOGCu-qs^H9#6tO08R1`mpkWYYU#l7!#fCi%&y=Oc*{t)nRM| zEQ#XL+{)6Y+$jjAz^q@-9)@5=Xqdw%G4geKuWrjefYqH3gSzcOlqwhyECAZ=6ohV~ zRKcdOk?`ss z8miZ~a~=wj)Y6Va!`1a91A)a|0Wu1Z6&xJwBex$2(E|pJ%bTBY8|@k#Q16)(&=F%eyG5WB zJ+R(#zC4H@c79%`0wP2f^EKq?Ek$S#=&B_+o&stJ*dt?Buvaqm_ z=3eUnF7PE7A9i~Rd-G!!{i3YA99y@HU7hFw$K&$mOv44b%te?-=68>OT(nE-F@Vje zCzIl~%jPC_fHzZaJn$7siippiQUE96Ht|j!LhgBnAz9Hx&Q$(U))IiovtSjwHCcs9 zP-~9JTc3>KV|NNw6n%kELqFh$#&*A+_g_f1O^cEFhbw^;b>641Za;#ZKCkvOkb``k zoG!J^kGtnRHy%Ijuf*!Q9-~uC|6iYkY~9sr^d$|?i}RJH2?01d&*mPMCJ2nVS3k%! zQ6fR9Jk>lSAiz1;SGl8PJc>A(-EgJbv@ix*FF)t2cc|;UH~w~iHM(VO4f9Ywzh!7c zU#rQyP4^;$`ZPud7xSO@QQg*lZN3)oNNOu&za06k0XkcRNZ`K}*}{g-xa1J-yF2ar zHRE}he-^*m`T}@$0Isj+Wc9T;wo|ILwoXKJd4`&|@J*)nn?5lHemzs2v#XyfYlRgA zXe6AQ$Hr=vS4g3bWIyMCr~3^Y33k0;C+zj7&sRaJ|`;u$hZC%>_QH z?7M5TZfL55pOzMwNNT4HORk-pUlz+Vs{_$F>oF1U%1oTJ^+3?jWMv)_P@vzyU4i!} zHkZJ+kCnl<3#vAq(6KOVMbR!hTa1<%ZTR7!siR;Fh6R2~ibn$DsMaHW90bXyQo4UdGi>o$YbFYw7<3q?fF3U`IDKc7seyE1}p&$67P6GB?MbU5-RSmG%^0(>uBOT_qSTmst)1H$g)40#wk#aB7V08vIJO^JJ~R5=(G}+-LJ@AFRcVhq zN}tYcJj=_^@18)MMo(3l0F*Q$JSKvL!&I9d_f`~{vn zs7$aRHpP4}%RQQOUaWPg zs0PgUKgI2T%jNbj6QwqM9v$v`GqajFOM3IbpBV($0Krvx^Qy@T@13 zbsTQ~+FTK4Gmd?W>}_65A5f;^agjwgY1lvqx~$Z=N^H?F&K4RThwc7%Qj+isSsx?U z{sYlq!^o>yc{*90B@Tq-Z)+Yq9PK;C7B!yi<;A8xaT8kF%y*2vt9N-l+-Udt6FF7N zPq78!h8W~Pn_Py0i>>;;jq5kI8qNH@?U6IYN{2jS#`e-_SPG^F*QQEB634@0hMulo z41ZKKQ$Nt%tVB@q-`}WqSWG$7!6|8yUoN(I7S}&ya>wq0V`pe-ME!PDXX4G!=hz$d z;6VlJd1fM9u?_CtkQekCKzJCjYyvusHxDhC7>435Rx1M>e%3Ry2ygw=;cF+M78R{G zka4pVE0~X@Q;lMlH;&{_wZv>VI#|Ack>(8Q+nn!eXuOQJhx+bElnzzHa@49IqF}n(-rfARc^^0byR{ZP zhfOdidH|3^vN|X&9<3*?ObG2x-wZBfZ=IAwr7hifxUOLStazfr441B|A`^N{=7_JS z8aT9beCL1fTEw7znG09r;kOf&1kY0Y03iQTKyQPL-r&`-Mme<<^K9YutnqFYIkrK} zpgsps%~3sYmzx;Ue>|Q2kp=M^Am?a_g_$~+7`mPaW=(?hH{bT?z+$T13uAA!afwlYL9jd+`yTg^sZ{uzER%@T0az0VH?yngJf~&Nk6Z^i;G* z$Ic*G?LT+Y#f%|Eh(`Bx>pjsFqSx4z^AUis1xkSWBLWGCrjv&~d_OI7StNb&NSnG& z)x`B4Jgq~_XsPM_f6X4PYPP@5Ujhn*RD}(_ggudD$fmYhgEm2>Ji#_wGnFD9mQ>|5 z-4FPrZh(P{Osc8)omFYB?sV(<C|Ce-bm`6@atiVS2) z={nbQ#RT>;am&}$c4Mxq2F8@X3T(XEGSjcY+Qq|+{P%1yJY}xVzh@kTo&3Zw1;XbL zAk|>jW9_4xhE3<)uzMWu;}DM=_%Gv)9Eu2W`0)C6Ah|G+1IJ#{)z9z=Bi9Ir>t_9( z>02O!ZhGtnm)dnrrenp?kCwpaHd|q{J=qW7v1cV=gVNGJ(l2dq%X^*#?loo;dRQXTr%wK9(L;FO-)P5Jmm5=}7 z%_g{g4?n+ggtJ@LQ>+O(cJAA$J4h2pU4 zHyacEA`JXcmcM~cZBBs8NtG<~dtC_o7pn?D;z6B((z&T$kDBJb>WCP_l}IhQ4z-c- zuGMPE*tWrDB~r*U7KRnNE>PUuiLo<#!AzjDt4`i#1*ZmC*L;^jblz{v-=S7wa1CuO znl@AA@_T&?=q=^)2Gd-=wCSt|2Ti!UJ#Io>o|SOr8qsf=%IFx7d|6sP{ow8BB#5Vp zoTNzvlAHeX|BJczj%sS_qDQaSYeR*rph#B{kS@K0N|Rm_q$|Dm4gp04QHu25J0bLh znxN8q4V?t(1VRlxB=30d_x)ZOH-W)f3xuf@XGa&G1tS`pskt4OLWU+VinU9;8{XH%Joxmui z*;4O(AnL9_&^r6k#>b41U-LV_(xV^W2ZWjBDU3x+k;l6mw3t&2;H;Ep&IY$h7L5WT zzo!3jphO9#8q~JVu%~PH)`c{cf^wiuxodW7V;(hZ4E3=yBpX@vL&t(Aa0HPu|9V$L zbBZp&Gqj8rdu|pT*r$sw(@~_jXpvdE8z1VpEX?Yte;n~8N^gjqJpG&W3rrB%@|tkC zo^^lD>JX*@q&&UPOs0M>c~)AGa#~u}c7))6F(|=Z7SVo$OC4 zhHGW?ZHz4K^a)T@7k@*T#k`fwrj;sf`K(MORj}U#$c!KWL~6 zJm4+vAO86>@8rfo+|of9>d|&e%Lb65;WEKrYu1KK0k!lw59n&9?T>q$4ZqVdMb1+f z)&d(%+B}9RDI;8u@t7#ZPwzd)!ZcsPPX4}#C*(fop@G~BFQ#aeZU1| zPmUjG6_Y*Rf_zXd?7T91dGRe!!~O|9)qjiF6RPVms^tK$2)xC6UA08|D&rLxrp5JO zu+2lC3{WNSM}>wT;_W<&HLBVR#`}K70=~5d+Jd8X6=sDs?SFNQtEg%=`>h0AcpyM( z?)99;z^(yRE`o^4yDYQSYBm4l-zQXl3x5J$##8%ICkkQ)#j|+knw*<&@7!Nq1*mlt zE^J#y37Syos=;ZJ2){}6w+}}^F0AUW`=wdK@J2Iz;n*hh6x?JzK<Vti|PuKC{2xlr*Eb7qU$+|3sf9UIJH!P z+oP_L(bFAMi*tEOPGmYq9Q}i6JbRY*F5e4XVF-%;!g5;z5FO=u`W#Pq0UaxoNlDTk z1fw@%vRb}_k%5gYKP0Br00KSdslF1@5T6@e z6IbSadhA;SYlQ*AKaN#`O-6ipXE5OTGbfODeD4w>02TC2g)4TA_0`QX_j;BWr`Z1) z^k8n672nPTB5iI1n390b zzi8t_e!y5paRy1}dsnw**Hd@8h~ee^H@!nD9g`}3#F%1+mtbvh?TI8g6|Wtah@9*d zTzI86R8w8^ufQ?c#?m=9E~{{^NKs0NGJBG1<$Ladkzq<+r$b%9{D+at`Q=6BPCAxk z%h}wML?jjP@7<#~o&mLAJOG*wOxMQ5BI4>&5@zdg&x>?jXlkF)&pTJ9?Qz*ylLKdS zoq!Iz6dfQXyYa8(1w|ZUlxfa--^;I293l_u|G7@uq%B?s_4ksF5%SX&cf?qJBe3tt zpBYqEN`9F1|CD~dr&P^-j-92nZdS4mVWIu~seo0u!v7fqAzSrJ;o#!0TCYYiOS3o} z;cU#7-@b09x%@Zo_=gfq@tc3^{UpIRj)jq%PNp4W>i$)xOys zg}B~TUEMLO9)6PslYRRr7wdbLjT%ly0iDq@%8&Ccoy%c>WG>KJcAal|e~bJ09T+-~ zB&SsRD&F)wXfD-5L}y%&iu{5fX-vFDKI_tj56|`Y&kuBr<-f!~4B=M0d8P0PpR5XL z3wq|y>HCBblw!c<8z^P`8!UdxdSPsY6zlZ)HQ4XFT0lq9ELP*BJs&ik#dFPgjPrw2 z-qK-m{x*JZ)naQ3?mC{Lf~#t5^amJ1tjf3OMGLY$STZF&eQMkH|exFQT-|ZLuTmc!;#tPn{Ry? z-5A9gEyil@{KHr6)!t#3uyydVeUqimua4BWiI99pl`F)d5aU%sqa{u6H`Qdd*Q)1a@?Q#$Lz z1UMO&EdQElc*_g;VW z0VMPzUq^{$f`6*5k_}ifh60v`u7_7A{bX>z5gwBr@g>?5x?s|(nG>n2`dl;}VKIogbM4Zqp1E8ZR;;4UEiOXNk>Z=QAt4Pw=>|LCmo%O*F|I}_> zD9MwJ4t31coXyo(LSEDCo~SVCLE5(&Q67m2CAW2HnxpOl=y2@5O%%YnFf3!qz6Ch$ znl%K$L%-_bb8whMnL?A|F}#Euu^0Ucm>O20OA{`)PHO(4R+DH2S<#F42Lscg=MkW7 zh90BtK+~_E0V2fiLFssjSye>OKw=`q6`MfvgFj!cb>+)wUEW-}Sf-94Y@-z^+A5V=PLJy3l{LksB#w4wE%$|} zSB@JKvc>&Z(_Mp;QvbSgHIhXvFL06)C*^{zK0jv(k@H@43P->I*_pD0%cDNm%jmvU zx?w^KTQ81mPtcC&3n=p0Y4s7_Bdu^p<6;Yk{>;)!<}tTsHJYBaV1&56k~;bxt4GUk z4w72<7S{9uGWctI%gMSg`a_(Bm&0tiPZb%}_UP@e$wK1Hp`E>lSh=yD?yb|UN!RK8 zK@J&K*5z>t`*t6D>|D=?>|#Lwk!g5=${f;DP31Yb0{iH;CVcgmrLC2R{Q#h|za`<8 zH(WRa-?59r1)N5>BXG<<{8Vb&S=mn6w4*GiHf9GdV5;?6)0OGM`|7 zFVvTlFEf4CF!}J%GGF|#2^8%-gEB9k>?@J)0-8+^FeSlm;i4LXf`Zmp3ik(Mb_o6q zj@+gEQr{;suhrIi_EIs++{;dxgF2=U;?qo{KF0z1uDIr0f7RO&Z8w(p{`WDJ@QGQ2$1EYFu+RDeE?|P`4HpZUqXkRkZQxC za58%YwcK3V0^+Y5L+u?LLC5HDsu+g70pC`Vz#2bi!sq4O@a&`s>};jh-r-)r{zfvo z?>V1o(_^8S<~w8pbxf~R@cu@ zQJT2P(qACXHE9-33IElXxoT|GX9j(Skw+C#v^=IEK!XN^RieKN$Xx!b9SCR@&b9qy z&2U72+pmI7GLWV2JjH3SNioh<);4{-XOj66psYkkd3sef-^pHIx2wzwczD~=9^d!< z^sB*>Or6P>tAA57g#6O=yO4aZ8qp{z)xnXG5XBh?%iSdwx?Q{0d|v}sR;!_=MkN9H z(FM(u1g8;nU$4*I%C~T(A-U5u-7jnUaS7MR#WX9N+U)17Pl!^@+J00#=<9-2pC?r~ zr}Z1R@U1Gf^gmlGgz6z@M8-klzQ4YVSFUfctGw?wK|~}RZ|;R0ZC~bdIEYy~`B8nW z|Kd4g!!af_a2$y%pP3k$K=_ovC$=46Of%Nu`C!-tvVQUVrxNiW1K-#JT}iD=e{(+m z5~4pEjKTC7@FlD5Kj|{fa?O%C14T<56B1lpqz24N z!ftO>;ENOlulHm<*%(GaRX!N4cuXohzIVd~H&MnOI50l`8f9$Vb+7i0Fu>NbpRB2_ zsE96)33~PC2l*A={bBS-ECl9k^;^wd8iGvQU=dG$q}&}iSTMv00Ky3w32FIUZ3UEY zhr_V%IkRnHl-~jdI9N2G>)JmiG(B^IM~2Al5qfFw)q)KT4dt&MJ5SW<%uWo1%gE(q zAOWxLvUbKD)P+Mcf_LQ?CO0YzRisNiPSe;(rE4Xh!Ac`~-jR?Mrmb+o5PH9}b<#QR ztfNLQn8$q3j6MC?2A-+;2$}`|np_!4e~{+$y3DIDiMX%|_U=M}ydCUuQnzuF8&lrz zhBiE9nti$U@x5G55l>kcwAaj%M)U0FzYR-yxR7&_W5ypDi@83$q^x99G&J2e)AO=z zN1>~|@7;S;u#?HpDB~X z7#RZ;O4>&7z2510QyEiW3N9r@{Dyd0gQV$E>vWFh;nazOVI5-ML_7B8y{4un1J}Pg zJK4J_+Bu$BTiLmcEaDdZNTX;#hoV7Js<+B#g-zwg?HK7`u1ZstVSLq}0v)U>8frqq z!o#O)vrodFCZ9*;fKVBDHgY;3lkHOfa^E~)c>8G$jnTc{HB-^N%1WTpc{HYG_ANCr zv6a1s8^+6PH8}H5J`BEOoo;ND*1^H9@?8h%P$=IOw2Z8Qv+)V^;vq-5#wvfR8ETWf zx#NW4cDnraH10KFroHJ88M=GqU&*WcZL zADd|K8$2E|N~FN04F-g4Y1Gz_Bme$pZIy$Y(+(Ni^8%#`TTQ7Q`S|SJ*aH~O&a-tv z8y&H_m8D?#{`(JRw0vgK5z(_v-t{Qd>KoMC4*{C@n3Q7tVV1={_?fz`>HOMHjjI`U zYnu2q$iZ0kk}N=B-I|9&)md4QTBn_z?GR#*!AxPszQiU?5#7#C2uvx;VTPjB4x8(3 zSaPrlJ=_m87zS@LawWukRXc&~eaTr^Fs=`^Qr8{53}yJwA{iDGepX^qnkNm2t@!K{ z6icHWDqjO)(W9CV57){Xd8J&g&dCN5BjYXoBRKliqu4gOvsHc=x%l`XDl|uEH|C&3 zIdSRXqZ@H@8HtH7(`%K~wrn%kZ&4(^g}eMZrEMH;IBAD9Q>tVd;%1AAl1`UkyaVBc z%>cZTw)U%(n4OgX_& z-=cBvXCILZui>K~tHrh^5Kq@v`E;7;ZkkQTH#^sEYvv!su&tGWI{-5y+}XgxR%pHE zNyg4pg+H5Zc(_X;pssXo^Sa&Vn@B9xhw?I}b)eIxm%=N|{-nW1ME;Xe6hv38Q zmukB(%~yW7QMCy&xOw5XFnNm9mmx@{+neZ8J-z5rfR=gnHXXx#=IN1q=TCFnq8c9Q zoyLZSF0*wW_%PTtX(T3)Ts!T#R%wnbZ~#qyk{g_@_S_kqzZQIEKa%W8OWd8`TFcr@ zODdA!H$w%Pxn?ePQgYiCOJwufHsAE=er>fDsbzkM1EJ1WYeOwXU1nrgG7SgPRLk^? zsgL~v?j878-$N+S{O$Hiu%2Ay3#15j7mj7Z9B2`a15Xc+CwIP4?|3^l-Yb(ZLwVGK z4|``QSr@6}_O+I1ALkla=>zW*6MUQUMzUnp4_2(4h@hls>mVZ&buseR3v=_K>FS_2 zZm*wf+XZh%hVv1WzB>b0I^NXu(@qEX>Db!+sxoPpt6_}dTk{2PdBZ7=1a&R`L|aB^ zy&|**K!dkGF_T{lIQD(+PWiJi{_l2O&|Rt-rm{R$cVEuxndq0+uAS|jeJuHjg^dan zBh~IReP%Vao_Ur_3=@#;An#TG(6j0sQJifL;9oWVT*g1sKI?*qKa1R{akAWKJkQ_KKVYR7h_giNhk(;v`}pz- zt#a?u>cDr+`KvB7H3tKeU46O1*_iqBqm9^TL#vj$^Y#mf)FH66~WMFa&SBfn^9 z4ZYG(JC^m_d*SH=#roEC%PC-70BHDaE{qSd?=WOP)B3xgEXO6(mh z9y9?AKl46Zu=0&RJ;w0ei0DZ*G_)N$U&FF0<)!jAQ`%Risi><9iU@!=abO?!Ta-}^ z9x@NPxuy7I-RF}ax`q6~AV3tt3bL)x(VZZ5#LR0xh%kdWd@6GHc25}BtD!Qi{wy^D zToMFb%;SpqXS?P?zB)BcHT*RlAQ)9)WB^tljpC}%?9$ANw^GG;5XU9tFGB(Y0RxL> z_t{H5B36#qBOdX1=oN>%uxY3vdIy!Y&U%wZ=0sP<{aLVmX8{2|2yka-m@;xm<<8x^ z@m7DPf_{iH9mkg|s6M(BwM+wS>W)^Kb)VM(VMR~}|31_Q4l9?ATfY&H!a{eDjI!}O zi)iuL6?PD&xECrR|d7s$;uJ^~6A_L>DIGfPm9-lX`j0!#=-bga)Wl=Ee4&G%9F7 z6e4pmF|diHxW(y#&}{{bOk*X0sza51aRdSAbw?nWNl)p;O4z(7mzVX`Zq`OSr>nqT zOJNEvPA@h#;AIAUf_$7Fx#%)+wR^`qQE7ipz(DBj?L+b6GDUTm`0CcdE z=n~z7Z4EP)kqnJdmA@_l|11SE`q}F}t^N-&s`IF>QGgw>! zGZj7f_mN(>+<5=0N~uH_aeJOtR8V%U|1i>Q;4m4l2I72yH$rnng}faZO$5ROkjxc@ zV6)7WwR%sOWh9Y7F3Vln1A(rqYenqr@$Av~5oEp*j1;ill|4wZS-~9!1Ozb6&!P@i zpQJVf&l?@IUMxZ@LtZyJWa@JV9IX?7^!r)`jj`XBJNG*y?t}yex1Th@9a|1299bZe z2B4#s^CL%ojSNAzZgDlWuXJNrX&<7#05vzVs$-Zl5wYhnDECBgq%o+c!tOdn4EKmJ zP&ckbHZ?YGT%hM8`_%_bLp=gjx;Z$Iw{Hx7qvKc z8>Dj+*%}QioY0^1X0fI$TqLWD=~GQHQE`0=8lo}$`}EHMf|q<^U~A&zE;%o5`}_Li zNt0vT(=4xZUHk9CTFRccmR+$u$axE_s6Vc#@D_D{iDa}%uH}DhRb5-l>mfTK3Ra9J zhj0*eP{OO_l5qjX1S!y z8EEJGU^vbXPFliT|4k47+KmMTk@7VX#|0-c>OaDtFIZ3X9Kb1emaNAgu z^n9<6`6zyJvxV5iHen5sCam)$K}F$vpd)#>joL6dVo=coK6 zog><6c}Hq*iQgiOZ=@Rjum7BW)uo(E=!;=$*V8nQdwNqLSYX4k1teVxFRgy-p&Is2 z$7ENlrA5~kVnUo7C=-9(G;I_xB;$SA@}?!QcIKc1l)s(;_`JKY!m z!TkUC-!&kz{hyB5^S%En11$cRYu@o+#nCVSWv2gMywFCes+P82#=lLlg;MT;cljAn zssH(PL_vY0O!Y?cZy^NW=l}7C{l9kC7Y2Vun+F>=`S)ppHX1!S_5n0gYQy1wqgz?~J^!HwDhs~2h|x&*{r7A5y(6oB-}@16&SSJ+Tt=9qypef?Q9h?) z1aktu9o)A4M0F57pXm{Fwkk?3@hmDma7s!)`*?t=Y?QTm=eVPfS#tVL<&xv1D_BN= zLDWUc>tsD&JvBbw^H8Y@r!va3X!Gs&^ZVXv0MBhV3({O8jeLX`QlKR`U=Z_QL z-RPEa|1L*9=y^tqr$ESZ;S@u5rfEiO{9(?G-srySOWEgq5gKaRGE#prKD}~I$ru;8 z)OE!idDN@M8g|>LIUpi(4&j`PlLO=*Y}ejX1G;&MzxoU=2fuDXg{Ajh#Itb}Zp#rL zTxoyeyd$CX)_e?9cQTCTR{nEB;InL}$x@sfBW6CUiiGqvzH@dxb z&72hcYd`ATJT7)$*GfMFH^L|xq(Atp8SFS*WgpJz+6q{91d)aJ_HMYU`&XLfBPa*q zO-BzJ0*5Z)Pcoa2M4U%@4sr}g32jO!VY~GQcsC!Pqb>ecM>N7X5f_CCu(k20QV`LshN;ya1*Abs``Zn;aN|5^yn>>(b~K352wB0IJm=j|3zkLZC<`* z-4ak&T)n+MR%#iDA5%><0o*PsO+0!QWBr+A9<>QR3mWg6iSWf^_}St(t&^j-2#K0>DZFr`YY)f;+-$Srsq)fyrKvYLY+V|~_ziKR@t1i9cfqtDlwqm41`E)}sc##NTehvLn*`t20)fs-;!v#E;}b^(oVz^!;2g zxOdMrzD8Gn$SC!?_?8oF%ZZ1U)L5$B`AEyUAQM(7&1@}O95hH^5k8x`vGd~c85nBh zKN7bxbjSn;4tWoc9paYq|>0YtFyzrX)F2Lw-mV*xT9F5S)Xy? zw0SW19VHJ{)kgTz(kUZx6U+L3yrpgoMY03Rk}lTebU z^WEL+0{P(^>+9;-dwbz5Lpm!H_8jVIk85kHtLX)thC9?qLot5tLuOk3r9l$sQR%*~ z>K?UmAhpi3nycQ9A=(uNu4vNopJqw~4t8{?goT(B*V+v*C{GU6)r`c{L(F;hD-E;e z;q>}HA)4pWf&Mk`-@h2z&b(QnWn;72P0Oy@aIn!O#K^==(>NH%bL#;iRDo(AgMbx~;>MX%Gg@~u``;tgbE-p#+Iq8>TV|5NU(G#*4OGB+ziGqScU9G}KV5j!=y_vf* z-Is<~`6D4K#7}(ymKpoXVtBZEV}q{D!Z+Ipwmn{*3Co13KfFAUE;2G$OwQ4%iC}M$ zE&^+aok~X{)Bo1UB#k7S^X|I2N;$4J*Ib0(VhG&*W*}f)R|}7w?$xzw?SAfE@6b^m z^4u*&{)!a7N`YY_Y4upk($ZPN-Am3>Z|NAzXo`mCEi?5^?%-yT19g*1N*(QZ=^PaY zKGPe!t@lrU7+Yc(RuGO=yYuzXA;j&RG8Yk80ad(#?NXW*{ z%A_zyG27$VaryguFY;+k)wx>&oGN^m*+4GE4Ad>F?FDQtV8<#|GqNZk$i+S$kmihTvmnD-}&jzJUrJMy1HNIJPWoo5Aw#J@d9mt zr{byYm#EDHjm^zNa8i|hdMZ!NLJp9FnUYQ>^(F=e21(pKEYojEPSxZwQ~Ed9=y*UzPh?U2?(GRt2MoSn;Y>U;E1`#cIO9y z;599nUcYTOQ8&`knB9F6+ZO(c>xqE!o^`@B39&xJm1ugfMt%tN=95;7I<#2TgvKY6 zKhe{#_l>C;xS}gT1F)J}8yP9M5!~H8CYpB)fP(}Dsj1_goq4bj5&kkD`NnOQ-Xz6GKx7REZ}4nHSUqlo`oP8NG<#IzQp)yr=)5icq$%xWKn$SNr*)p(`yNJ#}QfBrG!CBHS-q|{l0#_7r0 zu53&Rw8590ot@95C*^Bmc6X(inPz$)q{7%2#JA9`u9d82T~y5bfalz@3JV8^a)h*1 zVognr(k+&P%-Zqnu1IE-ZDsKgojKk{KunBbc&9N`F`lCJ$@P$m*GL<*F4Zw&CbpG` z5SAvIHNV@QmlSiII9Ydhr4tnpn7Ci7>+&I1$TsNa6~;=WEPnTw<=t(4Xi*O>WXLqZ zRBs+;M6#BOj=tx!Bk^z+J2kc5SGh_Xw9~$~1k6tWP}R}ah+(?RXBFzsm1~S_GZ52e zzixR^r{{09O-zEEoPvPSJS(fZ@^_AGw9|3f!6Z;C3i+fyU~|Br&egwN@V=JE4NpBw zkcvsG|M|EC8JM{5b5l(lu6u3&dJ@PoPm<*-s_LHkx}tM54$h8Sb#&thbzZ+~x_2R& z@v+GS??d`2{gC#JSpz$Lx9@U%Ii;pBhx>zBk7Q{rDwT^nTyID|7D`B_p8@*{@`~XW zlrx#q;GT9x#0fl^Z#M zn`8u7--QBfASiR{(aXe8p-S)}rYbwTm8-rjIs~;6uuT?Q>qi=d*H>33oi&mZ9Y3wI zt9BAG;}CmbFEc435JcVa<|a=4G6+bOEv zpj4ML1o89d&lc@PR>Me9*;gva$-m!hu!NjuHGd`%)P(#f zW`XP%TCoWgX~n)#lu~L|mE$#e5)-P|>!)&`!i7=5?Roap)D+;v9jfT&_SDtFB92TL z@in)y8ebMG#?OYl3VfvXqLr8E@)PPhK-!n`PvEva{#o)?$kxs%Q(BddrMG?R8T@1J z2hR=%;C+^Dv zWKH)ngBUCP2dk!Q&cr0;gH=S!tG?b>1F9?yzD5cTbQ!43GtL0$=D`IkF~D8N<``IeS; zAo0rD5aVzBrmmsFbnbG+`KlZ`>}H~=NUpq_4xZA)R3v%rq9j8UIgJt>UtYYykjJ}DvdHOZcER)66qoncVR1ra)T!ajJx2(OD<|)b70>mc?jkE2loqMY%kAN(wL180|>lhc4CNf7E$n+J^oryMVed}0UaS^1M%`2XI z)Eg_tuH}S9c%SYcU76+7@5h^r&7Mad+54SjzI*o&?=~-)2F((6##A7YpQftZ)kq8s zjq4UFc?1aTSv!uDdZ|8yD?(BKf6siJ7^j?xsZ@RgD*$m zyPG{#0r|!(WY)w4cxIuk4Qcx3&2AZbbGFXK#m;W3+@3Y!#wz3>f3mQh}9>>%gYHmmh3%utqXXn7aaN!iLp@(XsVlEKSxEOlSZbd z?11ctWKiL-R*T!Ytydg!0tAGaXx@E$gN@~$K!!E&^92m#8jwLe$$HvA;xFlwVzGT7d6{kkHA1J})PHi?4H(&5YuQYqviSDem z%AV{W4>Ti~ldw)(_9jCITrM)hSh=WkaQAeY zpKsyKy8SVpT23Ng!gH>z6!cPQf3s_q>8m(7o!oi5UK%uHH0|(XFCI)H5WEPf2A1k+PeXqfNp&4+$ExNBJK46Y=fgkdKIX5(^4Mz66``&aBvCFRwz(be7}Tb zsS|Y7@;`)676ZUc5TPXZ$B#!f!$eV42}C z%KqVTT2u`iJYL02Xk|XNTAc1@7BcA#z!w86V_HIb?_6%WwK-{MU@)dyV*2&!{wqxtApy~G zl!N`-vx$$n02lObc?sIzumD=y7Va8_Myxh?t$=yO43K#hZU4X>cCxW)XKK{VX)&ZE z-_59?o$(kQ`#m_1=^7%#sm#`-uWY|P?ecx|>6Vk;`&8lV7hH{jigxrP$6gK>9#O&B z8K;oW(rk4?q**aLjW2!N9`LG4O3|eQcNO^q+<1l5;TM3Ka!RILC`HE7=+NZ z2)~UlJ0ZjYEV=BjOS)G~iL=Ku2&xrEs2Wr7*x5Zn9+BOh`^31oxJBKb({tA~0@E{# zt($EjL^C~JY@L4>ka{R8DXB2|ZD+&;pL*K|D1}uHJpyfCNOfujcusC#?&;|fg!*Cv zG~yCsY&6D>{ORu8k=R4mNBc1)f2m)liD!?zIBDC!U7M7yt*ymo9{mc_FmC(#*y@Y~ z3FJF*iAEGhNcmcUU0Gy3)NMdOXc}S7&OX}SJazTj89y&L-dHkg-kcm0z8uT5g!h&6 z!zU)M4xTnFWQe(7?$gur`777+G0X9!K%GN>jWlZQ;=e(YwN{3lhld63cD2uq2Fy8BQ+&X|^nB7kQ`cYAMM-KR)zvTZL+_)uM0F z;7px$GYwc-SvfEK6x$#BRN?SVc95uV4-0o&+fGA+_ywsiXnMRxtlo(TJ(=e?$lf1} z{16`9Bres%-V|_n2H+`dlixn|3fE+Y5KoUCgJo-0lljgy0uJYa5~N@pn46*Qib<7N zV#1RGwCO%=VTVvBXD1PQOXl>Mtv|kcojQDPRGSPx!j}ZZ1)J0a+huFU_)%N`K7lFr zKkbYnA7Rb|i6>8~`5T*7+QwQl{3G^I@#~WH2DYk*r?h#%y*N+hR1o-O!$7lwPEoHd*97Y<>gyo zIEA<{NNfFxEv`g6Hv{oC>SgE$`GT{|kgxHffH)wdvPrW^MeAPS0#YADw1@blt?geQ`c2n)sVb91huNIE(?!Gmy@sgsY(95Yg3PTB1R zG@e3|3%?%+>C@G%^i`W*MC;l~X6r@byCo7Kck4gBlLhll19BAa)~0ZkdtgGuP=|>W zqmPZnzxa`LEyi^4w5g>2h>`M!D2RHK2(rQO1a_WsOxua7nRpunO=RDi>M;#;(P# zrN#OGbnO3lS5v@qmGMSuPyN`omE4%!Qt(~4f^B=3%@cj$8Zm?P5)ze)VXFJa#zq>U zz$sDZju%Mss$^5r)vHCgu2TZLwRKm4ZQ)wU)X7ub?Q81R&N-qWInA%I~J*i`iOHD1g4Pr}XHFuvjl#sCAq|yNPvw;EZ;Pt>>uT6yNvK)v=F0KN; zMsDvM6*-eIl~eaw(D*dr-c8l0&;6hEWGdSGBX{c(^ffL3>eSLVJiDK&{0vfh@@gp`y>1%Rc+?+<5G z42a~VOYe#0V@6vB87&IM>ywpYboeCDm0WGimWRE5|9+29y@oH3Ip}NcA6lKn?f;12@RX1j2XzIFwk8gNVUdQ`(0UrNzC50k zEv}~g`-R_2y5Dt6m37^tr}Y$hT&LjbAIW>_RH+`Jr6NQ7x<>H6$D-03Bbys#X|$nj z7IC`zempUUm$)UpzG`WRhn#!l#tLxm6@TC>VcuH5oxN2v(nL}*agR?)Ddz$u6J1GC zf}f3FHh)76?oTL17j+{Mu$J7|5hYIm>#(wU( zI@4OhPqft4f#%JSL4}##1y-7h*|5f|QT36k{;pc`#eHIx+lslPl^+`CjSW8jP`0e< zu6OxFY_FMKd)ajWKP+`eN1!Z(^VIq4SN?=vhLx}o z2e&|Td$?C$bR`t84YM!2dPVujF;MHu3&huY5tD)TzWJ&0C)r8Z6PW-?b*kExxbX9;Jb5W{{eM3^!x2(!R&bD@|1E zo5Fk|!IK^G@@dk=x&V^+IdRjs)Z1G~s5Xp%Hz^#ILuJRu>9q`5pXvz=gC=Qw9kV8SU$c8DP5SR1uD$$;YrQyi6-kV3TS>I?T zA>G2iV$5~{Xu2_wP-0knHk|umd4qI0i$pRgZaGcEMeyEvPDhiSvK5F)aeSZsUo4>2 z6ndA1i$)7~`W5U#^ux8Eo6j_zYXg%}z0*ce=021>$1K9m%C4l6D-f|Qy9GJQOD$DV zb(l%XMVe-Cb$91;Kzz64fRyw;L-w-<-{h;_a}+4WK`kCrIGZprn$gQJB|X|771KB! ze`XydZOyjud5Fio$z61Pu`E8F-Iv~%thU0t#nBi;@#V&}pUrV(lxI@(DOl?`g7J{u zs&Hx@D0KPARo{f(i@`4`Z_rO7Ecd80Zc;ie6r{Ad(AcB*YGF{&;|0G}BYlUbROqEI zS}Gvj#@5y-Q+(6%ANM~*k40x$|-%{y9H=?D(P9u zckLg|6nhBo9*iCW7PXxEg_DxL8vx7pOUGY~y^mkC%?w<-dVDDEM<|bicPzFdP)B)1 zs~bz3v|ds{Gs`+$Oz6r!QRYM0$)Zyv;n#~wmuc^*>d^TDfy;NPXN6#GD|yp|B~{Z24P@6S1fpM-OB3W^(*%gZ?d zJOBd|7_MKRs%R|RlJLY}cV=L8uBfNV7Hj1+g<}u`-?TuNo%Xt``ivpdtWLB#;TH=f z&*AMRI|sW_WJNfl%xxXFm&hTi#NOGESSdKYbVV6T(NT$r3f}mO(Id`bNr;Eejq~&p$*thD*h0hQSb3Hedd^tFjsrJ zrE*Oz-4Jrspay<`@`kLP3VDQjkymauV@b~)Xs}xt4?F6QLke9{ zkNap~rTa2@=8(e|_vou}@K#kW3bRlIvkC=qAVo7JQqy8b;4U9-Fg4*;nai@NDE)F_ z;kV|Zqew2t6E}an;Pdg>@!DAe59XePwloos>5yjzb%?|pva>Y^k-=Z4{Qn~`V&HG-dr3JB0Tp1Id&ej~;tV11nSLo-$m zX(LsM!Q_=4%*3xXjl(bn9+*8}CGe3lgHJtFr!f#oPi0ySNv`0P@!;ZJOmZ-|m<}YB4J~oQ-WoWMGYd{B*GynlvrvKD%m;@p3sX&vNZn3%sD@ zmuq~)#}*tk8_NLk4Ic4LgO#rGykA3Gwg$rDAY4oH(E9#t&?}SnbVs>pjcA6Z^~4QR zbC^yp?&RLsRZ(0cVG0X4*hABhoAu`*a&&GJqumxx&=;AGh%uD)kM|w4FE6+a4T5UN z)Lf1#(s=K?`D~1|wo&KJ%^RUsNuk^Et+_U@73#Q@3!PUNz#WEzN7uqX8hCq;qUi5Y zACHZC1OBSnkXphiDH96v8Lgw-m8PSk=Afnu7;F>wRaRE&a-Q4v=N6yq`GNASN8IC{ zqBUtx^(0Bj#1t1~;Qcw-SKbX`_UlU>JZ6TFOBGFX&b2r3U%f~DCjg?&^cA^o$RSf&Mm1G5HE zr!DjLMedzMZi()<*{P)=T?h>;$3D7erlw5TuLf*|x@!nI57)7&u}!ZZ@0MOv_MU9S zi#R_U=vPk@E%zVUY5I$iF4a&J{6DpKom`nnWBHC# zW1yUa*43M$0UL&M??1Gm&kivFbF2rKbC*)*Jke9My1FYI&0kw41q!Nh#3eU;`XrR~ zOzwP>!(*RXg6f$^2dUI=mQ~g}_hsl_p7QlzuOZCj?#3v@wApv!_!a2Zx&&ajAnLi;o&1L(eCu#9Obi%Xk|VFy%s@#~#%6RZ14j5qa4<&HX%yOW zjvgjWAAh?Ld}k-ejbJy>!J4fz=mv21SVvnj$QmsBKmg;-k(QJ&7U+4Ezj_vv!OK>T z*c@pd`AeYlS~9^%x#Jf%8RbYS|8}GE>S5kxQg!w4Fy?Ogxc=ZzJZ7riuTA>;#^~38 zpF`8Vouujnz2mBWQXKp3ViuRlEx(iIob2puXa+Z%OL|Y!;`?6{RbTu;&gLJpvroFh zYgABy1n2UO>l;p1u6zrSOP^moyb;9H7*UG*@pirp{jIXk%l;i!e}ai}QCk zw8k!@fSwl6%ul28D_)5mXe&?d@5?0Kb^O zR!~LI%2MYTh-NodGyL3rrC*b8xJp|1kaeY)mkqHV?KD#vXY`L*x1(SH$O zk;O|sdQ;x-#+a6x%tyyK9(+vh!11imupr+YMZ&0Dop)smhCZoHP^=yxSowND+fXN? zVV{ax|DnWeB9pdR&9kN-1=~t@IYmX}#Q3)#R@{?P%1Gm-nk7t>OkBH#zc*5H#Cd}HRVHDkB!68n)EIDyPLc5J;(73 z)Gae86b*-kwkVxNvAS1_OcI-4gK=%`P9K( zC%*10qqEyY|AC

v_4ET8PF}A%;f|V&!}=)7M5f$;%&;T_+|!^USt)!(y_l*Fd{H z=}U&zC@rQVzzh|T#4daORm9Q6QG_?0Sql_PlN$V{o0&~mi0ur!1bCl0sA{(Gq@Ti@ z9S{yQ3%=|;1sc@DH-LkM6r5$EaRSlUlqgCeT2GU8qD$m^y>(j#vi(6g_5s41>ynh?qz+j%dJVkQjbGSqW=W!8pRB6V)gs*P;4I z!B;6{L;=`#g%Q#`&=3^O#a~}oSg_LnfJ@wErjBYsg_yB>a&1r3_gJmiOGlqZRr0Ew zpgqK5wJ{^0-^Un6{k1|gSS;H3p)S)EOvy{zlNi}Hm z@E!pj+rU1{@BRREZFquv$sCy>dBW`Jwnn}NK`i81qLkNb#ur7AgXCVt|(uI_4J z(X5rJC+nxHd%p3ZNmhG0JSxeW6+jhqDJL<%oR~CqgjLm4#ao!{SSX>>Q&R=PRF;?h zu;wn?K@ufKZpn#IOa+G=Iw-heVgidoHvaH3+q+(K-c_vSl2MYK8?u6<&D1nHV+VfF z_k~x2rpldG)^K>V-@aje_L-BAlDu-t=1&7z>Q4GcU00wj9koF0HOv;F2e|%^wP!nEI zc&R`lkjT~(5e&-F4uj1ml&kqL7#*{SvS|7QD8T&D%#CWKw3lbf2h{}2t&!3&noplY z)qi}Sxb3)B&5IB6`MfA>-5E7q?RDD89{fFUtc$@iQAs_0XD`y)hClG}hTce9*WI9s z{`-*shtA3E?!7c;26d^Q`6E4QqB=5yDCugTd=DSjHZDPQv-+jQ{~T#ZLY1nrVC?Yh+Q(hAoe#cFWfbfh%kVCz@)6j&6BYld1tUa5zZv&Q_&rd<*BMO6j*=!@|G{OD2r<{L`3ZJt# z&n!%r5^`5H^sl$ByFU*1M~u7ZyQlL`x2VR=I%MjIrTaj)kb*x04=)i=kb zUt8GinL8n20SA>yd{`lVCmCtsdlga}53OEgMQ8#2ymX_%#0t=YU7g+XUXjW7`LrJy zgwipI*Bjk8KZk$#a2S2KoQztR?8~Y?u0;fcRSbEJwvd=P@^;s<2Dj^>V5phK@R7kqt~rSQ`4HE8i3QeIK#&n!!v) zLH3NWk^bTYeFE#9Yz`?)&{StCEt~4mP`Kd|MTZ{E!ZPhUr;p_(Qf-EzF zvQUz(ac(Bu+l39+p+NlPXNi_8w7{u(r&DbSU-r`OBar%V+a2)$F+4S`D^I)4R0aNL z6c~4OlJ$spEbaXiaBP+q9Eqya?3<+>Q@%se24P;*_=O=H=D8PATc5(56oW;TF z0qIvkwZz_~Euz!Cwb|3sXMP+-3RpgDsYBl;YP5hD;}NuXOcN15qU}N@P*6E{SP>~DPgiI^6cZNT*R{3$oO+@Fnh;|{|T6gl@Uop zUe~n=fL|7`&m!($L*-el>~0AK2BYpk$&V}F44D^oO7?v`Z1o+TxbW$Pa_IvAA4 zuPilJOit?ixGWV)VUPS*Bv378@qYDAxu^EV0Y8<(m<%$VpFs2Dv%JQnN4RRf0%FHm{(_84sMN6O^p^!SA(~Pnre|ZZxX*{#m6V7ee2#E$}sRw4qGp-l&sm(>9n|f zGO~SwUq)U6N&8{y^`a%BGsPLQG@a$am}As2NMJA8-pU4FQ_E$FhIJgm6Pxq} zSNMW!c>d_CE1fL+uIIuiRk$<*gMyi)tBo^{)&KUl$PvZJp-pq^Zm zgA%xp2sSLY)U44y^@7NI916Sl06tkQ3dtewc*qG~bl8nc`RE$fWY9eoo31@K)7qx7We29aR8xjFmhmK0 zf<_v;s|Gv7*0#B%k2Ea?*MMDX@U7YDux4@1+>6W^)RvSp3_Oxa9K18u!v#k89~Ax8G^@oG z-qVewEKjl9HIw?x^tds9`8`e!nIB^Ks<*@M@Eh^)@W4$_=Fp2@1caXG<3tWWZU-mN zz3HA0-K&@K?D}Dcof0N5)ho4}Pgl4&K_HsS0jpg)bR=(;=9t-L#<@-N-2+FE2{mtF zvq(!StE}`BhhcZzvdd)li^ZBok5gsLw6b!2h=M+HV(C*jJZzeF2H{atXQKEZUfld} zAA|rCcB~Y7do{Lmp3lcS>cP66gVolzBU4lp-#9otg={fsY2D$<(pkDJq7uVmARFb} zW+5XnT4K=F)#b4I_2QKG=AN{B8ECe7+FR-|F*8%;)$%`m;yAS0WhJa?PrD*?vN^F|ttVXzf$rtUuKeUU zCAmUIHTGS&%_yhr4B(82FCuv=y(e?BcXmM4=U;p7m%{*lSQuZdoNqJ{$i8}7n3H`4U_G#%wyd;ORo+Ag#VS}bzt{DI5xI{%L|c~ z1L>H$pB8Rnq64Y^e07JbW(2sCA+rviMpzK&Fi$DL?W0$$Now)D?Zl^mF`-> zr=*d{71qdyug zB(nnJbo89jtoY8MR%cva2(U+!SPOmOYG5wmJlhfoHv*2wS~>d(f|==VaFw0aao-7s z6!>(egkM=ezc9rq!8xLjkb$|9(Ho^9W%NL*Zwmfwf$dulU~qfQeOXx@MX9g@6<4XF zuM#V@ot%>Dy%u_M++b1`z~8(yc*%u9W6|s?fZ>A z#ZK(vWSr#0MIIj(O`H_ewOP1iUY1-#bo-D_a5B?9Mr=6nJC2mb_A&!rOPRRT^plzF za{E6Zwx!kEzdDVYpo3x2s`#6+h~hMfKOX&>96s1ev|D5Jdl3?D%1e*sZ)n);Sn^l_giD#9J?3S+GkldRWs9DRYisInEx`C%|lYt;Mr zmU_hQqf5Msh8lhF|naANqLoj8{X2M<{q{OT3mgCZ^oir(Gb522peMO9bi;BQE%b z<^IFsqOcCGS4*?PPv#4hoCpFM)Dp#0^=_67g0q{?1WZ0>`Q7aN_(4bHSj@;#?($tx z<@NJD=-j({+a`^7a*s4Fqh}ZH1lC9FZH&L5R~8^jBTmUl!%@9mpEy+wfMKzj>o83k zA~uSqZGpZ_QzIhNYWEU@ow6U19RV!!hvf{lKit!jSZXl~lmcp{cIg?xm@-R3#1eb0 zUPm2dkJ#5n)dDiad?%*@X%|NROesai8H-u)ItheYKe>~AFYDJMs0TBl^3S_>l}VhMW`K5VIwr}I&(sX*sTty>=%zT=!p{J)*kr3ren2Ojhu|2xBVaj3pwR}19RT4~c z3;}*EEn{&T>4zq&YkNw&Pe~{tUViocNQ&fsj@tFQuH|p0H{%8+eB^cLucxM^E2?y2 zVyns#29zhdKa=N!9D%x&JE7PdgA+;G-T4xH)Uma za)h6>I@~jQ;+QGm{s?j!a4zIjznK!!V!%vlJ}p_U?3irTX#Ef2YL{hQEx8kssf6YW zdwnHcnT3yF5Pl_GbrfktW_#w%{p6wla$*%2;i~VsNuC{hnplHqnz~>!`(1D5syW(d zz1l%wrJpZTPyYET~l3OS1Tkg0c(r^=&u|dK4ZW z4#-|9LV%Ptf6vR6r>U@zyWC8IB<=mOMxce?(b19LunZk^R_M*W*8<=6yYiIjiE~w zhoEEy<>ah7>#i?(G}TB2Q|90QY}*HA$*j~+&iqRwM!vnkUo;E(e4SIc;nvNy&3XO2 z&r`q|t-wJ?M+fM%4vh($cl!FZI%QMSIfZ3q@u|(fv9j8F{k;`9I+BG>JAZYAhADA! za@{pnlO;UQ`%EOUKjoy-%#cXFL}Wh-0EYKupG^Yuh-S<87Ya6ETE{l_u`=^s&ifrL zGCl;sivw4-_2SKt4NoC66;dP34&ChP=L4IwtTH~c$XmRap}uCNNISkc`DJehAoG45 zdCjO++ahzHmTrBdtZpaPi@li`e>nZzQ=?hB9)g*!pS}_sMOlV5XDClNpbb1fLc)wZ zybNbuF5P9J$5*T3b6fyF1!7vvtW!Hn-f#IMIbY31 z!VO>714H+`ksNI~8_8pL&HA1=m2ke5ET9Ejx{n$Nl{N+qOJ1BEY>; z2N9p3;2M5H+(|udkJU{{eC;@&`V;yr_8jG>J(E6?E?H8FQq#xuD(V%XXJm{Q>l!vE zim`Zw^!Ez6IX~tq{p1B}Y9B*pt91Zj`L8pN)E*9JU~gl8SM2AiVMyQ1;i-Q=9O511 zM=DUbB@(vHm@2k$I~v#N2_UgcUjo;25IP4 zYi^~u>%VkAo|f()-weEpCM`za?`E*Ek-AnR%TTd}OeHxb$}334W`B*hvv=iQ^NkY| zK+m=BqfLGdv6cxaT&?}4uG$|kpj9g&eheTG)833-IxjA*b4e5J8=o@?y45%-opp6* zgqFH9w7r1b5`D5vFd4^}FQuH>cahe+3Lssx-|`9q6krZM}x72fq7wvi7-TI zQyth8?j7Ajmu1fdy9#f4^7O$2zm*RQz%DP)cJme_ej)G>u(Spd`TF*_X3MO~U7*c2 zYxH6h6UL!Rib!dvKU^6rPwktgAA-as;&Ft_^i~dSk*^tZWNzHJZ3;KML$KwolQ4`~ zB79VMSSx%CqD#ctB1)P{vQsZHCrISHnvsV+kvV#WK^3WKw=@W&Bw-pBReReN7SfHX z^n>5BWBXA}JNV!assK@q)BN)#+UM74_WqbpoIdPG;%}11Ytn<#j;i(Hwr^GkJ}~-d zM>y~k+$q=}KjtWDn5)c;GaK&|z4g_$M#{Ff2!&c1j5ltqdL10`_8^C><~sR% zO}DBQ`|SonvSsm-7Ofwv7tB5%TI)Jld?tUgO>;H;U1Nrxjh$_GoN^GzB3{k6)`jT~ zYyIim?ih%P`)6aO>39|Ci@qu#BXPkVOYr4-aH(qWNiSnx&&SgutyI7t^eXE=ip#z) zayLt%S)V~>sBCzdkqF}};`tB;lk@z=+ozqVvz`RQ?unnfTOES|9_x#gZ#Fij=@~I> zL+Ua?zlXfauN~ytx)ewmB1xJ>u#~i7$9ci2N=Pg%{gOG7ts#IQ0px;_6STEKfSS|x z+V(a1PkIB5Z3@`mu0tiGrQ$89*a$_9-&W}JX4+t6T4P?n^NL+pR2LC8jDC@#MQu3{ zlUWb_Z9m13dhcF+$&j(7=%{4#ZEs@Iooy&}X8D3~TYF({9NKO7MacJ(;dEv$7;GXv zUG>pBo(Gh1uk%a*rtsDU%yj#fL2qFAwRm^&Vts>`)Vx8??t7y(a+-VwgXjZNu{AH_ zn(j@Dz}C_#^`wv6V9SEZk_45-k>1VCG-isahnhzm0Xg>r7}|PW{MJZLL~sOmo=XF2 zV12R*^SdOh!&`m7&#+FV;shi8r=rqHD1NlUSiyem4$l*QZoRF>RG)*W&@Gw(@IR32s~PMM+P$ai3#< zWAl%_-b+=@n)(xXZfPl{hnEdpJ8$V3LA3jOW4 zOJ_CmnJ}=#PmRab8{N29bN_7ElYC_5s_WnLvou|-kwF1YV|GwN#!9%CEcg+P6<_8@ zJqsW#(ovtkmi1f0Or&RIWIyvz{oHTaP?A@wx9SXREvo^Lc8Q*k=Y|ZPg=xrhH2l7 zOLzUx;y!Q7*=g)i7xn8N===d!eTiS#A4{9%r^L&;j1!@VW)x9+_ghHc4`wLOT&>H*A-Q>mzc3 zlaj%#eVMXjm1sMFKRf!!2p_Z?FR!VgnJ*{o_TxK;igwoC+Q4_0Ad&veZm6|}#wOP6 z;xU$etNsn8Ihxs!U4HAVeG|S|Y^@YV9!XFUfe3T}{{59cizAP@e?rHKWX?rO z$l?_KG^)@Y$V+WpgkynGSO7kyZsg)PTjJJ|%|?ydlh$kks9x_#?sP^xonj1RF2*=s z$*rfp5~Qdx!oWNGu;!dAQhb$&x?wHh1f|XngGcs0vU7Dwp=kx7CO{$qTfemyQsw50 zpH`El)>j9g+#A$0n|2W`Ui$Zk1yfTvyG>fDj9ab{M{_2S`-LWo9_KJ z(k>!cZ)E2so=N{Na-`4X*uM4nQb)OsRxJlk;Vjvh|a zBSMK5zoporMmtpPh&l^H86Z>cvOQbZlO^rF189XZB=)3K1zNi@LBVXoMPooluqv@n zOr?C4X*aB_%$anVAIfv$hDPx^Le9^ay=9nvcM(ZqGd)HXgx7+6I(k1hOyCJNEF26d zAyq$U{ar$7{gVM4eC@RtehVa4BkF=Jn_WH7-K`PFL485fu@27{?ru|1n{!F(5{1jx zvm#~m#)c7-d)8su8S>DD#p2k<)*}bmXLe;~6Wd`(_u;1vCZ%LH5mSS5x%sir+UA=#MHW{e`Pl;l?TFxoyY<$&y-`sD9%g2%r6V=eH0PchjSwG55+JY=1Kvlquo&0R` z&&bHp^-`}KCu2-UuOd$NL-t7m#R5IT_0xT`so~(Mos&kyq!G`PGw%NFSeAr&!;Sd_Ly1UhEMPwM<#_2 zZce-2Hb_@Cpv$oezm4>)->bVNYpOx<&@=Roi7u|HYgdbtDeyw50mp#zjFl_lwcK9L zj(cSgbyr*G-I;H6-j>q7;`7TJY}T-{vjd%UhzXO4XX%X9@Q|eZmUzLaJd}fJ#(igV zMxNrk8tWe;UOMg=Q3fA9uO-QVHpy0$5A9M?CflBA4n_8_0g3)M_C(v1h^~m4G@gB9 z7z-QB$QoX!CT=xsZo8b)=EUcm-lS?OIiE=SRvhkzyNs|4K3}<};Q#8ZB|{~0I~X?| z`Lngvvv)r|;fnjv^UIXUN+@UagbyJ=Z-7V&8C ztagFRYPZST>F}fcB7dnJLT!@yS0?Hmmy~$9zj(?+&>f3srUIr_$NP};uT0heF#|DH zS?e@$UzG0<2Vuy#$5>!*L$fubBO`bJsYZ+p^RSnmtZIw|xW1|WbJ9g>Cv0Ux*2y`t zDW$4v_dO+WIfz&u8Cerqv!CM8BIQl~rM_VK1vLdNii?X24Mk=Wwr85`y9i$|&7NnT z-qm%q=9CKB5f}AMuai~}AK-!?yEa)5S;@|bC+BQ#KGB3oK=6j>mq;pVS^~bN zB_K3x_WQ(Wp9C+jUt34HZZ0Uu3&TFDbLDe>)<;-)VICi8)+n zru?IV--H~fA9vupj{lzJ+_t!AL$H9D);e%Qty=TDbAnIe&}e*A8hhyHPg#e%x);w4 z{*tc)k3Xl?1jozg9+syA-8FnA;r*{0laZ{8Uq~nAb0rm8fJz)!sqisYq@UzES$T*s zd%+$^jAY^I>VEIamQMMTE2PMm6qUst+W=)-@?Oo_JFMF?GpV;r0_fp(fyXrfwRm_1 zfdX9${Va3GD^Tstt5n>~{-eFTywXub7wUa2h|b4kimt`{kr>cgDX&Yi6_gy4+Gj9J z&rszS^)FhEtjbl8Dkk)&H|;F=@PLJ_dG!{*Y#6@jDwC4WNOdrk)ed-pd)Azp<;6fs z_P8sMwD9P-idn8Q>~Q|uS(sKWWabyXuf!@)qE7{%0-#l#l0B;gywk z6jJgm-E=?tm6m|KnZ(lntZkY3B}~81Qf>F;!&`9W87Dt{wP`ogSb?p*xi+%gzcqd+ zLF4*WALXxKlWp?t@85Y(7D<~hQFZK2?x1TqcM`wFLzNXDswf#B=@gtDEZD7JYiGZ) zmVAFnwfmLi5N-?!eSJ6|pEJ8Y*;E)}ZE9B-{$dbo?+)F-Nc6ZUH=U8wuKm17uX;Q4 zKCd8lcwlGn8huXsTPyvi^lBQKNvruqQny9i9vEn-dB@_Ny4#7+oH(e6()S9X0$KrU zwlm|pe4PB%i802s2@*&-?rXZMox|&QCiL2?8^y#Cy5l!p?18|}`(%q$*391$FWd%* ztksaoquO2pKMRn>G`ss-aAOsDZ91!;4jQYbC|d2s!3eRl*|{es5cg8OKP55Q6?_yM z>lr7*B`1!Z9;iH$HNY-D5X4+XCNUNAA$N{ss4G~QhZd!-5G(LVaa(L>2KO1I@>F4| zyQiDaT@4tW7Yfb}J-?iQ)6<~2dTzp%&z;76XHZ@JMTPPmeXh8)BEy!O3P;RKz*E`=e`q3myEbuBBLLIMk2HUezSy-=amo* z0H_u}OL!HYSX=q_B==KqS*G7Y&TybNFyq0gGLEv((a*QAu((U9{id0b49I-oQO9QA z!Tu0xVv6*>Q^IUSWTa22q1VyQgZS6x&$b0WbRj($E?;Nm6}Mt$-eBBDo%S_}8Ny9R zYgCM+lA#M&AFjZ!jI>PS{X;`&{)TVLx8#2M}qK zD98?BsxxMDtmD?DGxRE5xams>Xv|xbuc{T8oQ-9f^!Po$WLPdMQFdDDVXp~a1RBpw zV&1I<`>;%lOu1CznzsCTa&|Tzp$@xxrw7UgOx$Ps{N%#7+?-#ywMeLL9|jX;AT4Mh z%0@YwS%BAx17Bkf{mqxjxxD=xOII~zqUlqtiTRAo{lcjhw4UR+!Z^y+VQv^?Di zS)06OY?-|?Sz_QM4BCk!aL=kG@r}8=m-v!bFA_f5Eb(Y$2AtHwn}-|R6s~**kCf>Y z3H%6-CO_#C&gyFmx_yljWoKj8bKuYQtW`sccfNk_!0)^_3W^=VOibNwv?=_0N~g3q zrdI8n7tHphN4UY%!cZuq0pzY$!V+P8wqGP5%OUpQLGfG`Xb&zt8WFDfwVGj*)jX-` z#iHWy!W{`d?!Li34GSjm&}#a3Uw#bl{3^ShO z{s&titush6+A~0sxDW`$XcL^uqPPQAP>_+4aR*tpx;n1X{KrFS>5kS`GUXU^;*bzY zAE(5_&3(tuIuWKIp$@PfRgLaHc%Vazi;iYTd=xi1mC)O;fpBx0J4ri6Ll z+hb09l~fK_9b9e5t1z5(LV_k&AwLSYeJ*d26jK^*5{`H0kjfKqCRb~qfxahaa?PWO&m(HD zG*c3`LQS>F@?rsXi&Tc~`8G!IKMR#@O4;_)+g9^a#}ogwXI2TEsdis_BWICR5vkK1 zGBwR?lj*(lOS(?t;>*4a+7UJvV{cNE`IijC&kcf|&e4C}ujV7b z9DK9KLEP-+pA32KswTzVCdB})(y^phK#C)MzqbF&de3fVjK1MPlz7rz4k_8ve=xwd z+G(yTZ`2C^GD%;!z;+R<`4?sI!u$Urwy$OJC6-2MK13f59nJsO$1c1s0@xZ?ZmFpQ z|M3S~aGbIH4OzbMgQKgrC;ssZ12r{!Woc;Z1LV65(M=k^71Bo`Mu6w(;|YVp26QAz0e+^{;KxzV8Y|? zwWjY@i`7EPqAMyYisL?n*fyF<61A%1{-VwS+&MXVhhJQrjBCXjQ3qTakILUG@bJKf z{V!vCdU}8sgckOTX3rCe;RHteB^!+VKVSL?f*Tys*>IST0zl_t9Ef~ofB*EmEG4?S z=@-L#`G7=u;pJC=878?aB;X&*dy%J?J{S>`T@e%h-IUoS@$LR~gSE>nCDTy~%pMMo zo&T_Ze*|YpUm2CYlxevfX!tQn@RJ}5-9eBlChsH)-%{eO$|e=*F zYyUoLz#aJiaEE(Fcj3bMd9{hWByfF~|9T-)UAS_ObM>>G#gD(%TvbU+vEt>MkN*Y5 COi^I~ From 7665ca8c84ddaf45468d9d578563ef61dc4c5c8b Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Wed, 27 May 2026 14:10:33 -0700 Subject: [PATCH 4/7] fix(external-host): drop coordinate-based click in Cowork submit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous submit script computed a (centerX, bottom-90px) click target to focus the composer before pasting. This was fragile to: - Window position (broke when Claude moved to a secondary display with negative Y coordinates) - Window size (the 90px offset assumed a specific layout) - UI layout drift (banners, status bars, or panel changes shift the composer's visible position) Replace coordinate clicks with pure-keyboard input: activate -> Cmd+2 (Cowork surface) -> Cmd+N (new conversation, autofocuses composer) -> Cmd+V (paste from clipboard) -> Return Cowork's React app autofocuses the composer when a new conversation view renders. Chromium's macOS accessibility bridge does not expose this focus to AppleScript (AXFocusedUIElement returns missing value), but keystrokes route to whatever has DOM focus regardless. Cmd+N forces a known-focus state without needing to find the composer in the AX tree. Side effects: - The wakeAccessibility capability is no longer needed in the Cowork driver registry — Cmd+N implicitly wakes the AX tree by triggering UI re-render. The capability remains available for other drivers that need it explicitly. - createNewConversation defaults to true on the Cowork driver. Each eval starts in a fresh conversation, isolating runs and ensuring the composer is autofocused. Manually verified: a two-case dataset (echo + Glean search tool call) passes with traceConfidence=high and traceSource=host-local-transcript on both cases, regardless of which monitor Claude is on or what window size it has. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/evals/evalRunner.externalHost.test.ts | 6 +----- .../builtins/macosDesktop.test.ts | 6 ++++-- .../externalHost/builtins/macosDesktop.ts | 21 +++++++------------ src/evals/externalHost/hostRegistry.test.ts | 5 ----- src/evals/externalHost/hostRegistry.ts | 6 +----- 5 files changed, 14 insertions(+), 30 deletions(-) diff --git a/src/evals/evalRunner.externalHost.test.ts b/src/evals/evalRunner.externalHost.test.ts index 3283d96..8cd888b 100644 --- a/src/evals/evalRunner.externalHost.test.ts +++ b/src/evals/evalRunner.externalHost.test.ts @@ -153,15 +153,11 @@ describe('runEvalCase external_host mode', () => { uses: 'builtin:anthropic.claude.activateCoworkSurface', with: { appName: 'Claude' }, }, - { - uses: 'builtin:desktop.macos.wakeAccessibility', - with: { appName: 'Claude' }, - }, ], input: [ { uses: 'builtin:desktop.macos.accessibilitySubmit', - with: { appName: 'Claude', createNewConversation: false }, + with: { appName: 'Claude', createNewConversation: true }, }, ], completion: [ diff --git a/src/evals/externalHost/builtins/macosDesktop.test.ts b/src/evals/externalHost/builtins/macosDesktop.test.ts index f7939b2..24dd294 100644 --- a/src/evals/externalHost/builtins/macosDesktop.test.ts +++ b/src/evals/externalHost/builtins/macosDesktop.test.ts @@ -27,7 +27,7 @@ describe('macOS desktop built-in capabilities', () => { ]); }); - it('builds a submit script that focuses the composer via coordinate click then pastes and submits', () => { + it('builds a submit script that uses keyboard-only input (no coordinate clicks)', () => { const script = buildMacosDesktopSubmitScript('hello marker', { appName: 'Example', createNewConversation: false, @@ -35,9 +35,11 @@ describe('macOS desktop built-in capabilities', () => { }); expect(script).toContain('tell application "Example" to activate'); - expect(script).toContain('click at {centerX as integer, composerY as integer}'); expect(script).toContain('keystroke "v" using command down'); expect(script).toContain('key code 36'); + // Coordinate-based clicks were removed in favor of relying on Chromium's + // DOM autofocus when a new conversation opens via Cmd+N. + expect(script).not.toContain('click at {'); }); it('emits Cmd+N when createNewConversation is enabled', () => { diff --git a/src/evals/externalHost/builtins/macosDesktop.ts b/src/evals/externalHost/builtins/macosDesktop.ts index 30c1ca7..9fcb577 100644 --- a/src/evals/externalHost/builtins/macosDesktop.ts +++ b/src/evals/externalHost/builtins/macosDesktop.ts @@ -248,23 +248,18 @@ export function buildMacosDesktopSubmitScript( tell application ${JSON.stringify(options.appName)} to activate delay ${settleDelayMs / 1000} tell application "System Events" - ${newConversation} tell process ${JSON.stringify(options.appName)} set frontmost to true - -- Click the lower-center of the front window where chat composers live. - -- This focuses the composer AND wakes the Chromium AX tree as a side - -- effect. Using a coordinate-based click avoids fragile recursive - -- searches for AXTextArea — Cowork's composer may use a different role - -- (AXTextField, AXTextInput) depending on Electron/Claude version. - set winPos to position of front window - set winSize to size of front window - set centerX to (item 1 of winPos) + (item 1 of winSize) / 2 - set composerY to (item 2 of winPos) + (item 2 of winSize) - 90 - click at {centerX as integer, composerY as integer} - delay 0.6 end tell + -- Force a known-focus state by opening a new conversation. Chromium's React + -- app autofocuses the composer on a fresh chat view, even though + -- AXFocusedUIElement doesn't expose that state to AppleScript. This avoids + -- coordinate-based clicks that are fragile to window position, monitor + -- placement, or layout drift. + ${newConversation} -- Paste the prompt from clipboard. The caller has already written the - -- prompt to the macOS clipboard via writeMacosClipboard. + -- prompt to the macOS clipboard via writeMacosClipboard. The keystroke + -- routes to whatever has DOM focus inside the active window. keystroke "v" using command down delay 0.4 -- Submit via Return. diff --git a/src/evals/externalHost/hostRegistry.test.ts b/src/evals/externalHost/hostRegistry.test.ts index 52c84b1..4550f9d 100644 --- a/src/evals/externalHost/hostRegistry.test.ts +++ b/src/evals/externalHost/hostRegistry.test.ts @@ -40,10 +40,6 @@ describe('external host driver identity and built-in defaults', () => { uses: 'builtin:anthropic.claude.activateCoworkSurface', with: { appName: 'Claude' }, }, - { - uses: 'builtin:desktop.macos.wakeAccessibility', - with: { appName: 'Claude' }, - }, ], input: { uses: 'builtin:desktop.macos.accessibilitySubmit' }, completion: { @@ -74,7 +70,6 @@ describe('external host driver identity and built-in defaults', () => { ).toEqual([ 'builtin:platform.macos', 'builtin:anthropic.claude.activateCoworkSurface', - 'builtin:desktop.macos.wakeAccessibility', 'builtin:desktop.macos.accessibilitySubmit', 'builtin:anthropic.claude.localAgentTrace', 'builtin:anthropic.claude.localAgentNormalize', diff --git a/src/evals/externalHost/hostRegistry.ts b/src/evals/externalHost/hostRegistry.ts index 95bfe4c..2f4b790 100644 --- a/src/evals/externalHost/hostRegistry.ts +++ b/src/evals/externalHost/hostRegistry.ts @@ -49,14 +49,10 @@ const EXTERNAL_HOST_REGISTRY: Record< uses: 'builtin:anthropic.claude.activateCoworkSurface', with: { appName: 'Claude' }, }, - { - uses: 'builtin:desktop.macos.wakeAccessibility', - with: { appName: 'Claude' }, - }, ], input: { uses: 'builtin:desktop.macos.accessibilitySubmit', - with: { appName: 'Claude', createNewConversation: false }, + with: { appName: 'Claude', createNewConversation: true }, }, completion: { uses: 'builtin:anthropic.claude.localAgentTrace', From dbee8cd866ad0a25be8891443dbe7b71f01a13a3 Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Wed, 27 May 2026 14:33:01 -0700 Subject: [PATCH 5/7] fix(external-host): verify app foreground before sending keystrokes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tell application X to activate` plus `set frontmost to true` is not reliable on macOS in three common scenarios: - Target app is on a different Space than the user's current Space. - Target app is on a secondary monitor while the user's focus is on a different monitor. - Another foreground app (browser, terminal) holds focus-prevention precedence over background activation requests. When activation silently doesn't take, AppleScript's subsequent `keystroke` calls route to whatever app actually holds focus. The external_host runtime then waits up to its case-level timeout (default 90–120s) for a Cowork local-agent session that never appears, and fails with `failureKind: no_matching_session`. The user-facing error points at the trace step but the real cause is the activation step upstream. Add a short verification loop around foregrounding the target app: poll `frontmost` for up to 2 seconds, retrying `set frontmost to true` each tick. If the app refuses to come forward, error out immediately with a message identifying the foreground problem rather than letting the eval timeout misattribute the failure. This makes the keyboard-only submission path (introduced in c43d4b9) robust to multi-monitor and multi-Space setups. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../externalHost/builtins/macosDesktop.ts | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/evals/externalHost/builtins/macosDesktop.ts b/src/evals/externalHost/builtins/macosDesktop.ts index 9fcb577..24fbd09 100644 --- a/src/evals/externalHost/builtins/macosDesktop.ts +++ b/src/evals/externalHost/builtins/macosDesktop.ts @@ -247,10 +247,31 @@ export function buildMacosDesktopSubmitScript( return ` tell application ${JSON.stringify(options.appName)} to activate delay ${settleDelayMs / 1000} -tell application "System Events" - tell process ${JSON.stringify(options.appName)} - set frontmost to true + +-- Verify the app actually came to the foreground. tell-to-activate is +-- unreliable on multi-monitor / multi-Space setups when another app +-- (browser, terminal, etc.) holds focus-prevention precedence. Retry +-- bringing the app forward up to ~2 seconds; fail fast with a clear +-- error if the OS refuses, since otherwise our keystrokes route to +-- whatever app actually has focus and the eval times out 90s later. +set activated to false +repeat 10 times + tell application "System Events" to tell process ${JSON.stringify(options.appName)} + if frontmost then + set activated to true + exit repeat + end if + try + set frontmost to true + end try end tell + delay 0.2 +end repeat +if not activated then + error ${JSON.stringify(options.appName)} & " could not be brought to the foreground (focus is held by another app); keystrokes would route to the wrong app" +end if + +tell application "System Events" -- Force a known-focus state by opening a new conversation. Chromium's React -- app autofocuses the composer on a fresh chat view, even though -- AXFocusedUIElement doesn't expose that state to AppleScript. This avoids From 94ed5fc0f8c989983731a714ff0b51fa84ad2c7c Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Thu, 28 May 2026 09:40:08 -0700 Subject: [PATCH 6/7] chore(external-host): remove dead wakeAccessibility capability + add foreground-verification test Two follow-ups identified during PR review: - The wakeAccessibility capability and its underlying wakeMacosAccessibility function were no longer wired into any driver after the keyboard-only refactor (Cmd+N replaces the AX-wake side effect that the coordinate click used to provide). The capability still used a coordinate-based click, contradicting the PR's stated keyboard-only design. Removed the function, the capability binding, and the test assertion. Can be reintroduced from git history if a future driver needs explicit AX wake. - The foreground-verification retry loop in buildMacosDesktopSubmitScript (added previously) had no unit test. Added an explicit test asserting both the retry loop structure and the failure-path error message, since this is the safety net that prevents silent 90-second eval timeouts when another app holds focus precedence. Also updates the docs/api-reference.md snippet line range to match the EvalRunnerResult interface position after rebasing onto main (L121-L195). Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/api-reference.md | 2 +- .../builtins/macosDesktop.test.ts | 30 +++++++-- .../externalHost/builtins/macosDesktop.ts | 62 ------------------- 3 files changed, 26 insertions(+), 68 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 97e8ca4..95541f5 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -435,7 +435,7 @@ The result includes pass-rate deltas, optional tool precision/recall/F1 deltas, **Result Structure:** -```typescript snippet=src/evals/evalRunner.ts#L106-L184 +```typescript snippet=src/evals/evalRunner.ts#L121-L195 /** * Overall result of running an eval dataset */ diff --git a/src/evals/externalHost/builtins/macosDesktop.test.ts b/src/evals/externalHost/builtins/macosDesktop.test.ts index 24dd294..4898742 100644 --- a/src/evals/externalHost/builtins/macosDesktop.test.ts +++ b/src/evals/externalHost/builtins/macosDesktop.test.ts @@ -5,7 +5,7 @@ import { } from './macosDesktop.js'; describe('macOS desktop built-in capabilities', () => { - it('declares reusable platform, accessibility submit, and AX wake capabilities', () => { + it('declares reusable platform and accessibility submit capabilities', () => { expect( MACOS_DESKTOP_CAPABILITIES.map((capability) => ({ id: capability.id, @@ -20,10 +20,6 @@ describe('macOS desktop built-in capabilities', () => { id: 'builtin:desktop.macos.accessibilitySubmit', capabilities: ['control', 'input'], }, - { - id: 'builtin:desktop.macos.wakeAccessibility', - capabilities: ['control'], - }, ]); }); @@ -51,4 +47,28 @@ describe('macOS desktop built-in capabilities', () => { expect(script).toContain('keystroke "n" using command down'); }); + + it('verifies the target app is foregrounded before sending keystrokes and errors fast otherwise', () => { + const script = buildMacosDesktopSubmitScript('hello marker', { + appName: 'Example', + createNewConversation: false, + settleDelayMs: 500, + }); + + // The retry loop polls `frontmost` and re-asserts `set frontmost to true` + // up to 10 times so transient focus-prevention can be retried before we + // give up. + expect(script).toContain('repeat 10 times'); + expect(script).toContain('if frontmost then'); + expect(script).toContain('set frontmost to true'); + + // If the loop exits without activation succeeding, the script must error + // fast with a message identifying the foreground problem rather than + // letting downstream keystrokes route to the wrong app and surface as a + // 90-second eval timeout. + expect(script).toContain('if not activated then'); + expect(script).toContain( + 'could not be brought to the foreground (focus is held by another app)' + ); + }); }); diff --git a/src/evals/externalHost/builtins/macosDesktop.ts b/src/evals/externalHost/builtins/macosDesktop.ts index 24fbd09..e679fb9 100644 --- a/src/evals/externalHost/builtins/macosDesktop.ts +++ b/src/evals/externalHost/builtins/macosDesktop.ts @@ -25,11 +25,6 @@ export const MACOS_DESKTOP_CAPABILITIES: ExternalHostCapabilityImplementation[] capabilities: ['control', 'input'], run: submitPromptCapability, }, - { - id: 'builtin:desktop.macos.wakeAccessibility', - capabilities: ['control'], - run: wakeAccessibilityCapability, - }, ]; export async function runAppleScript( @@ -95,63 +90,6 @@ export async function readMacosFrontWindowContents( return runAppleScript(script); } -/** - * Forces a Chromium-based app (Electron) to populate its accessibility tree by - * activating the app and simulating a click in the lower-center of the front - * window — the area where chat composers typically live. Without this, the AX - * tree exposes only window chrome (close/minimize buttons) and downstream - * findTextArea/findSubmitButton calls fail with "no composer found". - */ -export async function wakeMacosAccessibility( - appName: string, - options: { settleDelayMs?: number } = {} -): Promise { - const settleDelayMs = options.settleDelayMs ?? 800; - const script = ` -tell application ${JSON.stringify(appName)} to activate -delay 0.3 -tell application "System Events" - tell process ${JSON.stringify(appName)} - set frontmost to true - set winPos to position of front window - set winSize to size of front window - set centerX to (item 1 of winPos) + (item 1 of winSize) / 2 - set composerY to (item 2 of winPos) + (item 2 of winSize) - 90 - click at {centerX as integer, composerY as integer} - end tell -end tell -delay ${settleDelayMs / 1000} -return "ok" -`; - await runAppleScript(script, { timeoutMs: 10_000 }); -} - -async function wakeAccessibilityCapability({ - config, - run, - binding, - state, -}: ExternalHostCapabilityContext): Promise { - try { - const appName = - runStringOption(config, binding, 'appName') ?? state.displayName; - await wakeMacosAccessibility(appName, { - settleDelayMs: runNumberOption(config, binding, 'settleDelayMs'), - }); - } catch (err) { - return desktopFailureResult({ - config, - context: run, - state, - failureKind: classifyDesktopSubmissionFailure(formatError(err)), - error: `Failed to wake macOS accessibility tree: ${formatError(err)}`, - limitations: [ - 'Chromium/Electron apps require a real mouse interaction before the macOS accessibility tree is populated.', - ], - }); - } -} - async function requireMacosCapability({ config, run, From 5624b678da3b3566d5400d122ca40f910afe5aa5 Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Thu, 28 May 2026 09:53:58 -0700 Subject: [PATCH 7/7] docs(api-reference): sync EvalRunnerResult snippet with rebased source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The api-reference.md inlined a copy of EvalRunnerResult that included the function-preamble JSDoc comment, but the snippet range L121-L195 in the rebased evalRunner.ts starts at the `export interface` line itself — the preamble lives at L118-L120. markdown-code's content check correctly flagged the divergence in CI; running `npx markdown-code sync` regenerates the inlined block to match the source exactly. Co-Authored-By: Claude Opus 4.7 (1M context) --- docs/api-reference.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index 95541f5..f43edb4 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -436,9 +436,6 @@ The result includes pass-rate deltas, optional tool precision/recall/F1 deltas, **Result Structure:** ```typescript snippet=src/evals/evalRunner.ts#L121-L195 -/** - * Overall result of running an eval dataset - */ export interface EvalRunnerResult { /** * Total number of cases