From d2aa6cf02bb92be74b62badf146986b1071ef4e3 Mon Sep 17 00:00:00 2001 From: Jeff Date: Sun, 12 Apr 2026 18:54:58 +0100 Subject: [PATCH] Investigate pocket vetoes as implicit rejections (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-pass analysis over the bot_detection DuckDB to see whether counting stale open PRs as rejections improves merge_rate signal quality against suspended-vs-active ground truth. Pass 1 used age-since-created_at as the staleness proxy (forced by the DuckDB schema). Negative result: CV AUC flat, Cohen's d worse. Pass 2 re-fetched updatedAt for every DB-OPEN PR (4,069 still currently open of 19,502; the other 79% have been closed or merged since the snapshot and are by definition non-stale) and recomputed staleness as idle-time. Much gentler reclassification — only 459 authors shift by >0.10 under idle_universal vs 1,836 under age_universal — but signal quality is still flat (CV AUC 0.5489 vs baseline 0.5494). Conclusion: idle-time is the right signal, but pocket vetoes don't catch a failure mode the current bad-egg model misses. No scoring change recommended on detection-accuracy grounds. Interpretability case for the change is separately defensible but out of scope here. --- .../data/open_pr_activity.parquet | Bin 0 -> 59191 bytes .../data/results/pocket_veto_analysis.json | 447 ++++++++++++ .../bot_detection/pocket_veto_findings.md | 86 +++ .../scripts/fetch_open_pr_activity.py | 182 +++++ .../scripts/pocket_veto_analysis.py | 637 ++++++++++++++++++ 5 files changed, 1352 insertions(+) create mode 100644 experiments/bot_detection/data/open_pr_activity.parquet create mode 100644 experiments/bot_detection/data/results/pocket_veto_analysis.json create mode 100644 experiments/bot_detection/pocket_veto_findings.md create mode 100644 experiments/bot_detection/scripts/fetch_open_pr_activity.py create mode 100644 experiments/bot_detection/scripts/pocket_veto_analysis.py diff --git a/experiments/bot_detection/data/open_pr_activity.parquet b/experiments/bot_detection/data/open_pr_activity.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5545276ac775d45c052a2a79a4b49091a40366f7 GIT binary patch literal 59191 zcmZs^2|QHo8~=ZXp&154))OL2wy~wCRF)K?Xd|g?Bm2I#s*n~%t9GR%rD)ZzOki;PRmv=R1j(S#Hi znlL>)Dl|Gl5*lrzq2!{2zSQvO*yuD#oHR5g!NyvN;U$?Rg(gIVrb;5D88$-25^Wq5 zl|FrXbi(w=&~T|uT1sd_YGh(cyfh_M5&II-I4?TfCP_&sf~8aqce+G}rzEB(MyA<>rl%!NmnP^B?Bx=j5GhT; z3Z#-$X-bAP1vBxAt>^JPQ{xlkLUB-9su-ikN>f5ph5yJYS_vRU#K(o(L`;w485oDB zNHbl%yb{7~gbKVQ1E1*WQE3UVGVg%Ik}ru<)!;Q3gr>sWagx+1o0Rm(NODDJR$^RY zS{(F}gvTYON7yK-GD1TQQ97K!lj9AJ3r~!TPKXK(k4?3ikr>evr;LG}J>k<-8w~{k z!p7v3I4vVZg?ULkdQ8Q-JmMsY5Po{9v>%3 zOqp(z85NCVQ>??2l0r>%rCCXFiK)?vJTFnGd$x!-Rgj*L7^)%)3r&eN{Hh(19v&MJ zMpkhco{^cAJtHQ;CLD2=kP-(acq0u`q^YSfsW#FqIR#$311uOWjk8IRW~Eujq`FQK zN#jCO4ZD;CE~$vk@F(cmpMa78?f^c(L_^ZDyWC8i{1B*G^G z*JSJ6GNL4DWSZ-c;Nd=FC1ZVUFnL&M!Rfjl0L_Bfu1 zF&K1^&rizEQ?Zpq$0PhC@kp(>RIj7zDRDO8=_wxd;<01g;}Il`vSEoX;yG1LNn~O=N<*4!uC+GfI@4f! zA__wyY{Ba>^qH)2XgC~*Xit;!p7E1X=3Nj+Nz+qpbQE2Ev@u%ZAI}{`@Wmo%4$29) zikY;~Q)Y3}R4EjUPRq7Y5(}adryHJhZ5O4W94HBRW3^$aH11TZUeIR*42Jw2p>xq- zWOjT)XlitNJny`1T8dPP2uhXOL}Y|}I9MfxhDS+lQj_3+b;DfU=7h10TQiJD zgodMnWFs-m5AmgGp-Z%*LbKtTaOv`*q|mrHXuy<~B0Xk=`t2BWc@Q;^>%(?eFkmnrr%R?FoH_wW=p$|yRC z7W0%!WR+!AuxFl&ft7fGYN?D`kxb74>g#zLS(?{)TC&<{vEoY;I`uQ1(6 zy-5AY-m(T_0be1{&Bf5Wk8GLhqTzjtJspglOzchd``Vh>o10nKTUuDzO0=!*ZR~Ar z?QHGYeijZE{f!6oag=q^RbG3gJw#RORZK&Mo$M3($k`@?Oeu0{+>RCfl zcF~2f<|qfYaKy6@S)N1_K)U5c<@=VP(jd&k(L!Pe`1Ob@(D8?Pg zayn!+3V!i?G;)S;SGb(dJb}FlBgf-go6GeOFs}v(1^kl(@iE^S9~o{!0kdC8z#K&~ zZ*doD^93I05kp!zLMZ0Tj)bIw{n22ds(|qk8t`9@M(1aAe8FKlV{p>ru^3Bt96qK^ zz`k4PA23lU<_?Gsz&Mivv2RKc_Jw1g#$@aZzyN0;NP>mh%m@ezz)Dbx&VEybW{f+8 zDd0Fdw5JITm~jxIKn$n=XF&rXJ=hST3zH9F5gi>0rGA8APK|KPnThV5;3amsMPQ)K z5NOq;C|`<>gAlGkaEXL6Cm~z_H^JX!%1js9a)yxNs1?yT>K+=UF<3}58e3zrNPZmV za>7Z0fTnv6ORNMn*kPB1Ib$KL0BZr6 zOEDRzIzk{L%mj^~6}-aEUMWH+&O(RL(E;9pPv9qZ+NWaYKnOJ4DRlUx!I`P)P>q+t$N3Dy8o ztN}EG761kLv$KRnWC?DT6X+0S!zC;_CIK3W)|vrGjq~6E=mhTpdFU%nQkfx);?$uN z!vMU^t^e-<#xoHiebGrSqJZ-Q!Jrtd0;F3lxD9?|#GbQ+Axs|#!@*cUqsD>xU@<5J z%K%xX2`4Dc#tCE*8hs@20;2(0gic-pwuAppU~ZuO7LfjZ<_NPGBM7w61K=FE1YQGN zzqy6NIb5{!<_ZHDc?jBo7Hj~_!4N=pr$NZ}xgZbFj0hj*3Lx9x0Z+h3%xs<`EMcr6 z(21)-IiMt<(4xrcIS-a2?@kA_c=BdFAoXYs@CRRSKEi=QfkJE)px8+ONq{nNE?5pI zGme1M;4WwdZNR*fq0r>JtPvLRLpLLN_wT~T&As>-T8k*%h@E4OA<_06N7vC)_y|6O z{(PKVjeVOgAnWVu(cSkVJ|11d$AHT?AQ}hEzyVfQaKMPG=>FV@5A$o-M;epU-+&L` z7mkZ>LgHP9a0|dve100++Y%AvK2Vocp1T@1x_mPnQLLhfoJ;XBB zKE?{_pWs8~DVEV2`^XMt+^0Ck;TcZA68LVf%&2Jsj5)Jy4FXqL1AsXz29DNtWn!23cSMI1Vr+We|Uw0n|MNf&4n#5T`AIKq?IC z17R_Q1KgI;HVxSQak^qd)U+;v0_^hXxQAJNIKn9b0_{oDX64`+C-U;>tvd1NG zAHZ1uyGY3oMxhj-Lj(+fDR2ZsfeRo_f&m#8e&xzEt?o5IA}~5EUJ+0YC)d!dYXHr5 z1JFzqlwDA3_Yh$dmpbM{u|ODyN($d?7_62EVJ^r88_{Xvg2h%rxC%k!iX)CgfZ4eK z;<0!&AqSxq{0mH2Z8c^TD^}xb?jp8ZmAgGS>?w@m5AlMFXdUxF35JvPhTRMy^nnmP z3S+mho@#t0-~aMr0Ct4|nw|pv7^nxqY?c}`3ylJ>0r1!&HO}@FSrY*nUg5+6y68f>#>YNMDqK6jn9EjOkb?&}n zJp?GiSD7lz`QPQ0{8R>bY@IsiBNoCmP{}r^bD;|pW2zQ)lp}mlz&nE|KqeHiElQjj zj35O4w<81F(?EvpR_EexEQCoQ1)K!efhnu4!R1Rf1jI6zFtk8Hr zz(KZ7gS*cg!d7T7uIM2bg@RN-HtoSyYj6(}J`kpWEx?sdD@8+%E!Ftn%_f}&-MD!itDql_W+2eejy@K<5V z+Fo>!1sb;h?V##O>9Y@zpTH(=muRaYYzK4}4Mzrg2|fZxwn>AF zq_q&LfejgF7#hJK8jy#{Bg?@d@EtI$Hi8(99>5aVfC-=+90!!kG(5wKH5eH*NP8NI z421g1jnbPf`QtURBrPL_%~I!1{R%;iZQ$+^7##>?6$&>BX{vxU$7S#sXtRuvOYs5- z1E~77u%)U@9BZq}kLZl{<4=B%ItI1*s$D3ir0O6LgpT1K@FX?vBko>0KH($kGY+IF zIFc%ery}=20BZbot;UHS7QfVRJbX~mWGXO%*@6DW=CTVSKUO7A&eIKDV)hG zSs9hY*npTb|au!4L1rC_JADB~zS zv%yku1Uv(Bg>Br^q$Aoxfjgk%V?i8P0oH?IIL-}Q%0XX13QNIkuo!Ft-@tdktHMnZ zwgMmoieLyB4af)qfE+<(q5_-&7XgpW(&0|jhd>9BB2@fnkxM`ecnF>VG60Y5R_10= zfzS(>0S7=UjRNU_X4wx8g5%&kpzOK}7}igP8=x@9~}e1ATSs>0~bJc zpo%pY&_FA}aX?XY70^;1fp$P?`w5t`6)N0Cp9Uf8(?GO58fY3I?~#`i04aM(yR@2!fBDF*`Fu1E9jx& zet@6Y!z{+#6%T>{C;}5e#vq*oz+^yr#{n_|MLQXRbS(v>CrwH2Aj6ylf2W2q_`D^; zITL1muclK$^S(R0?@Zg??P!w?>BzAPxLLIG6)=0P-K5_zHXl-@tF2Y*L7q zC1h3#Kf1c4f($?vXf`+vt^%mW=P$!EfDQy&s~vC#G}A0VGc5r{;1Zyd{=rE~%ki|; z7eW}I_?!;rfxX}WAOlti+2WSqKV15oiEMFdr-civR^p z8E6Ez!5siixXk%M8m$r*aF;|wbV>keH3|d(%9wJn6I6r8;3=R`mS2tAYFdT^a0OF9 z4mb^k0w_*T;sB@hxT?ED@Bp-kbU=Yj zCL}*>2WP-_94xy5!A$m61*EGp7zro}f4{22cAuL1tV&25tefI`+d+sDmnm1K;|Vi*MUudX4?tK<5vO717BQ< z(m+{G>v91UOTNGl_ye-;B0%e^1s4I5m~Xlj@kGH#xkV=h0&-+HNCtEwrN&Y~X>khB z`kvtA>}{BJHiRYME@(%md>Ni`SqPM|BLFR!MoR-^;Z$rK25uym8!OKu+HbXskBy0$?1za*{d2+1ef*0U#bw1tRlS0#cql2`|FH-Mu&~ zQqoNW44UeK+k%OXv9*3hMEIt9h5l70U7NfkY%GZ8Jd&gi zQ`z_lWY`i-F3ZP52m+JA--&0SO)*UxQwZz0&8KT%g z3x2Amc?TE&1zLC$+)ouxP?(lqsfI@_qy~R@Pc&SWJh{u{Z=CMTR$IV*fn0H>=nqzk z;%*@CpiMUI!?sv(C3FzmWWgs2eA7nBnl96$`>4Bu8%As z)yd%Pz>00K;41hi2y_vlBZV}D&VL<4*6+xiF?ty{wi*%>DJ`21BHgwvg|2mA=Kr8;K$_N+&g9fRKlw$12Fq&-jpPSfi=n=66e|}vcfj|{riOu@6m(C-#ZK;Fob(-E3ktMZQ z2IaQ1o9W4x=wr+-JRku9#zwqQ{=WEz?61J;M}gscobX2iY=MsDl@JeP`n1<-2yp<3K6Bde3jGxQik?V zE9_>T6gF{J1|__#p_rkNo(AGT0-$giP}stS5&5ANjKKR1%2bNLiG{V?d@6;d+_Td# z^eqD+g%#X;m|tiI73Od{EJT}P+@-L9d+eiYS~j5Lx%!7~kHRAE#u(+5-&4iBf;*7v z5`u|8OALEi)9bGL=!v67CmP$>CLtFn6kem*E=4Y!)!A-E?s7@7eF=O6wX8ryd7;4F zJl3Pfn+^*^<2a8XL22GozfA!f zS{>yVuB-~&D;iS>RA{aP1-3(h^ZhXh7XcZU9P$v*CFv!QWz7`1#Pfm>3_?L9prI+a zk^t4WOhD$E31$Pz%ejC8jMljTkhbtYci&!!Ed|VSK%V_;W?CMVUosw9^gE#7gI`44 zW1$)ZUC>{xEC=C(H>J(${G0}+Dugi_tO}Ew6&_TxJRLqsz9cqu^Yy&WBNDNa?iQLl>Xdk zJW{IWs)xE#1@{`bKhBE~8?RN8gZmhP#EChITwS0M3FA3@j%SR&$Rf z8t9k^LI4>i6=VYn!ctHM{!Tz~^H+URT_?Vpd;AMRdoEZC?f{zHA-;{f5h4eF272)> zT-~7zBYRx}P=w1CQ#MM0d-q|D4i`XiwHq7)hXJkbB6tJzsMPdDV?4+LG#Z&@7a()o z1vXS6?9f;QB!y<2dNpWY0i-%j{R;HJD}8Nj4S)wA-}fkt;$B55qiqf>K`tQsDODwL zMWh++r$DnRhl?pP4SAI8Pf5?<3T1+=8JGa305Z`o=t5D@2j=OAtvg5q{g4F+^>EJy;& z0i9b3C~ZlLPv9$%tzx(vakAbh;16a2@^%T>i%gs<_e#ucj_+V*A)C&}doq0AZtf(B zZz`4yaD7?T3;9V>!2TK^Vk9r_bq|~1QnnGbb%fyk9T6aSx# zp|Kh?0a>;{iF-WU1A$zq#Fi>?F9*m_bR-#P8dwR)XPdzlV8?3rq? zUri>uje88HEFA*Ke?fpWhzF^FCRqwL0df}U=?zGBx->d+M@uO zF%ZN8Qk;~m0^h+e5Kz_4$u?DOj0otX&l1=JssQ&po48^+6~}#otX$Q>eT_uso(ism z;pjV!?HNEJOD5-6b#dQTgrQAFLG&>SxJ9HCB}0+{7XUIK1zRDYj3QUB2c>|{BUe%Y z)`Fwp7&rxJzP~~F9PLiu9PZ&q;G4xgI4ht}5r}{?5Cdai255Z_U=SD#hJsqw228Od@A8xF(5MG0ih;7N3 zU8P*;T}8)3K%S0ft2McqC?j5iw}6I_XT=h(Qd7*4>*;{I;0NfB=iPX0X&;3I^|yl$ zfKJvfY~WJU4((B39H2533n=g@x3_?Dupf|lkAV~55+JuWgFArclf@nTUz7T>QQFKe zw#Wo85<*P48>O#opb2w{J{Y#zAFof${OKDXf2NRS{P9Az&5s{4(7cVyd^gr*3|?cI zxib@3@fd!(yU>g)Ig}2_L;kqoLWBR^eO04lsfW;jUpxX&CgkY%K*C13bLES2Sqdn6 zhOvHQFewI~0O@374)xab?|tEwEvvY?qL8yO>Q9a!;?5 z{yVfDTP3#R&z;D82xPU3^ggNu4d>1*uJVmWI}uD~YkP9FVG)Ez@K@_hw&TwuR0%rC z$doHD0ENLIR$I(n1EL|2XQ@O{c+r6rK<2EUn7jIui-l~Un2U{(5YhoXO~o`eFj0*a z%+|&NJhHWL#x`TcA}$UianK@g7~oBx=zq1f03D0LaX`U3iFFZiSL&S*_JUi06d+5g zvbMdsNAD#N;2SF8d>^A|F;^>Qp(6*-JQQ3Hz$4HB?AgHHoFcR?y28|fUpU*qMEK{X z*$*8w1|^D0`RqgZ2)&+w88FEcz)o;w=qg+n*mamGOH6Xcz)nvO9u0dujZ~EN6|h3x`0o7 z9d}V}K>H?m1Ac@4xMx3zZH2EkcN;?cXrb2s*9~Sr$Vch+SDcnmZpp4_bl zWn%?61k|uVf0pTk`v`3&P?E)&nN~~gp}@ArZSP=g$AigWDu@Q80A(EoB)OI99a&En ztL0;>05kwChFoe0Y(al80I=}}+@(YzUYi?7ovj_uWYA0V4(yd@4T896Ag@!>Nmv6R zcL|*HUk7=Evg1C$ccVh?Mpy(v4Ag)TkN_{>18CLzcCF>(CNGM+kzutg`FJ%d=BB@g zz287jdd;nch7K?XxaV-3PMFn zFMpY7jDCCrcOTWr8YuIxoHOs{7W@{c{KAA5=W#6!gb)vsK`O`rd4QZsWpW3&0k9G- zd0${lj=}(3rIBNsBDvW1MTZjWslsI!1^f~4n$79U^p+GM&;wPN;gTfgI9sd4MF)P< zt;xOJr?4^x6j~I6LjhTK42S`X!5Pp5s8n?U(%>Wb1QaP9_0Z@84zt~MxRb84;}Y8$ zJ^tVjb(pYDqi_Ms@&0pj#=3ZOUgxtpUfk`%SR7aiD3W@!E?%hQ9iI4gbj3*20_~Cb z6@AGF=3iE<$P8g!G`Jh4HXJHs{WQ4gsWfb6TSVOLeJ}csj?z4yw4>}OW3wz6Q%N^x zX?r|QPC+N7%zi+LNTt)g%8YyJT8*}XQUhn7*=WB4wB{(K7B05RSb-+K{}rgf&-kD! zOBExN#b2qw_g5-hO_ifYB)X3S13HecRLa$($>^XgrR-$U*Be_hJDGs`6!0Xw8{1__ zFbaa3AdA&j;vOXEvS!M5XmO7|l#+fR9gw060EG@YjY`lm`X=NI+oH&*b_Q~0ia;;& zle=zG?ZLQ;+^J-wVn9jp6Z{5pofZF!5huuVI=eaV(Xp$*8bEnQmH#5R1ZV-xfU^HF zUhSwKx|GD9*@kSVNjku5z_MnsT!f#35Xoju z;+~1*=%G#>J%y4s&gd)Aruf(ZHiL3N>!f%&1P%jj+)-3uOLc4X8b#~xrm^Mp{f$n7=>#|H>yk^Uh4R9uViA; ze*!OgT9wSW`%8+LAly^@q|2oi-X6KZR65v6;d2ZB)||P|b_L++2iI+H*36Sxin|;u ztWviz#MzMkNDTh7wEeTi-!^ESrty5*|Mj;Gv<2F`dE8!Y!M;C7F`A6o|4%RXCk{Go zKEKua?@t_v<}zYN#+KpxG4e@_pomeZW<*Vlau*{O^VDp4>V7=UB%XE=kFDnEHu3bk zcm`sbKDIK(eln&>GG;|G7S%FVO)}P9GPYt_ds|rtKiL6EvQ9;^gQ{hnn`DP}$-0R7 zZnpg4e*6(he9t1jcQxOqi9foF?#Kzun}nOYgr#DIZMF*A{S?ZR6e@}o zDytRtG%4)uQm7Uy?zdGu=%;u%NwKy_@mRIui6+HUU5a&L(OFy3c|TEolBl6bbh%n| zwMlfXOVlJ*x?!ty%TMV}l2S{NQfsx+gC?a%T}o|Y<)^mF&;68NBq?_kDZj2(e%qw{ zu1mQ~tn$%T<+Gp4zey_HMJnH_Rem(7{KEHNs$v;CG2dS-pDY#>ixq0bqGqx32eDXH zRn1OS-CtERSyj7Om90_LZC2I)plYD1*2hlG*k8>wSYG?m4KYr*m`9p&xp>sH(f!sSo#8ACau?S*-3|qwdqJKKg^YpQ;A_hS7L` zjfu$`fyEj@H5$Rq8dE=Lgs5tU*=a`jYept(Mipzu)M&;vYbJcqOj6ZKvC~TP*UCuN z$|}~HQKL1hS!>P*tsGVD`F7g5{@VG;+6Behi)*x(Hft~Ypk1V@v%*eimA}rKWSx>? zo%J<38=G}Df6ysaWw+U}+x^+{WVWK1t*l}9G_!j@u+^%)_S^M3=-=ybaO-$ORo%08y663M>yvdGighp7=w5Bsz4k%3NmcKLo!%{fy*tTzEya4RHF^)4 z^&WlDYg5&KYN!9)U;jn2en+wX>l*#H&HC>?=y$31{%F_xvw!b@lY7tYF7ExUruUEL z-oHNdX4DL1><##14dhb{1S<>__8W+97$|=<5UUxg*&C{lHPlQo)Lvo8?l;uEVW|Jn z&_JzEANxMWWBZt<^f6n}$6|jUs~df+KlZUzGqSfgau{ngAjQaOh0&n>M$R{khJG}1 zQ8RY4Hy%FLctnb^=L%!*{l-2wj7NVo_ER$%Yi}}stjWX_lfV@wLHkXDZnrn>b_59^ybzi;-ozBAAEo&B`$Tp6=@ zeasfPndOC1Mt<%zW*9^L5+IH=H%!^wfNdjK$VI z7G-W0JHjk>&bQdL&0_aii>jv<`(!L@`dA)tvpf`Ld1Suj(QTH;&sv^*YI$15>P#Q2 zb8c1_!mKXNx4N{=>dING#-~=-WhBjgBsbk8x5FfN=S%KwliWWmdH7WFSjPHEAM0ms z*6m@|FXvmo+GhRctaay8>-RD?ANtsQaE5){|72&-)4tC*c_Vjw zlW_aK3+&Cy>@CmPOP<-=$o8`{>etV`U;prajtlw?EbBM;T)!dD`VEtHa5ZvpcX#jz zcNn?A!K=(+)Hw&=XAWaz`}-U9ALrhGLU{jx1A=8a{U??6pM0+Wnx_6!KlKmkIUuax zfQWGeB2x!Mg)2ngRERk+AnxXXgiiyKdOD``b4(lOn33vuB*!tk%yH&9$Jx&u=gKVzV5QUI15QhCIxYL;RMd0eihcuEjT^WobzsTLf$I+p+<0@~=1&7l zdk)&xZ_xH}gUV9}RjeFTd0^0-4~`prICXIC%E8AD3_fvl@TpIO z>v}q$?dN=coO6AubHhq!pKZ=p&N(+ebG|M+q}gc5P4^+U!-w2mFyvm@ko)I`JbX6f zvFy+%Mnj*u51rc{KJ?{+p|8q@zBxCv^V!h%vcowxKc?2Ew z2)^Yp^|MEa`iL-x5fS4@M5c|1S~Vi(;E1?eBN9H3NKzk};xIC8{K$;7ky)!o&Nw)7 z)~%6qK99^%_nhzGnLFMyKh3jXmFMDvo=b0eF8l0Rr0%uC!E4ocuQh32C9AyFZ#Um? z-fPoyuPuDh-nhcM@}T#gTi#ordhg?psxcmQVECv*5u=Xejyk%1 z)baD9PCg%Xn(uSQ*yr4Ep9>K_7ju0sZTGoy-ly@o&vm|Uv$5~Z;l8&ceDCJ^-rMeb z|Ge+R=f02mqn{X$el~n`d&KCMxuaigAN}V1=+5V(-}A?OFdp-1_?Ukp#(c>g^L6`} z@8`$-d_Lwk-;ZbFC+p!SC-v)*=O^6Zr+C3nsohUS&R^BUzo&=4hSXmx&tGSUf3FMv zdhPzb<;EJCj5YEYYa$)nH*c)@j(#!A}9+Q^NwGa1* z_Ho1H#=Dw~clQ|YAss(5Z@kxz@uMz`_iZ0PMs9+?$%Jtp6DCL}1msPav}3~L3lpZa zPnae*G1O#YxW`1PbmH{9iP1YI#$K2h-##%>E+E+?Ak`xvT^f*?7m&RpVCIE@+3f*y zH!5bvq_)xG-r``=l*$ zL0e6N$~=O0NP~9f1?}1qwEIF(ReR7rxydyqlMi@IJ|vxdByaN39g~k=n0&H*@@cu? zGbX|3Jc2JsgD>UF+Da|HR=HB#}a$7p(Zr+r8JEq*fFy&$Ul*e*Y zpO{R2<}tNhI`w7V)K@#EzPT{9vwiA&xoICvrhW35_K$Shm%M3TcTD?!VcO63X}{${ zc%~t;BSPdNLwe+g2+Knh>qC@Ygs8}es+xxO91*G!8LE{Zs#6}?t3FikMQCsNFhkQY zqY+^ykzsxF!_3RWEbGH0FT!l(!|hDN`;7?i9~thLA3m@=d~kjEkQd>@it zqZXA%Evb(xd=a%=KDyX6dgX}d)sfL_^P|_5M{lT)-t;1Ri+s#h)0namF*_n-cIL_zF^W4kBJM(D z+{OI3OXYD_>f;(;#7+1V*Q6Q0+(39^K>V$I;qA!yyZQ0=%H!|X$3J`#|5!fZiD|;K z5ee;)2`}>#UX>@jsZS_wPI&ikLYHRZ#{r3-Cno-zk=VT^@!R3VA9oUe{hP>WCCNA@ z@dJ|NGm`{slN635iS8yTe@PN+C963ms|O@&W+rQ|O=gcI>)uV)|B`H=mD0yC#W*0v zG&99)ZHmQ_6sx-_)?ZR=wNmXJQyl_Q2V|x?txX+tB-QzD>d-H#E?Q}Bj%mXK(ne&a zd9F?KK9c5hH*NHnG(WBMv5x8E1JWmErU$M~4?2<_d^dgSm-G;=j4;QHh=7d9%#5hD z88JsP;_hZ7e91`C%1m+0Obf`&$jr=In>pi1=B&G!bG~HeXl2bmVzuB%WNtuKer8s| z+N{M#vXTJ|NYNGp4VWA>_m>@}I$C7IUic4lvIFVqCq3_(44s$;T%>6fWZui=`-;T`v zad+;oFLN2~92ut^eqfG#R*s+~N1--H)RLpzog>zsr{*+IJ#e08);#T!d2H>xy=Uj? zbmN!0wl1@p}-=36$*mvqdx5iGDXThPyQLI0=)j-ED7B@1qqEg0Oe zU`WS;VS-#&vs`!2T#u;Skp;P46}h7ta(z2;#|ZNL&GN>1=1qvo3n<8&RFOBiA#X}Y zp3#@Q5bgXhr~HV({K%~QsFM7c+Wfed{DkiOB<+PMP7Bil7iMHF%qm$pqjuq}mW6Y= z7v^Xe%y%lt4J^pdDkvx^SX^7Mw54EKcR`W%q7_byRs}9vQ;@W_V9~mYMH?CxdE8yJ zxqDHm_Tp_$i?;_ZF3(zAQL?zQcJZE;#e2IKS8FfX@3iD#;F7~xOKMA&9IIV&qGidc z?j?2FOV2tjJs-HVK5J=1$p*OHg+t%E+}j^E4=Ahcsr`_Zb9L_io*L1g%3Ll z9}AW}F3%9VN?N*Dib8vg}>=vM%lAADxze4qX0k*7EL><*+vuA5t`vwqva^-f;vcSNt>xoG{aUF&yWT3_{Q{XXG_8uJYYyfz$)-f(2mhNHVS z9KW>T*i;Fg1+O_e@rHzfRHeMHQYBt|=(`(c1=uLMQZMwH> z)BQ`E9=_W2Sh)F#`Q~R{o7}+N$?@Yj1^Zh8Ek5ytkReZ0oytn|b9n%gfs&ueaGKl-UhhZ$GHaA*gJ?j54S7 zWrL2DIk%P#{Z{6}Zg(5BeR$CJ5i_=XuHWu`Y`ahE_R-(A`>{L54%#t3Xvf4EI|A45 z2s*YSxOKgZm&Y6{k83SY_*S08?o1i9Gc9Om#*Cd=>vzsL zwsThN&N<(9RthWTSyU|WuE>k2Sh%=iQDw!F%N2#ME0!zlDz@0Q(tFqHm|bfZ?^;*6 zYs2MTn_lnQqENZjqO#1paz{+%&c&6xDl2ziuB>`pxldttjm7Q*-n$RQ?6%9^UAunw zv17YWwC+CjZFe2J=j@<8=Y#gt&)Czje$VA&d#<+bx%O>O6I&H!R&~?6>UK=k-NjY+ zDy!~au6p>o>aoJ!Cl-63dGBqH+52+w-dB}--(23?`Fii*FMGS#eIEzy``lXaPt3kA zi}!u4-1q(RzMrr6{Z^>vSys!As+Nnb?y;m=xVu{MO109PY8AyARm+;5qiQr_YqXZs z=WmWRfTIy52nP{5Ky zlXf4PeC5!TH;1Mv9uBoU96su>H1_cHC5NMTACA3pIR4GyM8zY?mPb-Y9hnt$BxB|g zpT$SAcORK~<;d(eN9HQl&aDITw} zJbqx*@edaJ4#ggK%syVb;rOxL1;?)(Kl$ePX~h#~EKi(UQg(jwiTar*8aAA`eEh`K z`zNk_Ke0=Avf1+F%~28%ldUt4j5>Dm!TpnuzMpLCb?WKhQ_m-#dNK1< z$A(j{kDq#b|J1whr@DHb{y6yb=gFu4oq4)@!|89wPye`o`q%f}(6GdB+bT6hjUrY^+Md$^chzhHeS7a;_6lBvc|ZwYd@|w={DYQZXD>}U3tFX=|94PTapG{)DX0;*B>QkCEXE$qa zYGzM1>rUCC*V)`#>4u@?hLP_LllU8b3vZbJEVVd!!|LIUp097%Ou25WceC5-WXquc}+48gKb_-WsEH+h1~fobT-k z@wWpCZ%?YaJ-PArl+N38rzzbDmD~yUy(5jkGrjOmbk&{M#yjzaH3>iOBnL@huArTNYKdENN^h>}*-Cbgx))Z>8_O z)kE&AnSHNh)4kZrdm9?>ZR)(YMQK~AVrxUM)@?&tw@+y;pWRxqskQQC>z;?Ldw;f8 z>)qc!}a_Ycp$U%To4v6J^tJiLGE=lwdp2WN*oI6vh<{p<$~n;u*~`QYlq2iJZ+ zXwrLlW5~l>Qy$)#{jg=z!`71zA3S{c=;y;Wy+==nJbFIm(TmxSIyOCeee%)UhmYR< zeAK1)_~VerpQk+jclP7%O^?5weEj3#<6l1?Gx}{ZL)-XM+b&17^;p&>+}oyTotb*E zP09MK@~<{~#V4xPPkN4iqLJ`KYuOW>y-#{wd!qO5NpIz+hSpEr2R}7_q@wKq)O5~M zv&~N}l+7)#J(awBYNPzj&iYxu(a(4jo(-7u%xUwpL8qQMKYBLw*E5%?>8{q#-A6z7 zNO(SS*>kVG&qpO>_&j<(`qy(m{r0g#+s99BpE##IaC3XmoE4L=wNFt_o%*XiME^zD z&=(O?UqsG%5w-b6%&8Y~k6tAFdXc36a+vkY)XimS%8ffFz08>NGHdh88K+(zy!3MR zyO(p7JLc^@GJj}C?$nO_IUNO?I~JenSo)}A*{_Zw{Z}i7zFHl zS0BB-_UmnverKV1=grZbpMyK^%;{{|+}V1n^TDIeN54AT^xr)l`mW9UU3+{Lr$k{#0} zm)O-~d6#fsm*VvvlGu`i=S2|M$lMb3Zw4`7~%>`rzxI zhP?k|9G^E#g}x8KpYm-xEvwP8PS!``ha>l=e7hd=KBx1Tm>&XdJ9)+>vr{j>pFC4H z{F|Ns0LV`BJsSpM`+NVFqod>*=HL7M1S7DYF`Bp#$GJ=k^ZtQt&iq~)JO^wWl0O>u z$9|P-X8$-rR=>RY=V-`wyAmd1n{y^E__7bqdt2)_+Y#fb{(ETv9VZ^X)p8h)3!c=* zV7|I0$MM_IUw2xgbq!?ZR!eoxP{^_iE~t)yd}iv>As8nobbsJ37i<%(w$eN_k@_W4ti-(8P9OXC#9e}0DJ za~@wDJec(Sbb24@#aP}yPxHELRp}1*#P)8*)UiWw-2JfiDcCYK2PSPI{cgW#9gi*3 zXY-%qCp9itrt=(69(VG>@iSUBb9x<-gEnS?c3%k2GmYY`5`cz)5m<#=w&+FD*rghEPGwc}jb5UF~Ar^9-dBrBO0~6qv zHXh@;jQ>`)2K#dcTTb8^SmoTz7+OPSPWw4tPVB?78FUb1IP9fA8MC9Mqr}k9r~Yl* zEZCy%p>*J3bTfT^jGxaFN6j~4?_lXUEl!EEp<>?JO zb={fgWHs~m?A>k{EY+7C&&%>0-&9>U5~8b|u|F;9=Wtz~wrySi#!xwkb$-S&Hat&} zu}1A%!@#(b0p*Wzu1ne`n=??i&f&g^K0J^Uba{CSOfX?eTrg&0GK2Cr^O~%7Z1*}1 zJ(%EP!&si5NU5PM&d@+mn{6u>CD_V0<_W|uQFki&yb96D_rI-qNusdDrm8$o!N>!q zat3B(==xI845}7r0@`4i{S18 z3!b*P&eQ+vb6%imYvuk{o~>w_-;hLJh|h-Oi^;n=cf!Mxpk$r?%}qObW+Gnn|;ZzkscQVs?AS z81NcAL#)nDfU{kKcYBP4H5onWLSD7VMPlE=>lQQL<}XljZ8Q6L@YZ+C!uVgmXVNW- zP_KMxm}B*;c%K#QUbpt`q5{J%!NzZQTwTO{$}31K>o1xCbpu`%)DozW}-1! z7fKAxM5`iWU0jO<%4726oXlz>22i@xWggDOI_owp^)W0FyL1}-*dAIWSS4|z&d}x( zo}aibzBtaokP$dL-sW|Pa~6fxw{JBQEnht<*|5&ad~RirAR~Ur%ES3*lw(cC`hsf3Wet!^8TQvDb&~5Ddwxq`wLno_~ z?@!OCig9Ck!A3)Y$mV3`KGy=lrh`UuE`s_)M|l;3oZkzD0?{3I{{zE1QQ1FtUgQ6V zkYXj&7Uj1#&)|iKy!a&_;7L~Vh`OtjAo_hR&&8HGZBlTB3fR3xA>D=zqOsqmHNxrQ z=Yx50#Q)*w$^&BhzW7X<-0F0DP5VBRmPu2JR3zS5658xpYh(%e%9coaBPtOgWN)n5 zvzH}rtdS*qwlvuyO99 zTNfz~fugGsfq1!eckxk`9pR@&cyQ1hP}WPF1YudZL1KmBrU*$f!!`ArVyB8FYZh!5 zW3A^^oi)l3-!U|P6?WD9@LPf0j{JyBRwXg)u)4}bR1WbK~zFkpJH* zoZ7M$*IX`N9gmB6=JI?1b~>k1nyOVxvDhx8S;%2(1vnzj!EnySxIHNQ7AU~G1WD`Gd>+Q zYz?Yi{vlOu##{b!A&cwaHV$2ZT9AE{vucbujyNsN!};(LPw?BbU)LDN62q#l5`X2) zsrwDEEA~uDaRv&1N*bzM?wR+EOnk^sX)jR#dt+o6ZjNZ~oh}jr4-Q^?FHVDq_pV=% z9=6?2N{S(~gONANifhpgv5c^?Ocam)`6G-pDT*CCp9Kr18-kO>MgH8ijcRY4ps2m+ zog{Ir`{L`P2H>JZO>ZP24Cj(n@trCN_*Z4F&@*6DpEg(@0oR=x{BXlQ3|wMEO9@Q6 zHesb+0dJcM)@nrXS)V>$&O%zpvyQ0GEM&i<#A!r-C74E_O<9;ZC|-p&<=36%wAvVU zUn#7n5yr6L#HcELL8O19ekjh2KRj^Bb#w~YsqVuW9EaR@wiPqVO)*_Hg2{I}zHx@G z7_N>fDaL&ZeN@s7$MHyAwMs0D{3M^fTQK{d7tDDo7?cb4XO~i*z};sNbri|_ZkR_3 zgXLZk^~ergua5hU4`0OkBBK9X=k6qFyFq}gLwMJF+()e_1bsemE6_bin z?5VTtjF;ZcRc~}5q{3*1IDz3S0_LQvlc4+exsE6=5#vwX#LYjlclH{Y5TFE><{ncz!!t<|}B*dSq829~Id)(GFfn7h-6zEx-6lf&)%uxaM{&*D~IU!LH zKBYu>&u<;7_JrKymdLwFE7Em!g!fn={P29b(4jUF{}m0fH6|qL(OgtvTzGo3Nb)%icS!X6D7NFvcL*KNUL(BoWw!ADaZb2ye)EUf z*obS-Nn#1>P>rLo9qr6eyt#o}aaIn z3qE~?Zu9=lKo^O(%Ia~>9EGi5tngFxV4>saZl`3LB&C1%j8<;AeeaCUk5`vPSR1j4 zF1Wk#v-6%g;%ep{c+o4#&@>#?2fyWr`*n3qWJcxK3K_of{y=EDEq2#i3c(Hqlr!SiGoJwXPvH!_|F) z5HYN~a@GjLJl1gRZ_996XG+>Sq4pt*M`wv^7(OFmdzqStzMa~>)D)19dsSj4MRa}k zjkR*@Ik!)WwH1!rVd~{9JM5}Uh^D_>raV)l@29}q`a^MD#cBSXA8##`;_cA62_^0{ z*PP_O_=GM-j+dkiKVG>t;J7A2`QPJ8XmrNps!B z3q>lC^0LX-dq{S^>|x$yT4n0QpCYUDW4wq1w6d*RhJZ16B_ zEt<2|Ng{*eUxQu6rIB$SIWn|ktbA0B9QlzOQ9gPcs@@&DyR=djDIZ23>VrCnvspTK zgq(wLzUr#HGP2V5?OKdBMd$s@U3~C5$j57}swDYMAJ~bvT=b+pqjBrFTc_r&NB-qS zr{6zP`@>4x1Nb+Q?WiW8w;6b!N3ayiUjns_7HSl{q3YF*=YcG#wIKMWxWP*aR9{B>}ooC(NmmR!cUH01yC!Hal#(JoqQ4wu&t_@F}%yk+_U^d1;eRjdi4 zMaCq##fnxE%SF7T20sQjL5&7sp)a~Yu(IgY)$%lnc(Y4`sEN-!< z4VKD73-2vKkBXcBdZ>-K0Y)oc|53Mqw)6A3w1^PBTC2B zN!O6oIQ$xfFU=+Ja!C4WIS(!&%@=41fy7&RzvQ{#D|e5_=={>7j5-Xm_`RjW7t13c z_+kHc=yn{Ji=SvJVWDKtaT;L;?~Fg$>uTW8*?c{YmX-KRNfpMhNYtmTo&%R3R$Y)z zH{u(Y3I5OLP1auMT&%3mldD6>vqm)ykuvP2%L5N@VcP2h1U;C=3r7n<{Fj`XdQ{4N&-qe?DZ_?&bbKf-hGn1MxnrC1 zBq|u@KK~pl1-fnOak3{mL>YGC6XF8$r{^t6B{9@r6~d1U^CMvxDR!O`hP%dyqVp%+ zpvA^8Fl+2MaT$cS_0Fd$W^mvJw@yryMcdDrp|xa9q8&$TN=U#?2LYsz+*G=0{2?}9 zSEA;iY|G9J^jnN)H}#;^W~4{@WxV5;yVME3@;l{sbKV)%Zjr;#!@*RPqL|Aw!O(yqBQ2oO|`@OT6EiQRDknBI4(; zJ40XK>v^MA?!k`inOJX`2k(;qY`Idh-^I>?8E=vxHO)@3LBg~N| zao>*apu`x=lYiCf(%|nXr%~Es=oQ5BNVkTHtyzd-J>Ggl%*Bd+PTxHrS0k!DIXeUO zI~#vjyApZySoAWvzdx7L7#M;(f~Uitywa6IyPF^0qG($AeIAJFp8wkG!3UJ?s@@YX zptMAf?9f4+6dAr|b6X??+bSa91HQ^`@?iIfi%9c|O>&lD|I6|?sN>;Z8_RgHzjC(b zpaTwyUsKpIQCr3=tnPKdpC(Yn8nsOg^j zrB82#Ve!x72abyg$!K^cU_jGySVV@%YVtxaV_556=bfnxr7+E=0EN`$jBS`JWf;=j zy#q!KM|Yp%P}}_3vu?V+2;Q%CMBkP4J>VvTmn`b8XG!$zg@O&-$ekI7y6wzFD#5{f`a0lL*x3C!?Qx-p24|nbl}(>K zx}TUa6kSGlKjNT(DsTr5s|6jEV33IOXZJ?qFP9`^G4SF(t$Z^YmC59n+t6c;H9*tM za31?o*2-lF2z4w_A;^_}ZK}8u<|``hqRSCxw(~stfh-exuoM2sF9{7p1ML|cBTQ~t zW%M8%F09p?EHMvV%iI==S@?0V;UtEdvC%{IIJtcfdVEyXz|GZU8MdrGbHRx+gyegy zQ*(?c#dKu|ii>IYFZpPgSl1L?U&`6Tu~{Sf;~&XYp4hVO4%%@3+K+x?#R{}nv#&nT#=(Hp6&NLw?vJvCX$B2ev_k+! zK5zDYArQZK%+d+%g!dtz@0687y(D+3x){<*4^2SqgqT!J!FS^S96Zv7@^*%Wlhtty zn-OTU5(nu$+N}`X{+u*j5gI8zora))z`CBP{jO>vhkk`(c+9gOEAV}8R~!E;`WCqM zA>lf;vw)>oREHD#-ML&`&9F~uMjpqp?u8!JiF1L;AZuu>DNM;Ivq1~5R7P_DP=Gnh zmHQcaF>~1Cb+$s!6DNg%rUmBRN=_EbVBy5v#n?mEf*t#jtPkn(Y1GaE#t!S-32AP0 z@Wp1_2uG($6XXR@vhUADH1BSfOD&LGgZehFLt0+6^^Cz6pPLV9i%S2CsqT=v0N&mW z$`KdCw;6LbVUvUAcf;`oyDeEA@vbnnsT@J;Pd8p_Lm1JdfD^vRMbR(TNEmYYQxc-J zeDRajr3g1Y@>L&2MRk0W=%g3LLqT(wdui(6%b0POa1yo;s{Pa7(+8-5H;b ziTUJ5>oX(KPi8NSfkte@ecV1i1C10rarMHn;t<91Mdu{=Gk3-XO&Q@XjSwRB``Q(Y z(1&+yeR-3(K)HM0*l1sT!#V7dEp0Id?Qhq&$kA1I?H1?~;PZ+dUsOqocLw>-un{-@ zY8NpA<7Mqd9td{_jA|E-?yE=}|7aVn1StQw=@?D00`sTSM<8~0sg2VO9GoALEpev) z03Euuyg~|I&u&DBmb{p<1(9z)zNZ}sQmfxl&me63^Nh_@K;kZ%bXt62OExC<{qnOPrd$w*zlZ?r0 z8v}6(?C+y`sR@B%-=+RIgRmS?ElQ79OnZC0OY63F9&G>!JKD)n9t*opwyu*`gXg`a zrnJZ!I&Lefry*vre536eR01cf?q3!7*{A=l-H2j5ezOz@DP*4 z1bTVxi0*7K_CU zllQ*&k}^{0J+X8kj^KK!uA@4I?A%tSDPs6@Ni)VG^QLXAOh-Ebt7l+}AP$}l4%g5) zTk>>VlbC1NT?gMaAnjHx9J)=Mq8#<_&kkWB>yigfqeUj<T5I6WZPLGD-R-GAiPLBk3Gd;CP z%ql%I-Hg#85gFV+HlLPKsIS@bLj6TQ=b3r$l7_QEXl|^|yt|uU;>~ zUH(}R5<_`&WiOj2C1@@99)G`M=$cE# zK>{Z8=5ZNitzf7xA1hCS{A2UCBhUFhnW-vaL`(kMGr?VNlRZB{7XkegUmY|Gs9*cV zg;q+*UYl0JDt$lQ>4Mu3nPxs+n*v=T4h}F~Ld3PzT!9-7EqC1+{Zo$z;l(&YH`wA1V+p34>rppuHN_v~Q zNag;sV-Mki&wdPOpfU>xPTSl{l?#7=-!ixuNA|sWjNJ`fe!DBWS{t_g7NKfO>FH#w zE`ji>R)Mq>j{#fD1iiUp&*_2qV>v&*4oz3^*?G8+ob-tRlfD`mx?NQa?&FGv$m76t z{-5C}YUUpXtfXmeU_oJD$%OM0B)SgS9tG)@YIig%B2tyzm3dQsjO z!pnPhKyOFe@UI0gx-F|73yF&KEap5-uq53k9v65)^!rFhAv-n3vamHuBJa!QAf3t+ zb2efBe-0nnEp%K{_-zzrEMV{2^YgT^VA$||F}9UgZ}(9pf%?Fqtw_`RHHj22*^jRO zLIyYA@q0%K%s&@<1o>>_Rr@d+AyuBM++cyI1v@EajyIZ}hJuOGI132yo-Lz117_=7 z4-1-;FPmtmi2%P-?(S#`)_I}au+TQ`*byYr)!H*=D996NlMi;pX^xy$stMQ*yxLYy zVD|p~k5JeTG^h7KH*V^)&B$Zy{G+4oQB(@jE)K`{pFXs&!T`KVTJ##TGNSNpdZ8eo z%jZr)^6p&Yj2TFY)|HE+aCYBMoS+3x=>Jyi+=+sFV5W~3Ep=qhZ1q`&i?JgkzRaOz zQrPm}`pGCm?psVXXx&cw%=n43jqTU^FM1T5WkWb>64R~c9@02-s2Cad0H<=Xjs1C= zVgVWU4mGM4(rbbWr@)mjU8*f+_;o#t*V80R@~Qe1vWY0R^SCnH{nlT*KbQN%r)Hx+ z=!@2-)<)s3&-?t>9B2Ld*5@o$EI5bXjmELwetS5C5-CiwPU?$mIokWlYMluFdS%7o z8>6>~%ym@?T+yu`gX>$rzkiah2@*dun~+%~58Sq4?+fz7gq1Y?cVw3?4sK;8_)`X8 z?L5N_Z1cWx*%F#-312t*oW{2F;E6V>YUtBDxEP0A@^hLsvQGMtI6vA%$n56TnABw5 z-`OtL)q$hgt8O&T3OcV{UV=j#y{&b|go^19AvMaa++aj&aFYg0vJcZr8*n(wBw_da z4<}EcybTlvv`R+{lHcI^pgR@WkcHb;A=Z;|dZi=@=Dz&V9*Y|0>{iNA3g#Vo8;G2m zc6C6VI2Qcu#vI0ZMYma)N=vLrihc{aTB13nXHUu*uF6D)?j;~x1OfhflK!JYsiL&j7%w!G2EaJ8|}3W_>AiMh_cp9mFG0) zUVx9KtFoU3 z;8Z~3qj01VR?wtQy@xw%u5EKqTu0vcXpmy;u~MO2gZp%R&vCrX$((To#m%9{ekWpp zY|HUg-*KP&-&nSY5?APQNygF?a~NVedm2r!hQa&l*Hb%d=%K!Q*5B~gV~Mv^igb;m0YA~n$=EijEX^f$5fTea0+I- z9(noF@K+0kxPNcoDbh+u<%By!ru;x$iyKpDLkrAdbKhw@P}E_}?(b75OJQcBZ$fJ5peRdiP~RY^3WSS4A64!C7*A15F_?3bu)bowE2e{WPtn zO7B@3LrZM;*4uU}Ej)>KQ)>U+&J;QBv*E!Pw2VUB%@jPxteCdTwh_t7dEEFd z6;X&@V%v{4IKdYO+Fpy#PrCeJB zZ+9G@LuC|XIiC`!ojDjkEV+)bH$39yK9n&h-_1Hx;*6C>r~3=KthsD=7b;?4&$>g8 zXpK4ax;Nn!as#g#H2?#(UkfHDA|qe0NS-Sst2{R6d(dJtxM|nmiAta;JM#-w06FrJ z3aVkNvALiy)@Qt(P2&i-pzD`nd;XvMD1 zY}!E}`}e(>C^i3CKi1*s3p?cxz|rp}Xsfj?U>CNg9N+E{B_*gQ2fZCI3Jp?G+XKy- z8Zr?K7B#@O4*z!JMrL>JFaS;9*B!@~BW%cLTZtVyApwNz1$$KH>78nOX(Rd}X&+Qv3Ir0 zx=pKNP=T6*e%nPqnq~$)ZSL2o3Sg1h#tt~QPS0+oqDfq6J>naU190x0@-TRF?1S~HrULy_x4S}<+(;8#2c6HpKj)1bH9{B>yvVC&5F#=$m<~?GN8uQQ= z$4pg4;MrqJU*wo|O+~jx-T1;T9XTvg@fJUmmmhyfKm)^VYR{_2I zGv1iPf%D{PYp}KQXPRE0OJ1d2mx_$Zq>3%$WDwc7;DbJibj}D=V?3g(xYGmobwX=n zcO*e>uUD@4nk{#Ixjq5TITa=%Bs+iUFB;;>aKpQSwJHJL+Mex+ZMdZ_lQj*<%pHzu z;vg`+%1DJpZ`-{#&^kv5y!hayo`qi>D=aAbnZFD!SgQPCt2W3A-?Yr(=Fw7Tm}9$# z5fsF2LrFK9!jR$>f>R}07@s69^)?J55*dh2J`a-mTau{>8p`vKKBB- z!Oqa}C2srKwl4E&u@%OmVJm4J;u6E=8EPA#G;Z;3!Mp`*SUMdwjH$S20|p;eE0;CV zSO;?J$60Bv4VcKIW!e%L^R`i?D4ABvg+V`1^8(8nS-~=wHTerkt zc&TkU8iS^mj9)tj_ntqqP=KJ>KbcrZ0o&t!=h7N0axOF!y&dlC>+Y$-#2nijx5I!C zy4%?FSLeb)gTX7*3g}leZaKy{8M?mx5V*>WTfPOe%zVJMMoV1qj#Fmy5!LGPL{+Fu zAg9ld*CjFhz5tqzV-o4wdFnLixXVx2zxuDabEwD?*cT5TQ3e@!ySEu;*xtOG*^gFB zVaeS=!sJV2<^oQ4$vogF_V9Z6>>sK)@~zrT2%#T78*GT-)n2KM4YnM0J^qU>4QB5$ z4nwLr_oORyl@Pvt=NVcg1;vmF53$YXF12^@vL*3CEv-Wb{!sQ1T^R)0CZM5){aNEO zG4ZJkoDvx);Ay}6zj=v@?#uBGZ*c}!IzHf$rUNR;eT@wK^4U?8r_g%v$7UL90j@`% z=V|=m+O$|FxeVScxV2S~k=#jlf;4t;+Hta=`ewMK9M^=vyOQPeakRnf%2%KUdvI)z z#11Be40X`kk@z~n^XEN+J_n=E0xylaXW@{c&$WAmASodB=mFY7V1_F3rluHDmPcLC zNG|kk@V9+A{HhC$G)=aHLs!i>6`xcI$$5#!ZE#}Za#pR)< z4EC3$T@$kR#j;;j`ZO4QReFt9n=24yJ^{ztFVR zG{GF|10J2%7enph*-aV|I2E_+655~Ff}?iSCdO}=N_e)Md$R7XO0vi?ukcl zFV=8iA#Hx8i6dVN<_i6ZwnV4UmH>vkwDRo`A!xW&F$;7&EE#d;1gdN6#g_wc1;1r8 zN|0FQ@)1a^dcWL64GZ5dm0qBn8EB4c8YycESLXHiMi)TdDzr1rH3ntNiEYRSKleOd zK#2)>mESy~Yk)zgcYn~8!iV+Y=@?u68Z-9?5@u%NwhlPR>6{OHXx}u{4co#@X>Ylf?)Mxh=c-f2XK#)IS{W z>+O^Aw8aEIZ%=MUr146bpjo-(FB{R(;5+;_Xhc&xvcT%0oFfDJtPx(kT`{8=jeq_X zUx_~`PaUp6hEH=GR)gn_xF-3a0N3`<|D!m@=HRqf4l~ zT6r&OLfaJe%@GQ@tOxCmH{MH&r3XKHiG;k&93MyL8R(1gZ%la4|-4|A9Io=N>J zNDO~OA|r{iC$z|e;r3j%_=XBqd2{r{6ek?mq`%ujaVdoNx!YN7$M{(99N&dj1JS$} zWYFik?^r65k()p85CCr8yNop`?g1kk59nfnJ>2VrIu71DdfnB;!gXuYadHM$26sM4 ziygqoxx;+gKq3D6Mq5o0c|EXyPMl^n?0{-r)R2g)~?pI6rXTclIzE*q2k>0`v{a@&_r0qRgSr>*6A#mOh_?x1tc6U-7T;8qwuyw73K&n_cTgYuYH=r4Ne6@QA%p zscImu_DZ_AD9QS}&6@8xp=QsXWq9RMr|j7L8}HZ0 zyy}${Ib&kV_pDO#Q5|-d#~2C6y10$Cvott-F24 z3JLbYy{EhJp0izGb6#wh9J#yjrw}q$Ja2e|z43kQyo<3n?$FO`b8)qMyv_}$d>gQu zurH7nn?qFocUuHkVcN?NAM~aE177A|7GvvU&z7Gvp-Fbc*`yWj{QK@09L2gr(oXiJ zstE=#hhv1``pmvP4z$5Xd8cib#01B5U(sVSGDy1xz9VQ_Tj=^+qS7=ed-mSd9wSWA z@$3#8&~i_&XdNl^^|$v)M|8HX%^x%ieetOpoA;;__LRpI2sJ`s$L_|VRMgStN3@J6 zpNIFqiCwVM-o`Ihv(V8l?W8<`@x1331sD&L_6t35Y4J(XwV1@n8X1GQEq}+%`im;p zuy39*0|VuEdZKQ=;I~vM&V`8LnX~0d@Tu{JF)en3qE&a_(*hr|Eo2W>U`TbMb=^#r zf*CdG+CWQM7ocz(aTIe>BFh1;>uG@pGknlNh zgHV)osAW?q<|be;w|64e4#V$ZgL|k_VC}{uZ!mcIIwI>k+NiA7yHQo&Z}c_B+A-yh zxHlmJ?Ah~xaVU**CMT|J!Z#WA@QvI!+T;v_o2ougnF-wYx0NFfLxTS1BVE}8>B~`X zaqH8wF;E<9wn(6ua>>zjO#xVk1e@WMXPMlk8lEJq`YL9D^=eomSdos=Yp6oOi0Tyf zZfMu!o$TFlMr*3K;0^z)ki%}cwZ3V+Fq<*HaV}aBw%3R>9O2EKwZcz_J+qEIk&0xnL{IR(A)pNAY7^1cx{zOZhVgHl4dGa{euW>WLQW01et5pg# zsCnB@F2v5yzivU(#Myfe{U*3O-Ifn%gGyHUq)9JS<2^s`f@NyPurl-EC^Sz;-B~rR z&cFAWgH+}K-Xl*2(Mmi6L&ip_QpnppXAI#O-W|_(3NJeim@x|FeXw>;XRR!9ZS?!2 zm=oed_ci~*(puhskmnC7GlNy$X%}!q*UY~m)mAF|dUw2n6`A~rwP7);I0y>O8)xbH6L4%M!?l`bImChJcm;MS+!`T0rP1brLqd#Nko{FCK{>H-+KH!&8I z!sOQVZMaXO-mGzdnrlPqC$+)N>sL4%Y0UogzbbIajhRj>^dT@Y|6YQw1fHT(El~P0#vU22W=W4_ON9(GZ^L2D z92$^%QPO^u9nAgg{#C_5VboJ2A@!F0$7c!_WV+~vN@Ja2=)!(a4Bra9#!y zQPlZA{Fgf>l%Dgl(KaeQZx|3D!Fq!|1=iBC&Lk#s3npE;To-J{hMS{^i-w#&fK@|N zCTteyZv5TpwY<*oO$3(8@9ZQra&r{=tuw`@{HEWV(7!r0qOlf5cW=6xQXNb7Y(iLr za2}aLGi?G^$`|2w0w4Kc1E#>(<6ZGNA0WdCvEDtyB58{!jI^_FkJZ$0QF_^r7J0(6 zryF#b@h2feL#2ph)Ha7;2SKslx}p5stuS%1r3!ztRyLa^1;Dn+?dMXv0ODHjfGo+c zEdT$>n`ayQ3A3M=VsI9{H#j}Cca%04bSK3D+7fatS%N>Z15UoXgxvGXwA~$?*mjX4 z`klpfJtZtyJ%1diEhdG(oCW7q-14Ua168*w`$B_8~=O2sq}$=kw73l;+S7324eFoJ z?X5adnH5Yw@oP2WC-ZWKzQW~I-rN?5$rVUFd~UV28sZ#n@Bl8czJa4MqMS)BrP>-; zS~KOYwua<)v6m`r!Fzmuv$~Re3yYwop$zZvdciMT-0)>#18I{K&NkC480s69d}>W| zO^NNDbbX28lJ>;^hS5uhFz!BVkumh**vPSgG+}m8jdisQe-M z#@Ne3b4bLWf<@-V5_tOcerp;*A+*`aN-*)^ z9X87_G<90xwUP2tFz)Tz9$l0myQj1O=l0=HhKH&JD#AxvAxsbYr&qkNl%BbBGmx4@tZX)$g>Vq(M;fx}KNEsj)GW0~V;f8>v-BbPbMYA-`XiMM1#Zw7yJ zz?kdYN=K1~fyvGeV`!W`c;(-Ff<4^t|8t-KnSQI

c}S;DJW3aCBSdv=P3-FSVM9 z`fKaIlv%XNlDul3I*x;i*B#gDli(-csWm1WV9J_))#_r%*uWO5lNe4l&8j1n1rVp4 z3CLMCZ%?=25@&rm*hd`#O1E)XWn=;xyScKC=cIQA*e z@SgG%5bxTUsE#8G0{=>5J-}epj8eo0tk{E6g%$kjERLov?!@$duoP>C`np&dsH+Wg zPT(0Y_RW_IPtk#B)i7#}n1gQR{2z4TuSP^8T#4nS@9*yv`@^u}l~F1?Fuab3gbAb# za5ya-1z0Ng3dL=L{l7Xo2(H)H-R(jIoZ39>;6!mOX#1G4np}8dlI1Oc*+1=M0wO)K z?baq-=yG$hR256M{ud%Hw&IL_#r6%RDIO&FK!2KGLMkW42`0lK<;OE^8tK;bo7Olo z+=UKjg#xX68JgFcSlDUeU4oD{@!7kJR=Y90i85G$CvwQwJWA_OUw(;v5v@uhmlu1Y zDRH;yH30oS!$ot2V{ISaC39#qh;F%(GIB=boZjvOO>u(xjg6;;isHTpVgv`gNA3OV zSOf{CyN_?8)eg{aW$FanoL5=v#o7{x{W|WB){czZ{gJjH8DA9;W5m43E=xh>N*9cs zrB%R|6J3qe4PbWHULxr1$6wZT#pOZ289QnTDba zpzaR-y@(cv;~{{GEsr#55UDy`L?bLw1uqAhYey_ggbEJP#g4qRz>@Sodrew{Dd;da z6YOd6v#(nzF$KS_jYHJ_5!}Kgc6Z8yP~Wy8wgns?_%{XS$bqDp0;OG zmTMQ#CJMP7PGNf1m{jc#qy@gfH{?B}4LuolQq*%(WNn9gll6!d5vMk1v6P!{Wo_(F z=5~F72T(vX;(Ub^%Siroe{~Fzi>vb`1qgri$mmRY0MYxumFPimJ_UD8)h+OQ+2x6} z%oYkQRxihX+BJ8kvIs0BdUqLlgU@>VRmz3J@#9hDas{kld-YP6!c))d@yMBuWxu2m z79bg=-G@%fwDrkI-@mhMw6sM5#o?dJbh!}xab5)F9HIVqa|x>S31-zd#SO5v_S+cR zWCOm-jfz!vL^=JBx{_h5D^^z`$^Oi3n2DF_#%1Sqh`WZxx@qyaOjG+uY6Ud!og#?1 zcv$G^=>%_ySlqCMdJBvh^EyRz`8aJpac_(@0OZ3sztzF9C`^A02s|)MC2vl zv;V|>bqcuMiOE7#H?LCJ2mABv_IQVeKt{iujp72>eIPj&)l}-Rjw>;G86bK6Q84~o% zdpIsY&w^yUI*T&aFmmgXPx=Do$ojos!qIJ!9;xs|BF(EdEcO`%AS``=d;szqc-JC@ejF#Le|RXR+tx(`h4 zTl@j>meu)K!_2w8F(%5bmuiM$KIx;bNtHDW64 zgN+h@!rq7%eEBz;K{Yt7mik$NG{p_3esmj2Yov^J%78BzjD(WH2Q7GrBRzppn&Tom zpDz=z+1q1M+tbn@7-?;qM-`S(#2nJlIt%wBU!O0s!e;jSBngCLrV1YVh*LO>XjfRV z3xeIyU#(|pQy0U{S0%{Lsl8gGR;Psft);RK_!Wt%BlC3)u>8|(B~2nQb<&X0`UV&e zb({5d(5<1(6moQi>A9k6VK98K>|6FQ^K+_Jj} zh3s6t8jy`Ij*hU((F)8kP~Mb-ZI4EdfpIQpA7j&#}BUfR$O0>f2=R=0!v)MPv# z1@&(|Hwb9h0zb<_EZ8Ma+J2=49f141VihfN^5;+VciDgi%B+!d)o4u$aamxpsE%~b zZ!0MvCk=bkvTh)fSvpH2+QFWFzS}e$cuw9LD97{kMSem`Gr=`8Sxca5Y3Uig9a*`n zMk)({y9F!eATci=&%a3!i6R2Ovzv3y5ADu@9x2I+PB&Y2ep)_KcI(;g(GT12|fIT)J zN=m{6ih=vdFM_8&|9jybv@rqC1LvS4dpom*KvxG(CrY-tJu*J1R1U>hC=q@VFA}JNzJk#yOEU-Kzhe&-s;^ znJ2G-`^)v1J5$E9&&*N8`3+-l45DdDSaQAXA(ete`ivK%!>@l#Z=-Q8ef8c_l?Jhs zw3Rf?om^I9b{^}#gPI89^S|GcpOlkA!JiBL(N!d2na(t>EyKqRKQ&sw1YY}dozdR6 z`7yPFz8cl#2N(3I%H8MC%H9kcqmeXfu$($!X``M8)z4=Zni9xLbKRn;WERZy9UVan zx{_^;dvOw}nS=Ks*~r$U{m3rfp+bn(@7Mq66Gvl z#7ToxZ37tHi=3~ifv1zdF2#Q0)(_#-5kw;eT>%0W&G-o(hQ#M}*3|&V7N*jOC`NRn z^r?rwR5{x1*aL);d9UbL4Q;ZE$Se*X@(dB(z&x%6fogtVYvFX$yYNgSC)#48aD240 z0o7LTwaybUiKjfV{BXoaR7zd0mm1J&J7vW^k1qH-H*$OKR$9;{_Icuw`ERk8wa+Hx z;7L0^^XP1=_cX+Q?7{+fe2**ixHKHEA|}iC1Ex#35i13Ume;KoKCr?bf8H#d`;$C= zpY#cTJW-PU7pKa#eI7UXCwAsOsdF-3*~z9Qe+9)^x#{3Pyov^{Y`qK-ZvM-TE<^yQ z&wbfDUqH{-`gmOsuA-{q55KV8@ClQ;V!0HX&zu+{^w&w9J_)a!@@79vd{6X3c4WVx zahsP+cqtS_Z4bO+-wHMH$l0q`3fHXjX-5#<7CH23vqM6wdl3DhK-$m=w%n~|j6Yz{47tbUg zK_vyg_q^q$P$f!Tgq5lA*X7?ToeYv~^h2nfJ!y5*f~JKsqN=46_t1te@F~V_H5ExA z^>1aqP!4i(^vU<~Itc%^E10rw;B&tS#en?lSugBLc#9!cpfmiO{C{=mrX`nsMh(aI zUGFMQbAw)s{=CG1sIf)2N`S$7U%osPc`oy@u_Ofky?v~wC2pjpPZ}*i+>JBw?@nb5 z@pcYCA!Enw=%N?Ot>R9&f6|G_9%;6eHv^;FGxlq9VP47AGul#EebT~1izwT$mNIP| zykoac(#6952MtlQ1R>4m1N>-N8z}WUKZ7z>WNoaGw7M!`3X=dY*G{hzXo3aw|NX>MI10@B zzwC^d8`(W1n~IbSHzaQT78JBufA=3jzu)lFGr=79_%s_$f+?K+;qqBmgdeU9Eyq*J zhQ=50M|PHc51vb~Adv=$iK9ZzJ~9XoYeV+Uz40^_$f3XeG55yex%~w+#1^UsWW7Q^ zEZv|Xg_d{#ep==}9*l+eHy{0fmXWW1e?nB57&Bj_#2cyy^!|&{ee&Xz6Ij6uk@ATn zFe?4tHuSi*2)ec2G#x9I>bZ#$85sGUKZ3!ja_Y(r9WX`4DccOqL_^SK7ycQTI5~W* zT3Cy-1G?DYB@#a`a_?#(fH{^}HeDM>+D|UQ>>k6vK6bJ@;9X1` zEFfLCJ5ii#cx)Pc-ocG{hDD+5hoJ zVig`*o-HmmtSXe$F>HUoiSMYt4erc6KLGOHyZf^ zs}K1%E4E@yC;3nBirt9jY`CB*V0hD0OVInVAOrHhqXlKyPOI9C#e*?0_W3;xM#7u6 zWzM4Lv|pOD@W%ho++Xm_K!dQ6 zrtat#f@K)W9O;A`sC(JIAsyY+G8lMk>r|}TCg;XBOBH^we$V>Og4YoIb9fAmP{4*W zHb6Zp2`5JB6{U z2ObxaPc5*bNrTwD^`+?;{u)m2ffVK1eR=$ZmQdKbV|9B>i|+}vm9{W&{7+>s4WfXX z=D!rzL3bW;g8#Mm-%+oPMW|@g&3Ux>llFTxM zCLx)p(4k4FWF{1K%hhlzbE)37)qS3OpL?I*U+?>Q|M{KII%}`J*4k^Wy@v1luJ5$T z+wpwA9`Qgev+c+fN^>cO*^)0D2g6%C*t1Y8wN7S==oTnSVjAb*I()DPNCBmG7Yx6n<5V2C9wMU-P_oZ;)XRYL%D zZq>i3jZ&U6E>Q9jGk!exl|9c;gCTK^C3@f(vEQFL4w$2X*UL0@z{yKbH_Qeg)X*^t z)IuD+cFSWVO#>!KNcEu14UQq87srOBGw0<0ff}Ivk}yF9ui+|OZx}E3@Pc}aoMsoRKv-+xNA_IQrToM8Z&#Wu2nQ&$d z^+o9`k^pWgHme^8SHkIrW)XD5t=e_Sj#v%dP+Gmqfaiu2A^}EQLc2_f&4oWO&z%AU zXo%GI4$vU67}JzQU*x$UeCi@43h72*nc__bfc&^Fvgri?32~AvXwx2^OF)WoFeam> zk^|7Sr^oG)RZ%QMX4Yj*ln-W27l=hk(DAQ4Jv(!u{_zEbKm`g3zY|6C1O-2s0L2IS0QdC1Bq7}Uv_$=Qxn1OE9A?}ZO8s<0(iuL!WxjQtKFA|x!v zLc27M*bRr^7oRc_;3LajEyU+=@Zr5;fSiHd`TUF)1tFV2ezm(00tDME_TLmGLpiAr z(PPZmd=RNq7MeP3+R4Du-GYbuI};&ub%B{&U>M{s>nX7CoG4E}qg#M~-|j`Q9>@oZ z0Do-@bWHCdQbxoQ>ZJBRA=*Osxw-(c2Yd!pr3@(p2O!p5asaw)P71_oh;~D+)m0p6 zB5?7P*nJ`dSAX3n2@b!&KFS5gq>)t)*i?0W2?&0~3v-?`0z+{6Ate#4#x4i>QLv^4-E88(;yH5xbWRMD^vj|tQ5{dxTCMaa*L?LMg(3p4;kTs(> zOu1?U4Kv)wrox?oKTI$!If4Z3#xgc>cJD#F%;ER@fFuyv0RCKR=7Y%Yge&b2A4JUb zVbef*1%l;;BdrEi5&Aaxq9`E-9O%W^>yR?Ah9}b4#gJ;Ci9a={fIu+JHZo-j#8%;J ztxY>RMc_TJXDUUj;kmgx%_tM2A((6pV9sT-e?LOzg`R^|hXHL1Sx)&RDhqe}ZOs?O z!BLkg#}T?H{#&rpBQSdzhVSkG%>>z~0oojp-eHu?JM6+BiB8b`rZ8k|tC2*+mvsm% z_f^NN5U?3CUXSoYNRTj`H+V!3%r(w4{hJVHaWe1D^PjoF>?eGfK8jk8vL5O{Rt>P2 z2Wvv_5WudIS@tX}`2dRts#K%Q?U~Zk?*V)eeid;tHVw8Bl8T+-gZs4)Hi<&;Da_vO zNb`1hOIcqqD&HL_L5FGvaXoF~W3wW@0jf z7{OJnkRp9@p-~(?#rc06;rs+hskq>HYt$&rjl&1}0WS;YZdHjw%5d=O-h^*6r*^h*3_zQpa(SA;SUBfCjb=-;tkOOP_$pGPey1{)FX71B|lo&VW3R zhch|e&582CO=2BDsv2U{*CVQcIJiw$EDKTH4f!7hCdObowjMS8*WIfF3S^9h`{~P2pw<1C^JaxJ}9?+qL_0MNV2h<6kx?dg!_^}Ct`2lez4NEuxwES z9Y#+cmt>NK7!T*r00SqbvgGRajpyl-0J(sS79ofb|`UB$33$IP&o1x}5 zqON@bb|%^34nFN3aF3lx5YvH?2yuw}j1Un6lPX1?blSqdZS{NsKz^itAU7Z=U5qYe zG6R-e2NPIv(01HOCP z(r`;fln#F@AB+PycUM~AD$;ETQ=EC4OGMy4wes%(Ls6Yev53XO5YO^V=NvvU>31GH zcrxpu6fl~w6aM|s?^5X2?_BF{iS85L&Y9*w#ThTJx;O#^h%jmoCQJbC3$rT$Yoz_# zAOl_p4p|HMWl&%UiYdRJIXDAMFELCnB!C@{6aI~;4R$8{`t>u9_JHQE|F*{u5bW&5 z2l`5Ia9TzR;J|SDnIlP#fbfICx6`^&0#Eyr(J4T{AsDx^YoSt;9jc#L0cuYlk&bI% zQiY!jcL4bSX}CuGx_=2!c?>sW2ZuyRUQlC$iu-V~KLvTt2R~bIZ0)p#3@YN#JvReN z1VhK)O`{@o2zK=nGDC7<7#r0>>MVx)Rc!@{bC6m24WwHV3#?#jUO<_W>zrHO02_aK z63_Od=G^>FzSd_-gnS>%0PQcJI>UQ23xFy1Kye9_sv&nz_aGvIhe@L1S+Rh#qA~1$ zc@>wj^aYTR=-VzhCC35 zi`8vEgieJSm}6&=TmlxeQ~TAgOd`R%W_i*GhnqNROrCXNqA&j{hzoXwK`3 z%yxc&1D)Ho0R{q{u6xA6X94jBt$~*(lumRZ7{oLF^dE9lj9=yEHn@(>Xvb05J8!gDWqJ^}sz za5X?kWjVfTh<24W1WGguUsJ+flte)*AUuGplgq z*Zkdxhy}~S@hu|Q%Gq=YAkN9nlFNxiP#{9eJ@5i_^1%6UR9Z$;KJhCP19oB-xQ3(& zLH*sRAe8c*Fw}w_EPSMl39%J{IQMe08i?77nnVLUd9)*mD_YwgBlc!_O#`#Z&<( z+qocq1)#Gm9YrXd47NbwWP=Mjr=}loQ$!+!VLTUBCl)9SB`$9VP^oUD>;bq_x$TyL z;Dmv4xwJblg#c?kxOx#;<$?p^CQONR*fRe2udzY+$7YllD2+GIB=r{9qwpe5Oy|J% zONiQg!oj5)dN0R{^aGV4{=2^bzc?1F)V$+@yKfwM!!(ZKekcxV9_R2y;(sr*%P57P@L%{^k_q;xSX9s z=D;EKin1xnS2*!m!W@*kr*@tfapr&%1n;AA7n}tlLnyB4^eO~U3bFL6sk;>bv|p9q zaTyhU$DVATWCF6BgZ3O_((wE7NFXoC2}NXx0CO0Ojt}Qip$d&XckTmZiSl-*AA&*P zv;oN_B#{dW`>hSZ2>`9=2Nw*$rs{<+1xtJzocf5R4!wTVVFLsNE-2M=J4YEn@I^O~ z0kG%XSDBA^ZHJR~-1P+KuueI*2$ybnYJsO0X@+6tt?picO^-VhOw0s)R?Nap64LAg zW9s&JfEhs`{|eZZuo5X!R1^al2Jb<-Nm%+e1tfxL6%tBPMKCb;0>iEtXQ!Vc3HTBt$lR%2V8q%0H1j>pqu}dHDTKV9Xni+tSZHJ{| zF8~!ocGy8ykbg!M#<1bv6N6xb%!^I2{V?f5K-3HbQ%!GL0R%bwR>DxMDs28it=%^>PkhCrE z8*LQ-0A$92HDf@5*pzq|h+4sfjLIT}tPHH#9n=Dn;85{g7XTcHFerfo#&kE7gAt+I z?J`DI_rU~h!~7X=W{)9p%9H@ns-+&JSO?3vH~DE6APQfbiVcul9(b9atIO068z&2N zg1)O=Yov7AVvAm${GE*0IzcKNJ5fcrp{glEK$%cf%!)%~x#8P-E9=BLRfy*11gz0u zz8P#kZm<~xK3QLNkh}TdUp`rnL9JApZpQFIn4kI~JQy7G_t=e3){n|8j__>4U@$#8 zOt1&$e?M8@|G%HC|G)gn`ZF_pJgH2Z&d%_i{kY*~_!Q4FZAxR}Kg+}qeBhMRm{`wk z=Re29kNST&@S%GU7CFbHozBFQ!Ni&=$Di@Tne%7M@#p+;%aOsfDL0n?r<3Ey<}wN9 zF&#f&!hgPoANBv_3ZC#6edI6RE>J2baLHWY$`OIGa{}cx0+mk$sy+%-Z{JZXx1-*C zN8^zl&F6Ns*6e6|vZLeU4#sxDE;+$F=7RT*2zH+n?5Pp#dm{MgqhSAbp#eFeA#IuSJFDKmfRN!m}l4;Eky8 z+l`oaGwAK_8{t`|ogdM_&RMygi{?9*MTJ*lcg~*MIa{(5hE;+GX3Zz|1uDev?cBz%_(&*ZyHM>iP zc{fpPw=QY7p6l)dak~vlcN=!?HlE#W$}Da!CT>X*w{{h`i4(Ug6+hS|?l>#%#4O<= zCgDbs@Nktl94FyjD&gBD;Xf-8$h_x>*q#v5o>134;ccFU5%5vRw`A|C3SsP>ISoPjhJ*DNxH#Rx+zY&rBwQ6mvsBA^etw5rx^Y=34hlW ze?J)CosRFR#rF;1AAQ32bI1(H%M4k_j0DS!rOQmz%1jQ(On;Jj&arn^e(y_*y|04z z&ZY02uig80VDI}+dp~gO`y{__(PH0n@V?dbeV=Rhtq<(`_Gur6Q$bbBHXI zAFISvImE=xI{5IMdKIe|Jk!9h9U1vwE;c`*g~-InqaA@aBkd8s;i z{Gj~a1$kLc1$hMpMN0+c5Czo?1@$@w%|Qk21qA}9B1u7Uzonvnh$1;dky586xeU_m*E zQzckIg=VP|7NQc7p>niNC3;Zh*n$e3Q#DROHQrJ+Aw)GXLp7;RHDypWbwM?aQ!QOV zEz?phJ47uvL+yN>TEU=N(Slkrr+TS^`Xx*CDXn1)RSW9XoEo(X8ugYM zjUgJ%85*s18f}9b9Sa%^PR%X_%{!Kw_d+zgGcU>+!!Eh0n6$va>gv~Splu2N#C%{7lWRZa7B627axvYphG$LOnQJ|hEI7Ad) zB#Lm6#1u)ptw<6y5-yV@RZqeXk@hZh~KB?KfH6Z^or(p{QqNrAMXd*=Fk5*Xub9=^a|sbLP@_Rn&L4 z()Xn4du8hT)a&~V=?5(82XP$;Ry;tnIuJ%X5RrM{X#Iicp##Sj574>Daf;-4D{=yj zoR~>Yswbxmky97RX

BiUyfh2H7-&+)RV>^#%n)21Sbo#axt9Mam^B$`u-=ER#}R zPpKTDR4r1fxeRL+4ePB88)=5knTDxlCpiO9<8w47VAxk{OG&+2&AS`qzxD!3-WYLzc|2+~yog z=3Lh1JfY@%S>^%_=7PiK!b|2N+!kU=7Q3x2Btk84Sr$?a7WiR{y-OCd+?Mi6mWtMv z%AuC3S(fSzmYTzs+Dn!MZYz?K)qZO${ZK1%mKCMJO3!H6%4EsPjN95m$=b@=ni^_t zn`Ld^VC^t$eQ3$rnVagWM0K~OdWKTHvZy`{RKHW%kF%GUBR$j z(UM&;w|%LS{UvMrE1~vfS@z`(_Lal-RZI5O+y`rw4%S;AYz#fvoOQ6Z;b7bF!H%Va z3~q-mC5JoK4);PGy0aX58XWqD9Ud(?^m987C^-&UJC1}pj%7JcG&oKUJ5Db-KIc9( zt90n4^`Tdxhvu>l%{Lr+JACN<(xDIBPM?&V7OkC@L!DN$oIW==tq(hWTXMqiI5R6d zvrwHkhdD#p&TNg&@Q5?A?2P4c;ZSzrqPp;ex$tGX2sF9~j<^UfyNK|(iYdG9rn*Xm zx#F^2r5auFBd&XwU1fRPWyw1_uRCA|H@4R`2TXtZaT|u8q02k zMl_(q<4)R$0bxF!&6{+U-S;cI10Oh??&jlq8zFs9_kX)EcXGBnrP1AJ#NA}s-HgY> zLfONL>Ol?ju+8?cZ}e~&@i?^X;mqUds_f}b_4Ew$^vd@1Y4r3P@eEk@4B|N)tbCY8 zJscKxI3oM-(Z<8kBZrSIAExto#VLEmQ@s+xyb`m$k{Z2IM!Zs&z0!ER)0Mq5sovRP z-nrS{=Nr8XM!buby^DE#N|k*sQGKq2`IKe*lsEcRj`&n9`&9Gz)++nfQ+*r5e4Dd< zTN{1bMtnP#eHlD{UCMrUsDAgt{JOLKdK&%uM*JQv`}Onq4=DQ&QT<24{KvBWCmQ`H zNBpOk{h#v$%qj=Gqz1eS3z*9em~RYtI}-4IIp70N;3wt4MQY%3Sm0`Q;OEA`^^w4D z%Yhi)AZC>y7Mq~W;XzPN5L;6aJQ{?o1YvoPaHt&NvN^&NeuOXQh(Oa3!Og~ox=LiGO=Na>WNuF6`KHK%(a55e$YS23r7A}+*&MwRezYv-XnE7o z%F&}$D@Ut&qiR*6>TRML!=svWqFS4x+D4-~R-zcZ(OoLhcWk2Xg-3ViME5jB_l-tB zT8ZxGjTumh8M28P36B}eiJ54MnH-IoUWs|mdu&$a*h`yZufmVbXFE>`8IaY8iR(Lg5gfC7^HEy?UoJ2$%E;ml9ISxM-w|6y8mhZ%IdDRn& zwkMP$PN?RdP;Wk=Id(#G?t~5$PsGLRlH>Kf;`Jip$+_{A=6IvAc$3w5Grp7NxRaLT zlh$4*Z4yt~m7P4;ebRC6q!R?F4+(DM1P`x-!-)yrWeL9B3I1~lfzYWVxKknIQ=wj` z!V^zLmYs^~J{2=}>Nu1bi%UE~PCV(Acq%dRbXj6@cjB43#Iw-pbGXwP=2PclrX9RDw&oNKU%!m2@>R=~`J*MR(HmxuhFVat$uIj-1@!mE4q=+)|c& zvpczcF8LOe(uqsCO-{M%m2y8ZtVONQonhW1(pfj^U^ zmbu?9Q$I43oR>*y$ut_zG+E0uB7#ng7IsGPahOc{Z{yFOVNyE(F=p3 zU%iW7pDz0CTG5*aMelwq`hClVkCGP_3@$8rUsySPVeQ(5FApw!{q4f{EyYYy#hWO_ ztUkrRBo%KdFJ|v4-a22rjjd$6R0$`ggxjZtH>rfbyktjD$#J3ilAUa&qEe;1D5c^) zrF)V}CCf{tdrD>IOZTx|l#{xsK)I;ob5SMfqFVVyjh>5I^A~m4E)k_J=~6D~`CK}X zbjhInl3~v!eJU z<=5xSZ?IL=NLAEPDjIw$nvyD7$}4X6RJ6}m++wTjl&ZWe&UWLY)Qtto4In4Bl5}IO{Kl7_8(-&dd}pg>lCIulSk3BN{Y!H7 zmWpcj-s-Jys<*M%AUkR}-c)lM)^PjQ@Fv&rSJdq2tr2=tvy;76RJwMTVXe4t?VjXX z$%r|5K)bi`p^MCs46?H$|%{q-goF)iV)M-3Loz{la zcvGi!vrdP-o+w>U+HhJw0vh%E57z5P)syq6Q$`mMq_voaC0gikAG| zmclnJ7uZ`%q+2fhhi}>*v$sEyZXYykANFk@O>Q5rXn)$Zz~wxR`R>8l5$(E^0r3bZLPPrb>KTh{2g7R zJ9>V14y4>MsJvs?cgOhc9aH$OIsUGt(Oql5yEZ9z?JDmc?7Qpu_O26r&jo+a&FG$o z-@U^r_q;3b`S#uOe|s+wzJCONKg8&MsNenY==%}T_rnYBhgaT@0s+7K(UbSX-rhg9 ze*f6p`*gwXIL+>Ohwg;v?!6#BR9Uf#yKgcb3aK7z9!Q_LY z^#{d*J*ApGmmGSoME8^x^pv;tR8IDYRlz%NtoKyC?Wq>*t<~(Ucj#@5?rkpUZEfpq zo9yja?_~(~b!qn9ap=1j-Pc{v*VES5H`(`Sy{}*J;eh7DA%}+}(GSN89!|79oSb|( zz5ejI;G7y^4M`SMX@Q?a|xGNAK4keGq*7N%Qfd!{g=X$EyX8Kes(zpM3mn z{V_(UpINJ)#j$^LOg~iE&(_`#PxT{T`>{e#IJBN{IX>ZudBRutM4yVkyu!YvJmE$lq zX4tlH*uH(3`f%9s-LTWv5htM$SFI6u#}Ut%5wF4#pY{>IsgZ!MBSAu=!CIp<$I-Bu z(TKv)qwS;7Q=`Yej?#t3;MtoI$kU^QK~g@$#LRJ%tTq?M0xu}<m8pq#yo8PK@ zgZ>DiH1j{g|Bl_DnX}`563m%;bifzj`5lxne49huygd9}xPe7+Se%)UCD;x}5nk#UQs2hGjPDIhTX&q+Kz zI{bqCMiLl7ZkS&Qvl+|(qY7?jkizp#l!viHqJu{Q5>&?EwuOa9L>~1C*yrZw;Svz+ z$A=KT-93V3hz9#y&;WP`Ae+hpdVv}33M{y2@?@88;ore#R`ZoX0HDy+e=>;;UfVXgVKsW8;d!p#Apu*ng^1@*nH_Q{Vikv$=qUr*(Og z#AN1XiRsVGpwhpyGX5)r<8Lqe&z1hW(|^?2#l+LYB+>nGvqbL4&7j7=PlW$EJ^qIp zdDauRfZ>vWcqTq=24ldtImjiz-6e!ClQl}rJHXu|!pSYrFVsIEL`+@epd(&H?8mLs zMw-We#BSV1i3Pa$dw}d>=!6pkk9c}}__;g%XY7xPoM@3j8<_)LeLUP~Xm&6ygChSG zA3_U8BmF&SE?|PXfN}#u{rqC^BLA-E$2|UfP5+MnkGi}AXv&Izuj*gX|5IiEUMGLA z@ULnAQ7O8-Xx^acqkpgYk2Gb)gP|dgX!@U;5G95_b~G?xqo1s;bj4!+^7{W;oquo8 zUy}Y`wg;UVE;L%Ow`(YEV^&0gCW^TQd$`a7gTa_ZiTQcE2D=1DqVpms(j_=JFdS`e zm`8AkcVGY-E3dFucCVZm`rOZ!{weWi8io%hT3DFcsQ&Wu zNF*u}Nr#F0MB-th?%|C%66r7L;bx*8% zJ&{N;2l+`V2L67b_6Mknw(=%nu4F3BHNe>0)SKjLYZ-pn!2f5QpIVTszlER20qRi$ zpTNU5aJs{3L)OE{`xd~O9N$)7HMxw3N!FV%dii!w~6qzl~?j| zwXp_e6TRL1so}1X251@Kb~Z}BcD5#-?qt94pU;Nb`3L#gDVTd3=@PwN1E@4NvfrQ6 zkGhi$ex%t@W$kP%yxb4yp=}Ozw}}Yx1nu^Sq=eep$oZjdHSkvgZKU|Qd2iI`?_xs@ zvDa7oZ|VQko*>r%b9ocpjpsofa^W6SHCYc^($Dh!Lf!Y<$^Lj(IY0LU`rv&mywJKq z+e2K*ps&b(=^Ht>18AG6pilIFKKJ)NaHc9mE9@os)pYx78o z+TZ)s9=xYXAQ;my_aEc<5B>VLJpcdNPu-ZWUhZUzNEaK59O(OvIqRYDV;+O~2*yJ# z5*;%zcS1m(NIP4K(#HG=ASr=xq^pA4MxVL+QzPBv{lZ+m(RpqVX&k8-ZseoqjrNg2 z0Lcri6E8P8HFFEuZ>7I)ZfUzaZ9A27khuO4&=Q&UH#P*(EA9p4S0esc@(CgAb?2J z2V)totM?z}`J%~xDi6RNBJnSC58MMl0orAUUI850_<`UCklBykQK=UDiN-#lXjO2h zr@K)p$bsH$_#2)b@X<9Y=WTC`7IT;g9tHt7wA`OR0PesKJYOqrn3ks6i0Nf%!0kg;tkj;;xl>c&= NV{90V8PP)S{{V{e1N#5~ literal 0 HcmV?d00001 diff --git a/experiments/bot_detection/data/results/pocket_veto_analysis.json b/experiments/bot_detection/data/results/pocket_veto_analysis.json new file mode 100644 index 0000000..4138096 --- /dev/null +++ b/experiments/bot_detection/data/results/pocket_veto_analysis.json @@ -0,0 +1,447 @@ +{ + "universal_threshold_days": 90, + "characterization": { + "state_totals": { + "CLOSED": 34637, + "MERGED": 146033, + "OPEN": 19502 + }, + "outcome_totals": { + "merged": 146033, + "pocket_veto": 30894, + "rejected": 23245 + }, + "state_outcome_crosstab": [ + { + "state": "CLOSED", + "outcome": "pocket_veto", + "n": 11392 + }, + { + "state": "CLOSED", + "outcome": "rejected", + "n": 23245 + }, + { + "state": "MERGED", + "outcome": "merged", + "n": 146033 + }, + { + "state": "OPEN", + "outcome": "pocket_veto", + "n": 19502 + } + ], + "stale_threshold_distribution": [ + { + "threshold_days": 30.0, + "n": 179748 + }, + { + "threshold_days": 51.44804763793945, + "n": 7325 + }, + { + "threshold_days": 62.094017028808594, + "n": 4728 + }, + { + "threshold_days": 32.073368072509766, + "n": 4034 + }, + { + "threshold_days": 30.195825576782227, + "n": 1035 + }, + { + "threshold_days": 73.39346313476562, + "n": 790 + }, + { + "threshold_days": 30.832239151000977, + "n": 426 + }, + { + "threshold_days": 40.00093460083008, + "n": 398 + }, + { + "threshold_days": 33.224021911621094, + "n": 296 + }, + { + "threshold_days": 104.15167236328125, + "n": 197 + }, + { + "threshold_days": 34.333587646484375, + "n": 176 + }, + { + "threshold_days": 30.21098518371582, + "n": 150 + }, + { + "threshold_days": 33.63052749633789, + "n": 150 + }, + { + "threshold_days": 41.0978889465332, + "n": 137 + }, + { + "threshold_days": 30.72226905822754, + "n": 130 + }, + { + "threshold_days": 39.4652099609375, + "n": 127 + }, + { + "threshold_days": 58.54113006591797, + "n": 122 + }, + { + "threshold_days": 92.92449951171875, + "n": 118 + }, + { + "threshold_days": 64.95616912841797, + "n": 85 + } + ], + "repos_total": 96, + "repos_using_default_30d": 78, + "repos_calibrated": 18, + "per_repo_calibration_check": { + "mean_delta_vs_2x_median_ttc": 31.311005377063044, + "median_delta_vs_2x_median_ttc": 28.69724537037037, + "n_repos_with_closed_prs": 96 + }, + "open_pr_age_quantiles_days": { + "p10": 72.75832986111112, + "p25": 138.96756076388888, + "p50": 245.59493055555555, + "p75": 481.70465856481485, + "p90": 923.2952430555556, + "p95": 1096.9740763888894 + } + }, + "distributions": { + "merge_rate_v3": { + "mean": 0.5567027952888478, + "median": 0.75, + "p10": 0.0, + "p25": 0.0, + "p75": 1.0, + "p90": 1.0 + }, + "merge_rate_universal": { + "mean": 0.5368757212868577, + "median": 0.6666666666666666, + "p10": 0.0, + "p25": 0.0, + "p75": 1.0, + "p90": 1.0 + }, + "merge_rate_per_repo": { + "mean": 0.5332918174688098, + "median": 0.6182486417385746, + "p10": 0.0, + "p25": 0.0, + "p75": 1.0, + "p90": 1.0 + }, + "merge_rate_idle_universal": { + "mean": 0.5514260290930824, + "median": 0.7, + "p10": 0.0, + "p25": 0.0, + "p75": 1.0, + "p90": 1.0 + }, + "merge_rate_idle_per_repo": { + "mean": 0.5505710864785152, + "median": 0.6729415904292751, + "p10": 0.0, + "p25": 0.0, + "p75": 1.0, + "p90": 1.0 + }, + "merge_rate_universal_30d": { + "mean": 0.5332379062475792, + "median": 0.6153846153846154, + "p10": 0.0, + "p25": 0.0, + "p75": 1.0, + "p90": 1.0 + }, + "merge_rate_universal_60d": { + "mean": 0.5355565622443484, + "median": 0.6363636363636364, + "p10": 0.0, + "p25": 0.0, + "p75": 1.0, + "p90": 1.0 + }, + "merge_rate_universal_90d": { + "mean": 0.5368757212868577, + "median": 0.6666666666666666, + "p10": 0.0, + "p25": 0.0, + "p75": 1.0, + "p90": 1.0 + }, + "merge_rate_universal_180d": { + "mean": 0.5409542251449829, + "median": 0.6666666666666666, + "p10": 0.0, + "p25": 0.0, + "p75": 1.0, + "p90": 1.0 + } + }, + "shift_analysis": { + "n_authors": 31296, + "merge_rate_universal": { + "mean_delta": -0.019827074001990234, + "median_delta": 0.0, + "n_dropped_gt_0.05": 2370, + "n_dropped_gt_0.10": 1836, + "n_dropped_gt_0.25": 895, + "n_unchanged": 28493 + }, + "merge_rate_per_repo": { + "mean_delta": -0.023410977820038193, + "median_delta": 0.0, + "n_dropped_gt_0.05": 2663, + "n_dropped_gt_0.10": 2111, + "n_dropped_gt_0.25": 1094, + "n_unchanged": 28179 + }, + "merge_rate_idle_universal": { + "mean_delta": -0.005276766195765481, + "median_delta": 0.0, + "n_dropped_gt_0.05": 607, + "n_dropped_gt_0.10": 459, + "n_dropped_gt_0.25": 241, + "n_unchanged": 30434 + }, + "merge_rate_idle_per_repo": { + "mean_delta": -0.006131708810332761, + "median_delta": 0.0, + "n_dropped_gt_0.05": 705, + "n_dropped_gt_0.10": 529, + "n_dropped_gt_0.25": 279, + "n_unchanged": 30305 + } + }, + "signal_evaluation": { + "n_labeled": 31293, + "n_suspended": 739, + "n_active": 30554, + "merge_rate_v3": { + "cv_auc": 0.5493659354900655, + "fold_auc_std": 0.01149119403429272, + "mean_merge_rate_suspended": 0.5136005337296283, + "mean_merge_rate_active": 0.557734499146874, + "cohens_d_active_vs_suspended": 0.09623777997840265, + "n_suspended": 739, + "n_active": 30554 + }, + "merge_rate_universal": { + "cv_auc": 0.548848295654899, + "fold_auc_std": 0.011339429192468128, + "mean_merge_rate_suspended": 0.5011117987116154, + "mean_merge_rate_active": 0.5377443527572696, + "cohens_d_active_vs_suspended": 0.08114560040346083, + "n_suspended": 739, + "n_active": 30554 + }, + "merge_rate_per_repo": { + "cv_auc": 0.5486495526055911, + "fold_auc_std": 0.011542410042848126, + "mean_merge_rate_suspended": 0.5010423916595617, + "mean_merge_rate_active": 0.534091457487316, + "cohens_d_active_vs_suspended": 0.0734021937818801, + "n_suspended": 739, + "n_active": 30554 + }, + "merge_rate_idle_universal": { + "cv_auc": 0.5488917644689147, + "fold_auc_std": 0.011813844138263785, + "mean_merge_rate_suspended": 0.5128096256371848, + "mean_merge_rate_active": 0.552348716801441, + "cohens_d_active_vs_suspended": 0.08659711664734189, + "n_suspended": 739, + "n_active": 30554 + }, + "merge_rate_idle_per_repo": { + "cv_auc": 0.5487741351566113, + "fold_auc_std": 0.011853501921961123, + "mean_merge_rate_suspended": 0.5128096256371848, + "mean_merge_rate_active": 0.5514730120143265, + "cohens_d_active_vs_suspended": 0.08473831668514915, + "n_suspended": 739, + "n_active": 30554 + } + }, + "example_authors": [ + { + "login": "wuhang2014", + "account_status": "active", + "total_prs": 9, + "merged": 1.0, + "closed": 0.0, + "open_total": 8.0, + "open_stale_per_repo": 8.0, + "open_stale_universal": 8.0, + "open_stale_idle_universal": 0.0, + "open_stale_idle_per_repo": 0.0, + "merge_rate_v3": 1.0, + "merge_rate_universal": 0.1111111111111111, + "merge_rate_per_repo": 0.1111111111111111, + "merge_rate_idle_universal": 1.0, + "merge_rate_idle_per_repo": 1.0 + }, + { + "login": "simondanielsson", + "account_status": "active", + "total_prs": 7, + "merged": 1.0, + "closed": 0.0, + "open_total": 6.0, + "open_stale_per_repo": 6.0, + "open_stale_universal": 6.0, + "open_stale_idle_universal": 1.0, + "open_stale_idle_per_repo": 1.0, + "merge_rate_v3": 1.0, + "merge_rate_universal": 0.14285714285714285, + "merge_rate_per_repo": 0.14285714285714285, + "merge_rate_idle_universal": 0.5, + "merge_rate_idle_per_repo": 0.5 + }, + { + "login": "sahelib25", + "account_status": "active", + "total_prs": 6, + "merged": 1.0, + "closed": 0.0, + "open_total": 5.0, + "open_stale_per_repo": 5.0, + "open_stale_universal": 5.0, + "open_stale_idle_universal": 0.0, + "open_stale_idle_per_repo": 0.0, + "merge_rate_v3": 1.0, + "merge_rate_universal": 0.16666666666666666, + "merge_rate_per_repo": 0.16666666666666666, + "merge_rate_idle_universal": 1.0, + "merge_rate_idle_per_repo": 1.0 + }, + { + "login": "Copilot", + "account_status": "suspended", + "total_prs": 438, + "merged": 177.0, + "closed": 38.0, + "open_total": 223.0, + "open_stale_per_repo": 223.0, + "open_stale_universal": 204.0, + "open_stale_idle_universal": 4.0, + "open_stale_idle_per_repo": 4.0, + "merge_rate_v3": 0.8232558139534883, + "merge_rate_universal": 0.4224343675417661, + "merge_rate_per_repo": 0.4041095890410959, + "merge_rate_idle_universal": 0.8082191780821918, + "merge_rate_idle_per_repo": 0.8082191780821918 + }, + { + "login": "iycheng", + "account_status": "suspended", + "total_prs": 423, + "merged": 295.0, + "closed": 110.0, + "open_total": 18.0, + "open_stale_per_repo": 18.0, + "open_stale_universal": 18.0, + "open_stale_idle_universal": 0.0, + "open_stale_idle_per_repo": 0.0, + "merge_rate_v3": 0.7283950617283951, + "merge_rate_universal": 0.6973995271867612, + "merge_rate_per_repo": 0.6973995271867612, + "merge_rate_idle_universal": 0.7283950617283951, + "merge_rate_idle_per_repo": 0.7283950617283951 + }, + { + "login": "amd-jmacaran", + "account_status": "suspended", + "total_prs": 105, + "merged": 105.0, + "closed": 0.0, + "open_total": 0.0, + "open_stale_per_repo": 0.0, + "open_stale_universal": 0.0, + "open_stale_idle_universal": 0.0, + "open_stale_idle_per_repo": 0.0, + "merge_rate_v3": 1.0, + "merge_rate_universal": 1.0, + "merge_rate_per_repo": 1.0, + "merge_rate_idle_universal": 1.0, + "merge_rate_idle_per_repo": 1.0 + }, + { + "login": "harupy", + "account_status": "active", + "total_prs": 2771, + "merged": 2151.0, + "closed": 327.0, + "open_total": 293.0, + "open_stale_per_repo": 293.0, + "open_stale_universal": 281.0, + "open_stale_idle_universal": 0.0, + "open_stale_idle_per_repo": 0.0, + "merge_rate_v3": 0.8680387409200968, + "merge_rate_universal": 0.7796303008336354, + "merge_rate_per_repo": 0.776254059906171, + "merge_rate_idle_universal": 0.8680387409200968, + "merge_rate_idle_per_repo": 0.8680387409200968 + }, + { + "login": "baskaryan", + "account_status": "active", + "total_prs": 1494, + "merged": 1295.0, + "closed": 153.0, + "open_total": 46.0, + "open_stale_per_repo": 46.0, + "open_stale_universal": 46.0, + "open_stale_idle_universal": 0.0, + "open_stale_idle_per_repo": 0.0, + "merge_rate_v3": 0.8943370165745856, + "merge_rate_universal": 0.8668005354752343, + "merge_rate_per_repo": 0.8668005354752343, + "merge_rate_idle_universal": 0.8943370165745856, + "merge_rate_idle_per_repo": 0.8943370165745856 + } + ], + "recommendation": { + "decision": "Keep v3 as-is", + "rationale": "No variant beats v3 CV AUC 0.5494 by >0.005 (aucs={'merge_rate_v3': 0.5494, 'merge_rate_universal': 0.5488, 'merge_rate_per_repo': 0.5486, 'merge_rate_idle_universal': 0.5489, 'merge_rate_idle_per_repo': 0.5488}). Cohen's d also fails to improve (base=0.096, best_alt=0.087).", + "cv_aucs": { + "merge_rate_v3": 0.5493659354900655, + "merge_rate_universal": 0.548848295654899, + "merge_rate_per_repo": 0.5486495526055911, + "merge_rate_idle_universal": 0.5488917644689147, + "merge_rate_idle_per_repo": 0.5487741351566113 + }, + "cohens_d": { + "merge_rate_v3": 0.09623777997840265, + "merge_rate_universal": 0.08114560040346083, + "merge_rate_per_repo": 0.0734021937818801, + "merge_rate_idle_universal": 0.08659711664734189, + "merge_rate_idle_per_repo": 0.08473831668514915 + }, + "universal_threshold_days": 90 + } +} \ No newline at end of file diff --git a/experiments/bot_detection/pocket_veto_findings.md b/experiments/bot_detection/pocket_veto_findings.md new file mode 100644 index 0000000..813af4a --- /dev/null +++ b/experiments/bot_detection/pocket_veto_findings.md @@ -0,0 +1,86 @@ +# Pocket Veto Investigation — Findings + +Investigation for issue #51. Does counting stale open PRs as implicit +rejections meaningfully change merge-rate distributions and improve the +signal's ability to separate suspended from active accounts? + +## Dataset + +- 200172 PRs across 96 repos +- State totals: {'CLOSED': 34637, 'MERGED': 146033, 'OPEN': 19502} +- Outcome totals: {'merged': 146033, 'pocket_veto': 30894, 'rejected': 23245} +- Labeled authors: 31293 (739 suspended, 30554 active) + +## Staleness definitions compared + +- **v3 (baseline)**: `merged / (merged + closed)` — current scorer.py. +- **age_universal**: open PR is stale if age > 90d since `created_at`. +- **age_per_repo**: open PR is stale if age > that repo's + `stale_threshold_days` (populated in the DuckDB; default 30d). +- **idle_universal**: open PR is stale if it is still open AND idle > 90d (`fetch_now - updated_at`). +- **idle_per_repo**: same, with the per-repo threshold substituted. + +The `idle_*` variants use a live re-fetch of every DB-OPEN PR's +`updatedAt` (see `fetch_open_pr_activity.py`). PRs that were OPEN at +the snapshot but have since been closed or merged are treated as +non-stale — the close/merge event itself is activity. + +## Calibration sanity check + +- Repos using the default 30d threshold: 78 / 96 +- Repos with a calibrated threshold: 18 +- Per-repo calibrated thresholds vs 2x median time-to-close: + mean delta = 31.3110, median delta = 28.6972 (days). + +## Distribution shift + +Mean merge rate across all authors: + +| Definition | mean | median | p10 | p90 | +|---|---|---|---|---| +| v3 baseline | 0.5567 | 0.7500 | 0.0000 | 1.0000 | +| age_universal (90d) | 0.5369 | 0.6667 | 0.0000 | 1.0000 | +| age_per_repo | 0.5333 | 0.6182 | 0.0000 | 1.0000 | +| idle_universal (90d) | 0.5514 | 0.7000 | 0.0000 | 1.0000 | +| idle_per_repo | 0.5506 | 0.6729 | 0.0000 | 1.0000 | + +Per-author drop from the v3 baseline (n authors, >0.10 / >0.25): + +- **age_universal**: 1836 / 895 +- **age_per_repo**: 2111 / 1094 +- **idle_universal**: 459 / 241 +- **idle_per_repo**: 529 / 279 + +## Signal quality vs ground truth + +2-feature logistic regression (merge_rate + log1p(median_additions)), +5-fold CV on 31293 labeled authors: + +| Definition | CV AUC | Active mean | Suspended mean | Cohen's d | +|---|---|---|---|---| +| v3 baseline | 0.5494 | 0.5577 | 0.5136 | 0.0962 | +| age_universal | 0.5488 | 0.5377 | 0.5011 | 0.0811 | +| age_per_repo | 0.5486 | 0.5341 | 0.5010 | 0.0734 | +| idle_universal | 0.5489 | 0.5523 | 0.5128 | 0.0866 | +| idle_per_repo | 0.5488 | 0.5515 | 0.5128 | 0.0847 | + +## Recommendation + +See the `recommendation` field in `data/results/pocket_veto_analysis.json` for the machine-readable +decision logic. Text summary and follow-up branch sketch below. + +**Keep v3 as-is** — No variant beats v3 CV AUC 0.5494 by >0.005 (aucs={'merge_rate_v3': 0.5494, 'merge_rate_universal': 0.5488, 'merge_rate_per_repo': 0.5486, 'merge_rate_idle_universal': 0.5489, 'merge_rate_idle_per_repo': 0.5488}). Cohen's d also fails to improve (base=0.096, best_alt=0.087). + +### Follow-up branch sketch (if adopted) + +- `src/good_egg/github_client.py`: extend `_COMBINED_QUERY` with an + `openPullRequests` selection that pulls `createdAt`/`updatedAt` for + each OPEN PR on the scored user (or `totalCount` if we can push the + staleness filter into the query). +- `src/good_egg/models.py`: add `open_stale_pr_count: int` (or similar) + to `UserContributionData`. +- `src/good_egg/scorer.py:256-261`: change the `_score_v3` merge-rate + formula to `merged / (merged + closed + open_stale)`. +- `src/good_egg/config.py`: add the staleness threshold as a tunable + config value. +- Tests: parallel coverage in `tests/test_scorer.py`. diff --git a/experiments/bot_detection/scripts/fetch_open_pr_activity.py b/experiments/bot_detection/scripts/fetch_open_pr_activity.py new file mode 100644 index 0000000..2fa964f --- /dev/null +++ b/experiments/bot_detection/scripts/fetch_open_pr_activity.py @@ -0,0 +1,182 @@ +"""Fetch updatedAt for every OPEN PR in the bot_detection DuckDB. + +Used by pocket_veto_analysis.py to compute idle-time-based staleness (a +better proxy than age-since-created, which the DuckDB schema forces). + +For each repo that has OPEN PRs in the DB, paginate +repository.pullRequests(states: OPEN) and collect (number, updatedAt). PRs +that were OPEN in the DB snapshot but have since been closed or merged are +by definition non-stale (the close/merge event itself is activity), so we +don't need to look them up — they just won't appear in the fetched set and +the analysis treats them as non-stale. + +Output: experiments/bot_detection/data/open_pr_activity.parquet + columns: repo, number, updated_at, fetch_now +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from datetime import UTC, datetime +from pathlib import Path + +import duckdb +import httpx +import pandas as pd + +BASE = Path(__file__).resolve().parents[1] +DB_PATH = BASE / "data" / "bot_detection.duckdb" +OUT_PATH = BASE / "data" / "open_pr_activity.parquet" + +GRAPHQL_URL = "https://api.github.com/graphql" +PAGE_SIZE = 100 + +QUERY = """ +query($owner: String!, $name: String!, $cursor: String) { + repository(owner: $owner, name: $name) { + pullRequests(states: OPEN, first: 100, after: $cursor, + orderBy: {field: CREATED_AT, direction: ASC}) { + pageInfo { hasNextPage endCursor } + nodes { number updatedAt } + } + } + rateLimit { remaining resetAt } +} +""" + + +def get_token() -> str: + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + return token + result = subprocess.run( + ["gh", "auth", "token"], check=True, capture_output=True, text=True, + ) + return result.stdout.strip() + + +def fetch_repo( + client: httpx.Client, owner: str, name: str, +) -> list[tuple[int, str]]: + results: list[tuple[int, str]] = [] + cursor: str | None = None + while True: + resp = client.post( + GRAPHQL_URL, + json={ + "query": QUERY, + "variables": {"owner": owner, "name": name, "cursor": cursor}, + }, + ) + resp.raise_for_status() + payload = resp.json() + if "errors" in payload: + print(f" GraphQL errors: {payload['errors']}", file=sys.stderr) + break + repo_data = payload["data"]["repository"] + if repo_data is None: + print(f" repo not found: {owner}/{name}", file=sys.stderr) + break + prs = repo_data["pullRequests"] + for node in prs["nodes"]: + results.append((node["number"], node["updatedAt"])) + remaining = payload["data"]["rateLimit"]["remaining"] + if remaining < 100: + reset_at = payload["data"]["rateLimit"]["resetAt"] + print( + f" rate limit low ({remaining}), sleeping until {reset_at}", + file=sys.stderr, + ) + time.sleep(60) + if not prs["pageInfo"]["hasNextPage"]: + break + cursor = prs["pageInfo"]["endCursor"] + return results + + +def main() -> None: + con = duckdb.connect(str(DB_PATH), read_only=True) + repo_counts = con.execute(""" + SELECT repo, COUNT(*) AS n + FROM prs WHERE state='OPEN' + GROUP BY repo ORDER BY n DESC + """).fetchall() + con.close() + + db_open_keys: dict[str, set[int]] = {} + con = duckdb.connect(str(DB_PATH), read_only=True) + for (repo,) in con.execute( + "SELECT DISTINCT repo FROM prs WHERE state='OPEN'" + ).fetchall(): + numbers = con.execute( + "SELECT number FROM prs WHERE state='OPEN' AND repo=?", [repo] + ).fetchall() + db_open_keys[repo] = {n[0] for n in numbers} + con.close() + + token = get_token() + headers = { + "Authorization": f"bearer {token}", + "Accept": "application/vnd.github+json", + } + fetch_now = datetime.now(UTC).isoformat() + rows: list[dict[str, object]] = [] + + with httpx.Client(headers=headers, timeout=60.0) as client: + for idx, (repo, n_db_open) in enumerate(repo_counts, start=1): + owner, name = repo.split("/", 1) + print( + f"[{idx}/{len(repo_counts)}] {repo} " + f"(db_open={n_db_open})...", + flush=True, + ) + try: + fetched = fetch_repo(client, owner, name) + except httpx.HTTPStatusError as exc: + print( + f" HTTP error for {repo}: {exc}", file=sys.stderr, + ) + continue + relevant = [ + (num, ts) for (num, ts) in fetched + if num in db_open_keys.get(repo, set()) + ] + print( + f" fetched {len(fetched)} currently-open, " + f"{len(relevant)} match db-open set", + flush=True, + ) + for num, ts in relevant: + rows.append( + { + "repo": repo, + "number": num, + "updated_at": ts, + "fetch_now": fetch_now, + } + ) + + df = pd.DataFrame(rows) + df["updated_at"] = pd.to_datetime(df["updated_at"]) + df["fetch_now"] = pd.to_datetime(df["fetch_now"]) + OUT_PATH.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(OUT_PATH, index=False) + + total_db_open = sum(len(v) for v in db_open_keys.values()) + print() + print(f"Wrote {len(df)} rows to {OUT_PATH}") + print( + f"Coverage: {len(df)} / {total_db_open} db-OPEN PRs still currently " + f"open ({100 * len(df) / total_db_open:.1f}%)" + ) + print( + f"Missing: {total_db_open - len(df)} PRs (closed/merged since snapshot" + f" — treated as non-stale)" + ) + + +if __name__ == "__main__": + main() diff --git a/experiments/bot_detection/scripts/pocket_veto_analysis.py b/experiments/bot_detection/scripts/pocket_veto_analysis.py new file mode 100644 index 0000000..512c414 --- /dev/null +++ b/experiments/bot_detection/scripts/pocket_veto_analysis.py @@ -0,0 +1,637 @@ +"""Pocket-veto investigation for issue #51. + +Analyze whether counting stale open PRs as implicit rejections meaningfully +shifts merge-rate distributions and improves the signal's ability to separate +suspended from active GitHub accounts. + +Operates entirely on the existing bot_detection DuckDB. Does not fetch from +GitHub. Does not modify src/good_egg/. + +Outputs: + - experiments/bot_detection/data/results/pocket_veto_analysis.json + - experiments/bot_detection/pocket_veto_findings.md +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import duckdb +import numpy as np +import pandas as pd +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import roc_auc_score +from sklearn.model_selection import StratifiedKFold +from sklearn.preprocessing import StandardScaler + +BASE = Path(__file__).resolve().parents[1] +DB_PATH = BASE / "data" / "bot_detection.duckdb" +ACTIVITY_PATH = BASE / "data" / "open_pr_activity.parquet" +RESULTS_PATH = BASE / "data" / "results" / "pocket_veto_analysis.json" +FINDINGS_PATH = BASE / "pocket_veto_findings.md" + +UNIVERSAL_THRESHOLD_DAYS = 90 +SENSITIVITY_THRESHOLDS = (30, 60, 90, 180) +SEED = 42 + +MERGE_RATE_COLS = ( + "merge_rate_v3", + "merge_rate_universal", + "merge_rate_per_repo", + "merge_rate_idle_universal", + "merge_rate_idle_per_repo", +) + + +def characterize(con: duckdb.DuckDBPyConnection) -> dict[str, Any]: + """Phase 1: sanity-check the existing outcome/stale_threshold_days columns.""" + totals = con.execute(""" + SELECT state, COUNT(*) FROM prs GROUP BY state ORDER BY state + """).fetchall() + outcomes = con.execute(""" + SELECT outcome, COUNT(*) FROM prs GROUP BY outcome ORDER BY outcome + """).fetchall() + state_outcome = con.execute(""" + SELECT state, outcome, COUNT(*) FROM prs + GROUP BY state, outcome ORDER BY state, outcome + """).fetchall() + + thresh_dist = con.execute(""" + SELECT stale_threshold_days, COUNT(*) AS n + FROM prs WHERE stale_threshold_days IS NOT NULL + GROUP BY stale_threshold_days ORDER BY n DESC + """).fetchall() + + # Per-repo: how does stored stale_threshold_days compare to 2x median + # time-to-close (the hypothesis in the issue)? + repo_stats = con.execute(""" + WITH closed AS ( + SELECT repo, + EXTRACT(EPOCH FROM (closed_at - created_at))/86400.0 AS ttc_days + FROM prs + WHERE state IN ('MERGED', 'CLOSED') + AND closed_at IS NOT NULL + AND created_at IS NOT NULL + AND closed_at > created_at + ), + repo_ttc AS ( + SELECT repo, MEDIAN(ttc_days) AS median_ttc_days, COUNT(*) AS n_closed + FROM closed GROUP BY repo + ), + repo_thresh AS ( + SELECT repo, + ANY_VALUE(stale_threshold_days) AS stale_threshold_days, + COUNT(DISTINCT stale_threshold_days) AS distinct_thresh + FROM prs + WHERE stale_threshold_days IS NOT NULL + GROUP BY repo + ) + SELECT t.repo, t.stale_threshold_days, t.distinct_thresh, + r.median_ttc_days, r.n_closed + FROM repo_thresh t LEFT JOIN repo_ttc r USING (repo) + ORDER BY t.repo + """).fetchdf() + + repo_stats["two_x_median"] = 2.0 * repo_stats["median_ttc_days"] + repo_stats["delta_vs_2x"] = ( + repo_stats["stale_threshold_days"] - repo_stats["two_x_median"] + ) + + # Open PR age distribution (relative to the DB's max created_at as "now") + age_stats = con.execute(""" + WITH ref AS (SELECT MAX(created_at) AS now FROM prs) + SELECT + quantile_cont( + EXTRACT(EPOCH FROM (ref.now - p.created_at))/86400.0, + [0.1, 0.25, 0.5, 0.75, 0.9, 0.95] + ) AS quantiles + FROM prs p, ref WHERE p.state = 'OPEN' + """).fetchone() + + return { + "state_totals": dict(totals), + "outcome_totals": dict(outcomes), + "state_outcome_crosstab": [ + {"state": s, "outcome": o, "n": n} for s, o, n in state_outcome + ], + "stale_threshold_distribution": [ + {"threshold_days": float(t), "n": int(n)} for t, n in thresh_dist + ], + "repos_total": int(len(repo_stats)), + "repos_using_default_30d": int( + (repo_stats["stale_threshold_days"] == 30.0).sum() + ), + "repos_calibrated": int( + (repo_stats["stale_threshold_days"] != 30.0).sum() + ), + "per_repo_calibration_check": { + "mean_delta_vs_2x_median_ttc": float( + repo_stats["delta_vs_2x"].dropna().mean() + ), + "median_delta_vs_2x_median_ttc": float( + repo_stats["delta_vs_2x"].dropna().median() + ), + "n_repos_with_closed_prs": int( + repo_stats["median_ttc_days"].notna().sum() + ), + }, + "open_pr_age_quantiles_days": { + label: float(v) for label, v in zip( + ["p10", "p25", "p50", "p75", "p90", "p95"], + age_stats[0], + strict=True, + ) + }, + } + + +def build_author_features(con: duckdb.DuckDBPyConnection) -> pd.DataFrame: + """Phase 2: per-author counts and all five merge-rate definitions. + + Age-based variants use (now - created_at) with 'now' = max(created_at) + in the DB. Idle-based variants use (fetch_now - updated_at) from the + open_pr_activity.parquet sidecar; PRs that were OPEN in the DB snapshot + but are no longer currently open are treated as non-stale (the close or + merge event since the snapshot is itself evidence of activity). + """ + has_activity = ACTIVITY_PATH.exists() + sensitivity_cols = ",\n ".join( + f"SUM(CASE WHEN state='OPEN' AND age_days > {d} " + f"THEN 1 ELSE 0 END) AS open_stale_{d}d" + for d in SENSITIVITY_THRESHOLDS + ) + activity_join = "" + idle_cols = ( + "0 AS open_stale_idle_universal, 0 AS open_stale_idle_per_repo," + ) + if has_activity: + activity_join = f""" + LEFT JOIN read_parquet('{ACTIVITY_PATH}') oa + ON oa.repo = aged.repo AND oa.number = aged.number + """ + idle_cols = f""" + SUM(CASE + WHEN state='OPEN' AND oa.updated_at IS NOT NULL + AND EXTRACT(EPOCH FROM (oa.fetch_now - oa.updated_at)) + /86400.0 > {UNIVERSAL_THRESHOLD_DAYS} + THEN 1 ELSE 0 + END) AS open_stale_idle_universal, + SUM(CASE + WHEN state='OPEN' AND oa.updated_at IS NOT NULL + AND EXTRACT(EPOCH FROM (oa.fetch_now - oa.updated_at)) + /86400.0 > COALESCE(stale_threshold_days, 30) + THEN 1 ELSE 0 + END) AS open_stale_idle_per_repo, + """ + query = f""" + WITH ref AS (SELECT MAX(created_at) AS now FROM prs), + aged AS ( + SELECT p.*, + EXTRACT(EPOCH FROM (ref.now - p.created_at))/86400.0 AS age_days + FROM prs p, ref + ) + SELECT + author AS login, + COUNT(*) AS total_prs, + SUM(CASE WHEN state='MERGED' THEN 1 ELSE 0 END) AS merged, + SUM(CASE WHEN state='CLOSED' THEN 1 ELSE 0 END) AS closed, + SUM(CASE WHEN state='OPEN' THEN 1 ELSE 0 END) AS open_total, + SUM(CASE WHEN state='OPEN' + AND age_days > COALESCE(stale_threshold_days, 30) + THEN 1 ELSE 0 END) AS open_stale_per_repo, + SUM(CASE WHEN state='OPEN' AND age_days > {UNIVERSAL_THRESHOLD_DAYS} + THEN 1 ELSE 0 END) AS open_stale_universal, + {idle_cols} + {sensitivity_cols}, + MEDIAN(additions) AS median_additions + FROM aged + {activity_join} + GROUP BY author + """ + df = con.execute(query).fetchdf() + + def compute(col: str, stale_col: str) -> None: + denom = df["merged"] + df["closed"] + df[stale_col] + df[col] = np.where(denom > 0, df["merged"] / denom, 0.0) + + df["merge_rate_v3"] = np.where( + (df["merged"] + df["closed"]) > 0, + df["merged"] / (df["merged"] + df["closed"]), + 0.0, + ) + compute("merge_rate_universal", "open_stale_universal") + compute("merge_rate_per_repo", "open_stale_per_repo") + compute("merge_rate_idle_universal", "open_stale_idle_universal") + compute("merge_rate_idle_per_repo", "open_stale_idle_per_repo") + for d in SENSITIVITY_THRESHOLDS: + compute(f"merge_rate_universal_{d}d", f"open_stale_{d}d") + return df + + +def distribution_summary(df: pd.DataFrame) -> dict[str, Any]: + """Phase 2 deliverable: summary stats for each merge-rate definition.""" + + def summarize(col: str) -> dict[str, float]: + s = df[col] + return { + "mean": float(s.mean()), + "median": float(s.median()), + "p10": float(s.quantile(0.10)), + "p25": float(s.quantile(0.25)), + "p75": float(s.quantile(0.75)), + "p90": float(s.quantile(0.90)), + } + + cols = [ + *MERGE_RATE_COLS, + *[f"merge_rate_universal_{d}d" for d in SENSITIVITY_THRESHOLDS], + ] + return {col: summarize(col) for col in cols} + + +def shift_analysis(df: pd.DataFrame) -> dict[str, Any]: + """Phase 3: per-author shift from v3 baseline to each alternative.""" + out: dict[str, Any] = {"n_authors": int(len(df))} + for alt in [c for c in MERGE_RATE_COLS if c != "merge_rate_v3"]: + delta = df[alt] - df["merge_rate_v3"] + out[alt] = { + "mean_delta": float(delta.mean()), + "median_delta": float(delta.median()), + "n_dropped_gt_0.05": int((delta < -0.05).sum()), + "n_dropped_gt_0.10": int((delta < -0.10).sum()), + "n_dropped_gt_0.25": int((delta < -0.25).sum()), + "n_unchanged": int((delta == 0).sum()), + } + return out + + +def cohens_d(group_a: np.ndarray, group_b: np.ndarray) -> float: + if len(group_a) < 2 or len(group_b) < 2: + return float("nan") + pooled = np.sqrt( + ((len(group_a) - 1) * group_a.var(ddof=1) + + (len(group_b) - 1) * group_b.var(ddof=1)) + / (len(group_a) + len(group_b) - 2) + ) + if pooled == 0: + return float("nan") + return float((group_a.mean() - group_b.mean()) / pooled) + + +def cv_auc( + df: pd.DataFrame, + merge_rate_col: str, + n_folds: int = 5, +) -> dict[str, float]: + """Phase 4: minimal 2-feature LR CV — merge_rate variant + median_additions. + + Mirrors the 2-feature baseline in scripts/refit_bad_egg.py but swaps the + merge-rate column so all three definitions are evaluated on identical + labeled-author splits. + """ + y = (df["account_status"] == "suspended").astype(int).values + mr = df[merge_rate_col].fillna(0).to_numpy(dtype=float) + ma = df["median_additions"].fillna(0).to_numpy(dtype=float) + ma = np.log1p(np.abs(ma)) * np.sign(ma) + x = np.column_stack([mr, ma]) + + skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=SEED) + oof = np.full(len(y), np.nan) + fold_aucs: list[float] = [] + for train_idx, test_idx in skf.split(x, y): + scaler = StandardScaler() + x_train = scaler.fit_transform(x[train_idx]) + x_test = scaler.transform(x[test_idx]) + model = LogisticRegression( + class_weight="balanced", max_iter=1000, random_state=SEED, + ) + model.fit(x_train, y[train_idx]) + probs = model.predict_proba(x_test)[:, 1] + oof[test_idx] = probs + fold_aucs.append(roc_auc_score(y[test_idx], probs)) + + mr_susp = mr[y == 1] + mr_act = mr[y == 0] + return { + "cv_auc": float(roc_auc_score(y, oof)), + "fold_auc_std": float(np.std(fold_aucs)), + "mean_merge_rate_suspended": float(mr_susp.mean()), + "mean_merge_rate_active": float(mr_act.mean()), + "cohens_d_active_vs_suspended": cohens_d(mr_act, mr_susp), + "n_suspended": int(y.sum()), + "n_active": int((1 - y).sum()), + } + + +def signal_evaluation( + df: pd.DataFrame, con: duckdb.DuckDBPyConnection, +) -> dict[str, Any]: + """Phase 4 deliverable: run CV for each merge-rate definition.""" + authors = con.execute( + "SELECT login, account_status FROM authors" + ).fetchdf() + labeled = df.merge(authors, on="login", how="inner") + labeled = labeled[labeled["account_status"].isin(["active", "suspended"])] + labeled = labeled.copy() + + results: dict[str, Any] = { + "n_labeled": int(len(labeled)), + "n_suspended": int((labeled["account_status"] == "suspended").sum()), + "n_active": int((labeled["account_status"] == "active").sum()), + } + for col in MERGE_RATE_COLS: + results[col] = cv_auc(labeled, col) + return results + + +def pick_example_authors( + df: pd.DataFrame, con: duckdb.DuckDBPyConnection, +) -> list[dict[str, Any]]: + """Phase 3 deliverable: handful of authors where definitions disagree.""" + authors = con.execute( + "SELECT login, account_status FROM authors" + ).fetchdf() + merged = df.merge(authors, on="login", how="left") + merged["shift_universal"] = ( + merged["merge_rate_universal"] - merged["merge_rate_v3"] + ) + # 3 big-shift authors (most affected) + 3 suspended authors + 2 active + # high-PR authors. + big_shift = merged.nsmallest(3, "shift_universal") + susp = merged[merged["account_status"] == "suspended"].nlargest( + 3, "total_prs", + ) + act = merged[merged["account_status"] == "active"].nlargest( + 2, "total_prs", + ) + picks = pd.concat([big_shift, susp, act]).drop_duplicates(subset=["login"]) + cols = [ + "login", "account_status", "total_prs", "merged", "closed", + "open_total", "open_stale_per_repo", "open_stale_universal", + "open_stale_idle_universal", "open_stale_idle_per_repo", + *MERGE_RATE_COLS, + ] + return picks[cols].to_dict("records") + + +def write_findings(results: dict[str, Any]) -> None: + c = results["characterization"] + d = results["distributions"] + s = results["shift_analysis"] + e = results["signal_evaluation"] + + def fmt(x: float) -> str: + return f"{x:.4f}" + + lines = [ + "# Pocket Veto Investigation — Findings", + "", + "Investigation for issue #51. Does counting stale open PRs as implicit", + "rejections meaningfully change merge-rate distributions and improve the", + "signal's ability to separate suspended from active accounts?", + "", + "## Dataset", + "", + f"- {sum(c['state_totals'].values())} PRs across " + f"{c['repos_total']} repos", + f"- State totals: {c['state_totals']}", + f"- Outcome totals: {c['outcome_totals']}", + f"- Labeled authors: {e['n_labeled']} " + f"({e['n_suspended']} suspended, {e['n_active']} active)", + "", + "## Staleness definitions compared", + "", + "- **v3 (baseline)**: `merged / (merged + closed)` — current scorer.py.", + f"- **age_universal**: open PR is stale if age > " + f"{UNIVERSAL_THRESHOLD_DAYS}d since `created_at`.", + "- **age_per_repo**: open PR is stale if age > that repo's", + " `stale_threshold_days` (populated in the DuckDB; default 30d).", + f"- **idle_universal**: open PR is stale if it is still open AND idle " + f"> {UNIVERSAL_THRESHOLD_DAYS}d (`fetch_now - updated_at`).", + "- **idle_per_repo**: same, with the per-repo threshold substituted.", + "", + "The `idle_*` variants use a live re-fetch of every DB-OPEN PR's", + "`updatedAt` (see `fetch_open_pr_activity.py`). PRs that were OPEN at", + "the snapshot but have since been closed or merged are treated as", + "non-stale — the close/merge event itself is activity.", + "", + "## Calibration sanity check", + "", + f"- Repos using the default 30d threshold: {c['repos_using_default_30d']}" + f" / {c['repos_total']}", + f"- Repos with a calibrated threshold: {c['repos_calibrated']}", + "- Per-repo calibrated thresholds vs 2x median time-to-close:", + f" mean delta = " + f"{fmt(c['per_repo_calibration_check']['mean_delta_vs_2x_median_ttc'])}" + f", median delta = " + f"{fmt(c['per_repo_calibration_check']['median_delta_vs_2x_median_ttc'])}" + " (days).", + "", + "## Distribution shift", + "", + "Mean merge rate across all authors:", + "", + "| Definition | mean | median | p10 | p90 |", + "|---|---|---|---|---|", + *[ + f"| {label} | " + f"{fmt(d[col]['mean'])} | " + f"{fmt(d[col]['median'])} | " + f"{fmt(d[col]['p10'])} | " + f"{fmt(d[col]['p90'])} |" + for label, col in [ + ("v3 baseline", "merge_rate_v3"), + (f"age_universal ({UNIVERSAL_THRESHOLD_DAYS}d)", + "merge_rate_universal"), + ("age_per_repo", "merge_rate_per_repo"), + (f"idle_universal ({UNIVERSAL_THRESHOLD_DAYS}d)", + "merge_rate_idle_universal"), + ("idle_per_repo", "merge_rate_idle_per_repo"), + ] + ], + "", + "Per-author drop from the v3 baseline (n authors, >0.10 / >0.25):", + "", + *[ + f"- **{label}**: " + f"{s[col]['n_dropped_gt_0.10']} / {s[col]['n_dropped_gt_0.25']}" + for label, col in [ + ("age_universal", "merge_rate_universal"), + ("age_per_repo", "merge_rate_per_repo"), + ("idle_universal", "merge_rate_idle_universal"), + ("idle_per_repo", "merge_rate_idle_per_repo"), + ] + ], + "", + "## Signal quality vs ground truth", + "", + "2-feature logistic regression (merge_rate + log1p(median_additions)),", + f"5-fold CV on {e['n_labeled']} labeled authors:", + "", + "| Definition | CV AUC | Active mean | Suspended mean | Cohen's d |", + "|---|---|---|---|---|", + *[ + f"| {label} | " + f"{fmt(e[col]['cv_auc'])} | " + f"{fmt(e[col]['mean_merge_rate_active'])} | " + f"{fmt(e[col]['mean_merge_rate_suspended'])} | " + f"{fmt(e[col]['cohens_d_active_vs_suspended'])} |" + for label, col in [ + ("v3 baseline", "merge_rate_v3"), + ("age_universal", "merge_rate_universal"), + ("age_per_repo", "merge_rate_per_repo"), + ("idle_universal", "merge_rate_idle_universal"), + ("idle_per_repo", "merge_rate_idle_per_repo"), + ] + ], + "", + "## Recommendation", + "", + "See the `recommendation` field in " + "`data/results/pocket_veto_analysis.json` for the machine-readable", + "decision logic. Text summary and follow-up branch sketch below.", + "", + f"**{results['recommendation']['decision']}** — " + f"{results['recommendation']['rationale']}", + "", + "### Follow-up branch sketch (if adopted)", + "", + "- `src/good_egg/github_client.py`: extend `_COMBINED_QUERY` with an", + " `openPullRequests` selection that pulls `createdAt`/`updatedAt` for", + " each OPEN PR on the scored user (or `totalCount` if we can push the", + " staleness filter into the query).", + "- `src/good_egg/models.py`: add `open_stale_pr_count: int` (or similar)", + " to `UserContributionData`.", + "- `src/good_egg/scorer.py:256-261`: change the `_score_v3` merge-rate", + " formula to `merged / (merged + closed + open_stale)`.", + "- `src/good_egg/config.py`: add the staleness threshold as a tunable", + " config value.", + "- Tests: parallel coverage in `tests/test_scorer.py`.", + "", + ] + FINDINGS_PATH.write_text("\n".join(lines)) + + +def decide( + e: dict[str, Any], s: dict[str, Any], d: dict[str, Any], +) -> dict[str, Any]: + """Produce a simple quantitative recommendation.""" + base_auc = e["merge_rate_v3"]["cv_auc"] + aucs = {col: e[col]["cv_auc"] for col in MERGE_RATE_COLS} + + best_name = "merge_rate_v3" + for col, auc in aucs.items(): + if col == "merge_rate_v3": + continue + if auc > aucs[best_name] + 0.005: + best_name = col + + cohens = { + col: e[col]["cohens_d_active_vs_suspended"] for col in MERGE_RATE_COLS + } + + if best_name == "merge_rate_v3": + decision = "Keep v3 as-is" + rationale = ( + f"No variant beats v3 CV AUC {base_auc:.4f} by >0.005 " + f"(aucs={ {k: round(v, 4) for k, v in aucs.items()} }). " + f"Cohen's d also fails to improve " + f"(base={cohens['merge_rate_v3']:.3f}, " + f"best_alt={max(v for k, v in cohens.items() if k != 'merge_rate_v3'):.3f})." + ) + else: + affected = s[best_name]["n_dropped_gt_0.10"] + decision = f"Adopt {best_name}" + rationale = ( + f"{best_name} CV AUC {aucs[best_name]:.4f} beats v3 baseline " + f"{base_auc:.4f} by >0.005. Cohen's d " + f"{cohens[best_name]:.3f} vs v3 {cohens['merge_rate_v3']:.3f}. " + f"{affected} authors shift by >0.10 in merge rate." + ) + return { + "decision": decision, + "rationale": rationale, + "cv_aucs": aucs, + "cohens_d": cohens, + "universal_threshold_days": UNIVERSAL_THRESHOLD_DAYS, + } + + +def main() -> None: + print(f"Loading {DB_PATH}") + con = duckdb.connect(str(DB_PATH), read_only=True) + + print("Phase 1: characterization") + characterization = characterize(con) + print(f" state totals: {characterization['state_totals']}") + print(f" outcome totals: {characterization['outcome_totals']}") + print( + f" repos calibrated: {characterization['repos_calibrated']} / " + f"{characterization['repos_total']}" + ) + + print("Phase 2: per-author features + merge-rate variants") + if ACTIVITY_PATH.exists(): + print(f" using idle-time sidecar: {ACTIVITY_PATH.name}") + else: + print(" (no idle-time sidecar; idle_* variants will be all-zero)") + df = build_author_features(con) + print(f" {len(df)} authors") + distributions = distribution_summary(df) + for col in MERGE_RATE_COLS: + v = distributions[col] + print(f" {col}: mean={v['mean']:.4f} median={v['median']:.4f}") + + print("Phase 3: shift analysis") + shifts = shift_analysis(df) + for alt in [c for c in MERGE_RATE_COLS if c != "merge_rate_v3"]: + print( + f" {alt}: mean_delta={shifts[alt]['mean_delta']:+.4f} " + f"n_dropped>0.10={shifts[alt]['n_dropped_gt_0.10']}" + ) + + print("Phase 4: signal evaluation") + signal = signal_evaluation(df, con) + for col in MERGE_RATE_COLS: + print( + f" {col}: CV AUC={signal[col]['cv_auc']:.4f} " + f"cohens_d={signal[col]['cohens_d_active_vs_suspended']:.4f}" + ) + + examples = pick_example_authors(df, con) + print("Phase 3: example authors (most-shifted + labeled high-volume)") + for row in examples: + print( + f" {row['login']:20s} [{row['account_status']}] " + f"total={row['total_prs']} v3={row['merge_rate_v3']:.3f} " + f"uni={row['merge_rate_universal']:.3f} " + f"per_repo={row['merge_rate_per_repo']:.3f}" + ) + + recommendation = decide(signal, shifts, distributions) + print(f"\nRecommendation: {recommendation['decision']}") + print(f" {recommendation['rationale']}") + + results = { + "universal_threshold_days": UNIVERSAL_THRESHOLD_DAYS, + "characterization": characterization, + "distributions": distributions, + "shift_analysis": shifts, + "signal_evaluation": signal, + "example_authors": examples, + "recommendation": recommendation, + } + RESULTS_PATH.parent.mkdir(parents=True, exist_ok=True) + RESULTS_PATH.write_text(json.dumps(results, indent=2, default=str)) + print(f"\nWrote {RESULTS_PATH}") + + write_findings(results) + print(f"Wrote {FINDINGS_PATH}") + + con.close() + + +if __name__ == "__main__": + main()