From 8745db6e1101332773fb646fd7d94b01ea54041f Mon Sep 17 00:00:00 2001 From: 51616 Date: Sun, 28 Sep 2025 15:46:47 +0000 Subject: [PATCH] iclr 2026 (tmp 2) --- ...ist_train_fw_qa_v2_2k_len_level_0_tiny.png | Bin 0 -> 12045 bytes ...ist_train_fw_qa_v2_2k_len_level_1_tiny.png | Bin 0 -> 12042 bytes ...ist_train_fw_qa_v2_2k_len_level_2_tiny.png | Bin 0 -> 12042 bytes ...ist_train_fw_qa_v2_2k_len_level_3_tiny.png | Bin 0 -> 12042 bytes tmp/icl_results.py | 204 ++ ...ist_train_fw_qa_v2_2k_len_level_0_tiny.png | Bin 0 -> 15365 bytes ...ist_train_fw_qa_v2_2k_len_level_1_tiny.png | Bin 0 -> 10448 bytes ...ist_train_fw_qa_v2_2k_len_level_2_tiny.png | Bin 0 -> 12662 bytes ...ist_train_fw_qa_v2_2k_len_level_3_tiny.png | Bin 0 -> 13750 bytes tmp/packing_debug.log/debug.log | 2997 +++++++++++++++++ tmp/test_chat.py | 105 + tmp/test_detokenize.py | 35 + tmp/test_emb.py | 51 + tmp/test_logits.py | 121 + tmp/test_mem_alloc.py | 151 + tmp/test_pad.py | 8 + tmp/torch_profiler_test.py | 69 + tmp/visualize_ds_len.py | 101 + tmp/visualize_long_samples.py | 133 + 19 files changed, 3975 insertions(+) create mode 100644 tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png create mode 100644 tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png create mode 100644 tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png create mode 100644 tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png create mode 100644 tmp/icl_results.py create mode 100644 tmp/n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png create mode 100644 tmp/n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png create mode 100644 tmp/n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png create mode 100644 tmp/n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png create mode 100644 tmp/packing_debug.log/debug.log create mode 100644 tmp/test_chat.py create mode 100644 tmp/test_detokenize.py create mode 100644 tmp/test_emb.py create mode 100644 tmp/test_logits.py create mode 100644 tmp/test_mem_alloc.py create mode 100644 tmp/test_pad.py create mode 100644 tmp/torch_profiler_test.py create mode 100644 tmp/visualize_ds_len.py create mode 100644 tmp/visualize_long_samples.py diff --git a/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png new file mode 100644 index 0000000000000000000000000000000000000000..0f8e0ee34df8051b9f846b629b9817fd48cc09eb GIT binary patch literal 12045 zcmeHtbyQSq+xG_SBcNg+2pC{dA|fRW3dWGqIiPrG327M01d%unDbn2|-Jz(6bPhFu zB9a3r-QTs<^Pcm3$7g*{t>^vcTT2&fF~i>b-uHF?u3ub_ughPh-NCd2K@eK(H7O+o zq4q!!s)21=;5P!zy`SKRi0x$!TV+c_TZfz028i5ETdUiawzp0658E49+n88d@bmDU z;o;{xY;0?5Wh27NYyLmq;IXte;x**R^M!BT~u)l zr;y!kRfZy}zbIUKx67^W^`^6u!JAq??ogDX`el%~U+&*&7pM{n}Bsoq@6dLZ^nODL&ZdaqB+K=#gUS&Gz5cOJ@}mi@S&pJp#XVsgE?9=9b{ z^~f`9Du?Q{Gxs%;wuN~9_n@32*sCdlA)=#z~alOn4 zGHvjn>ZZZ3zjpQy9zTlcdgRxZ%O?dO+YpHZx4u%e4Rsfxmd`N$$F~v|gf#8LiBwFK zx_SmL#x*g(vx*r0dsTwjyfw~+9A#xT<$7RnW@$+XkvMmTW*dUsU^GOU&2`ed#0-ZJ zgv>&Ilvs^ftGC5&cnNI6|&PJBgUH~4!?CJ)70g8N?LEPPCJQo^-x@XzM5NwUKxHtO-n26 zbKSGJ)+B|iZMVtAs>iPS1y+4%`t69Mik4QlrX-vhuhdZd4#$N_vWU%RpS9xRVy*tQ zNe8>#SSg)Zl2`o> zHM=g5lOo0NV_DU#0qiUn9IwMO@Eg`Ob?2F>Dk{d9v?iJiRNno%;wWAyWL%V72l-7>AJN3^XC5uyV2 z6B=b--ZFPrxL5SK%*YjyEpV=L?arT`UOi{ocR_4*x=LDBwk=gPEifo3;Y6h4CBnJe zooCGZ%W<{A+{X}V#KVI3GY?f%Mp2O_Z~0 z)VQ(Gc~;DK0%FB-BpM-ljvqG5g`T-MIy%}s>CK)JA1}odQrma-?Acf{2ZDGIxeRFr z+&327$Lj_A+j31>CtO+muh!Ss!vpvvXAp@Y)q`1P4X&pPDRX+McSIthL|0B=-@JLV z{6YjBCp=QczV<>0B{2HC55p<|L9h zv0t6bx=S30OiWA@VPC&m3rENs7#Z~l7_r4=BvKfosy|r08@0W18=f8~C)-EU33$<- zrp}4M=xyJ=(SJFlHX%hh$#j0AOB#!P+B(`sLx15gk(8Eot6G}y)}M_yF>zZi@`9#J z2#@L+#=A?a4N%-OI*CgcBhkAW$sueJ9IxM!aI>SW;TkNwdfXZrh z@k=&W8aZFL%z1P)DXODY0kc*##dg8*g)W|w$Ghq?N46sa<42LB$0L}}Zev6c zoxcx0Kl-Zz70sg?UyKn6=704m(Y*>#5N|G9=u2yBQ{mv?Fj<`b(4AxCPqxqu-p`|x ze3xb`91tv=MG>n3iPwC7tYWF9r5B*|I8K}}8*57uaa(nis>e^2B{5r;Dwa9VDP6hp zD3}M=eQFRk%quZ;Fmi6o6`L_uj{uYgo?& zyFMD@$x3J(Ze3qnfzsv0D(UJ*`W+IEg>ui!%^g`^nIg@Ow&q!oMX*XjhdLI&3ZdI>}4D4E8V#!0rpdU8O?E5We9FdwR&Uu zJr=E)jTsE5m>ABCl&NZCle0djx1sv@@#Er;-kd6`stJ0l9}jGj7~m8Ue9fly(O*dc zLG-TF;i%UsmhNeArSZv%N^^U>UcjY>IT4_ziui?|QJG z+O_SZuy7ZhJ0TDjxVY#@>CUUoEE-LG9vB*GOj>jiZ*OZG$!QcfE3lN4liT50wHw;G zggV-o2K_M|?}Iq)x(q^0WoB zEX(GCVVU1h6X&)xv|Hg26sN5q-@;xiMa8F?C6gjz^PRVrODa}G%hu+5oaeiY<|c9) zr3oKDepJ!WNRHZA9WI{8uAMNr(bC##Hyq(0y*%UdA)}zrR8Pdh#H2Z0GY2i3grp=H zLixtplv#7^xsv?K#8ey^9*De8XU*8N81o zZ>##jN`ogvCg!@=3KcK%A#>#>iv*J8Gd0Cnx)lXQmM1{|fG~ZB#%&((t4BwmRU?d# z{%aqEJ}`Db$UO-&wGBNE#*Z;O#Gnw&L#h6vDy--ai{S7Jn}JC*0E0up{GWK@P^f8o z%<2o+a8lAhg^B>4y>AahE}%IyV~dPAQ-rrHbyQfXlZFVXVh>g zLflm()^NPz+j%%d=lN*o!C3gHfsqxCVA~rVBru*cv+xoyv8)10PeG6Q<;$0y=i1ft zZdPqE@5z?~hTie|F#~0u->Bi;)Iu8YA5r^>CfHUujj0cRInXVzm7X=Qe~MIiXAnh| zSFbL~JwGq^T+ni|?G9{)l%(wXUL82J7g!seRwQG7(*>rIp}(}&ZtnLr zHCbk!bS$ZLb+VhOsjy0icQs9as2Z{_p)f@?CIQoOU77FtovA=gku~e;5x;}{R}vBu zidVnZ+I@Prn_DaQnqZ$}Txwbx3W%Yh$)7%b@>(b)TWr365v6iI+uY45=H_}T40N7$ zUq4}?U*+`_BqkmQV$nsc`Y^rOH&3+K)=e|F1a7f@5B8Vgxa^A!mv&Wkle$noCg2Di zM7@oom+3iyfq}+wMinVEz+YDU@1revY&&xJI4@Ncx3I9L`x?nT!(V*)+Mftr3Xxba zb#T`4!m!xB-uY$TcA;ByW!Re&oawtQ@eKN}`Axw)h<6pZ!wJW;WSdv^;R1ezuOk z(Rgin4j7pC=O#FMBA4sPG>e{?)r-)u{MQ$&F{YIsRK*`2G3VM0%hNKRks##W?mGJe zji4KWuyFqT%xGd^VWe4-SyQBjE>stqw~ zioWQ)zBH_f>Mh124cT0r-4P?HuRgrc;R(4Evn%BO2E786r0E6~Ju5ZUF3!buzn8$( zaBLg95RcCNHVY_1Mc?{gvL8?}i}3%Am_0+!?*TRVpNG!lpJ@gN95Q}`(Hqh3^0lu9J}JLlb}U-YHBLlG*Oia^dsWDH~_P;R~O>Ax_Ae= z+i1~nWGhFAj{X$=!%NN|9v+x|qPKbpMXoEpObI~y&`Wn0*(9-#{*GK9pYV>Mu=QX& z!|c0b{v*Kq)qs4ln*jHKDr#X+xDbQF7Rch_@yrsa#_VLD11N62zKw95Jo#Jyl)AjU z{Ok4i@1=o9N=>kd*e0Ll=ij$&6Fc^3(D4I1mu%)$Xyt$wa_7Jk%&KtVz9PFb6*)=RaMo%GVZFs8j$U^Vek!7fmgXH z{G&((kQZIzU4%zg=EjpaiKS=t^`gXG1H-~hr=oxY88${{1P1N}$sjgeNdsb!Nz^`_ z-2`OZ`-X;&Pc;E~wxddb@Z1_NBsOo}49`f2Kge%5l9CkJ(pzLBVDNsgc~|!3aS~Bo z^WZ`r73ef>;PfF`KZTrmRxxMe!MDCB=>_;(o*j)_U3KBsE$z&#dH-HXQ}Y=ZDY-67 z)?kWVYs74nLW?{12wriM$w#i*>{#;S%+rVHYKN>(?cH(E33FJ1YEWM^_ca@Z2RY+0 z+TX8qAc9KDcGn`nJgffXcOA*~ze$$ZE&J&@xAD$QSpnLTbQiMmSUi;Wg ziHX6g5+Xsdzx~OV{>t^B0~J~iN#pF=RiTF9AMB*_+8l{8ubd>G5*;l8{>fNR0nysV zMpaLb?T!!7eK4iwCdt|+CMIG0Mt<^9;w*|ecH?TkhlG{rJS8O`?No%1jCRr^$l)6E6jhNy!NH~Uck4=S2m5SL^-5{?j0FTN-NUY9bEna-?RbIaqz@%bK0(uXQfAT}J>G_{PD zQm2_&0Hf1PUv@xoLtF18i3M-4P>ybHvzB zG48zbpdx=hKc6>syJwY%)7N+cfzVA+S5^)P3k#bdeXv@*_jWt<30gj38poxfkhIiP z=@NTN3%vWFs3_vv%EHwWZSV=6)(&MC`1tr3JHWRGIG@xYp}^vx)u56DK5zEgeGEN& z;8a2fNz2OOi!CoN*9D42`1HlG0sr@WGKUyR088Li2a(2h5OqNLe3=BK#qjegmoGmg zt0^nTLA5!R>0$|_@peG8_NJyAq`;7n+ELrnqM|*S!YhpC0_N3=7w9~zbR!eZQI=+^ zKgnA8@uNO!@I8C>OvHX7v?D`130Sl#Y^8OP4yxb&qu?bNN(11<`V@z(Ubt0Jan+x-yYNn&VMF*SC=YSh z<&)s+fo#k-%g)Y5o#EKnqo~ZhzlFACrV+y%)kqp0MBmzNoIXpXMtcNlw|)`r>(d=u zHd)0K&nzs2v~x_NlB)8}ev!5^ryzwCW+z9IG~YrKLfu%;~F4NaA4p_w;55j!fh=uI5-iSx-w{q-2y)qg0YGJ zsF~l4$^fro&>sL*%AhDwCHCFIGKRhK69=c^;7b49y%=B54&`#o%nF2QC7lf@>-dfOeXT3w4}xLB+|l@Ph6%dIE8r zxBZFocS%uhsX+Tsr71|Z4Y(BZ+m@Y--z3gHYQ7Y*iI7w!#E0BT=G`!mC+Pn`mio^+~) zhKWC{E@QAA?H2>s|Ed)!CZ?v~^!cHld~sMP9qK}%OP?d@5rJzc5%&JUzqAT^GH#qP zCWW)CcX#)6mnR=TL^lougEF7=hjY*4=<^Nz6KQV1#sFH+L5_2|gp&{tSOAN`r=_Jm zu@8aolxK&oaSv$b7@7c|a30SnI0s%dzM^_i5vV7i`!PeR@$vDYFOCyo@cu+UI+`nN zkHXnZ22{<>%|U~dmX?Uw5;-QwKVmb(@vU{^pI*ZiRgp~FGoH&;nR|89KFbfPnh5dUSc znp0F%8v>AymIN8l+j%!$QiFNa$?u2;Mal|d2}L%;rtbq!b!Qtq0o6>*J{$QP;0ak9 z?wP4mq!e@gTK`gm!#VJU@hj~_HvBFDe+Dl&9KMWvmlQ61c@XKo?zH4m{(Dg`*&(+y)p1w7?2zV&7j%a` zh^c9A?s~HDD#57o@VkrvhMkeE6Wc_M=v~GaoU_toM9LZeRr&QHL}F)CA1?>IT^qjd z+2Q|#-DzyuB2i#7*>V-lG20jrFS&mPul@Ic(7Lwoso#S%04jhs9@^2E}w z(o!V`nkY}S}7R!~>TkfNZ@ zIJoeXkr*nkdosfz5d~BpHE@?mp*xq3UGZsxfQ4+a0#%Zy|7J&qRvy~=m4M-Ynpv_P zYdckPoR?Pt{32y1r=s%~y~zcA&N2{k7W7noc!8!$K*Q3&kMPH=7qYr7-V|Gz$YBxx zCJDWN0M<1fop4>ZB?Ge4a0Frf!b%93Ol%vA)yp*sZbz|@m4RPZwr4xJYSXOhFi589P z&HLS%Hws71<(TX5`o_k_cw8i)sw2{CH&=q!2OR8Mz6g5Pp3=?nBVUBzQU&NYF2|sD ze_npRDY$0#_1czY34q%riKV4FfLl%_A?_SC|5weTzX zI+dF@W8BtfqQ)Y zr4m@6ac|+B`SGqCBN+muh^pYB0(Omo!-{4UfYf$Xw1URK{>R$WLe?Vl%t)rFQ7{y& z{Ueyn(8zja;ZUUGJm@Qkx+#irS5n+J*5Nqae64jC1zN>AA`N4${vDPhhR9uBPdkLl z=FP_i2cvQ8Z`RYmtDg{=P3Y_XG5Nv}Fpf}5TorYwVGn*tbz{%pBzBMnN>Dbu=|Ub- z*{c5pJPN^?7vBmt_ekbXOz@xFnTw|NOEj&~A<9 zJc>|PZXQ^;M`e5Y@6J0^NghV5p2DDjheq{xxd>0STjF?ca(ghaV5n~1G14(`};pNUWZ!M7w$yMI|Nw`4P-XBs@MQ^!QTc-AM9Y8ARj z0_@_r5oYCCpVKf4+7iU06d)pMo@4@wuKxb4cbzhokkY4up>pR=Rv9nQ zZAds_z<^So^!pJ6xi;Fkam4OXM^s%kqS9 zq9ZxCXa-1%Ixv}%W7I>`*4BojtC*0Oc$$ZY$6M_F?67+?4Cf?gW;S%5TGM#D^w>N+ zs`KqqQ6X1t3po4W#5yHOG05jgKFjCw()3a#vU(MuS{1i&t2b zYTW{|EO!)?C!C0KQ92}!`($mninAxxI<3`36)rm)jT7tNh2rQY^^>$azV)@{8)Rh5)t&4^`c zU`s1QdOBG;Baa4xZQ&wz(qI}%%gL$sm%9c%e~$0_*!Q!%rkLMT{{e?g@JVQw49>1i9@T=lA01e z&>A`bdomyi1nK;~7#cx3BqiezU)$G%`=0LOvoj4mH#B)ir)tK*#LVeYh!Qu)ISmD= z$;CrsnU8ZLCuaBV#rmO2Zy>W)oqKt~Qo{BsWk?1r`w*ZRC0{XAF znGH8YbhxipyK8_?lQjvRt?|wjZ6|>E3^1F`fD(5=XU_mXMgy{)ykK5E3zC&v)6;Dp zBWGbkg$vsOssZ>7t*bY8=h4HNh|LD5zL77e%byV$J$WojPa*d#8&iP^G~zT*KsH(l zWK8HJP66CqQWMX?&4%4H(%e71 zmBnACa^&nkC{uPn!LNL?ULe>bGT{4Nfiy#gcQt(L;~56?iZzO*fb*;|878p68E+LJ z>BJ33;u|b5j?>ZiUfy`Ubmv>fFZ+54D=&{y7)sb5Tp~g+)M5noUKv~s@zrTbY)i>h zc^cfv?A3$q1R>0b<-#-S16bZP^DJ&o>Oj@Z}TU$$I^ib5!Fy@*y ztQmyllgtGcmvx;-Im%`lM7v=K%?a>hWEsq^Jh1ZWBgM2g7X3Gp!N(rIidlZThk*g? z9LtTh{DlSg`6aB3%oAgUYuBzzOYaySrml~|=s|dl3O6)$f*?R_c_a>kzyS6JdXYm^ zyKtAj3#vS*2V;lL8%dd&@@P@QSHRRK8ar$qHa0hJhc(Z2-txy1&=Ck#b#*h;!*6W# zx1M?dGexKg4#8^cK6Ra#)YQ|k%+q3G5i|GYUvk`dUHU_eEt99*k}lrg(zPPS>JLh} zy|a^1QBO~g*T4D}DSV~Pq-Q*z$wdt208EXIo8CP=7WCV1(cqb*iWtK3>kypwS_lrG z@J7k<#KeS`D{AvWW)i=#v0^?owJ|Z%+SIhG*=>2&ghk9baK`2?I=Tm3(yS#Oi%y{( zSBOjp!v&oEYadsm2X`a8p)<9>6u>M5`eqRIgJkwLW&=!ho}WnZ_TGWl`-`3KyBPH= zaOQCNVJAJmMN>4;r;tn3LEF!thY~{P0Ng*XN;QJib??i@vbinBz_I4S!lhx1;;|40 zb_1xmw-CnH*B5$Ou37%=u2`rI--G;#3|E?6*G3iGm0VmtsawH@!7#+>$M7C^QIf8E zH1gYgmIaerQ@g+-59Vw1E!~$}qb9<@ch>+ftOJH8(t%fL=n_d_A==}6&K?`A_AdZl zmk!!r69#y0JbOxKJ+E9D{ATHKB8a{o>1)?su)5AFj)UsxfPst*u;H}PsXkoJuQhu( zVU`6|sJB3O;5%3xdYdQD9wX`4_yQT@F$49`VGx3Qd-m9Yi*9b^bL#GM>hovMRIJYt zGf=#PX^1>CDF17lBwll2DHiA)kC}Yr1(UTZ%<6qRm?P4F;?6H$D4WuR(EYJc17xJ> z+Zm2)V6j-WjrFw*R8FYL$jF@Vf4g&div)mw+45Lg2b>-gh(OIonqoR&mWH{xue;k+ zY_>@<7|1#V`*yY51sdNRA@|imwha7aQ8nsu57~-jtV~t(14?Na(Zy-Q^i9WodU0a| zLqjb{aI(w4_*!aU;PoF1#z1)-d!yN*S?dR$P;}2A>z`QRf5UkDkKdzigL+Q|{ev%6 S!Kf#LU@yx{WnQ{*@BaY(9Qcp` literal 0 HcmV?d00001 diff --git a/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_1_tiny.png new file mode 100644 index 0000000000000000000000000000000000000000..5818be73cf970491c9af0cc6d5155a6c6078ed0f GIT binary patch literal 12042 zcmeHtc{r8(+W#Ys$j~U6b}2F@MCM9q5kkluw#@S~WNuezqL!&-C}ZZ#EU^?(DKjgJ zMP;@ugk{L|yPw+oJ^Q?8pL6y(?|HBHpI_JN>dLb`)A#;Y^L6Gg3 zb1FIrvLP5j=ss=T3_p=-?Hz=FWU*&1VRc=uV7)Eetq~0itgE97*3r(A*UQ@7!_LK7 zLR9>ysDv=DEf(wQAuA^4^k3f)b#b>5yK<=HHk@Uf>p4RY1Yxm2|I$5E%CJKaQ7??j z$%{VmRB{mZ;!r}vSFKZTSOeeJZ9H~5W@FnI25l8O#({i&jeTytojb^{&z|2jerJxIvvWKX-){dedo>{FK>9g2A2(E!nqZ) z=H{@k8y=t7dlr#r31#NeIeYeOaV4xp^)))oieu>SeykNpFO4dkEYFa+!StFMp=Bb)jb69=N6@-|KnS^Re{L-{D$@Q>(>cIyLat6hto`y&-~?=4Qc7=SuXw6 z?>>Cc3kV3P6SsG87+P5Ftn_wCOG~qxpXiqL9KWdYAx6Yl(!N9B1Sj)$F4>E4C_zdE zp7Yu5ii}`yDk-f!L#ro$VUnCZ6|}yLpKX#mggJBOjG&NE`!^b8$h|_+WB9yAyo~nx zQquYnjBLQS;*~&xJa>fG#K)(#p)8a{ze(-pLTDn=69 zNhJ=R1NWrWaQgXXxu0u89Y&fGIHaB9&4N}vsr>|Vmlt0{dY#Mi%D*ru%(a}3lk?8o zBWm)ju1+P_s6@@s(9n0Hpn(qq_ePwj_LMkwA43;*bl!4u$MmDP$B${R7{zSgMIC{{ z*45LCHZn4bij1@(GVMQp{?VgH3>!D#Qk8k1upK=6hU(QLrI@7k+4a)$JhjN7N&c*{ zMvpkGA-?G0V>yLZD&+sA~!Yf98OJ5 zZP2}9Lge7VgCtX^(M6Kwab#^FXuYh@XJEU=eL0?# zuFRZ?-ZaVJyzz-N3zG4k;p`4rgrtW|!Dul+;Y(S+i}D~rsi!Y4`V0*ZXJuw$VCA$F zXi?<86jc_dkIzp^TzP+=NYP7D42Vif%71-lZ;uod3lzGpw)TSpkAI>{b)+Hhc15-HZg*&@BRJ<^6XRow83JS z6=~5sO8+l|(C=Yu@rVr8VMQKrcH{uEL3>VvUhw4}#8t6;BNK{kR}eaX5$Xp;Zj&z& zr?e2He-|CA_}Ik#lQjtP>DA{QNB<%8`7p2_-FQcp9wGg^cli)rCr{8y!DRKaGXR(1 zkt3LcXYaO5k}IiRjfKURUvAjWDRTjX!MuC>wsi}SKdal_gR0>IVq#jF8M<%ZzKxUh z%nIj{+oKHtH8qN-ngPZ_kMES4;v-wIdD=4cYZq2MCIZ z5kt1`FLvrVPWJ8(@bZ)4>{|RfAmuZCSx--ItW}k#^VLm8+FZMq_qY!-s7dyavvA4IM z8_P&4?I}x7KYX+Ha~J^DlWb^7i|`ukhTcWv7r)#ov1>p4;i33xrKPu8b1SpW>Nwcc zd~c~zUmYyG>wy_g^q>Kwgy*)@}*buC%n&zT=s; z@8APf>`=79c=hsRdqmKR6)cw^$yC?Pt>Dzt`&FHtoo6c5VP}~ezu#1IjeZNjlbmJ$ z^ygYVV`CxAjE_7n_1m{^VQ~PAl^+JaK70D~r{vX+OkG*`!E-7aeho6S5>3OdGQdp^ zLe&K49Uk8BNs4Ha0>~T|Fhy9e-ps-+JSN6+!$+Q5@8ZReRA>TFEi=AVIKF&hHlRL~ zLbudoX%~G5hwS=rE(fAxPpO88h=`ZQ)vFbv6@$1i{%|M`{Nsmy4CT+Boii{GDB-?3 z$hJ!EUrp%qpVd%Uook<;5q5KP6BHKi5DQvTH}(Je`rW&CZP!^8p8?Av2QE`_Ysaj7 zWBVn( z?xTCM3@hc+sb?^j7Hg&h?{1HnSx!!lu)^Bnz`MrAB8TDkk5T~TDDakbt*lb=s^*QH zddsz|d})PHdxj~GBy7ghA3N*#`@eAa@IbdsC3(nwj7Bs!IC=8gS!f<>tIG~Af7v8V zw=-qyPY7Zn;}&CW*gC0Vu$ws)*uyS{o8>&y@~o41o-VqLtc{Cn;V=DAg5FZ_=ZMzt zG44P98<;S!EiF}lO$}{pnRx#O4E>PSF;b-%s0=JOK<)l)FCt^5x5cQj6M< zZYsZ4;zf9WI=Z@@WiIAuzpAcQ{K8&RQZlo=I!>-EYinzh^qY4~ko6QyzS0;Y!XafJ z#mO8R!pxNghnm>Aq1$x4HbJ6N)y1kVhO+RKmQj1kdYpi{`NFO#4qWwhBIEUdp10NA z+DV-UlGix3kX=z>DCafNGTW?_-`1uha_MsUXwSGU~9RAB~As}chc9XpoM)3SQcJ3^(?-T@V94lzxj^x|UqiE7o=0YsA zZXBFllJ(4i>m`UBwn3B@KhdXUWKfLzzs1bV%v?mbbXHcDj=K6CcX#&*61>AB5)!)C zl$CM2wr-TPtlL^acCM@+nLWbE>;M4n_2E&2ZMK~p=m}--JsuLlRt@2Lc6OPtD|c`y z`d_-UN2JY@Xqr*ww;;&QBq%Pf4IPGU%B_;ju=zSJ_3}}6wlxx|hgALSFeftwULYWq zeVL1fh6cc4p?PTS=UEO1Z^@s{42vt_o~u;=gSPEF{4uC{Wu{TeYvR&&Zuv`;uH5C` z%Uy=>yT!Jx$fQ28$-{5vg~`@3PuWk z^B2#Yxk)zrH`$yhvNmC()ZUeA>^c4EB=uv#X{E)NEFD0i$nehGD@@Q%0CKxvH*)VVwZhP_Lvs`6_V;j>h2-FRITCdfuxlanJ zxgM&F)G<^OtXwyY+2P_LpOB>K=G`6Yi+}!B?DFf&| z(J?XU-9JlA;JGN!Sr+2{tROzfe6zsm#lZ z0m{W8Zg~U5tLeb((k|xE5|@5O5N%e={))@&R6Bcn9U0E>k8n18+zngrCkkTKi^+Yy z6cH+^gc4tR`6w!K5ApM>PI!NZD!{qV`MFiO)I0XCK0nutmwB3+n(CognVPBs5cjQq zVB9(6z0e)6y-*<1tbl`_o}Rv+u5Bs4@__?1jSz`^>w<|%&IN&=UEoOTBp7Tp#62E# z&$F`Bwr}5#Hk}MQ5-D>B~-S4651106yzyDyLfw{R@M8iRpZjH$A z?w$YEXdka>Q&|**gN6SqYNa7fP6c3khR2ioP=5O2h5T~!uSm6;7#GLU+12%qLQ})N zFv|z$A|)hbb7lIA7bZadySuyp3eR@~M9fo&TCt{HhIQ470pA3qq&oYQ!0Q0VLH)>) zBkr;Z`1Q5rA)2q0bFU1h4XAr`LPGA_NPdU0wlqNaIHjO!C}p@uJ`6NQ;!Z93rzbpy zQBQ#M;%X5fUq<1vNe2+OWRzXzTXq*(-;RhdaCXiEdI+s*u0vO=oBFRp4p<l4fQv@TyH6nOp(h-JAsfUDoYK|AR06Ex!ZG zTJAoiW*WFO1WO4YAl<&y65_;sB)?V`w35us%<-PGd?2KH=H@(Zcfqq*S`DO4l8r%S z#3}fdY9`3;)-JMc+;fjZN(~whIGND??QED>z-<`^k-^L%c}_h>SR3pguc^M={Vp%G z%gA1j!7k#*2nq_e3FW^?rcsCv(8Fc1qxTPM#@iCs!I=Qqv3GXv0wIdZob2M_i;R1O zpF8~OdhK^EKMBGFW%J;A!6#jlM+_*4&7B`@rusaXA>09T+GMu|F0`?A<(X#7O_ra= z%|SP`Tb%xk0tD)G`BA0FzLf9R2C(3>>&((aZ7#*7!JXhS*#nKWg(sNjn^D z#u8}1`MvqS^)bGT_Nx|KxU6l&-Y}}rGoGX$-AZ0sUe3Nqb5duq#mL&uf$Tmi!fO)>`=O5og`1!kLnrPGv_d zYwusR9&tp1XZHd>EVF6mhOV0+e^n=7dA<`UX)2Happ!zsQY)HsnK6h!+%Bfg2Va>R z`x*D_?F2J8LB{PANPg;kHrO@5uI4GOGX7=*;G%|wJdCG%F^E$~5&qpfHB?LU%;K5mNDbpuZg>yh=!T3+Y1VFS)Iy%YOd3d$e zZ!}#aqc~^6s#j-6he}60?3qoQHmPZ7=z(yFe*8H5yKV7jAx)L#H%M~^xd$gARd{58 zvhQqbOa400K!M)pr2+P)+0~)lx#k2j<1**Yr>8?D-M*+)uKqiK#`RZ#_J%*nKe@PA z64jbS^PnoQW;QdiKPhn_j)%)l-sb~p;K!lpU%ug5aQ(gg8ALNbjj}%9Y-TQ5zOo(> zj8Tbw^l(!`Hjpe8*uE&lf&EeJJ!Q6BbXX%U6 zHXrlgfm@#yK-s9HI2Xdmz<@qh6_{nMlehUY%gZl4z3`@Qe0)5ads`?#Vc&b<8m*!HoD zIyyR)0q!!K@EK7+4J}(hdgaDK)G(?7*AV43g@uI&NdK1K4_kLM5A4O~(0oztKNYwj z2ko)B+6KHtc-CJsu@9AaQZ>R|{iA zO|#%7{*Z(kOvVFWYB^0P6>Nb;s z_y1l9M#siF48D1QhD37{Ma}OARF0PaOXBZGT(L8f&b6o1>Dyz%(thw`ghWL(#Rx0zDwWoTM$a!_+sw@QEH#yH zXmD^89Lun0C)@|az^(z}9}T`nrf$ag#{w%hsi_yU;dzylvS^C3Z{I%1eC$+Nwkz*q zLKE4|$YHeCt}MEt9UFk)=Sr%2tZ3)$-9p9S73t~gXMqx&S@M?!{b?E*8=C56!Xk8UEr2rS3JFN4y>D3 zENdGqO6g^Y;Qc^TQjWq#PfdmIij)X89|SQAk_1wZf(76-_qgQrf3>y18*&U)Lh}%9 za9@G%QRAS}`QwK(!DJYLJvj%4z+e25&?aJ|mnb%9_6mH|3J$akig zpN~%fASwB1NNL+x5ev}#-8o~qlBUP;kVg7s6i27Z% zEK0@ZJTk7zK(`@L(7tjdoq>^&)@Z(_v)A0e0FZ9(({&a>1qE`Tjt}r^Jb)%`fM@*@ zs^Y;#ywVh(iHSUX1R@sPTrJPq1=JBlNtpMf36)%F0L&3}C6mPm8AJF3>5!V(TW^ta zzXfNqYwh-Vieb?e3~pw2R?o;N3xwk+bXh!> z>i0)3iHc~iN??4jlz9+&qK4h-@&fUfO^gS+7|?DJZW+i`b7{{V%2psF{!%&gTOSJT zArRz@V=K3S(SwYH{$pPB@SzEOPnOiJez-uG35ooxzzz)q?bi*k<-$x)D*pgGYQf%` zK2a{ls_gyxPiZxaeGf(Rs;q%CBh_2~fCWv`Z4A!;JhAM&WG91|vH?am(G&JEJrVly zcN0hM4RpbbclmG9)ye)~Zm1uRJsNYHj@9A6CM*!vP1FwziT4aoAzU|qU?;m0{o-Jx z);Q!+dPmtipfYI`^L3;Cr6%uWvIw%MR&>M0s*AP8z+#T+6WT;jyMVCxT5PV%*u@Sy z0)ULYt!D5&9pl^C70v@~ceHJIO+uSTn2a8Q7CYw>a-1Zp5Jy7Nyz zy$@qDp&k2-9xGC!n(11qe%_Ny;FapH>pN4;d_vS!kyKr_J&|5Q8d+;zUy0BJXRNG; ztDv9iMFFsJSWtld4M9-riDijn$7E9@fW1_}d=`a8MOB^UBN11?HLB%-7DuLYfK_JXRTVR{KWyOWgu3_m7OcDil@+&Ysws8L#lpr4Fo>f@RO|k{D_a{-$_aPq#tAxoXo0q)q{hB;o}FYf68coyUuIH{+H8; zdk*K{E6}3j2l)zu>N_Bml$OHW!6~zR>DvkbL8F)In-H}<e&24}s{YT>nxychHI( zmr|e!psGu8B}&aK?lK1ltuF-SqqGt9H{_ZfyA4d9!f-+2?B%Wa&7rkV%~!tc9WhB) zb^Nqacf3irrT zyZ7uNEz2mi(!MtEP$ny<^zH4lUhJ(9&;tN?Q{&?~1EybYVQY~2 zqehyb5am1ql-(D;0!bujMZSRLp8l@WEDF26_7unSUJ&r_fidwqc4&V};LhG7pVBe2Q3py}rFzzB@KwRIW|HlHDA+Cs&Z z>C=^0-r^2{zg38KMGAD>(zSV)^)6uegC?q8ksrXp{aZskFsY6NnHI#OCzT57}{(clZY1vTK_y?ZgWpP#R-_OFu*g&Wbq zfOGWF4u6lo za;st2eb0J~D%{i1v?yq$A*chj{v(rCP_g%}xw&jTuxEK$X?7*`FGS_$tVbfKfDBie z97y>x0cg;PGw#4e3*ZEp9V1(~!jUl1ryxZ97A8C4hEvG9SCRm>6sBwFVQ|9$8gDY0 zD?Y8HL=T?1^S z5vZ%HNmm0oI5-ZFU8~O6AB|zE`9PS2SrC@2H!@)075EwTHn`Y1Ft0n>~SYN z(#?Fw4mkmPf;_M z`48Rr_@vPan3-uZJU+I%(2sKILzo}z4JwF}IU6#xFDsJAk3;15F0UK(q>-^PWSt*1 z+0QJ{U~KA$qT-BDIV_mHlT&ov(n)?`hypbukHoD)U=AP$a=T04bLwWr<@b4yK=?&J zdSnOp-;F{MnDrw0M2E665s70k9)>~}IvFE%xdw^ZH^ytDxXIBlKFQV>CLwZf+}KD# z!9fvHVsy$-AHt_x$ZcNc8A4tK(!&vUF`=mM>IfI5w8z=O`_TjNbKA0Va*9h``km1s z3tQZ`MTl_9g&!&MNK_qdM9QW%x19EJe6OKGK6 z>C#?P>;rVZlCni<@tP+rWY_?wyiZHqQHx&yzkjAp_4lJ{!X7v*cEKOjkX$!{R*)uRBfs~V*%77nZWIJ_AZ9bt%xK3$=gWoV z^%_2kPji)VL`DRq{?<{^kDx&kX#$ literal 0 HcmV?d00001 diff --git a/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_2_tiny.png new file mode 100644 index 0000000000000000000000000000000000000000..5818be73cf970491c9af0cc6d5155a6c6078ed0f GIT binary patch literal 12042 zcmeHtc{r8(+W#Ys$j~U6b}2F@MCM9q5kkluw#@S~WNuezqL!&-C}ZZ#EU^?(DKjgJ zMP;@ugk{L|yPw+oJ^Q?8pL6y(?|HBHpI_JN>dLb`)A#;Y^L6Gg3 zb1FIrvLP5j=ss=T3_p=-?Hz=FWU*&1VRc=uV7)Eetq~0itgE97*3r(A*UQ@7!_LK7 zLR9>ysDv=DEf(wQAuA^4^k3f)b#b>5yK<=HHk@Uf>p4RY1Yxm2|I$5E%CJKaQ7??j z$%{VmRB{mZ;!r}vSFKZTSOeeJZ9H~5W@FnI25l8O#({i&jeTytojb^{&z|2jerJxIvvWKX-){dedo>{FK>9g2A2(E!nqZ) z=H{@k8y=t7dlr#r31#NeIeYeOaV4xp^)))oieu>SeykNpFO4dkEYFa+!StFMp=Bb)jb69=N6@-|KnS^Re{L-{D$@Q>(>cIyLat6hto`y&-~?=4Qc7=SuXw6 z?>>Cc3kV3P6SsG87+P5Ftn_wCOG~qxpXiqL9KWdYAx6Yl(!N9B1Sj)$F4>E4C_zdE zp7Yu5ii}`yDk-f!L#ro$VUnCZ6|}yLpKX#mggJBOjG&NE`!^b8$h|_+WB9yAyo~nx zQquYnjBLQS;*~&xJa>fG#K)(#p)8a{ze(-pLTDn=69 zNhJ=R1NWrWaQgXXxu0u89Y&fGIHaB9&4N}vsr>|Vmlt0{dY#Mi%D*ru%(a}3lk?8o zBWm)ju1+P_s6@@s(9n0Hpn(qq_ePwj_LMkwA43;*bl!4u$MmDP$B${R7{zSgMIC{{ z*45LCHZn4bij1@(GVMQp{?VgH3>!D#Qk8k1upK=6hU(QLrI@7k+4a)$JhjN7N&c*{ zMvpkGA-?G0V>yLZD&+sA~!Yf98OJ5 zZP2}9Lge7VgCtX^(M6Kwab#^FXuYh@XJEU=eL0?# zuFRZ?-ZaVJyzz-N3zG4k;p`4rgrtW|!Dul+;Y(S+i}D~rsi!Y4`V0*ZXJuw$VCA$F zXi?<86jc_dkIzp^TzP+=NYP7D42Vif%71-lZ;uod3lzGpw)TSpkAI>{b)+Hhc15-HZg*&@BRJ<^6XRow83JS z6=~5sO8+l|(C=Yu@rVr8VMQKrcH{uEL3>VvUhw4}#8t6;BNK{kR}eaX5$Xp;Zj&z& zr?e2He-|CA_}Ik#lQjtP>DA{QNB<%8`7p2_-FQcp9wGg^cli)rCr{8y!DRKaGXR(1 zkt3LcXYaO5k}IiRjfKURUvAjWDRTjX!MuC>wsi}SKdal_gR0>IVq#jF8M<%ZzKxUh z%nIj{+oKHtH8qN-ngPZ_kMES4;v-wIdD=4cYZq2MCIZ z5kt1`FLvrVPWJ8(@bZ)4>{|RfAmuZCSx--ItW}k#^VLm8+FZMq_qY!-s7dyavvA4IM z8_P&4?I}x7KYX+Ha~J^DlWb^7i|`ukhTcWv7r)#ov1>p4;i33xrKPu8b1SpW>Nwcc zd~c~zUmYyG>wy_g^q>Kwgy*)@}*buC%n&zT=s; z@8APf>`=79c=hsRdqmKR6)cw^$yC?Pt>Dzt`&FHtoo6c5VP}~ezu#1IjeZNjlbmJ$ z^ygYVV`CxAjE_7n_1m{^VQ~PAl^+JaK70D~r{vX+OkG*`!E-7aeho6S5>3OdGQdp^ zLe&K49Uk8BNs4Ha0>~T|Fhy9e-ps-+JSN6+!$+Q5@8ZReRA>TFEi=AVIKF&hHlRL~ zLbudoX%~G5hwS=rE(fAxPpO88h=`ZQ)vFbv6@$1i{%|M`{Nsmy4CT+Boii{GDB-?3 z$hJ!EUrp%qpVd%Uook<;5q5KP6BHKi5DQvTH}(Je`rW&CZP!^8p8?Av2QE`_Ysaj7 zWBVn( z?xTCM3@hc+sb?^j7Hg&h?{1HnSx!!lu)^Bnz`MrAB8TDkk5T~TDDakbt*lb=s^*QH zddsz|d})PHdxj~GBy7ghA3N*#`@eAa@IbdsC3(nwj7Bs!IC=8gS!f<>tIG~Af7v8V zw=-qyPY7Zn;}&CW*gC0Vu$ws)*uyS{o8>&y@~o41o-VqLtc{Cn;V=DAg5FZ_=ZMzt zG44P98<;S!EiF}lO$}{pnRx#O4E>PSF;b-%s0=JOK<)l)FCt^5x5cQj6M< zZYsZ4;zf9WI=Z@@WiIAuzpAcQ{K8&RQZlo=I!>-EYinzh^qY4~ko6QyzS0;Y!XafJ z#mO8R!pxNghnm>Aq1$x4HbJ6N)y1kVhO+RKmQj1kdYpi{`NFO#4qWwhBIEUdp10NA z+DV-UlGix3kX=z>DCafNGTW?_-`1uha_MsUXwSGU~9RAB~As}chc9XpoM)3SQcJ3^(?-T@V94lzxj^x|UqiE7o=0YsA zZXBFllJ(4i>m`UBwn3B@KhdXUWKfLzzs1bV%v?mbbXHcDj=K6CcX#&*61>AB5)!)C zl$CM2wr-TPtlL^acCM@+nLWbE>;M4n_2E&2ZMK~p=m}--JsuLlRt@2Lc6OPtD|c`y z`d_-UN2JY@Xqr*ww;;&QBq%Pf4IPGU%B_;ju=zSJ_3}}6wlxx|hgALSFeftwULYWq zeVL1fh6cc4p?PTS=UEO1Z^@s{42vt_o~u;=gSPEF{4uC{Wu{TeYvR&&Zuv`;uH5C` z%Uy=>yT!Jx$fQ28$-{5vg~`@3PuWk z^B2#Yxk)zrH`$yhvNmC()ZUeA>^c4EB=uv#X{E)NEFD0i$nehGD@@Q%0CKxvH*)VVwZhP_Lvs`6_V;j>h2-FRITCdfuxlanJ zxgM&F)G<^OtXwyY+2P_LpOB>K=G`6Yi+}!B?DFf&| z(J?XU-9JlA;JGN!Sr+2{tROzfe6zsm#lZ z0m{W8Zg~U5tLeb((k|xE5|@5O5N%e={))@&R6Bcn9U0E>k8n18+zngrCkkTKi^+Yy z6cH+^gc4tR`6w!K5ApM>PI!NZD!{qV`MFiO)I0XCK0nutmwB3+n(CognVPBs5cjQq zVB9(6z0e)6y-*<1tbl`_o}Rv+u5Bs4@__?1jSz`^>w<|%&IN&=UEoOTBp7Tp#62E# z&$F`Bwr}5#Hk}MQ5-D>B~-S4651106yzyDyLfw{R@M8iRpZjH$A z?w$YEXdka>Q&|**gN6SqYNa7fP6c3khR2ioP=5O2h5T~!uSm6;7#GLU+12%qLQ})N zFv|z$A|)hbb7lIA7bZadySuyp3eR@~M9fo&TCt{HhIQ470pA3qq&oYQ!0Q0VLH)>) zBkr;Z`1Q5rA)2q0bFU1h4XAr`LPGA_NPdU0wlqNaIHjO!C}p@uJ`6NQ;!Z93rzbpy zQBQ#M;%X5fUq<1vNe2+OWRzXzTXq*(-;RhdaCXiEdI+s*u0vO=oBFRp4p<l4fQv@TyH6nOp(h-JAsfUDoYK|AR06Ex!ZG zTJAoiW*WFO1WO4YAl<&y65_;sB)?V`w35us%<-PGd?2KH=H@(Zcfqq*S`DO4l8r%S z#3}fdY9`3;)-JMc+;fjZN(~whIGND??QED>z-<`^k-^L%c}_h>SR3pguc^M={Vp%G z%gA1j!7k#*2nq_e3FW^?rcsCv(8Fc1qxTPM#@iCs!I=Qqv3GXv0wIdZob2M_i;R1O zpF8~OdhK^EKMBGFW%J;A!6#jlM+_*4&7B`@rusaXA>09T+GMu|F0`?A<(X#7O_ra= z%|SP`Tb%xk0tD)G`BA0FzLf9R2C(3>>&((aZ7#*7!JXhS*#nKWg(sNjn^D z#u8}1`MvqS^)bGT_Nx|KxU6l&-Y}}rGoGX$-AZ0sUe3Nqb5duq#mL&uf$Tmi!fO)>`=O5og`1!kLnrPGv_d zYwusR9&tp1XZHd>EVF6mhOV0+e^n=7dA<`UX)2Happ!zsQY)HsnK6h!+%Bfg2Va>R z`x*D_?F2J8LB{PANPg;kHrO@5uI4GOGX7=*;G%|wJdCG%F^E$~5&qpfHB?LU%;K5mNDbpuZg>yh=!T3+Y1VFS)Iy%YOd3d$e zZ!}#aqc~^6s#j-6he}60?3qoQHmPZ7=z(yFe*8H5yKV7jAx)L#H%M~^xd$gARd{58 zvhQqbOa400K!M)pr2+P)+0~)lx#k2j<1**Yr>8?D-M*+)uKqiK#`RZ#_J%*nKe@PA z64jbS^PnoQW;QdiKPhn_j)%)l-sb~p;K!lpU%ug5aQ(gg8ALNbjj}%9Y-TQ5zOo(> zj8Tbw^l(!`Hjpe8*uE&lf&EeJJ!Q6BbXX%U6 zHXrlgfm@#yK-s9HI2Xdmz<@qh6_{nMlehUY%gZl4z3`@Qe0)5ads`?#Vc&b<8m*!HoD zIyyR)0q!!K@EK7+4J}(hdgaDK)G(?7*AV43g@uI&NdK1K4_kLM5A4O~(0oztKNYwj z2ko)B+6KHtc-CJsu@9AaQZ>R|{iA zO|#%7{*Z(kOvVFWYB^0P6>Nb;s z_y1l9M#siF48D1QhD37{Ma}OARF0PaOXBZGT(L8f&b6o1>Dyz%(thw`ghWL(#Rx0zDwWoTM$a!_+sw@QEH#yH zXmD^89Lun0C)@|az^(z}9}T`nrf$ag#{w%hsi_yU;dzylvS^C3Z{I%1eC$+Nwkz*q zLKE4|$YHeCt}MEt9UFk)=Sr%2tZ3)$-9p9S73t~gXMqx&S@M?!{b?E*8=C56!Xk8UEr2rS3JFN4y>D3 zENdGqO6g^Y;Qc^TQjWq#PfdmIij)X89|SQAk_1wZf(76-_qgQrf3>y18*&U)Lh}%9 za9@G%QRAS}`QwK(!DJYLJvj%4z+e25&?aJ|mnb%9_6mH|3J$akig zpN~%fASwB1NNL+x5ev}#-8o~qlBUP;kVg7s6i27Z% zEK0@ZJTk7zK(`@L(7tjdoq>^&)@Z(_v)A0e0FZ9(({&a>1qE`Tjt}r^Jb)%`fM@*@ zs^Y;#ywVh(iHSUX1R@sPTrJPq1=JBlNtpMf36)%F0L&3}C6mPm8AJF3>5!V(TW^ta zzXfNqYwh-Vieb?e3~pw2R?o;N3xwk+bXh!> z>i0)3iHc~iN??4jlz9+&qK4h-@&fUfO^gS+7|?DJZW+i`b7{{V%2psF{!%&gTOSJT zArRz@V=K3S(SwYH{$pPB@SzEOPnOiJez-uG35ooxzzz)q?bi*k<-$x)D*pgGYQf%` zK2a{ls_gyxPiZxaeGf(Rs;q%CBh_2~fCWv`Z4A!;JhAM&WG91|vH?am(G&JEJrVly zcN0hM4RpbbclmG9)ye)~Zm1uRJsNYHj@9A6CM*!vP1FwziT4aoAzU|qU?;m0{o-Jx z);Q!+dPmtipfYI`^L3;Cr6%uWvIw%MR&>M0s*AP8z+#T+6WT;jyMVCxT5PV%*u@Sy z0)ULYt!D5&9pl^C70v@~ceHJIO+uSTn2a8Q7CYw>a-1Zp5Jy7Nyz zy$@qDp&k2-9xGC!n(11qe%_Ny;FapH>pN4;d_vS!kyKr_J&|5Q8d+;zUy0BJXRNG; ztDv9iMFFsJSWtld4M9-riDijn$7E9@fW1_}d=`a8MOB^UBN11?HLB%-7DuLYfK_JXRTVR{KWyOWgu3_m7OcDil@+&Ysws8L#lpr4Fo>f@RO|k{D_a{-$_aPq#tAxoXo0q)q{hB;o}FYf68coyUuIH{+H8; zdk*K{E6}3j2l)zu>N_Bml$OHW!6~zR>DvkbL8F)In-H}<e&24}s{YT>nxychHI( zmr|e!psGu8B}&aK?lK1ltuF-SqqGt9H{_ZfyA4d9!f-+2?B%Wa&7rkV%~!tc9WhB) zb^Nqacf3irrT zyZ7uNEz2mi(!MtEP$ny<^zH4lUhJ(9&;tN?Q{&?~1EybYVQY~2 zqehyb5am1ql-(D;0!bujMZSRLp8l@WEDF26_7unSUJ&r_fidwqc4&V};LhG7pVBe2Q3py}rFzzB@KwRIW|HlHDA+Cs&Z z>C=^0-r^2{zg38KMGAD>(zSV)^)6uegC?q8ksrXp{aZskFsY6NnHI#OCzT57}{(clZY1vTK_y?ZgWpP#R-_OFu*g&Wbq zfOGWF4u6lo za;st2eb0J~D%{i1v?yq$A*chj{v(rCP_g%}xw&jTuxEK$X?7*`FGS_$tVbfKfDBie z97y>x0cg;PGw#4e3*ZEp9V1(~!jUl1ryxZ97A8C4hEvG9SCRm>6sBwFVQ|9$8gDY0 zD?Y8HL=T?1^S z5vZ%HNmm0oI5-ZFU8~O6AB|zE`9PS2SrC@2H!@)075EwTHn`Y1Ft0n>~SYN z(#?Fw4mkmPf;_M z`48Rr_@vPan3-uZJU+I%(2sKILzo}z4JwF}IU6#xFDsJAk3;15F0UK(q>-^PWSt*1 z+0QJ{U~KA$qT-BDIV_mHlT&ov(n)?`hypbukHoD)U=AP$a=T04bLwWr<@b4yK=?&J zdSnOp-;F{MnDrw0M2E665s70k9)>~}IvFE%xdw^ZH^ytDxXIBlKFQV>CLwZf+}KD# z!9fvHVsy$-AHt_x$ZcNc8A4tK(!&vUF`=mM>IfI5w8z=O`_TjNbKA0Va*9h``km1s z3tQZ`MTl_9g&!&MNK_qdM9QW%x19EJe6OKGK6 z>C#?P>;rVZlCni<@tP+rWY_?wyiZHqQHx&yzkjAp_4lJ{!X7v*cEKOjkX$!{R*)uRBfs~V*%77nZWIJ_AZ9bt%xK3$=gWoV z^%_2kPji)VL`DRq{?<{^kDx&kX#$ literal 0 HcmV?d00001 diff --git a/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png b/tmp/ctx_n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png new file mode 100644 index 0000000000000000000000000000000000000000..5818be73cf970491c9af0cc6d5155a6c6078ed0f GIT binary patch literal 12042 zcmeHtc{r8(+W#Ys$j~U6b}2F@MCM9q5kkluw#@S~WNuezqL!&-C}ZZ#EU^?(DKjgJ zMP;@ugk{L|yPw+oJ^Q?8pL6y(?|HBHpI_JN>dLb`)A#;Y^L6Gg3 zb1FIrvLP5j=ss=T3_p=-?Hz=FWU*&1VRc=uV7)Eetq~0itgE97*3r(A*UQ@7!_LK7 zLR9>ysDv=DEf(wQAuA^4^k3f)b#b>5yK<=HHk@Uf>p4RY1Yxm2|I$5E%CJKaQ7??j z$%{VmRB{mZ;!r}vSFKZTSOeeJZ9H~5W@FnI25l8O#({i&jeTytojb^{&z|2jerJxIvvWKX-){dedo>{FK>9g2A2(E!nqZ) z=H{@k8y=t7dlr#r31#NeIeYeOaV4xp^)))oieu>SeykNpFO4dkEYFa+!StFMp=Bb)jb69=N6@-|KnS^Re{L-{D$@Q>(>cIyLat6hto`y&-~?=4Qc7=SuXw6 z?>>Cc3kV3P6SsG87+P5Ftn_wCOG~qxpXiqL9KWdYAx6Yl(!N9B1Sj)$F4>E4C_zdE zp7Yu5ii}`yDk-f!L#ro$VUnCZ6|}yLpKX#mggJBOjG&NE`!^b8$h|_+WB9yAyo~nx zQquYnjBLQS;*~&xJa>fG#K)(#p)8a{ze(-pLTDn=69 zNhJ=R1NWrWaQgXXxu0u89Y&fGIHaB9&4N}vsr>|Vmlt0{dY#Mi%D*ru%(a}3lk?8o zBWm)ju1+P_s6@@s(9n0Hpn(qq_ePwj_LMkwA43;*bl!4u$MmDP$B${R7{zSgMIC{{ z*45LCHZn4bij1@(GVMQp{?VgH3>!D#Qk8k1upK=6hU(QLrI@7k+4a)$JhjN7N&c*{ zMvpkGA-?G0V>yLZD&+sA~!Yf98OJ5 zZP2}9Lge7VgCtX^(M6Kwab#^FXuYh@XJEU=eL0?# zuFRZ?-ZaVJyzz-N3zG4k;p`4rgrtW|!Dul+;Y(S+i}D~rsi!Y4`V0*ZXJuw$VCA$F zXi?<86jc_dkIzp^TzP+=NYP7D42Vif%71-lZ;uod3lzGpw)TSpkAI>{b)+Hhc15-HZg*&@BRJ<^6XRow83JS z6=~5sO8+l|(C=Yu@rVr8VMQKrcH{uEL3>VvUhw4}#8t6;BNK{kR}eaX5$Xp;Zj&z& zr?e2He-|CA_}Ik#lQjtP>DA{QNB<%8`7p2_-FQcp9wGg^cli)rCr{8y!DRKaGXR(1 zkt3LcXYaO5k}IiRjfKURUvAjWDRTjX!MuC>wsi}SKdal_gR0>IVq#jF8M<%ZzKxUh z%nIj{+oKHtH8qN-ngPZ_kMES4;v-wIdD=4cYZq2MCIZ z5kt1`FLvrVPWJ8(@bZ)4>{|RfAmuZCSx--ItW}k#^VLm8+FZMq_qY!-s7dyavvA4IM z8_P&4?I}x7KYX+Ha~J^DlWb^7i|`ukhTcWv7r)#ov1>p4;i33xrKPu8b1SpW>Nwcc zd~c~zUmYyG>wy_g^q>Kwgy*)@}*buC%n&zT=s; z@8APf>`=79c=hsRdqmKR6)cw^$yC?Pt>Dzt`&FHtoo6c5VP}~ezu#1IjeZNjlbmJ$ z^ygYVV`CxAjE_7n_1m{^VQ~PAl^+JaK70D~r{vX+OkG*`!E-7aeho6S5>3OdGQdp^ zLe&K49Uk8BNs4Ha0>~T|Fhy9e-ps-+JSN6+!$+Q5@8ZReRA>TFEi=AVIKF&hHlRL~ zLbudoX%~G5hwS=rE(fAxPpO88h=`ZQ)vFbv6@$1i{%|M`{Nsmy4CT+Boii{GDB-?3 z$hJ!EUrp%qpVd%Uook<;5q5KP6BHKi5DQvTH}(Je`rW&CZP!^8p8?Av2QE`_Ysaj7 zWBVn( z?xTCM3@hc+sb?^j7Hg&h?{1HnSx!!lu)^Bnz`MrAB8TDkk5T~TDDakbt*lb=s^*QH zddsz|d})PHdxj~GBy7ghA3N*#`@eAa@IbdsC3(nwj7Bs!IC=8gS!f<>tIG~Af7v8V zw=-qyPY7Zn;}&CW*gC0Vu$ws)*uyS{o8>&y@~o41o-VqLtc{Cn;V=DAg5FZ_=ZMzt zG44P98<;S!EiF}lO$}{pnRx#O4E>PSF;b-%s0=JOK<)l)FCt^5x5cQj6M< zZYsZ4;zf9WI=Z@@WiIAuzpAcQ{K8&RQZlo=I!>-EYinzh^qY4~ko6QyzS0;Y!XafJ z#mO8R!pxNghnm>Aq1$x4HbJ6N)y1kVhO+RKmQj1kdYpi{`NFO#4qWwhBIEUdp10NA z+DV-UlGix3kX=z>DCafNGTW?_-`1uha_MsUXwSGU~9RAB~As}chc9XpoM)3SQcJ3^(?-T@V94lzxj^x|UqiE7o=0YsA zZXBFllJ(4i>m`UBwn3B@KhdXUWKfLzzs1bV%v?mbbXHcDj=K6CcX#&*61>AB5)!)C zl$CM2wr-TPtlL^acCM@+nLWbE>;M4n_2E&2ZMK~p=m}--JsuLlRt@2Lc6OPtD|c`y z`d_-UN2JY@Xqr*ww;;&QBq%Pf4IPGU%B_;ju=zSJ_3}}6wlxx|hgALSFeftwULYWq zeVL1fh6cc4p?PTS=UEO1Z^@s{42vt_o~u;=gSPEF{4uC{Wu{TeYvR&&Zuv`;uH5C` z%Uy=>yT!Jx$fQ28$-{5vg~`@3PuWk z^B2#Yxk)zrH`$yhvNmC()ZUeA>^c4EB=uv#X{E)NEFD0i$nehGD@@Q%0CKxvH*)VVwZhP_Lvs`6_V;j>h2-FRITCdfuxlanJ zxgM&F)G<^OtXwyY+2P_LpOB>K=G`6Yi+}!B?DFf&| z(J?XU-9JlA;JGN!Sr+2{tROzfe6zsm#lZ z0m{W8Zg~U5tLeb((k|xE5|@5O5N%e={))@&R6Bcn9U0E>k8n18+zngrCkkTKi^+Yy z6cH+^gc4tR`6w!K5ApM>PI!NZD!{qV`MFiO)I0XCK0nutmwB3+n(CognVPBs5cjQq zVB9(6z0e)6y-*<1tbl`_o}Rv+u5Bs4@__?1jSz`^>w<|%&IN&=UEoOTBp7Tp#62E# z&$F`Bwr}5#Hk}MQ5-D>B~-S4651106yzyDyLfw{R@M8iRpZjH$A z?w$YEXdka>Q&|**gN6SqYNa7fP6c3khR2ioP=5O2h5T~!uSm6;7#GLU+12%qLQ})N zFv|z$A|)hbb7lIA7bZadySuyp3eR@~M9fo&TCt{HhIQ470pA3qq&oYQ!0Q0VLH)>) zBkr;Z`1Q5rA)2q0bFU1h4XAr`LPGA_NPdU0wlqNaIHjO!C}p@uJ`6NQ;!Z93rzbpy zQBQ#M;%X5fUq<1vNe2+OWRzXzTXq*(-;RhdaCXiEdI+s*u0vO=oBFRp4p<l4fQv@TyH6nOp(h-JAsfUDoYK|AR06Ex!ZG zTJAoiW*WFO1WO4YAl<&y65_;sB)?V`w35us%<-PGd?2KH=H@(Zcfqq*S`DO4l8r%S z#3}fdY9`3;)-JMc+;fjZN(~whIGND??QED>z-<`^k-^L%c}_h>SR3pguc^M={Vp%G z%gA1j!7k#*2nq_e3FW^?rcsCv(8Fc1qxTPM#@iCs!I=Qqv3GXv0wIdZob2M_i;R1O zpF8~OdhK^EKMBGFW%J;A!6#jlM+_*4&7B`@rusaXA>09T+GMu|F0`?A<(X#7O_ra= z%|SP`Tb%xk0tD)G`BA0FzLf9R2C(3>>&((aZ7#*7!JXhS*#nKWg(sNjn^D z#u8}1`MvqS^)bGT_Nx|KxU6l&-Y}}rGoGX$-AZ0sUe3Nqb5duq#mL&uf$Tmi!fO)>`=O5og`1!kLnrPGv_d zYwusR9&tp1XZHd>EVF6mhOV0+e^n=7dA<`UX)2Happ!zsQY)HsnK6h!+%Bfg2Va>R z`x*D_?F2J8LB{PANPg;kHrO@5uI4GOGX7=*;G%|wJdCG%F^E$~5&qpfHB?LU%;K5mNDbpuZg>yh=!T3+Y1VFS)Iy%YOd3d$e zZ!}#aqc~^6s#j-6he}60?3qoQHmPZ7=z(yFe*8H5yKV7jAx)L#H%M~^xd$gARd{58 zvhQqbOa400K!M)pr2+P)+0~)lx#k2j<1**Yr>8?D-M*+)uKqiK#`RZ#_J%*nKe@PA z64jbS^PnoQW;QdiKPhn_j)%)l-sb~p;K!lpU%ug5aQ(gg8ALNbjj}%9Y-TQ5zOo(> zj8Tbw^l(!`Hjpe8*uE&lf&EeJJ!Q6BbXX%U6 zHXrlgfm@#yK-s9HI2Xdmz<@qh6_{nMlehUY%gZl4z3`@Qe0)5ads`?#Vc&b<8m*!HoD zIyyR)0q!!K@EK7+4J}(hdgaDK)G(?7*AV43g@uI&NdK1K4_kLM5A4O~(0oztKNYwj z2ko)B+6KHtc-CJsu@9AaQZ>R|{iA zO|#%7{*Z(kOvVFWYB^0P6>Nb;s z_y1l9M#siF48D1QhD37{Ma}OARF0PaOXBZGT(L8f&b6o1>Dyz%(thw`ghWL(#Rx0zDwWoTM$a!_+sw@QEH#yH zXmD^89Lun0C)@|az^(z}9}T`nrf$ag#{w%hsi_yU;dzylvS^C3Z{I%1eC$+Nwkz*q zLKE4|$YHeCt}MEt9UFk)=Sr%2tZ3)$-9p9S73t~gXMqx&S@M?!{b?E*8=C56!Xk8UEr2rS3JFN4y>D3 zENdGqO6g^Y;Qc^TQjWq#PfdmIij)X89|SQAk_1wZf(76-_qgQrf3>y18*&U)Lh}%9 za9@G%QRAS}`QwK(!DJYLJvj%4z+e25&?aJ|mnb%9_6mH|3J$akig zpN~%fASwB1NNL+x5ev}#-8o~qlBUP;kVg7s6i27Z% zEK0@ZJTk7zK(`@L(7tjdoq>^&)@Z(_v)A0e0FZ9(({&a>1qE`Tjt}r^Jb)%`fM@*@ zs^Y;#ywVh(iHSUX1R@sPTrJPq1=JBlNtpMf36)%F0L&3}C6mPm8AJF3>5!V(TW^ta zzXfNqYwh-Vieb?e3~pw2R?o;N3xwk+bXh!> z>i0)3iHc~iN??4jlz9+&qK4h-@&fUfO^gS+7|?DJZW+i`b7{{V%2psF{!%&gTOSJT zArRz@V=K3S(SwYH{$pPB@SzEOPnOiJez-uG35ooxzzz)q?bi*k<-$x)D*pgGYQf%` zK2a{ls_gyxPiZxaeGf(Rs;q%CBh_2~fCWv`Z4A!;JhAM&WG91|vH?am(G&JEJrVly zcN0hM4RpbbclmG9)ye)~Zm1uRJsNYHj@9A6CM*!vP1FwziT4aoAzU|qU?;m0{o-Jx z);Q!+dPmtipfYI`^L3;Cr6%uWvIw%MR&>M0s*AP8z+#T+6WT;jyMVCxT5PV%*u@Sy z0)ULYt!D5&9pl^C70v@~ceHJIO+uSTn2a8Q7CYw>a-1Zp5Jy7Nyz zy$@qDp&k2-9xGC!n(11qe%_Ny;FapH>pN4;d_vS!kyKr_J&|5Q8d+;zUy0BJXRNG; ztDv9iMFFsJSWtld4M9-riDijn$7E9@fW1_}d=`a8MOB^UBN11?HLB%-7DuLYfK_JXRTVR{KWyOWgu3_m7OcDil@+&Ysws8L#lpr4Fo>f@RO|k{D_a{-$_aPq#tAxoXo0q)q{hB;o}FYf68coyUuIH{+H8; zdk*K{E6}3j2l)zu>N_Bml$OHW!6~zR>DvkbL8F)In-H}<e&24}s{YT>nxychHI( zmr|e!psGu8B}&aK?lK1ltuF-SqqGt9H{_ZfyA4d9!f-+2?B%Wa&7rkV%~!tc9WhB) zb^Nqacf3irrT zyZ7uNEz2mi(!MtEP$ny<^zH4lUhJ(9&;tN?Q{&?~1EybYVQY~2 zqehyb5am1ql-(D;0!bujMZSRLp8l@WEDF26_7unSUJ&r_fidwqc4&V};LhG7pVBe2Q3py}rFzzB@KwRIW|HlHDA+Cs&Z z>C=^0-r^2{zg38KMGAD>(zSV)^)6uegC?q8ksrXp{aZskFsY6NnHI#OCzT57}{(clZY1vTK_y?ZgWpP#R-_OFu*g&Wbq zfOGWF4u6lo za;st2eb0J~D%{i1v?yq$A*chj{v(rCP_g%}xw&jTuxEK$X?7*`FGS_$tVbfKfDBie z97y>x0cg;PGw#4e3*ZEp9V1(~!jUl1ryxZ97A8C4hEvG9SCRm>6sBwFVQ|9$8gDY0 zD?Y8HL=T?1^S z5vZ%HNmm0oI5-ZFU8~O6AB|zE`9PS2SrC@2H!@)075EwTHn`Y1Ft0n>~SYN z(#?Fw4mkmPf;_M z`48Rr_@vPan3-uZJU+I%(2sKILzo}z4JwF}IU6#xFDsJAk3;15F0UK(q>-^PWSt*1 z+0QJ{U~KA$qT-BDIV_mHlT&ov(n)?`hypbukHoD)U=AP$a=T04bLwWr<@b4yK=?&J zdSnOp-;F{MnDrw0M2E665s70k9)>~}IvFE%xdw^ZH^ytDxXIBlKFQV>CLwZf+}KD# z!9fvHVsy$-AHt_x$ZcNc8A4tK(!&vUF`=mM>IfI5w8z=O`_TjNbKA0Va*9h``km1s z3tQZ`MTl_9g&!&MNK_qdM9QW%x19EJe6OKGK6 z>C#?P>;rVZlCni<@tP+rWY_?wyiZHqQHx&yzkjAp_4lJ{!X7v*cEKOjkX$!{R*)uRBfs~V*%77nZWIJ_AZ9bt%xK3$=gWoV z^%_2kPji)VL`DRq{?<{^kDx&kX#$ literal 0 HcmV?d00001 diff --git a/tmp/icl_results.py b/tmp/icl_results.py new file mode 100644 index 0000000..94728b3 --- /dev/null +++ b/tmp/icl_results.py @@ -0,0 +1,204 @@ +import json +from pathlib import Path + +all_results_path = Path( + "eval_results/google/gemma-2-2b-it/20250918-111058_734998c4/all_results.json" +) +all_result_modulated_path = Path( + "../train_outputs/runs/Aug10_08-29-46_sakura-003_2000_f73050aa/eval-results-100000/all_results.json" +) +with open(all_results_path) as f: + all_results = json.load(f) + +with open(all_result_modulated_path) as f: + all_results_modulated = json.load(f) + +# Find keys with test_metaicl_macro_f1 and test_metaicl_no_context_macro_f1 prefixes +metaicl_keys = [k for k in all_results.keys() if k.startswith("test_metaicl_macro_f1")] +no_context_keys = [ + k for k in all_results.keys() if k.startswith("test_metaicl_no_context_macro_f1") +] + +# Sort keys for consistent ordering +metaicl_keys.sort() +no_context_keys.sort() + +print("Comparison of MetaICL macro F1 scores:") +print("=" * 140) +print(f"{'Without Context':<40} {'With Context':<40} {'Modulated (No Context)':<40}") +print("-" * 140) + +# Create a mapping of task names to values for easier comparison +metaicl_dict = {} +no_context_dict = {} +modulated_dict = {} + +for key in metaicl_keys: + task_name = key.replace("test_metaicl_macro_f1_", "") + metaicl_dict[task_name] = all_results[key] + +for key in no_context_keys: + task_name = key.replace("test_metaicl_no_context_macro_f1_", "") + no_context_dict[task_name] = all_results[key] + +# Get modulated results (assuming they use the same key pattern as with context) +for key in all_results_modulated.keys(): + if key.startswith("test_metaicl_macro_f1_"): + task_name = key.replace("test_metaicl_macro_f1_", "") + modulated_dict[task_name] = all_results_modulated[key] + +# Get all unique task names from the base results (this determines ordering) +all_tasks = set(metaicl_dict.keys()) | set(no_context_dict.keys()) + +# Create list of tasks with their differences for sorting (based on without context vs with context) +task_data = [] +for task in all_tasks: + with_context = metaicl_dict.get(task, "N/A") + without_context = no_context_dict.get(task, "N/A") + modulated = modulated_dict.get(task, "N/A") + + if isinstance(with_context, (int, float)) and isinstance( + without_context, (int, float) + ): + diff_with_context = with_context - without_context + modulated_diff = ( + modulated - without_context + if isinstance(modulated, (int, float)) + else float("-inf") + ) + task_data.append( + ( + task, + without_context, + with_context, + modulated, + diff_with_context, + modulated_diff, + ) + ) + else: + # Put tasks with missing data at the end + task_data.append( + ( + task, + without_context, + with_context, + modulated, + float("-inf"), + float("-inf"), + ) + ) + +# Sort by difference (with context - without context) from high to low +task_data.sort(key=lambda x: x[4], reverse=True) + +for ( + task, + without_context, + with_context, + modulated, + diff_with_context, + modulated_diff, +) in task_data: + if isinstance(with_context, (int, float)) and isinstance( + without_context, (int, float) + ): + modulated_str = ( + f"{modulated:.4f}" + if isinstance(modulated, (int, float)) + else str(modulated) + ) + modulated_diff_str = ( + f"(+{modulated_diff:+.4f})" + if isinstance(modulated_diff, (int, float)) + and modulated_diff != float("-inf") + else "" + ) + print( + f"{task}: {without_context:.4f} {task}: {with_context:.4f} (+{diff_with_context:+.4f}) {task}: {modulated_str} {modulated_diff_str}" + ) + else: + print(f"{task}: {without_context} {task}: {with_context} {task}: {modulated}") + +IMPROVEMENT_THRESHOLD = 0.10 +# Filter tasks where with_context shows >5% increase over without_context +high_improvement_tasks = [] +for ( + task, + without_context, + with_context, + modulated, + diff_with_context, + modulated_diff, +) in task_data: + if ( + isinstance(with_context, (int, float)) + and isinstance(without_context, (int, float)) + and isinstance(diff_with_context, (int, float)) + and diff_with_context > IMPROVEMENT_THRESHOLD + ): + high_improvement_tasks.append( + ( + task, + without_context, + with_context, + modulated, + diff_with_context, + modulated_diff, + ) + ) + +if high_improvement_tasks: + print("\n" + "=" * 80) + print( + f"Tasks with >{IMPROVEMENT_THRESHOLD * 100}% improvement when adding context ({len(high_improvement_tasks)} tasks):" + ) + print("=" * 80) + + # Calculate averages + with_context_improvements = [diff for _, _, _, _, diff, _ in high_improvement_tasks] + modulated_improvements = [ + mod_diff + for _, _, _, _, _, mod_diff in high_improvement_tasks + if isinstance(mod_diff, (int, float)) and mod_diff != float("-inf") + ] + + avg_with_context_improvement = sum(with_context_improvements) / len( + with_context_improvements + ) + avg_modulated_improvement = ( + sum(modulated_improvements) / len(modulated_improvements) + if modulated_improvements + else 0 + ) + + print(f"Average improvement with context: +{avg_with_context_improvement:.4f}") + print(f"Average improvement with modulated: +{avg_modulated_improvement:.4f}") + + print("\nDetailed breakdown:") + for ( + task, + without_context, + with_context, + modulated, + diff_with_context, + modulated_diff, + ) in high_improvement_tasks: + modulated_str = ( + f"{modulated:.4f}" + if isinstance(modulated, (int, float)) + else str(modulated) + ) + modulated_diff_str = ( + f"(+{modulated_diff:+.4f})" + if isinstance(modulated_diff, (int, float)) + and modulated_diff != float("-inf") + else "" + ) + print( + f" {task}: {without_context:.4f} → {with_context:.4f} (+{diff_with_context:+.4f}) | Modulated: {modulated_str} {modulated_diff_str}" + ) +else: + print( + f"\nNo tasks found with >{IMPROVEMENT_THRESHOLD * 100}% improvement when adding context." + ) diff --git a/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_0_tiny.png new file mode 100644 index 0000000000000000000000000000000000000000..80fa0243629e7f860acedd6b2f8b422875bf4ae7 GIT binary patch literal 15365 zcmeHu2UL`4wrwe98!_7kz<{EXw4p$WV!}cRk~1n2OM*lN2`#O*0nw76D1xGZM2SVl z7Ht$nkWeLAB`1+kV-EaEN^t_q3X6}9K-t}De(iW)t>-*2yXP>?Ie_35siF?7C z1r!Q}n|5fw28A-yheDb0Vcs143-w)HGyafvKA`8UX>Z}|cHGgNqH^5X;gr4eDeDvK zUCkYxtnKZ@MR)EH6&GG_BrPUp`@g?K)ZWoj%wlue6}-xPheP^K6bjFA^52YT zg(zzZMdmbZzoOQedp-3Y4wanT=|0`1?^QCo4~o(b&0l+(;LiqDhw-#E5F@s|59%Y;uY zec8Qt!F++nMt;>i^9>8$|70XzTKLB8;-pu@_-Nj+_>m!MV|}mNKH=izS-3#6pVHm9 z@MlKSJRkgVaRX&0h4TGH$~+3?*!MHnQz$;i|Nj^Ml_R-r*Hgd8l2uvx=$4>bj9YKR zg8B2UdtE7%XVj;{hb`aUJ9_F=+}?d9va1eKCg$FgJ*#p0^y!{@RN}}S)qA@h{Kz#o z-ED2pdCCY+dwn+Zk=Qg>rcr|HXMBUv)jjLOGctCKd)Dor^~`(EKw+L}@KxNpJ%VQa zZ04JZ4{Px@$7b!pjV)MAnL(j^w~vCGzT>+Yi}8gW-@cHlPcce5abfS?y{X53nd#9u zI?xt)|9(QA=Xi{6qKUGKiZ*_g4Qv+@3S=-I^2xbx-M(FU;VRL}mseN$+6;ZJk3HS= z>+23@oA$B?MI|MMh~CpZYO;Mlx%M{LvQJefAGmYpjzRjVJB#?`<93?; z`l^c)la)0V;z8a$IZns5_08(8AVa&P1q&9$^?FZ_vntdSZ1sc8t5__9`fN95x^1lm zJ8ybwJV+omBjbItaIt%npHy5~nR;>CKsB|Coyas24c0a_wZ{*M7^J?+;;;;TukZF8 zb1v@U)c6ak-m`if6;tJjuw=V`v!}#T|_}xJ_ERCZW$gcfqWg z(yraNjh#m8OcL4mBOW|Z+`M_SBK5(82g<6d>GXuh!9kU5w$S2vvuttGLsep@me;kG z+;tiGay%F3u6u3toc!)}iw+z-IH$+|`t_jFwiLQ3w|7ZXiJ7VHvU#(dKiBQ{?0dP& z=G3WENs&1@6CobfG*#8>L1Ly;flcHk6=OqZ$e#Z6(|xZo>)f%<*abX1_VLy-0daA{ zqmy19GN(W7_h6#P9O?~3C_w=M(OFprO~3kSGVSUs;|--&tXR?UIM_r3|GS)*H~!+q zIXL+_a)~i9$~$-NWIVck`?gTR3tO*qX={RuAOHOGJWdx|?fCKIA}k~6!YX&`%M~2~ z#VreJa4FJvbr$+)Mjm;GLee%cu(59{^k=`hBi`eUE;GJx;lhy9_U+rX^!2y-7jB~U zpOKYzAG9i~%9=fUwqvG|@}Wad>7t<#nUQA~T`G9+;>C;bkjTg_D^{+opENqP{#GqTf&HWB46>Y`lvh-=I7@G)Ikmod7eb|{TD{vSH#O2!lVa(gZdV_Z zlOsExfx1;be7ItyuZfByj5kPAc5!jRHwOv4?r}9u3%+jkijgBUrgX&D%maP8J>X=|Hn{JmqE z2@X3eW^r+H+>qVF6x5(&c76Z&q*U{`&%GJ5C?ELUA{z5$HhL}m{x7KVcj4y0%@^X= zP}qs4i72k(7RT54-PC8rPB0!CF?OqeeB;KAaND={b}Kq*>F5xkl5-za*U}1DDQXmb zb(Pqhs_y!1U6lQ;ijK;%2m1o=-t{;j_Gh>_F*=|O5NOQvtcukYdmI@VgI}DL7|G$j zHdT~e4j=Z!>m}_)mrFah-|+h4%ilgV-pfuhE9?#t3_GV0r5c*d3>4C>G4u2DQ>M|5 z?%U@()Z3Wow^nMql$36Db+v0rUsEB0?$@^jV^dSL7OoU7Tc#%a0BBcBHB3Ks)WRY% zNW?I*;1Uo0&F!82?zu4u3F-77e)wTm%1#?79a`dQF zs^hM8203Sv`uqEFbQTKszg}A%6Cbbb<&`I}ZQC~5vMoo-Q1${e6%~~$SFcv!fYMIB zUJ7jC^fsE_m9#DQ<&ob?+b$-?{&~qJf{(OYTZ}8{3x6+5bY!FR<%H@Ro6qWYkDolC ze3yR}#V4WpV7|B8e%m45XTSb>(6%8*AMFK97@E>G_0*mm<5ax5xhXI(@NxcRkNuF| z>C;IE{nnUYQlQ~HQ1L_hFFt)bu~|@XE-!i|hCp*=Sy?!P!3Yn*Y`_<4skld34j=cb zmwfqHK(#7cpN$b69^1*PS%#sh#pkcoRvLzhnw#o5?z&k(;1Gk)YvedLpR8U|Qj)Y| z^Uo{AOyaIwSupw8E&rCl=()V`FM_YVx(7z{(zh(0*It+DH1_GLSmpco?@y?lJ)0?a zso^k*+M_^D;QSWdUGk-F^yu4gX zOY6~3Kh2WkRBF1`M<0!lIQER9a_5Hw{GNk1^YFo$u0rvmf`UrL!D9BklDN}T8{(&y zJT3TGMvD0M?dW^=*7>a#KM3fsOOnLhVhnz~#IMXaapFWm#!}|!=_5zpuvr;Kq6hZx zk2cNsTCi{-J*l;P_nI|pv#~BDh1mzx>!CQe3&i4Y8m;ov4N)P?J!%2EtPgGa^+(W5~(%zuIJw>G=4Z zd{~gEam?VL^X^bpTKT=*PN^0pYrqE%`72Ci40TsW0Ef4W_4ew%vaqn&E+V2rhSQ77 z%hGJBwpUeES+nBw83N4O`05l(A#rg{&#}Rq#VzG1_Ljag38M>+fq1`-(dIhl;*!Sa zIe0Ww+C|xpX&9m8&-z@S%_-oYQU}v0HeV>}&}(4Ptkt(IeHu5<)`OfL?** zkR~oFE#=AD(fo3isg{*hw1%c82#;7IJJzL(mG&C~kh_*}Z5-=D*{B2mo^Rm4>mP8< z|2E+N8#T6}80CBA9`xrE2`=7@-~NRW#@VjbO{A0E-LuAqdSXp--O1tO*S|aqe&fgn zRLY`tyVQkr;?!xh%ZqrWAMsCon41Mo>eBswCrtx4(8PPv1&mqcz<~qOXZs)Vo%t|h zeDvQu01{XWDU?0BD^BPKRRzs$uZZ$DIrDM0+{g#t&c^(FN!yx7bNSDyGqg-h5|(T_ z7!63gwMC^o!8qHw(Q~+?`SqzgPZrOfK1U)-!8G@*Bm1i>qwtL)sUG zUuU0a6co~^naBW$Y?d0E%-&>^(o2z)l(ZRWEs2SZy;azB6bC43lo8(Cd@|c&_`bR5sJlGHa+d0lZTJue&(g7tecdx#a5AtG=yP3D*A%tDgG|ok-RrN9oWIam(xJ z_KgWZhrf{PDkUW)^b(D6PTo{~gwAKrH=Y%OZOy6eZAVI}yr!9%*;Q)r+tX<#1|a{N zn6?{35NCDMeE9d{6O)@;RcVyG@c641*zI;>Q<%<_ar_<~P z0yBeIMqX)l+S-9wXf|%#xE#pw^y$-;AjM5Kxv4KYoQpb~O$u!YM$3M~=q**)Eq3la z3hJPH{P+Voov%=<42nr(6gCka?wi08*}C;caf`;@z2}XcI@Xm?OPqGCTeNx0mN``t zRO*wbPs$ltq8nx=LuK_ zCV=(S#3&iq+Dy$TRc*XH-`VCR@kW^%@?mO#K5g`)XDi`k1}PSPD@6<*74mxY;*;+c5-x-IR1PV z2fNkLqenv|Z9~8^&)a;?>Jq(xPO`=F73Nkv~;3x`aW?_ROt{neh~Q zLq|u4`*;siTU|XIl7>}hp7*N9wYYZ}tB_}tOP0>Qpl0Kw-diiS*?JjaIb^JzoT^`v*5O8( z^vV}_g1uOA+|-k!O_q18&LH|hQ$x)BYO}w_-S+oMX09rZj*eyb&`$U&;}Il*z<_{R zRX<(4xPABT>I9tcy9*0fowvF8O&6)AqCf%zW6P|MytzrKW}N@uMW{24 z{sW;V+gO2B3gtF*F&t|yBp=1`F!#=F-3~v0Z|q) z&CAKH#l?#+LkQT0D$xYKsh9jIPOGb!n?Kgj(5Q@1;=y-We|&x+>rCGX^%DFv)vj(A_AWwr zGBY!E2UA>o47gYSu>a%7kAzA7;yX7ndlil}fxPL_qc@?yafWK`W5dJutP(XULkVi@ z>RPf-o;=ykVVkl6#AGpsR?+$6T)wA;kZy@y0wC&X%r{*tWv69lmjGF;Cj?8J5X2sQ zrw-a{rLbPO;QcduK@igE>8Gk6qC!Os(~m&FWr5u`bKJF!jXg`gPV@s`6%*=N@Zg6J zACk9|>CMsf$~T=HA7)3Y+#USj%cIP|envgwGWSPrjfl=0B}r{>8}d& zZy@|fak~1W8Y9OfapsRc*Lh_u*hbPyPAUTgvbJyAHa})QH@CK{Ej+fUU9QzWTW>tD{#&gx08m2hl6L}Qe;-TJC(oK&}Uc232TZR}+Ie`L?1!GB=S z8EXKxyh?XxKf_)Q*s=!X+PeI_RupSL^!EnXDeoFY(1AuwJKLQd1Tmf3RC{WfYG~-x z;ldX$q9LfKrKKraHb8}2zHFHpn^5F`#FdSpGkoRB%7nYg#=FlcOBODbXgzuIBqlL2nB7&C<3umu z%G<)c8xk)2`|FyR+=uj->dvv@#u6-+Kaq^uG8TuwQUyvRBV!1sTilYy41~G4Atb)IHlk48Sccd|5Ln7YfLcvM)}EhB?S#)7Q(5+3555NL*B4q zgJNyHTO+TQ)7YSO$-Uj0qQQr6-nQ5K)!l%*2tkX#_I(tA2 zqHYCL$`x?v;4XFeNj1z{!xTOYwU~%+7~=u=?#1!QkA%Cd#C!R5cnM$+SIKb!08+&!{x=|glc9Yka(j~oVCSNGoPQ$tY5g>WvtIRn>Gwqi8MS!^Qg zsVPiqL2NaErsFkM5mwYT@cN^}Ax4m}{zG0F*XW!aBZdSiH0X0}Z$pRbh@gRqo(Az+ zKj<;sJh_24;$yGXyf)I{e?Z^R!CbNjSP;?6%XWaHkY#>@?e`x6WXUSA22?BzmL zc7K(hr6y#2Jcd7O!zsAPy*4t_sjYLMRDoFtO=89B)z*-k0kS%f-0-Q7vFpyZg}RR1 z62RU7U6|N%->CBGT$9AFDh903;L^5*F{iWZG53<`IGU^&JE{I-ZjGKyA_~1MEbNN# zmoJCp5KGo<5NzdQ$3})v!Ufakbv)Q~TU~P4g6l9Hv zmf~Qh8$4*r^bOZs?l&b;jd5O+V;u!N3IRlifHf!`-HLTZ+H))w{Ifd*s2d*ATBsYF z={!;1mZnv~_=~Oh!X+0=y3h4+W3CTTVO}w%3e!ehbH2MF00s*WG3L>u7YR^P{X;@R z0s;a~vb%xnnD$TUs7eVa;q}fHDy}N963dAB+uV0XfCi!rO$0LvwyMJiALELpAfds< zD^dxsmISW`IK{$b20$r~^`E$kziR8~bV8Oe$Z&Yj;5BaRiA8a{uyEz%aAP8NQBgiUr^y7#29od?^G%8j*0T=Dbc&azqk z6$Yz%DSXn-hrmzL`K2x;r>088o93G+U0?SO`?9v7VJzNX1#@6}s&^WM+ghQmO4qas zgOKbm7qD|fNYD@8PnjYIEaGr}F|1m)6&t@FH~z|e_?Dv-V>navYEKZ9pI>`R{E z8O=-|9Q3t6$t3(Y(~19hxkqB>Jo6kM>QGV`yLrOQY{k;01Y2hL83sM>?mor+{f<3L zmMlS+v!I?X{ueY)y|58SLZ;uhB+gVcT+DGC(glphweF1W}|>e_v%dmAuh zpbaJfY>!HqhguLkh)n?B(Y^CYRCgb=$tRjoHeWvc0@n6!p;%6M+kFq!{IROU?hsJa zt_PR+%2D?m=ngix|3t8X$&wKJm1b{!kt`gmvSrcycSAk3RNEQ}n(6)1SN;5|#7ku( z!X}yp83Or@iGY@t)>a!RA@0+Y!(<60LM145umGIqt_F|ZD$>^|QBt&|Yv4mSJM0pm z9S#zH2W|GRgI0SWv@;a=#8O< z#T~L9kO$>_V>dYNrsQ0Z$8~068$bI4(4*L6{=%slj|vnUSNr*yV>0!*^RTZ`OIZWj=3PFV5Z*Rw_utXjoLk z&&zA;k9WcLGTYCsi=_fGJwL|U`ubu2K5c67b<_CZpyuXg2rIRQMn-koCJ;9E6c%Vd zV{iCd!n$@P#q7ih0UGfc#$#h*x|43-{vKP*TtCyCGe@J;!WRA2EAI+bMNTN3o2=@QhRQhvy(D(F^F)oq^%`{iXklfSq-f=}g70Pj2UC zQ1%#p_zvLt|0T|TU;(qR zG)y58TN&ary1DKg4I3Mb`k3>r(f_o?rXg~(1|<*HMzt;%0Ea#d1|6n)U-k)5kyZ{embIHIhq0ZvTFEnFJw(f*dK zrt31$f42Y@!FgW2dKKQb|L=QaP|ei8dSf^+(`t`DsIX%uvP8FcDB$`4R%ABqG1&f& zh&>;E@lAt7h`>QzyKBPXZ&|g51`>ap*jF55?&4E zml$rp?e%<#(b4Kqtq=zLoBYDI4I6&>{q%rqOP4M4wZT2q($lkIfA!j09A|%x?3Hly z_$Z9NRwEJ-807x(;a;}C_k`&rn?WNvRDJ0qO_;B!c zpqFZtCPTvV?K;nizCyC+qr%ec>ZNV#v$Ubly7D7`HpWTxjz@5)6Ks|Z5ftMkNXPHQtO9jLyYbE z^bgQAW5M@mte9h$g?wjF+PClG`kR>aM;V_CiwOh#yD`^{#f{&uLJGWW_Y%sxm|cev z%q>)+?7#a-cmsA=o*9d!)>+PJ*hHpm(HX(nBf~YoDPDFkzU~WHLzQ-dXMvh zb4R|T?CIJ-d06qu>;#39w_1KCI&Q~zmXvqqUv}=KP}mnKKa2YLU!Fn9SUO|z!H?t( zX6^Aw`H4blT1@$|IMq4mi{nrGDSIp}&QXlAUE_(B4{AcdEPv#sn8fK%FNW+E@~-3G z%tetFkKp=%L|6cJA zHHZH`Jn4FRtzUejwkN2uH@LXPhBK8%3-`G*k3n(hsex}VzGLXSW5}(codW^`&3hMq zlMDfthUZj&hX6>W<-yZ8YxZhRHrAXpahCJ_AwA7)Av^w5CNtNNaxA)a*w(l7$2)5& z3gg}4RK$m+rKPRBAtR#F@jBmt!RzHzMn;BY$CCD!e;vSAn~6zJ0PqesryMZb^IGFGS)eujm*4mGh8{ zZbcA_j1UqgN7NmDe{#I3!QEG{U%x|Oq%zyho|psJNMfR+=f&*yp2~sPPmCu{XVqZ{ zkGgdeImyWynEIXXA}*nNQxVZK&N;I)+M>51ca4ig4jn3!JU8OtagUQ#ZaO`tJ6W7Rah22yQp@>;YU)S2}k(DRD z5&6F$0Xhy&u?Z2Nr&v$Y%nBEE5jf|lO%JN+oI97}f|z!8D|WGTY*S{rlS8#EveFyu zyR;~c&%J$vM_uo*A*^9C<>TR4c1h)r`mI>9KXzZm7W3mD~6EW*@7be#ps-&2N?Y@tgYbD%<0 zT3M)nB`z+K!zT>_8N!v^QJ~B7H8R=W$p*O7h#?x?5nP!$fy$G2@^i!laxn{%L zIhm*e!bh48_>sNkuQuPJj}GstAiN0fll-Y&S*-P1U@@aIN{yeU@hxAqIMo6qpB=8~ zd!BWwGWw{Zl)~^I9#$YJpwP+blV)ZSlJ*U8Q2ij;vAXDwqN3oYwOnDp3`y%os#hK2 zwq4$+1yQkK-!SwNgK~yy=xY?ba;8SZru7jN8ACWW7R|;U+T}f7yJ*H!u2C#s2*sXx zDR%Q4uT0a_<=r>XHJz2#H+t?0gwr*>toXZdLzZb(l&=fsix9__TYi4_Lg!jrrgihH zn~c13a*HnQ15Z>;3r?)AVl~CufK9`zXRj2CQsG5-7b^<`Np2GJ{1)DQYGQgE?h;9M zAiR+9y;(^DM8`*XMg^NG9T*~wh2nEY)1uJbP8T^gzaWbghx<()J$VPVNP_v3wfU2G z@nFl=%eY~$+r!RtWnKB^TXn=U8*F-Yq;q}?f#JAGDw)_M0I(su*nQH!J%F)d`En#) zkTU34{yDRa4{-z~e;vT@93H+n5^q80NZ(X;R7CmO7=$#H zn>kDVRit5P3Yr8`GSTVjlF?KB)a)AfbJ~8zEu8`ORybgQlQu%SI4=o{H{ZiY58>D- zqJUs05knU-HkJA;7sPjOnz(mtCjScri>ub&fACsk3Y$=O2vQN~=p#~4jJe&Z@dba^%$pmJX1{F2ejI!N_E`S(xI%GqUe1a=(6%_^jnfSzZSDdMj(JANr`KyZn}slN z91y5Dt0lfywh>B6jT$5PpWYiVx1CS3q9 zR@l^_pb)@~tZVSuZS_}nxn%mz7bSf-5&Jcn(lw z?4_93aqcw|ukbVi3rWg&W}{bju3^@OhI0cdhs#$O!tpfMH*Hw-VR|xu+L9cGYE(tY z59U`ewU$T}!mhLCT;bqia}q<5c4|GMUJ^DvVc7-cn+aaQiMRJGHb^@0($D>J ze9YrC#>w3NA|clj_&#yolWgxayR42au}c1YEVAJdLrrnu92^A}*>JJZ>H$bn?w@U4 z-9;V@quS?R(Gq25ageW2T_rLdAUt76%Swm^pj>9_;2*fVsOBb{d z2>L(-g6{LSE$|c3w*C=#kaD}I@22f&>E?OO#R8#z&CSWq(ap~0`cV%H7grlc2b6%Y zkO1oRQEN9hCs!#!LHob|hJd4sm7wJb+~5u2Pa5g9*HM~@kmZspgyNq?|yhxARb51ArPm}Bk19W zcM+Qqh*yks+Yksd7WzF1M9rB^M-hlXi){+$bhCtGnPMg+L^+VuhuA7PGaqwGXml#!H3hzo|;NPd#aGZ%>xpg+MqlXe7vF z{v9aUh^Yf8$aoGp?bSK(VRri`N ze(gzt({7mYz^pCL@d^r7Pc4YseM)j3#-BQN?5w1uq?FTVg*xhN(`M)_p{YoO%#CkJ z{#0_}^aJVHkWW><6qm}yUQBP1eeB}wK$dp8#^jr=9BzxAZf-n+f|@fkGv2E+#S7mX z1+ywDj82?5(GtD?qz3J&r2)^xsi`R)ef^h89DX|G9y2a@yMZ$F#fulczkfLCzWnW) zdV-9>O$CMK=H`KfozT-s7T%wF3apxQiU(~oGc%jItEuh5QnTC?08U60q*RpOqDJ>rE2Mau^?mAD)XYDiVX2@ag4>n66LhahkZc z#7T1R*(TANu9@htW5;t;Tbp zokAX8m`qR1Oi4*;6s`~FAT1h|X0&ThWC^eDzZIqfTsRGQpGIIXn6Cc*Y^W4oIXRQLxw+nQ&%A^Hs@J>v z`o?i`MrEZj9Kc@r?#|8^uqb6M6_p0G-jIh}H31o3p8_}%#)49Gc6K&?P`ZZfex|C| z?@$cr@bmU2^AQyr)K&NSi7Q=QU6SEnzuGm&inmgT=kGFJad*d&QM-2SvbD2&LL^0T zQP%yB$a;$m*$!l8@_NazTUc7c&C4r}N4J=l7ztQdSX{(z53bG3&Go?lW?^CMDge-b z8y)Vq)u}FI?=uPK>vcufYVX?2Pqeix2JK|deEITHiegwq{csy}#OTths5zcdxxFa{ z6&Vwg0UL>FekQtO%WdzWdna`+Up`vg$FF>~(E1S}&onQRPjBjx5_{Y1K>5N{R$gXu z@>{QgtFnK%qC8N`myIPsbB{H1hNF zpUL_Z_4f82*tTi@?M^|7)#0I`Lf3H(ev=A)|7z3PySudg{i_oES8S}*6cz9C^70l! z)8!ZzYnazR=n56AdZ}>qGd2cg^*|$0E+-;l-{#wA)nm?FjEs!j%Y4@e^Yi|(!uSh5 z`xwwX2O_$s(QuxfRZJ4OCC7&-N)}#MVxS}#E(R>GAF*HKH-ioPpU9XrdYUi*JkRZ2 zykcN-z!Hdmg|Fo=W{v+vf+0 zCb1{KTG|zO8j1B5LDWAulJ)xa8Gx-Rpp+G|{D!OFQaP~;L;#m^W0YVwCa*HhzOTrJ zG*~GGQi0v~+u8APX(_2EQBmoDca`glecqJOD71-5E|jcQ)4lyCX-}=rmTS7ZyYrqt z-I)?DmuXz?A!%OZFaVu(T$VEOSju<#6}M(Ww!+b*u`x@lCfpd*fRO$>q`8`7Rn5yQ zDf3?oTBn`|v$v~@54->nnTCHya2=8NH!3Fjm)qLfia>W)&f!^C7K^KMfUQ1^la(XH z?0a9JvveotCp%n9jjrNsBcaQ5Oif7@`2JGYE|5754Jyi7un%eKv1tp_-Kr69zq|tZ zQ|LCSbk3x`{H<~sS{G7nc?cNT%5>m`%GZdfoaoLe&V%v_GQar!oh4u>4I=bgF z&jFW~EA*%Pbs;XKSQ=FE@o%-U%r3fu8t=zJ!#pH(m#(EnGScycnYzQ?s+PLxjIL=>reZc9bT1UQM;30)Z9@ z3`{af^Yd=;vOc=8;*Nv_S)W2ow}4lr3BOUvWl$}LMQxwJ8lMUn^Hvi{M()zCV`{)o zw$wGs)dI^f*yw~CL)#A?I3Qs)NIMRwBg|++r1!`pr9z)2hsu?ytjw3o8(G&v@85rt zn3xANsGcZijOotpDs>$%Gz|83cS$cu8z_3l?7vFaJ^om)$JUBS}~ zvTJ^89NoWb*8Va}s}{b+%*G~zhHQH|6?r5j4GJuqwa%YEUl+ZTS>R<-k^(dg;oFy5 zQJ+OS(9{>PHgb2KkYtUL+Ls&`Yp*!%D8_{nmBR5wgC@9E#zD)@SEo#+7 zfOgl>d2uq)SBs{hM?O40xNTEse}DhXj~Z5lY@x$&t2|VB#N)>}Pm;-i52@1D-oEQE zq7lM>h(h0wXukT2~bQ{z3&Z-rSLxO(ckSrpdoxOb|SJ+`ul?WX} z!>mwdL0)n3ZWh$Fx0_Pa)3d;FMCfR1rzk(>mYh4O3@+;VtZU@O)1Btw zx$NgxS!CDKZYbC`uA-_sy5JqJboJ`h-_6DeMAT`6|?Qu7>hIRUO9yng>2TnK6HJue-&dGqGp3a!>3n(R`wf|^Wo{X3{?)EsGyj_77~04S;pgSm z0*mmvpg`1b#VyCUTz9-BzT?*BopF-JKjr_G@GYOA4NoWsELsH|%3n{7GHG9`-thZp zOpy;(t+&)Q9Z3B&Fedn=Q}Xhrn)2&~C~i*9^RWD@RlcRrRp}u3rrt4|>KxtUgois& z0tc_<(H5Z48hV&UpmQmTv|n49j&H@_4xW}Z$1&30@{ASm)xwPrlteQhTPh> z+7Mx@mbkVGFEaQQ2ryoN1)nNtl?z3Z2YpDB7}M2f`}XaF+$BvELogDJqIn2fl5*+N z#}~?v=U1llOqYIWpcK$RuH!8>-$y?5zPffkYPCDtAZ0u;z$8}8zMZgSTt-4Ho1diuHD***QOkbV4kXQXjsQf%nb+^8xt9nLs}?o@MGa}sS+_A1m>8PHhg$_jNK02gOfO{O zaA@}MbS1(}sFBxmMaQAzdk{5dUES5yt0{7;yFY`&0{s>l8#`@B_O5LB@Zrku(G*w# zn#X_xx^8Zc3)t|tYN}isoUTjcf(;ZE5t-=Fly}D)QKjO|rA*U*n!xOpuVO;rBqB!0 zl;cIKzH(NkLP#AK7neHf^70L!CEn|hR^ZX~6aC^YBbPSceuJrM!Hu;?LiPsKAlZ8U z`A6mob1*8@ds%^b=Rtu0LXt13s)jB;fridCOx$dyqH>=rBO~L~bsv!b1%kjo5C!gC z6*Mk;0ZQQlc5(_CRgLFhb)`^gR^XTHYD9DpP+wuA!-_8Y(WB>(Nul-hr2Ec2f4E2@ z3RV*ZmB9_z+SpuyxDI6rY_KD^7PiD0|Fy4JlUdNj4X=YnPZsq#bi9AW87~}FtpWK` z_hgGv*^o#1=-iFt?65`#b%0f?M8CAMGAS?&$}6?+4uKnjq)G=`V8_m#3*j5f;VaFK z{wW}#rhJI22@s*j^94CpuY35BV~L$i{@)mmNH{5MP!cyXPN+V$p6xG%LWtyAUohRk z83{xk5-@rRF^|nOA?IqbL;U^w_v5@y8y2ZSxU(`IK((UPhCJnirrX5FWBdlMFGQ66 z#9CUwBB*h5)?P{x}m5 zyp0Nk)BMkmhrS5j{CwA#5_Oh~Dg}cl20E)>2Tb9CZ7I2H`qANc=nxYD5)8kLrhGSH z z+Z2x8FL@(^IC^+njeoYZGik9>m>uQjvhA$Qd%Dt&?M3|k-VsL`4fS{%wtjm+fk5fSE^X}R9 z63qUG4y+uPUQXfxwH zPMU+ecez?t3B#jaQ!fg(?&fD54^*hvPE`6(bB7;N~@&YoE$5vlz|VY z^^eHp^bc%>L@YBa3j*{M2^5^@pv4S}f-)S>pG(tZS-|H&6LWV&%b%&nECI&JU9c8S zIm!r{0a8c)b_$cJj@DK<*g@c&thMg9*?-dC{3ZH{gc_xZ^die0+<9MSIr?Vj356gNN-euy*g-MgC`(x6d zW~NoXeCs9#1~J#KUxxwKgq^v)V?QZi!)LvjxE7`u&YlA!5A|mv%E7yjxGb`o>u6~u z$2wLS+`D%VQogQW4u7OGmUei96<%NhAX;gE8?V09sa8gf7N@{+XV?!T88Bk80 ziPXu&DGurvj^or^K1`qN>f%g?XBGKKYFZjEH@Bj+$JdZgMD6M2e-u#ji-xs0vEW!Vb)!o}qn+94gfBRxNQ0Cr(Boz73hJj>ISh2LZpJGH$m7h-?H9@}~ zBHLjyr3q9iZ3`%)f&n&Q)|y)bsN^c|*a>>NPxWd4I2u%R)@4w2buD=G8;m|NSmCtP z)OU<5XX@Mxn zjgF+ObXU)AI*Rcb*3G$UFzy|RXSlVh}lGl zQ|SSAdC9CR6FmkZXL{80j^JCR8o!q?$OCmoOQMV3;qj0t=mB0_iR-CpheSe|mT+=q z6cy=$Lc;2IZ{N?&BP%;#(EaS$VJP-fL)^r_=M2-eJxiLYu-dTR-Cuf`muS5kfe(VS0i08NNQ_i;6oXl{~&d6v3>B<+?IEvlAiQd-Q z`eS;cf3Fg0i{9%M!JG3qz>^Cw+tpi7-J95&4l$LI4V<9_o9?x1&u6Pv9n|9_u0()L z{@Zj_nDDphYT4DL7C*7R*A}6>Ipnf$Z)GyUh2tunt5*j|h0eo@V2CG`P z6HIUuj9YF)Hc>K#j9*oJz}CY@u4m>%2qzslaNuQCl?mLjZejWvo;VpVt;VPP+E6%P zn384V&z@l0O`AGy%xi9NDVITmpiNEl)MLfKy302duzoV}TR_uBy)?~MwbGRbBT`yQ z271%N{-(na=Vv()%O)hO$=H3f1qg14;FCqNZkm6HI6-qwgT7>S$b|_#2@w&=z=gQN z!VB1HUt?`8tuiCnog%PMXy}GCjrdHk05BQGK)~lsrHmis;1En~t?wO7qG4c6Ow8Us z2T;JGVqz*Q8#qdG025pa(m1yK4H&^nxsGAGFy+_Z?SQBs1A_ROh)u*;YhvDLV^kJQ zA@*VVw7qSQP{fl1jw4nd{zzk<0`tV%|IF<{*k6e=*s#R5MVwGA^jr0q8LV8*4Bz+q z^=pye-aia+x3{*wY;K-XkPFjQrEvDJRQ^phd#KgXL z>lqov0ZVcbiqhvhrB>GcA&L5f*%%%sTHjwp^AzDWYIXr2J6^0OBNfqc15X-0=8^>p4 zXN!mrxIJDd=ACR9)uUsjAp*nJ-O+GZfSaw+O!8|VE1*6sVLUiSq|OlIgkhS_ed^SQ zH9#gYQsq(tW!AI0Zx8H0^9W{LxE>D#hlHl;0f8_P~=2o@q*OOULmRQOcrT}aBh{mUSQrbrO8H7O} z>ryW!kH%z&i~H=k)uPS(=R!>{d}Ru*9jlk8=Fd@l@iQK=tE6h8h=|AqEQPqSZc2t^ zAofdbFl%_d`JvPLW-SS_eNm1n#O*=nEaCKMtL5V2ik7E5mKTO~+^Dlx)bE@1A~%bd zuPuqfP9Zy7rFXwpnwV}L6V>Z70?0!B^4PINkz^G=Qxy5t4-|f?H^zGZ@jsDSNoHJw)cCEeEn&14we0odaI?Zl|-3Wrv zU~Wh$Aqce>f>8DD+z#InZ0H(*e?;x2)$Nt7jO?B6*cu}8ckHdrt?bQB@3J@=+S-{~ zS@Lu9@pAK@V==L}x3&}I;j#Fy-{7{gHRds5&wd6^*=2o0!wy0A-$DPUN|H!4MG)R^ z7^y2N&Jh#c_pMb7qc&y~x#M<)Oj^rbXOlF#le^Ov!;%$KkglQW|2+NTo^#bwWj1%V zu-<#p>GOK>;vE~OZQkJ_n(uH@+^Vd*52|zS<$eBAj#u_pO#0#;R!7N`-Ci#wmzUP~ zg82ocb}4vkM=hj`smHdpk}c)Ci*f1lfro z`p2kQ5X8%Hrx*O>)ye<-;D56v-;NhN7&^zeZ|r+RERI}UbocIEhJq6aa%+D%eWa*^ ztWJTWs(EUp*`2MWC7&8%uLlzd9c>n{{ZoCvzhoV4Nz*dOX*e6hgp@3P+e3;H(+U^1 zY7r*8#p)KhjtqwAsmaJZiIYg2+_vZF_b+sMDu#yfmc>g(Y3lE;7TsTPd~WltY?pkv zko<9BOBHkTRG9##Gq#ev|ue-XmC%Ox;I{EeuS*DEwW-X~cx-A)c85PW) z+R>itd9DkSX&Z|H8}&L)-GhFui6K*8i{0;IJlA&&I2M#FL{K4KW@nD<<`5KYc45xJ zKaDwjMj@0*%rz7qUO(Pb)af?gZfrExk~Y++DB*xR%B^j3nJJ+C?VX1%8*2-s0FPPL zz*nz~YNN!)n!0Q%P8ij{zB1J9L>Ba1{LFkaL$|0V*QP&WMxWQDP&eb4ghwIA`SXGq zMe_vA&6@&s5*vlelnROUhO#eT!lZqVhV7SFt*;3euFJwTS213+?oD=G;{K>#f8)YO z{ppM_Aq26enOV2Esj{VaygicyJ6z&$nQ>ouP|y)d4S(Im6;3WL|LVaD=m+qXhoVor zapC^LqytXKs>dA8u7*y}?T#Ox(bA8>glpdW5|WcSzje7U&)P6;G)k-mR@T-&zvyXX zl(5w5xz_5sV9UO`g&KLi)VJ0dn&d&5AqkUJ;J`dq`$i zfuy8{hDNP;Kt*g#%}rWbS`7Z#vu8m;K?830&SBt3&;0$HC%X$G30hV~s=IdWQt&5F zVKp?u6rwJVG-dr$1X_4$H!~t*~Z)2+Qzsf%e;(^&Nvh8_WoTbAje{g zl*6{^{IPW9jrBFw`MI%H#lgWrd>rgGMoC&)T1sA8`Z512JU+&t;<>b+Uas|(GYSzt zNyIUFy6FB+95+*m5>E)>l2iI`A75vl5V4J`6@_<8I0x@G6mF#kJ5p90AhE74<~sX= z4Hm+|&My7<@#9+Ovq!$dJ`GZq*f8SJiowCboZQ@wxDQt3bXY#@ZK6?Yx{j#ldZBWr zMW@i*NaOVrqW0D}ZgE8m`3aZN#_D)&?vO4^r=PDqE=75U7rQ?ny13{}nQc@IR}^0= zD_kB;Nr18%$|+e(Q_i&RyBZi2bg##`pN8^r2XpBCsiK_P&eFZdE|j4qP`EbVncy_l zb1>%tF*Q{{H>34tDDTIhSFZ?yW-XyY79HA-$A3OPdSfYiC^kU+!k(k&k|;vF*wMiP zYgzzm(*7y(?P3}~zH;-f+=jjMyrsF?+>JR_dX!Ilc%=NiKi{D&_6Ho#Xf=bC=fTIx z!|q6tTJ=0F%KeWUcVwFxcUk6yU9^c2aTryRBCSl7SjjwRG(~@VeR+)Z>G2+(F~jbB zhaqo?)i;Cz|(5yB}Zu;e2`b_J?ka zdl5FS;lb0Z$1^#JT;ba3J4-;_i;B1{MCP2&0jUy zu%zv!XQG_1^6~K<)FKzVN6y#{*K!C8W1(Fr6*x`^xXzlWs;EdY#ugT8x3{#?U2{Q13 z$?54Wm!=A1vAQnODQan7SC=S}gwNjlxh`GO%IS6tmz9+ja9_3;Tb=&gk?)YAoas2B z36*d9@waUqC7vbQX&Bf_$Qe-d%>^h}9j%sX_-)&sn%7qzjxwe|eWYn-Ul?o6aGH0Y zZMZIj=iubLP)<*2(Xt9_Yg0eGb87^FmP?I3r_&-iITAY9 zT5W&dXYX)mY{NG4^8O z>rt%^1S&RVn_<^_cSzJP&h)G5>T+WMIa-z|!+;@!P4g3-*zN)+?Cskx*@#I=H@589 z7t}iDfLrw_x%%eCKBmjx0fqelg$-1&*p%Gdi!rB#gix4Lkl);D;XckIQgmttZ*J-@mG+ zMkRnrEX%T6l>PK+8H$SkDG3QZfU$c9IWv*2wH&$TC@H31o5J#FN3y~ z6OPx-;WJ3`DiYM|UwAy;+@H+54qc%h_*Az3M`~D^X;-cdAQnNu6#wJLkEVA9*Ymn9 zQCp-v1)D#EpzPYpLhGYlhqFu?PQm7r!i5#{^73r+wDED3foIG{zQzMyzhZLz ze$s+mtOJxr`N~H<@#UK8YGnn5=NB$q5O7usY;YU4XA|BQzNS z!>S`oZsOS%ooXT?BCeyv#0WxGmH_RMBPQj3$28yHdz$YsmJp)rit#8R6OTa+JZF|j zhfB+KV5iA+@5he!hH_<(Phv;6A;|0aXFzMpEo1Wj3&H*8ZLD?Z+_j(iN`y(5i@g0SlIdd1o$%gq@A&eQs66@1jcg{q3t=9Ib8E&Ffa_ni=p^H^I( zkrOJeeuxzAu+Z&<>Y^X6MfkX2E-Y?-yj|2|HTTe=Ljw2b&EbY8Wo0#q3VOWkW0y?jR~f;5hBp*!~usvT>xJM_SQ5sjhl^7w*Eq> zqchTb2z0}QiSTK~$-sAcw>}&S900&N?YolPT=n6)=zumd@F|C>9v$GO__&yuQ#6bh z>vwtL2NqUn0eYu$DJ0qSbu{X0pP*N+};g$lWhvRWpCb8ftyQ9M~5{geW}P*;&HIa(*`g;cpT+y zhqDpqm5H3O(g#~OIXG-SfFST^mgrfUQcKe$zI|KoqVP}R`Drn+wP(K|b7i0Y%KU_& zeyNv_yG7>*)kKBJh_%4e@=Ch8QKF9HXLCB!;M^;RDI(bJ=l>*jr#~V6UF^m%1q}bO zmZF^lvQh5y8ehuGacBuAC!=-nW3W1M2D-dOM;1mo;pWS_VP$Ap!GVFXiBYaqM2wuA z@|7zOjA}yPpn_W~$I^8qJ_IgPB!SPM{5uc~f`n%=h_UVanNs0uOKH>F#_IyQY9=R< zchsMj?-Re;wc`LBXycKdlHAG%Zx(95!jac zSDg4?;z*rNrn3I>j24B;LB|60Kgv>;3&qHIwJXoAibO*1SW~*rC4VL{O_1LQE6(!p zU>zMlbYz*v4h-BQMTjWVaclb|wZAj)ezNZbXGnECm@uZz$*Li9@6FpU%#F7zLs_bU z{0AUA%%>au^EhLG&s%oqn?hee*<`W%%It#Y#zOu=&wU2~70g$^WBge*{W58KCAycK zCYyk?IDj>BsV;*^ybT|poRTtEdui%>FjluASv5t#bKNa*Xu9unRc&pHMNW4D9)Cza zT3iRbinkvuyGNT+bqds6TwFBY>OYvDB4=EAxV>^}VGJ--ffW3c@e}X*@g6Yry7E?f(A#`v?MTW-{0_0AqMII5N}T(gzBQfvCK=b#@^2)KOYm zX~x(O3G;u3*gs2c(S&&#aO}sDo2b)d1nA^M(9AkO1hE**Q|LorzZ|(eHRlT4oDK6p zlI7HD^Yhl{?pS&77}_(99exa8nwy%MH0-|DpB$#gj>dBXg9CuDW%9oIhN}hyhGgU6 zIoeWj_vNIxIH>}ssg@uPr+Sy4vN}*Sv3}9qr zq^hp26_iody~;yCy_53uHDQAf&cVw9X1`>E?`b=-TTb+IHRpC)0=-WLvG~l_HzjlE zn}IelB)=x|5*7@w_UDWuN+lRn@LgqC759PzfrZK{Dq?|UsHm%d_k+h0K~Ta5;|C~9 z_OO#aDGoUnou>h5*Vk4q3DPnD+(|H=;Lie*c6op1Ja^6>H`*wt15(q@H@pL!taLngb!S4gwG8-7ZDy8SK6b~H1vT#NQNRH~p z^!^PNj4POww8xL9K<%X@C(C-2=v9aCI7Knj(ItXNFs%yWfHmSeKcs=tHQS`DC?aTT zYRY-`Y&}}La5N5ME%H1?vv`n#%KSz(^d}@dv?;@NSe66zy&oTD8&<#a*>f$u=_hMy z?p>fVuOxhOf0VnQ1wpWh9EG03LLPhQ|E>7-VR9QyZ^xKOV()u*)UJ7i+BF9H+x}ft zK$)=PmZXv>yWI>0z=i9n-%|fga`6<0)IjyTz4N&VENdXzW5p|z%0G=UU`;{c;VBA{ zqC{}Q8N^((076dTK!l=P7#z8?;Lh?J*ET)gLw5@2INtW=%{A|;L0cj=Lst`ItKr%? ztq(WB6;uHk7#tQxgfF!ne%>{CxNEhMmxAH(sK5hC&_&Sl0+_M3w(n6Eh?>X$GHTxXk5cnN*}tadD#PjAG%mE$sOvD^23XJDy1Z7vfvcZF zwm-arQ33~x9rKUkeB8qw`$hjir$1rlC?fxbnbRCaKKH#%P>fCww<^}yvVHFhHqiR) zY;0#aI5=Wj#^yK=?7S#Q=Uvh<)A4R9dDr8sqW9-hi%@*P$+{l&JpJY z1+$|v0aWYl1=Numpb(~}#FkWz##@QGLx1Ign2x^!7)sZCxO(XKCZck-Y2)dMj%+#b zTkYl(|AOjVR-3SHgX1pJ%$cw1@-;)hh>5vsnWOde=~KXUWo>QkNO;+P*&=PYkClVA zI$v(WvjxO%>re@ln~XLQUBP0=c3*J>M_k%3F)=X(q#laY^Zz`;|HQ2$&9tRps-ZN& zs&M8>M~>B$TSzT*a6Z1`Fx!4ai9`^T+b=fz0yXFXNQtj|d9am3 zgM%Dr&Rjos>==W%TkcUF-Bc(?lW%2D6%;p}4EAR@MEl*{$7~~0>yuNb=1W$8RJ%@p z+`*2i6JJTaYhWOV`lPC=Kz__1tx>Sh){cO}j^j{H_7avc2 z{aO-$yB^Oj*OaCuTr~St?#IfS0x6WwzKK2Mq~sTFa~RTBo@8<|+Lw_S+A8 zcD#-67x#h_km%a+f=mq^|Lwm2fmd?4S&#Y(#bHD~A&*Z0S`{4^JnLVYy8dL>0&}2* z+lx_hbrr}JONTnuzptxv)6GEGYbf^riV8~gSd+QD!2bR!w$K%uu2aAXR_1Xbb02!) zauYBz@+b4hf$yGR|PN^wC>=k%p1@NynA#rel;|otq--0WVgCtdY$W z77i&~!EB<$8U_~C{EU`sw#^`bm8K^FPKfgW8wiJ4SfO z_|d=l_;82WU{&ycH&;}KCJMBd51JeFv+9E-V4p@V36C*zJor1)c*TxNfzrb6qNKrY)MQ`u5w}A{kXWF zb_*1zeAo6NPHs&Z+324q%Zv_y$5=&%1dwmf%W=x?LJlX(PFLVkc7XakHxpn#sTlG9;RUM zZsO7KVyfEO<4xvZ83XBt@I`?Hx7U7s)g{pg$iZl1VgRhY_b&l%NIqnHdEJvfn{*n7 zTy|g1f=oq4N$J!`9i0|Pk|ZGN5D%edoyS~?-U!6XO%Jy0_{neKgdNh4OoD!!%h;+hb+om)||tWmhNC5A6fZ1yZ2B#6!b^JqB;Y zqpXP9Du3iO+HgP#H1c>Z$<0l6YeCW1dr@tPAZ*@|LxOGYh()P8k_T)Wl6G+~?VnCb zq?seLWY%Sev>Wd^cY+G5_>-h@3OSX`y{5}T-nA8ZJ#uH0+@>Ni8okd95G2N%RuZtd z`YpAOS^AseGrx15M3A}9fBl?3R(%1CgL}k^I}X_U;7%`-TU$jCguo>Jvl09<&E2;8 z9rce+*n;%h{xUUaCDr4_=_20B?nF#R{xK--+P*zkN+;)8Ukfh;{Es%5w&~D+zFsOR zHXZ&6{XVkOZNHo10i-@+<{&TFfqxdQa#K^qJmPvxg^;#U9WY_h=GcpvBrop#8v*j! z33*`Vvqu_15^Rx)l!Fe38;9C1Dx{(OzwO2Ar~L+Z5y`A~n`B%Kd2rM#I&ur5 zO+&4JSS>nW-OK9p^$Ntx3Jmmf<^DvfQUwSU_NmcOmF{cVQ3^6?B+NMKHmD|$ytCot zRT+pCt;RoiolZGfLXgbe>SgwM6v_RSCju-1zBGON;ybvo0zPcQz`#Jc<^H#|3%f!d z$soC|eC70wZSn%IAabKQE_72aggcn+@#j172?;n*odh5hw9(AF9}mnpAKAMd8bxdm zGu;Ek-`9Fq)=iqV^=?M>0%~>0zP^3CONzYI+D-=w5#S3*TJur_Onv~n5D9qr@pYWU z#x>l|zGN$gfm~!Xam8=Wzxxwyu=t48_)#yT#CiA5Z-Tt{xAcN<8&4oGxD@sHtm$T8 z0dSKkBB1|Wp<(fKMuP*uB1T;r?0i(%afc+uVYG?E<#JTHW$bPB<48#W>h$vP)VVP3 z^LBut9{4c^FaYb23V_qpqyhsHWgUaVPd;@C?H$VQJTQ$C7Z{8OD=&Ar!jCoBdgAaci94Oi#jgFhAuF%EtB z^-4Xd#i(rd@=smchyI)PXU?WRtNI7?FxUBUA^;B;I8pcK$AN?svQ}5!(5o=fnS<&f zw4DM-NjF7Ya#Db-57=`c>H&cbpywDk_2Y-Fp|NoaH~_K=3N1BZ{0`vZnJp<13PBd4 z{G%~Z(P^G9hfId4pf{dNA)dFvLPS{zfH4evx!30M$$$;eisqeVWl~_dj|3`;KfPEE z8zJDa>I@kqL;`5@7lk}H+k-=177}YF&s7_AoCSEbCq>vZ7R*A=fJGhb&<77V#CSB^ z1dvy~dDBnGyzKxVQ<}?a@5BA})#r1PfIh<$nq#DW8J6SwWcQ{M(Fqf9lPD?S7BbnsML$(8w=DUUCuwx3^;n3njPqy0d=h0OFnlUoC6-@; ziOz3a7v+FMdl|n;Jqrbnj3BYvw?6@txDkZ9{xf)jUt+XJhy7?%rFlnIC8M}o8}PZF zEVI_ya~ki^|o1#v-qoZRPqioHDUjP4l;*I65r5IBHWO}SaLL3|z zP?rBNt$jf_Mx1{njA}O$0q6$sy$50#^y%B+xBl#r5xJlEeJCz-EI^T1Fyap z+b=Q<9Q9x}K=T91QXBW9Kx}bsDs*UqwYl3dC?~brTUHc1LzUZi0XjTn!?O(I`r+Wt9go^I&Q$ zf~3rtwmk6q`t>XKC7B6zI3tEQPd8y|7D6!aq2O1qMkjK}mg8!gkAQQ2hhl1mOVb4A zXSrpfG`O%;zyDsJ;kjmp`TFAIH5E* z^`$Sd?)yz4;^LjTE@%i7r&QyT}07(mZbXb@RjjnX#EZqCELP9?#xVX6$fcGD=JAS@_a6P&%5%zurCZ-h7 z(r-#sWEO{URDXBcXBOZa89xp7p{RJUj50Ug&t9> zcYODLKahEU!ICoHX%*g6bYIIU1uqArKXfdkq$iK2X3AsXQX`B3@td`rOMH@-77c16 z40=Ny+8|``OH&)`R+OImQ(+qm8|%@0r$s61fNL<+8^(L*H*}~MMvwU*OYIZl_4h;C zc6w04$)V`mLHFW^!5S3#g&gJN32~x1FgVpt6aimuDaUg~xbe-k9WW?!oDu+o%0tl1 z{4UlGM|)aM7E%-fLqkm<*)d1OayYbNy?$)ttPTSL7t^Z0+#hXX<%70i0BQs}UFVZ` z(P3a1eeQz!tA5+O8np6Y^63<=czxN7ALe%&As+-BCxJ2~0fwg!R-4#@v*`kpYVy!F z<-tK9?f018LIHs--DUI$_JO>yzTmL#PO7ZrE0?ljB71JQ^V<&GKp#KXs%Hfg(aveQ wr;4Th_~^PtVKN*!&qV$=HfR2^L7-JnuFJJj!F@Wv!3!Z6X$7hDtNOqFA0`5AU;qFB literal 0 HcmV?d00001 diff --git a/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png b/tmp/n_token_hist_train_fw_qa_v2_2k_len_level_3_tiny.png new file mode 100644 index 0000000000000000000000000000000000000000..0d4aec830f187d622becee63309aebe0ee86a4de GIT binary patch literal 13750 zcmeHu2UJvPx@}>=1Y$xE1Z+i75s-!g$p#R~N-8oUl9NCIl2HUBpe0aBkR&-OxrmB_ zN)A;;MnOUWMM=(Zzk|-)nfrR~oi{UYt$A;~wYrxr;hZ|>|Gxe0Z|~g?ud2#3?qu7E zLZKKHuE?sRP@DWvD7vrP>ESEr=I-zChd5SF2dm*=j&(C}GDE4DU~kzvU~R2#9C0;s za<+1?7vdK@%P(~1h$R+#%UN7N!0yjK!SCQ?Az*$y{}Ehe$E_>6&L|X<3Gy#picGQ< z+~BB!>;=u+QPdvaTbi?F>ysFt90inHRW^4L_tS&fdoDcP(!Z2GfX0Va_o~0y9Nwv! zv%CBJoxGJRk1tR|ZcP?!+WMNLKM<{W!TfZl?8VlrvEgA{?=-0sjb4$1! z^+TJEz+cR^`@z@3e?x6Tp>zWO|2O`bSMrcLN}pvGclAn5Fh{Ek*)TaJB_+##8w!Q> z6TF0inGjd- zU1$14b#-%uLyJ9j_g!pDOH0(eBl}*IOdu)5yT@LXZV$@O=1IMA-`i<_2ROyYdr@>K z)R{}DEpXZl@Lg2p?_18pH)>^tQM)SR3KEx78^SAV9c%=q|=xS0R}uM0R~-!-qY zsV}^pc~;L|C%RKjYl10j%QUxFeJw5Va96absLlLDkH@zF7EL|9wEq4Z$wQx>9s5#S z+uGySmpM9WT^}i`79u(IpunzcsLi&!=vqKPfF1#xy8H0C*yaTJ+Tm%+!sLLeif0)- z^o&l@{`zPMve&3a%siKDCTGhsYD2=F1eKhx%I+ET75s;atL!)@TO;kdrH8HmfK1-wZnGrmnN1E#L@u%=g(&OST7%zUoa4Xz7f1V$&^zh+B6PM4QKkwYKNwq%l zTeeH7%eXOdi~oga2{+Yb)o67kC8bF8eI~AnTLN}g*4FSC&!^9^GORsjhRcq;5~nhJ zU0sHmbs&A4RXa{T!5soRnb;o;#VA|Di)!5h^be!eRMT(e`w>w3tUWoJ!JJZ^4T^AYtW5MiRE+PMJKXM+fIwphJ}fK|&Mfuq+m;TDzJ5A^Ansmfq<|xj z5#RZ5tI8r}rldUWqH3F)CuulS4rih>or?_k&CJXKi1#l3rlqYdj^>~w@S}n*FlyLDGxmqUdeDMQ!v#*G`w?(W4Jxh7Tg z`&n2lpiAoY9m>o&y=SgHfANCV)y2`X47&xhmkxD{`fauA#RKClQ^?zMadC-|_FghG zRg+oGi%&>Ex)F^=e;%H;qITvt+*Ve8QefK=ubXTBd2BXKEk^oFVD=FOWa z85zpZ%;Ya&yy)9Mu1n2#J5Mcosn&;{(!lnA*ul)g!nuyeL@*Mk)`$S`c=*l`JkX6!#T&Z2)OXJa`aY|uvKhj_`h~u z+$bT?$rn&GLQzPd&%SeSDJ-iCj0+@0vB`_bsTmRmlPC}36TN2hq@N>j?UggoDM zAB%3*1b{3<-_=?8>Q(5|UiabX95K~blBrg$NuQ~%yn=$)gv{%jUJ6;fx9;_tR_TEs zcoY(X(bJ0v7clnG6^InEYx(i5ikXWm`&z$WFT=0G%PFBD?u(szx$)nOc~R z$u{aFxTN=bjPZG|FZW?8d{(0(25a#)#566vLi^~s@s0#2s80ZgD~r>r^?6pU*pa5+ zv@|s>Ys2|E^KI}*Oe3Rq zBFn*RfF1X`y87~Q9-j2PJWY*cRU&jsW?h{!wX0CAqoV`&dTPqadq^mzIc9B!Cow5$ z^rgkC=ECw>zO9?+iVgUO+apCCNZ+dNA6L*wQf}_Zwa7QCJvh8j*z1vA{p3)B%NWTe z1wRM4<}S&J77_VEb@iPISZ3{9qV_Qj6#CJj7Vb~{SnDAk^`QTOuUJWZo0~g}3?i6p zgu!tAnE|f#_CqpG-ydwc-KfCM5+^gzznpJ%3n_bIr^xH|xJgU*pXSJlPF+>CLULyf6 zQd`^W>t|UD5WKayaZF72BEsjm1EfzMlWB!G+Dm&1K8JirpJ#TKkuUM?t;3{LhohtL+}lrB#Y)v^YobSam)8v z^^jT~&pmzexAFwzwJE&#?H{q_D}w^OXtWkoEpBFBC~&^lGrN3= zm~9Q1z0^bVNrqywf%Aju7RK#8`PewLcCh8iAt9to?0{DAVzM+%OyXK=yNjK*jEu5q zzU#~4UJE&#zAKK@z6#@9i$<88NLyZ$t=N3dzWZF+%2e&?)2CYimQ$ZRVaU{ZbFno^ z#o78Gzd;K8^k}m@m)50A{u4z5yTkB!UTNvxRQJh`ENIU;+aBjesYk)Vr0<`f=UWp` zTXYXhj$kU5!glW584BYMW@cJZ(YbM>JB2;2k)&+n?Bhz|BawY8N%lqG=`6etYdv5L z2LSmd$nAG_=G0X20-eTS75( z7CR>+q<@)K(Rawf%Icbt(W{8a$b8Rv`?4SJcb(?r({gltqv8Dh(_Y{jl!+3Wwu?*t zP<>>wJg2*CSAm_gwTOLp%KP^h`3wn~+jbvr8fi`#oLl2^=5p;<;xd<{ee0umr+fmn@dX#aG}8X9S5qdW=Dy* z*H=%z&dU?7e<>snFKyzg0LNYi`oYQDZSCzR3%gdIk*Nc$grba|h)A1PDMadRW%r>n9D$r=%_!F zbklw{$v6pGsRl4CAjt5Hipt6io~lkxZ#S@c&-Qx(6{Jbh8(*#c4D>D&-D1(=E{c47 ze0HS6(q8Q^6UvM_TL70~K^S&C%W7*O2QWa0c|c$Ja6gaudiml|B(e_ub}4Wq5pSe^ z@!~~pCgO2LMa7v9`_-^6KtWMlO3A(SCx8sB&pE6ox%GKtPMtbsZEvrA>Cz<~6>sk{ z7zx}8@HM>Q!>*;GGPc65%zO8?Fdja4RZvjSjs#nQXIWliO3D?WY7MS4BhA5lz~jz~ zwxrVisCy~2R-F| z8(bV_a`I4%n4XQ^@U3D4IPab5I|44IWn^Hj#aop0`}_O9yOvamz-nisA}h0Lty;7^ zuF+>M7hw)aMJFoZf!1F}_<|&72tpa4NM~t%#`cB!&O>!D3S6=X(ZHjB7ZRFny2NC) zI5i0GUhnPAhX8U1SXg)zBE+1Nw6l#e19q`kFU?RoVOkh@P>zR%tqxeX!_@|dhj}F= z^apA}$d8zLv2)|PQY$~IaBXovHz9caGOH9dk00;0&5|qDwEXEATR7ahY>zxUB`7ke*9?Fo?`-dSvJ04L&|7wio1j`=BTK{)SePi z+tOnj1XUJR$$4s5GS}*eJOR40YOXOx#yGol>XC?D=WAGnN88_+64uw2qbI||55fti z78mQ5xY0<_RNVOF95mR&CL0L5hAsMNBR#8lfavjhpL>Hy`(LIhlb3xw6p>cNrhKbl$=WWxxi@I za`SZMnD`@Fjra1Lp~%7QTfRFLhr(uDAR{KGd*^&*YO(N2esE#%YI?-=vnkHQ4HTf* z)>c+lxkK=JI3^PLR8&;pp`AFl`M>kkgKldwlpAOG3epeSZNi{zq5m5i#5`BE9pZe!;~Ab1@(^F-Ve9I z9^1Ri4_5Gy-JFssAXr({HZ+i6{fhFPA0;XQo-{aH+t@%U@nn;E6#$HZtzK^@5`bre zK=Gxn4r|RHosgg8vKwEUn30i@69K~&;5+#QFK>E}QRhCu%EB&=pBur^IgZM)tP)k= ze+*sZWHl~E*0YrzGN#oG%m)vq`L3_QB_o6^f=i9Eor%maOFDzDvMo+>K+jYE^ z%QdeD;YKS)+B-BRCYvY5t0ndGr%zX*kcS#$vcq^W)<3?!^Oz_ehN6m!IHi#^0wm-# zKfgA7))#`Nnnp&Pt|nK0YI~Y{E0|(c2vy$qikObT@g_->`nsWgcCr4a=J{^`cz>Y8 z{vY~O${-I2HA*H#I+W|e)P;YHvN8IF_Ab?cdfCv=P#fo=?TD~tXc-%GiOB!mAe(_l z+mI|rn&J?t0?5?dL=U@2?@tLb9tJN6aW;VJAR)AXd^Sr1>2RrgSUUOA{oM*UCT3=5u6pv2^u%Lp}AW-__E|4@J+T(49#hhf_-Q5Yx6D3-P zhMDfuLzHLooXw#5Mnwd%i>Bq|TvfvJ6JakB_L#ARU7SP4r#saudqa9Y{e{(3d>Xa7 zJWtk8lS|OQoZQsy4|A6{ZJ_qvi zm)L>~J@+7J=H50Y&QK6c7pI4bu=60BDgccp>=p+6_@!Z(5aIVfY#(+Kb>FwIf#^+FaeorySI&%qv_;N}-{l zp9Ut+!FlUp3W2WN`goKRWyp_57OISlGhv~jmcU>Ujs>*0?H2?uEiHK@zJ7fL2Vdha z*;hF0=!u4!-mzl`+qqk@ptN>o6k~(uJgk(iU3&(zG@1QqK&ps4n*TWOnFcLuZp zNr&shhO&XXI(By8AdG&W^*F8otpV?Zp+v6`%n7(GC?p`{IOmb_8ot3q3JYV_-oG*E;<}^YIK!~b;A$SQ~ ziP64_if;SDp}LpHvdLoxE)pj1x0somkH+2SY?&IYh1Z>T^YcFD0|!p>^Q+>7orjb& zjmq@EZ)lsLwB~y+yOzzgD4phAt_fg(tAW6S0pX`Qge#kwS9@fL}%7Ej8mdP|MA-Vhfi-l!) zKj2AJFkM$8aZDULkg8vF%i%oAkMCbdfPYfi53VkoM>wIzcX_U?y)ES*g|fnm{G*!g z-!wf>6r{7CuVFMa0Ludki_=fY4<7MfX_Y>-M4dTxs=3H<0Ndk2hK)}F6f1Q*J3F0v z=vFNl&UBmUQylurBP8A5j*g90SwjgUu-RMY$qPTd>)@HDkB?cZtX;?3UKKgvdqoTC4tjrR9Jq667d3dKGv92a|E;Rb z<#a@~$TtPg25m^4r%j}2RBUbsWkUh!@W0Y3g#K}@VlfBWa;yKl97YuCs~)ULXAkT| zL`}l#To>Bu`xOQCBYU3K4ypt2>Dc{LDm0v}7Hl{iIF>PJ7&gNV(XAkUvgCuar3*$F zb#}_+FD$!vL@3-5X>mT(!2<_Ul9Dd7i`b4JwjPM5@Wj}H1o;rxUp%=$=tQ)XX940k zflg}$l&oP5q+Kv)!Dsw!yiWh8KoJ$eWSrUMVK$M7<|@}KEH5|uiO-K7KD?%^tQ?8P zH#CEUeUODExu~d!R|lq^-vm#*1u94|0xoalvuI}5S8jOd&>@onU0vN#VmkgbFK@JW zetv#hdiwKU>M@7p?X%<7J&zvkEbg%f{1itntJf-iA6|yGwziH6DB@;jiGOlL;~%`D zI|K3_GVUK6uUsS>x$lOdrfwFaNKaZ-da~*2} zkPXYvxZ?<&=TTUySYMU_jo1e8Jwn_it=DJS4!jH-NP0jAgmbU{xG&RYAw7L)wQ+qV z1W|SS9}A>`s3NoYSq<1UwpB$m6~=?i!aJs@*dnJv#U4*u8Im#KrgZhK6@Yz`2lWgT zJnI*)!BCGrh2{3-p6w?=Naz@zybZeZF|g*q2>n8v2S2qLB#6++$n=pBON4sHtgj3Y zg0KyJ|9Q|!3=M+J`fA^LCYX<@#uYv=K0vWykSK2)c9zCXT=}c?gHHyU@=)AqF8Hx1 zV5J_04@JL;PC*?;XZspRFcxRvQ^16_l@t-thV_Ce9d_3u+hEcgpVg&Pd$#9!Mj>?c zmjyRxd?stB^l5t3*Mx5(XECuUDJRG4Zk%O>T#FuyYn$`l3Al$OhAMIvE{S^$bT?-e zjEDvuu;+rgWH?29Ci~^0CEb6|NXiuBm#UUYFUFRZ{g?eeOAl2C`!oE?26F!I`);#i zB2Mf^Pk=aW>U)8J__VFAfMjr zhzPb!gJN|g^56_?WOx)pGI{0et2AEV7%-li;HJ^~A7$m_^1F74HxgHF&sWv-ZvuvY z^e}3k)|XvEEAMjeUh*zpS@8lQVhsQ-?m3r%INRV-j08(eT!l0Sq?4kDOFp_anR*8w4bkpf{vr;& z8Sp-kK*>+D=lALtS@!*U%zrXFZflh@eG1si#bp7CE@pYPM56!nb)A6a10Vu33v2=i zIzN9N_4f$2VBRh4nPA?I8ln6|rN1w#_f6WbQogbb?nM6me{wrG>N%tMoz?t7rUt_-g91`$AGgrG+=}|u#K?+g2CAb3> z%S^W?hqdqc7L>$ZRLPs*QnAg^K~J}%q?qXTRMyO0TWj1Iw--eSo~645X1d2{YVv{Y zTaev<194#TmJs-2YX<6$qOct4o4>+luBD@s0!4U=hew`QJL75Num)^@z%K~|sY;J$ zOKe*%>_fdr%rcBW`uh7T-sLyQ`nRh!Q*Avv#BXFuiQc3Yj<5QY612)IgkTDh}hi~6BU&K zq_=P}3WwW=^bMY>rld3$%1L)*pCm7VusPOK+fsEVMw3}@qylYkQr$NGl>4KmE2)e?+X!Pv zu70ek;e|T!oS&eMh7)d`SP2|soEv?4&J^W2*(u6meE7Ty#k;}$rUP)Nc- zyvJ&NZM6XsGvH?s)|SSQd|Y?8LD_VJgo$&Mq&wn=LJC6r#*KKyQ3D^rfQoO)1cE>E zriN4F^5wvN`}WPGs(CB+t<4oy`80;*+x2A`mdtcC`e;M#AubD;$5qwG@&g^p7n3GR zr&W<>1y2C5FWyL0X{2vr}lkXe4IaOKKbnWg5CEK;qk#CAVS z2LO-?xLDIgiHRS<7LI_j;U6FEbXr+(Job6KySSCrR2iZl6xeVO4yZ9b*X6*jP)(a> zH(guXCLk`}xn&bw@(_fd1nmpE05}YLSeojiq`f=4VbCVU$G`VlYK?K-bYyI)`HWuP zr*6j@j&-c$ft@d5Et`hame?{I8=Ep@6c*v=&7TMk!9o0vs{>0dDgxM41dxwudhaOK4w)eVLUDxS8d?b))RExd@QxPCm>Jx_2cr4nBvWThDQP?O@x&pR-#S3h}G) zn@~98!M96VNae7em$pk`}P46!^4AOolrKQW$yC{MnQ6t&6GMcv<4qMAth$sLc znP~X<_^`OS*?CeD@iV~Htsx^qpd>{iln+$*Q)qNp{?@ELTy(~A(7c+;%5`R#YorZt zEb%dc;9B=eN?*Ugw%NgFnw^nTvb`e)hl>Y6AL!RTqA?*0sI6U==h%xNs2m8XI3fx1 zH*elN3JM~E6@X-&K2}$s7885vvbsI1V$B2g`#>r?n4lnXaj%*ht`Yn0K#7CAFKLTF zin#mK>-@3P8>$!l6_Iw3O^Qis<|$$`0CFp2*m28RGcI;Dt?or-AKmcK&LE+*Wq1Wp zA%n9XB{{hh^{fdH02>>`8EkQ!4^15KnC zzHYXFtFH~U)kTT5zD`b#ger`L&4oRiYy*~{P_IV`4z|S?Alqvg8QIe_vfZ4eDt0rl)w;J{J#6v*-f7x{#-BMiK|vX4iT=FHD`o(rwpQ(`$*h|HDGN6ptm&o;yLCKm*r}Ro6M(L76hQcsLg0O`OLx|iQkYT99B8HKj zDd6G|jA0>2ah~AkZ|H^$UlK$jGlsH#SJT}=gja#JIH-#%f~Y}a-b*t?uR}emXIX{8 z)kae&loOBu^qP7eBiy${DtvE`eh=>fL5fIUjp^y>Ai-z)RekGg^Fy%4oP`???T=Zw zrwf`K(c|HW-$_&c;s9}w^gJZXH&lziZ$J3={C|8F*Qt2tw?Bps', 'eos_token': '', 'unk_token': '', 'pad_token': '', 'additional_special_tokens': ['', '']}, clean_up_tokenization_spaces=False, added_tokens_decoder={ + 0: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 1: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 2: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 3: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 4: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 5: AddedToken("<2mass>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 6: AddedToken("[@BOS@]", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 7: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 8: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 9: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 10: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 11: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 12: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 13: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 14: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 15: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 16: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 17: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 18: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 19: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 20: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 21: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 22: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 23: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 24: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 25: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 26: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 27: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 28: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 29: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 30: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 31: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 32: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 33: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 34: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 35: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 36: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 37: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 38: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 39: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 40: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 41: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 42: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 43: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 44: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 45: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 46: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 47: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 48: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 49: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 50: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 51: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 52: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 53: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 54: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 55: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 56: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 57: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 58: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 59: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 60: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 61: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 62: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 63: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 64: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 65: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 66: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 67: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 68: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 69: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 70: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 71: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 72: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 73: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 74: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 75: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 76: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 77: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 78: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 79: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 80: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 81: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 82: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 83: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 84: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 85: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 86: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 87: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 88: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 89: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 90: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 91: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 92: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 93: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 94: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 95: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 96: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 97: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 98: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 99: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 100: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 101: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 102: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 103: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 104: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 105: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 106: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 107: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 108: AddedToken(" +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 109: AddedToken(" + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 110: AddedToken(" + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 111: AddedToken(" + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 112: AddedToken(" + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 113: AddedToken(" + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 114: AddedToken(" + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 115: AddedToken(" + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 116: AddedToken(" + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 117: AddedToken(" + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 118: AddedToken(" + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 119: AddedToken(" + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 120: AddedToken(" + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 121: AddedToken(" + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 122: AddedToken(" + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 123: AddedToken(" + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 124: AddedToken(" + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 125: AddedToken(" + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 126: AddedToken(" + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 127: AddedToken(" + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 128: AddedToken(" + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 129: AddedToken(" + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 130: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 131: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 132: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 133: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 134: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 135: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 136: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 137: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 138: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 139: AddedToken("▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 140: AddedToken("▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 141: AddedToken("▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 142: AddedToken("▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 143: AddedToken("▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 144: AddedToken("▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 145: AddedToken("▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 146: AddedToken("▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 147: AddedToken("▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 148: AddedToken("▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 149: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 150: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 151: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 152: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 153: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 154: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 155: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 156: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 157: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 158: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 159: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 160: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 161: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 162: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 163: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 164: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 165: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 166: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 167: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 168: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 169: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 170: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 172: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 173: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 174: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 175: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 171: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 176: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 177: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 178: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 179: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 180: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 181: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 182: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 183: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 184: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 185: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 186: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 187: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 188: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 189: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 190: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 191: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 192: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 193: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 194: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 195: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 196: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 197: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 198: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 199: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 200: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 201: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 202: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 203: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 204: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 205: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 206: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 207: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 208: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 209: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 210: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 211: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 212: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 213: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 214: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 215: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 216: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255968: AddedToken("[toxicity=0]", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255969: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255970: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255971: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255972: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255973: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255974: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255975: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255976: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255977: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255978: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255979: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255980: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255981: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255982: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255983: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255984: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255985: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255986: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255987: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255988: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255989: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255990: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255991: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255992: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255993: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255994: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255995: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255996: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255997: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255998: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255999: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), +} +)GemmaTokenizerFast(name_or_path='google/gemma-2-2b-it', vocab_size=256000, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='left', special_tokens={'bos_token': '', 'eos_token': '', 'unk_token': '', 'pad_token': '', 'additional_special_tokens': ['', '']}, clean_up_tokenization_spaces=False, added_tokens_decoder={ + 0: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 1: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 2: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 3: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 4: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 5: AddedToken("<2mass>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 6: AddedToken("[@BOS@]", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 7: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 8: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 9: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 10: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 11: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 12: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 13: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 14: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 15: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 16: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 17: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 18: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 19: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 20: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 21: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 22: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 23: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 24: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 25: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 26: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 27: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 28: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 29: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 30: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 31: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 32: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 33: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 34: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 35: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 36: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 37: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 38: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 39: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 40: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 41: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 42: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 43: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 44: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 45: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 46: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 47: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 48: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 49: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 50: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 51: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 52: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 53: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 54: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 55: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 56: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 57: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 58: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 59: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 60: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 61: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 62: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 63: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 64: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 65: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 66: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 67: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 68: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 69: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 70: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 71: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 72: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 73: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 74: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 75: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 76: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 77: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 78: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 79: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 80: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 81: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 82: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 83: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 84: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 85: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 86: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 87: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 88: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 89: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 90: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 91: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 92: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 93: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 94: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 95: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 96: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 97: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 98: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 99: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 100: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 101: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 102: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 103: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 104: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 105: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 106: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 107: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 108: AddedToken(" +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 109: AddedToken(" + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 110: AddedToken(" + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 111: AddedToken(" + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 112: AddedToken(" + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 113: AddedToken(" + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 114: AddedToken(" + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 115: AddedToken(" + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 116: AddedToken(" + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 117: AddedToken(" + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 118: AddedToken(" + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 119: AddedToken(" + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 120: AddedToken(" + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 121: AddedToken(" + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 122: AddedToken(" + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 123: AddedToken(" + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 124: AddedToken(" + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 125: AddedToken(" + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 126: AddedToken(" + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 127: AddedToken(" + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 128: AddedToken(" + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 129: AddedToken(" + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 130: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 131: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 132: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 133: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 134: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 135: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 136: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 137: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 138: AddedToken(" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 139: AddedToken("▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 140: AddedToken("▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 141: AddedToken("▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 142: AddedToken("▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 143: AddedToken("▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 144: AddedToken("▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 145: AddedToken("▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 146: AddedToken("▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 147: AddedToken("▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 148: AddedToken("▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 149: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 150: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 151: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 152: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 153: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 154: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 155: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 156: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 157: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 158: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 159: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 160: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 161: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 162: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 163: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 164: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 165: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 166: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 167: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 168: AddedToken("▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 169: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 170: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 172: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 173: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 174: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 175: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 171: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 176: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 177: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 178: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 179: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 180: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 181: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 182: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 183: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 184: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 185: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 186: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 187: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 188: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 189: AddedToken("

", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 190: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 191: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 192: AddedToken("
", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 193: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 194: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 195: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 196: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 197: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 198: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 199: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 200: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 201: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 202: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 203: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 204: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 205: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 206: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 207: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 208: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 209: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 210: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 211: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 212: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 213: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 214: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 215: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 216: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255968: AddedToken("[toxicity=0]", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255969: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255970: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255971: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255972: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255973: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255974: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255975: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255976: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255977: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255978: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255979: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255980: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255981: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255982: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255983: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255984: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255985: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255986: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255987: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255988: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255989: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255990: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255991: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255992: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255993: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255994: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255995: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255996: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255997: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255998: AddedToken(" ", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), + 255999: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=False), +} +) +2025-07-03 10:08:34 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 10:08:34 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 10:08:35 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 10:08:35 DEBUG: Attempting to acquire lock 22432746895584 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:08:35 DEBUG: Lock 22432746895584 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:08:35 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 10:08:35 DEBUG: Attempting to release lock 22432746895584 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:08:35 DEBUG: Lock 22432746895584 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:08:35 DEBUG: Attempting to acquire lock 22432746892656 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:08:35 DEBUG: Lock 22432746892656 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:08:35 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 10:08:35 DEBUG: Attempting to release lock 22432746892656 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:08:35 DEBUG: Lock 22432746892656 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpzoas0zl6 +2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpfinusham +2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmplwzimi68 +2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp5cwifvch +2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpru97pwia +2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpm_5lsjk6 +2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmptiivg2lg +2025-07-03 10:08:35 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpcqzm9x1g +2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpho0ay9es +2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpnquy7ttc +2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpqaf0qq0e +2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpipy0xquz +2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp_godjfvm +2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp2fyweh9f +2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpgu73efxs +2025-07-03 10:08:43 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpgj89zh35 +2025-07-03 10:08:45 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split... +2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp94fjzt08 +2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmprs9180is +2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpzom3vgt5 +2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmptz_xedub +2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpbpow2msk +2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp8ytrp5an +2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpt5lyfg6j +2025-07-03 10:08:45 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmpq2ak9_ha +2025-07-03 10:09:21 INFO: Tokenizing 1499551 messages... +2025-07-03 10:12:14 DEBUG: open file: /var/tmp/hf_datasets-cq1wo3z4/tmp_f82tqbc +2025-07-03 10:13:08 DEBUG: Attempting to acquire lock 22446846621872 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 10:13:08 DEBUG: Lock 22446846621872 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 10:13:08 DEBUG: Attempting to release lock 22446846621872 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 10:13:08 DEBUG: Lock 22446846621872 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 10:13:08 DEBUG: Attempting to acquire lock 22446846623120 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 10:13:08 DEBUG: Lock 22446846623120 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 10:13:08 DEBUG: Attempting to release lock 22446846623120 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 10:13:08 DEBUG: Lock 22446846623120 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 10:14:27 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 10:14:27 INFO: Starting packing script... +2025-07-03 10:14:27 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 10:14:27 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 10:14:27 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 10:14:29 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 10:14:29 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 10:14:29 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 10:14:29 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 10:14:29 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 10:14:29 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3 +2025-07-03 10:14:29 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 10:14:29 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 10:14:30 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 10:14:30 DEBUG: Attempting to acquire lock 22478710121344 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:14:30 DEBUG: Lock 22478710121344 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:14:30 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 10:14:30 DEBUG: Attempting to release lock 22478710121344 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:14:30 DEBUG: Lock 22478710121344 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:14:30 DEBUG: Attempting to acquire lock 22478710120240 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:14:30 DEBUG: Lock 22478710120240 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:14:30 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 10:14:30 DEBUG: Attempting to release lock 22478710120240 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:14:30 DEBUG: Lock 22478710120240 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpctv11mcw +2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpequ_8z7h +2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp4tckogmt +2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpofu2jvxr +2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpzc5pm5p4 +2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp0mwsm8d2 +2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp10hz0psh +2025-07-03 10:14:30 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpo2ry1ela +2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpgmq_evz7 +2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpkkc97a11 +2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpwk8wx3qr +2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpalg7n6db +2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp6uze3tom +2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp93zj2oyq +2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpm3qxwbpb +2025-07-03 10:14:39 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpygk3d8kc +2025-07-03 10:14:40 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split... +2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmprzy2uuh6 +2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp6i70tn2k +2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpsc14yk28 +2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpoze7h04q +2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpl58x9ij3 +2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpc3u4cro_ +2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp95wrned0 +2025-07-03 10:14:40 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpb2pmdl4j +2025-07-03 10:15:16 INFO: Tokenizing 1499551 messages... +2025-07-03 10:18:05 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpbxaduedp +2025-07-03 10:20:16 INFO: Tokenizing 1499355 messages... +2025-07-03 10:24:37 INFO: Tokenizing 641765 messages... +2025-07-03 10:26:32 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpnaork6vj +2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpx9lka_d0 +2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpycqswfp0 +2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpxq16nflq +2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpsg9fv_g5 +2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpdkfzk1tz +2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpmr27xqzq +2025-07-03 10:26:35 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmpcp6di0wa +2025-07-03 10:29:04 DEBUG: open file: /var/tmp/hf_datasets-m6dw7r3g/tmp2mbztoy9 +2025-07-03 10:29:21 DEBUG: Attempting to acquire lock 22492809575296 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 10:29:21 DEBUG: Lock 22492809575296 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 10:29:21 DEBUG: Attempting to release lock 22492809575296 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 10:29:21 DEBUG: Lock 22492809575296 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 10:29:21 DEBUG: Attempting to acquire lock 22492809574576 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 10:29:21 DEBUG: Lock 22492809574576 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 10:29:21 DEBUG: Attempting to release lock 22492809574576 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 10:29:21 DEBUG: Lock 22492809574576 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 10:50:36 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 10:50:36 INFO: Starting packing script... +2025-07-03 10:50:36 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 10:50:36 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 10:50:36 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 10:50:38 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 10:50:38 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 10:50:38 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 10:50:38 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 10:50:38 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 10:50:38 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3 +2025-07-03 10:50:38 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 10:50:38 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 10:50:39 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 10:50:39 DEBUG: Attempting to acquire lock 22684547950832 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:50:39 DEBUG: Lock 22684547950832 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:50:39 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 10:50:39 DEBUG: Attempting to release lock 22684547950832 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:50:39 DEBUG: Lock 22684547950832 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 10:50:39 DEBUG: Attempting to acquire lock 22684547947280 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:50:39 DEBUG: Lock 22684547947280 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:50:39 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 10:50:39 DEBUG: Attempting to release lock 22684547947280 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:50:39 DEBUG: Lock 22684547947280 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpmv4ehy89 +2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmppkqcnfcq +2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp3wgvbrlq +2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpl3c1jz16 +2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmps63_85v8 +2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpbe55sax2 +2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpxlh1mbir +2025-07-03 10:50:39 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpxdp9hkaa +2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpbg2o_p5d +2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpe7w6ky_9 +2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpd1n7mexm +2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpttuwa6tc +2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpleql5td5 +2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmppmix2r6k +2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpuvvw86dr +2025-07-03 10:50:48 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp3qhazjh5 +2025-07-03 10:50:49 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split... +2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp5c7umr8l +2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp9hxunkv3 +2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpuow8wxzz +2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpb_fbepjc +2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpe41pr1lu +2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpzyo5rla9 +2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpzvlv57r7 +2025-07-03 10:50:49 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpbutjlxxl +2025-07-03 10:51:25 INFO: Tokenizing 1499551 messages... +2025-07-03 10:54:24 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpqyh4h5n4 +2025-07-03 10:56:36 INFO: Tokenizing 1499355 messages... +2025-07-03 11:00:51 INFO: Tokenizing 641765 messages... +2025-07-03 11:02:37 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpn07zwasm +2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpik5enaxv +2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpr877c841 +2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmptw93t2ov +2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp1ploazhg +2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp4to_8gh6 +2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpowqshns2 +2025-07-03 11:02:40 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmpd3nlvnuf +2025-07-03 11:05:08 DEBUG: open file: /var/tmp/hf_datasets-3ztr12n3/tmp_7ttid7k +2025-07-03 11:05:19 DEBUG: Attempting to acquire lock 22698589453184 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:05:19 DEBUG: Lock 22698589453184 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:05:19 DEBUG: Attempting to release lock 22698589453184 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:05:19 DEBUG: Lock 22698589453184 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:05:19 DEBUG: Attempting to acquire lock 22698589452464 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:05:20 DEBUG: Lock 22698589452464 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:05:20 DEBUG: Attempting to release lock 22698589452464 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:05:20 DEBUG: Lock 22698589452464 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:10:10 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 11:10:10 INFO: Starting packing script... +2025-07-03 11:10:10 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 11:10:10 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 11:10:11 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 11:10:12 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 11:10:12 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 11:10:13 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 11:10:13 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:10:13 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 11:10:13 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59 +2025-07-03 11:10:13 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:10:13 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 11:10:13 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 11:10:13 DEBUG: Attempting to acquire lock 22683137782880 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:10:13 DEBUG: Lock 22683137782880 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:10:13 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:10:13 DEBUG: Attempting to release lock 22683137782880 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:10:13 DEBUG: Lock 22683137782880 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:10:13 DEBUG: Attempting to acquire lock 22683137784896 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:10:13 DEBUG: Lock 22683137784896 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:10:13 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:10:13 DEBUG: Attempting to release lock 22683137784896 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:10:13 DEBUG: Lock 22683137784896 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmplqhp3tdc +2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpjd70q98h +2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmprctpi541 +2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp6pngkjtu +2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp6759ubna +2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp7n6ktlxp +2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpa2zcwvd_ +2025-07-03 11:10:13 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpqunuzogk +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmphbj_y95p +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpxy7oy1yn +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp7_ibg2z3 +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpvtwgcngf +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpywxi57z6 +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpiv59p0ag +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmptuaq784j +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpfi9_nrmk +2025-07-03 11:10:14 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split... +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpfjlodwq0 +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp9h8owzhz +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpxc586awi +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpvckodjj3 +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpwrspo0ex +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpxk994zcr +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpi1lw4894 +2025-07-03 11:10:14 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpce7t2fde +2025-07-03 11:10:17 INFO: Tokenizing 149943 messages... +2025-07-03 11:10:33 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp_4l3tj0b +2025-07-03 11:10:46 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp88m1h336 +2025-07-03 11:10:48 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp9h2cb1fu +2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpot41f_f2 +2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp39vb7o54 +2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp7a9xbrb_ +2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpx5q40t11 +2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpmxsmtjcs +2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpkaqkvel6 +2025-07-03 11:10:53 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmprnaz0x52 +2025-07-03 11:10:58 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpck4im9p9 +2025-07-03 11:11:28 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmptkc2cdu3 +2025-07-03 11:11:45 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpiq5wwgkp +2025-07-03 11:12:22 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp5ae0eci5 +2025-07-03 11:12:47 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpztb00fq4 +2025-07-03 11:12:47 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18742 +# Packed samples: 323 +Avg inp packing efficiency: 0.869 +Avg ctx packing efficiency: 0.978 + +Input IDs length stats: + + Avg: 7120.9, Std: 769.4, Min: 4961, Max: 8191 +Context IDs length stats: + + Avg: 16027.0, Std: 590.6, Min: 12720, Max: 16384 +2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 323 +Avg inp packing efficiency: 0.864 +Avg ctx packing efficiency: 0.980 + +Input IDs length stats: + + Avg: 7077.8, Std: 834.6, Min: 1476, Max: 8188 +Context IDs length stats: + + Avg: 16055.7, Std: 856.4, Min: 3077, Max: 16380 +2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 322 +Avg inp packing efficiency: 0.869 +Avg ctx packing efficiency: 0.979 + +Input IDs length stats: + + Avg: 7122.3, Std: 794.3, Min: 710, Max: 8192 +Context IDs length stats: + + Avg: 16037.5, Std: 938.3, Min: 1130, Max: 16384 +2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 322 +Avg inp packing efficiency: 0.857 +Avg ctx packing efficiency: 0.984 + +Input IDs length stats: + + Avg: 7022.5, Std: 759.3, Min: 3561, Max: 8187 +Context IDs length stats: + + Avg: 16118.7, Std: 570.1, Min: 8611, Max: 16384 +2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 323 +Avg inp packing efficiency: 0.866 +Avg ctx packing efficiency: 0.981 + +Input IDs length stats: + + Avg: 7095.2, Std: 770.7, Min: 4684, Max: 8190 +Context IDs length stats: + + Avg: 16064.8, Std: 580.0, Min: 10728, Max: 16384 +2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 322 +Avg inp packing efficiency: 0.869 +Avg ctx packing efficiency: 0.980 + +Input IDs length stats: + + Avg: 7116.9, Std: 800.4, Min: 1397, Max: 8189 +Context IDs length stats: + + Avg: 16059.5, Std: 815.3, Min: 4429, Max: 16384 +2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 322 +Avg inp packing efficiency: 0.862 +Avg ctx packing efficiency: 0.983 + +Input IDs length stats: + + Avg: 7058.2, Std: 772.2, Min: 4721, Max: 8192 +Context IDs length stats: + + Avg: 16097.6, Std: 486.8, Min: 12416, Max: 16384 +2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpotlzqo0m +2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpvjxl4xqo +2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpklqpwy05 +2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmp0w7rdusn +2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpiit8bd4n +2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpziwhqw2d +2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpcfiekslg +2025-07-03 11:12:54 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 323 +Avg inp packing efficiency: 0.865 +Avg ctx packing efficiency: 0.979 + +Input IDs length stats: + + Avg: 7084.8, Std: 832.0, Min: 960, Max: 8188 +Context IDs length stats: + + Avg: 16044.9, Std: 914.8, Min: 2392, Max: 16384 +2025-07-03 11:12:54 DEBUG: open file: /var/tmp/hf_datasets-mv9x78it/tmpj_i6ra_3 +2025-07-03 11:12:58 DEBUG: Attempting to acquire lock 22697169681280 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:12:58 DEBUG: Lock 22697169681280 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:12:58 DEBUG: Attempting to release lock 22697169681280 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:12:58 DEBUG: Lock 22697169681280 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:12:58 DEBUG: Attempting to acquire lock 22697169680560 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:12:58 DEBUG: Lock 22697169680560 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:12:58 DEBUG: Attempting to release lock 22697169680560 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:12:58 DEBUG: Lock 22697169680560 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:18:05 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 11:18:05 INFO: Starting packing script... +2025-07-03 11:18:05 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 11:18:05 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 11:18:05 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 11:18:06 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 11:18:07 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 11:18:07 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 11:18:07 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:18:07 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 11:18:07 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59 +2025-07-03 11:18:07 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:18:07 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 11:18:08 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 11:18:08 DEBUG: Attempting to acquire lock 22346090740848 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:18:08 DEBUG: Lock 22346090740848 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:18:08 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:18:08 DEBUG: Attempting to release lock 22346090740848 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:18:08 DEBUG: Lock 22346090740848 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:18:08 DEBUG: Attempting to acquire lock 22346090735280 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:18:08 DEBUG: Lock 22346090735280 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:18:08 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:18:08 DEBUG: Attempting to release lock 22346090735280 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:18:08 DEBUG: Lock 22346090735280 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp8g5era_n +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp714klyhm +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpf9voq0kk +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp062oeaex +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp8d_378dh +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpnkufhl8b +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpp6s1ma4r +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpawz_qg_b +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmppih93v5e +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp0exum433 +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpr6hwzfol +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpric2i8_v +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpbo8_439a +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpjipxs939 +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp7p_bvpmo +2025-07-03 11:18:08 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp2bi1y45r +2025-07-03 11:18:08 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split... +2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpo0utbnxr +2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpuc8v_avj +2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp4gbd5kzl +2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp_ismds3o +2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp9ucr287i +2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpjuzjhbnr +2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpj8h9ggpw +2025-07-03 11:18:09 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpfyxsh1zr +2025-07-03 11:18:09 DEBUG: Tokenizing inputs +2025-07-03 11:18:11 INFO: Tokenizing 149943 messages... +2025-07-03 11:18:28 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp5dbhiws5 +2025-07-03 11:18:39 DEBUG: Tokenizing context +2025-07-03 11:18:40 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpfzoaoh52 +2025-07-03 11:18:43 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmps01yu83_ +2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpi2xcwdqf +2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpivudat_j +2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpn3g8rgsz +2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmprsx8l7s9 +2025-07-03 11:18:47 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpx8ddazv_ +2025-07-03 11:18:48 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpae9325e6 +2025-07-03 11:18:48 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpu_fy6yaj +2025-07-03 11:18:52 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpjluh649p +2025-07-03 11:19:23 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmph97o06mk +2025-07-03 11:19:40 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp8kantouc +2025-07-03 11:20:17 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpxomh3ra5 +2025-07-03 11:20:43 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp42on9wrn +2025-07-03 11:20:43 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 321 +Avg inp packing efficiency: 0.867 +Avg ctx packing efficiency: 0.984 + +Input IDs length stats: + + Avg: 7102.6, Std: 727.7, Min: 4866, Max: 8190 +Context IDs length stats: + + Avg: 16115.7, Std: 432.8, Min: 12817, Max: 16383 +2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 321 +Avg inp packing efficiency: 0.870 +Avg ctx packing efficiency: 0.980 + +Input IDs length stats: + + Avg: 7127.7, Std: 754.4, Min: 4555, Max: 8186 +Context IDs length stats: + + Avg: 16062.2, Std: 638.8, Min: 8993, Max: 16382 +2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 322 +Avg inp packing efficiency: 0.864 +Avg ctx packing efficiency: 0.983 + +Input IDs length stats: + + Avg: 7077.3, Std: 715.4, Min: 5048, Max: 8190 +Context IDs length stats: + + Avg: 16113.1, Std: 448.4, Min: 12101, Max: 16384 +2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 323 +Avg inp packing efficiency: 0.857 +Avg ctx packing efficiency: 0.978 + +Input IDs length stats: + + Avg: 7019.1, Std: 827.2, Min: 2735, Max: 8192 +Context IDs length stats: + + Avg: 16030.1, Std: 781.3, Min: 5173, Max: 16383 +2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 323 +Avg inp packing efficiency: 0.863 +Avg ctx packing efficiency: 0.981 + +Input IDs length stats: + + Avg: 7065.7, Std: 813.1, Min: 1161, Max: 8191 +Context IDs length stats: + + Avg: 16075.7, Std: 873.4, Min: 2308, Max: 16384 +2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpfp4hgkwb +2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 322 +Avg inp packing efficiency: 0.868 +Avg ctx packing efficiency: 0.984 + +Input IDs length stats: + + Avg: 7106.7, Std: 728.3, Min: 5117, Max: 8188 +Context IDs length stats: + + Avg: 16113.9, Std: 541.2, Min: 10593, Max: 16384 +2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp9le6uwja +2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18743 +# Packed samples: 323 +Avg inp packing efficiency: 0.862 +Avg ctx packing efficiency: 0.981 + +Input IDs length stats: + + Avg: 7058.1, Std: 817.0, Min: 302, Max: 8191 +Context IDs length stats: + + Avg: 16066.2, Std: 894.7, Min: 1761, Max: 16384 +2025-07-03 11:20:50 DEBUG: Packing stats - Original samples: 18742 +# Packed samples: 323 +Avg inp packing efficiency: 0.877 +Avg ctx packing efficiency: 0.978 + +Input IDs length stats: + + Avg: 7185.8, Std: 770.3, Min: 3765, Max: 8191 +Context IDs length stats: + + Avg: 16028.5, Std: 710.8, Min: 7969, Max: 16384 +2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpuwzi6dvu +2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpak0dghww +2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpnbj_jby2 +2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpfesj1hsi +2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmpyume5l4e +2025-07-03 11:20:50 DEBUG: open file: /var/tmp/hf_datasets-j3p78bkx/tmp3d_k3cud +2025-07-03 11:20:53 DEBUG: Attempting to acquire lock 22360189877120 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:20:53 DEBUG: Lock 22360189877120 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:20:53 DEBUG: Attempting to release lock 22360189877120 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:20:53 DEBUG: Lock 22360189877120 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:20:53 DEBUG: Attempting to acquire lock 22360189876400 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:20:53 DEBUG: Lock 22360189876400 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:20:53 DEBUG: Attempting to release lock 22360189876400 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:20:53 DEBUG: Lock 22360189876400 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:23:45 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 11:23:45 INFO: Starting packing script... +2025-07-03 11:23:45 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 11:23:45 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 11:23:45 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 11:23:46 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 11:23:47 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 11:23:47 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 11:23:47 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:23:47 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 11:23:47 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59 +2025-07-03 11:23:47 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:23:47 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 11:23:48 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 11:23:48 DEBUG: Attempting to acquire lock 23209466157504 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:23:48 DEBUG: Lock 23209466157504 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:23:48 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:23:48 DEBUG: Attempting to release lock 23209466157504 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:23:48 DEBUG: Lock 23209466157504 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:23:48 DEBUG: Attempting to acquire lock 23209466156592 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:23:48 DEBUG: Lock 23209466156592 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:23:48 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:23:48 DEBUG: Attempting to release lock 23209466156592 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:23:48 DEBUG: Lock 23209466156592 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpfec7vl4l +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpysiy_p01 +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpbn_a1b62 +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpj6hgwiug +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpgt68oo3e +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpfhbs6tm1 +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp90i_36e6 +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpsryi9dhz +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmptd1ynpn5 +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmph4gy6_dd +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpf_cwbp90 +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmppkh_biyj +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpt_h8xham +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpeqljf18i +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpibzkd2ix +2025-07-03 11:23:48 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpo2041sis +2025-07-03 11:23:49 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split... +2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpgmmgkser +2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpk288x_lr +2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpuc9v6pu2 +2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp4s34rjkd +2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpz6ea_s79 +2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp1xrgrf6i +2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpxlxmzg_y +2025-07-03 11:23:49 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmphhykjz0s +2025-07-03 11:23:49 DEBUG: Tokenizing inputs +2025-07-03 11:23:52 INFO: Tokenizing 149943 messages... +2025-07-03 11:24:08 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmptbba2t_2 +2025-07-03 11:24:18 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpog17x3g4 +2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp4ey34hdk +2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpnk_ua7cz +2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpxcc6hmci +2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp88f6s6fc +2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpa8l78q6p +2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpl1i3283r +2025-07-03 11:24:19 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpmmj1jo7a +2025-07-03 11:24:25 DEBUG: Tokenizing context +2025-07-03 11:24:27 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmptbw3it3a +2025-07-03 11:24:29 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmphfkyd6sv +2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmprnu0y73f +2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp0b336u8q +2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp4sal5bqz +2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpber1v2pi +2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmptejpz18j +2025-07-03 11:24:34 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmpufj1vbf3 +2025-07-03 11:24:35 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp42cex_yj +2025-07-03 11:24:39 DEBUG: open file: /var/tmp/hf_datasets-9c7ppc1i/tmp0k5sjd0z +2025-07-03 11:25:10 DEBUG: Attempting to acquire lock 23223565334704 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:25:10 DEBUG: Lock 23223565334704 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:25:10 DEBUG: Attempting to release lock 23223565334704 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:25:10 DEBUG: Lock 23223565334704 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:25:10 DEBUG: Attempting to acquire lock 23223565335952 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:25:10 DEBUG: Lock 23223565335952 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:25:10 DEBUG: Attempting to release lock 23223565335952 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:25:10 DEBUG: Lock 23223565335952 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:25:26 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 11:25:26 INFO: Starting packing script... +2025-07-03 11:25:26 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 11:25:26 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 11:25:26 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 11:25:27 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 11:25:27 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 11:25:28 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 11:25:28 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:25:28 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 11:25:28 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59 +2025-07-03 11:25:28 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:25:28 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 11:25:29 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 11:25:29 DEBUG: Attempting to acquire lock 22885911361136 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:25:29 DEBUG: Lock 22885911361136 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:25:29 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:25:29 DEBUG: Attempting to release lock 22885911361136 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:25:29 DEBUG: Lock 22885911361136 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:25:29 DEBUG: Attempting to acquire lock 22885911358880 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:25:29 DEBUG: Lock 22885911358880 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:25:29 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:25:29 DEBUG: Attempting to release lock 22885911358880 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:25:29 DEBUG: Lock 22885911358880 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpy5fwoz50 +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpbrdl5yn9 +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpe3d_frcd +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpxnnn6fep +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp9km1e6nh +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp6pdnoa5u +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmprtvlzxai +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpsqojgc96 +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpcxr9l95w +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpa3ldk1_a +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpotvktm_l +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpns59o4cb +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpgd9a03ph +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpr8reidgr +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpas44yp08 +2025-07-03 11:25:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpqnqj9q_3 +2025-07-03 11:25:30 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split... +2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpugq344fy +2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpm5u4x1ya +2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpxbmgzcpx +2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmprf0sohhb +2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpp2dqadfv +2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmppp99qmln +2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp3hnry9yr +2025-07-03 11:25:30 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp_m7w_l1m +2025-07-03 11:25:30 DEBUG: Tokenizing inputs +2025-07-03 11:25:32 INFO: Tokenizing 149943 messages... +2025-07-03 11:25:48 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpq507t7y7 +2025-07-03 11:25:59 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpzqsm7c1f +2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpywr6xcho +2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpldf6bsqr +2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmphr9728b1 +2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp8tk2oetl +2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpkydnfjya +2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpn32ja7yo +2025-07-03 11:26:00 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp47z6fa1l +2025-07-03 11:26:06 DEBUG: Tokenizing context +2025-07-03 11:26:08 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp3c8wg3li +2025-07-03 11:26:10 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp7b92vgqf +2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpljenzlqz +2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpfw_j_cam +2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpzjzi4gf2 +2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpn17kcfan +2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpn0bo6r19 +2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpzb3qin2z +2025-07-03 11:26:14 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp4pesbzlz +2025-07-03 11:26:15 DEBUG: Unpacking data +2025-07-03 11:26:15 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmps7fx04bu +2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpxud5axcz +2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp8a4l7orl +2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp2nb8htq9 +2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpt5odwoy5 +2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp4ofpmull +2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp674xd6j9 +2025-07-03 11:26:16 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpp_vgnz1a +2025-07-03 11:26:22 DEBUG: Split too long QAs with max length 2048 +2025-07-03 11:26:22 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:26:27 DEBUG: Longest old qas len: 12709 +2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048 +2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmplizfckpl +2025-07-03 11:26:27 DEBUG: Longest old qas len: 7701 +2025-07-03 11:26:27 DEBUG: Longest new qas len: 2046 +2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpquc67_9k +2025-07-03 11:26:27 DEBUG: Longest old qas len: 7088 +2025-07-03 11:26:27 DEBUG: Longest new qas len: 2047 +2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp3zm_5gtg +2025-07-03 11:26:27 DEBUG: Longest old qas len: 10478 +2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048 +2025-07-03 11:26:27 DEBUG: Longest old qas len: 8086 +2025-07-03 11:26:27 DEBUG: Longest old qas len: 10726 +2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpme8uky3f +2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048 +2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048 +2025-07-03 11:26:27 DEBUG: Longest old qas len: 8052 +2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpr8qwayr3 +2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmptf17s1gl +2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048 +2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpspuxwo33 +2025-07-03 11:26:27 DEBUG: Longest old qas len: 9002 +2025-07-03 11:26:27 DEBUG: Longest new qas len: 2048 +2025-07-03 11:26:27 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpi4jw6nia +2025-07-03 11:26:29 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpru_vctzw +2025-07-03 11:26:29 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 305 +Avg inp packing efficiency: 0.899 +Avg ctx packing efficiency: 0.091 + +Input IDs length stats: + + Avg: 7361.2, Std: 520.3, Min: 6151, Max: 8187 +Context IDs length stats: + + Avg: 1490.2, Std: 352.2, Min: 739, Max: 2553 +2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 306 +Avg inp packing efficiency: 0.901 +Avg ctx packing efficiency: 0.091 + +Input IDs length stats: + + Avg: 7377.6, Std: 537.4, Min: 5115, Max: 8189 +Context IDs length stats: + + Avg: 1487.4, Std: 366.0, Min: 743, Max: 2707 +2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1654 +# Packed samples: 309 +Avg inp packing efficiency: 0.906 +Avg ctx packing efficiency: 0.089 + +Input IDs length stats: + + Avg: 7418.8, Std: 603.6, Min: 1933, Max: 8183 +Context IDs length stats: + + Avg: 1461.0, Std: 365.0, Min: 247, Max: 2605 +2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpxnd8oqqb +2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1654 +# Packed samples: 308 +Avg inp packing efficiency: 0.903 +Avg ctx packing efficiency: 0.089 + +Input IDs length stats: + + Avg: 7401.4, Std: 597.8, Min: 2003, Max: 8192 +Context IDs length stats: + + Avg: 1453.7, Std: 352.5, Min: 361, Max: 2671 +2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 312 +Avg inp packing efficiency: 0.900 +Avg ctx packing efficiency: 0.088 + +Input IDs length stats: + + Avg: 7372.1, Std: 593.4, Min: 1845, Max: 8190 +Context IDs length stats: + + Avg: 1437.9, Std: 349.4, Min: 298, Max: 2735 +2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 305 +Avg inp packing efficiency: 0.904 +Avg ctx packing efficiency: 0.089 + +Input IDs length stats: + + Avg: 7402.1, Std: 578.6, Min: 3633, Max: 8191 +Context IDs length stats: + + Avg: 1465.7, Std: 334.6, Min: 720, Max: 2465 +2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpmfgpy69q +2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp3v0a04vn +2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 318 +Avg inp packing efficiency: 0.901 +Avg ctx packing efficiency: 0.086 + +Input IDs length stats: + + Avg: 7378.8, Std: 554.2, Min: 4579, Max: 8191 +Context IDs length stats: + + Avg: 1414.7, Std: 336.9, Min: 737, Max: 2537 +2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpg8ekl0dk +2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpriqbx70d +2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmp4cexfgee +2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpt_xgct68 +2025-07-03 11:26:33 DEBUG: Packing stats - Original samples: 1654 +# Packed samples: 312 +Avg inp packing efficiency: 0.902 +Avg ctx packing efficiency: 0.089 + +Input IDs length stats: + + Avg: 7392.3, Std: 595.2, Min: 1850, Max: 8192 +Context IDs length stats: + + Avg: 1450.7, Std: 367.5, Min: 150, Max: 3108 +2025-07-03 11:26:33 DEBUG: open file: /var/tmp/hf_datasets-01osio7t/tmpb1qfrx_i +2025-07-03 11:26:35 DEBUG: Attempting to acquire lock 22900010351488 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:26:35 DEBUG: Lock 22900010351488 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:26:35 DEBUG: Attempting to release lock 22900010351488 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:26:35 DEBUG: Lock 22900010351488 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:26:35 DEBUG: Attempting to acquire lock 22900010350768 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:26:35 DEBUG: Lock 22900010350768 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:26:35 DEBUG: Attempting to release lock 22900010350768 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:26:35 DEBUG: Lock 22900010350768 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:33:14 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 11:33:14 INFO: Starting packing script... +2025-07-03 11:33:14 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 11:33:14 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 11:33:14 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 11:33:15 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 11:33:15 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 11:33:16 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 11:33:16 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:33:16 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train[:10000]", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 11:33:16 DEBUG: Dataset hash: ba6ac9db96f5f958bcea13130eef67a1d1a337ae6ef4d475f1a77504adea0c59 +2025-07-03 11:33:16 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train[:10000]... +2025-07-03 11:33:16 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 11:33:16 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 11:33:16 DEBUG: Attempting to acquire lock 23413206015936 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:33:16 DEBUG: Lock 23413206015936 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:33:16 DEBUG: Attempting to release lock 23413206015936 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:33:16 DEBUG: Lock 23413206015936 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:33:16 DEBUG: Attempting to acquire lock 23413206015840 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:33:16 DEBUG: Lock 23413206015840 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:33:16 DEBUG: Attempting to release lock 23413206015840 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:33:16 DEBUG: Lock 23413206015840 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp3iow_g97 +2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp4acu9dnm +2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpw806zxch +2025-07-03 11:33:16 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpa6sp_k1b +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp8omef5zz +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmptzeftfow +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp99_jyd_3 +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp_6aop2jd +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpo85hrh2_ +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpgwlhkeef +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpnd0lmu6j +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpb7vkkppc +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp7_n4eq37 +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmplrlyd652 +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpc6btfw7a +2025-07-03 11:33:17 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpskd9u5p_ +2025-07-03 11:33:18 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train[:10000] split... +2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpbaks5ics +2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp9la0c5af +2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpb_1cx56l +2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpv1irw8qw +2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp2_frssya +2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpft84b0js +2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpymr73n8_ +2025-07-03 11:33:18 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpwwvivbog +2025-07-03 11:33:18 DEBUG: Tokenizing inputs +2025-07-03 11:33:22 INFO: Tokenizing 149943 messages... +2025-07-03 11:33:38 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp_2p6th06 +2025-07-03 11:33:50 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:33:50 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmppfao83jd +2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp0_9qxggx +2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp0d4a_n7j +2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp6spvygl_ +2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpuqurh2hq +2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpohp_srcq +2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmptbmao1iz +2025-07-03 11:33:51 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpvy2ko277 +2025-07-03 11:33:57 DEBUG: Tokenizing context +2025-07-03 11:33:58 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpwe3e38xz +2025-07-03 11:34:06 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpm923uoci +2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmprjrxv5u9 +2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpiv4pycn9 +2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp2uf925qv +2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp2_yi8b47 +2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp6iqs7a5x +2025-07-03 11:34:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpq3cubq4i +2025-07-03 11:34:11 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp61crqj99 +2025-07-03 11:34:12 DEBUG: Unpacking data +2025-07-03 11:34:12 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpymuc2336 +2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpadowp4en +2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpl899did7 +2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp7s5m7k70 +2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpmm7wmug_ +2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpzfew9k6i +2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp5kt6v9rd +2025-07-03 11:34:12 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpm94cswgg +2025-07-03 11:34:18 DEBUG: Split too long QAs with max length 2048 +2025-07-03 11:34:18 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:34:23 DEBUG: Longest old qas len: 7088 +2025-07-03 11:34:23 DEBUG: Longest new qas len: 2047 +2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp2tqrekv5 +2025-07-03 11:34:23 DEBUG: Longest old qas len: 7701 +2025-07-03 11:34:23 DEBUG: Longest new qas len: 2046 +2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpxr1nve5w +2025-07-03 11:34:23 DEBUG: Longest old qas len: 8052 +2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048 +2025-07-03 11:34:23 DEBUG: Longest old qas len: 8086 +2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048 +2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpotwjfepd +2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpfjk164jy +2025-07-03 11:34:23 DEBUG: Longest old qas len: 12709 +2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048 +2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp6a7vbi_3 +2025-07-03 11:34:23 DEBUG: Longest old qas len: 9002 +2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048 +2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpuqhczdye +2025-07-03 11:34:23 DEBUG: Longest old qas len: 10726 +2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048 +2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpox70t9za +2025-07-03 11:34:23 DEBUG: Longest old qas len: 10478 +2025-07-03 11:34:23 DEBUG: Longest new qas len: 2048 +2025-07-03 11:34:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpl3chauqt +2025-07-03 11:34:25 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpur8k2bxl +2025-07-03 11:34:25 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 312 +Avg inp packing efficiency: 0.895 +Avg ctx packing efficiency: 0.090 + +Input IDs length stats: + + Avg: 7329.7, Std: 575.8, Min: 1937, Max: 8182 +Context IDs length stats: + + Avg: 1473.5, Std: 391.4, Min: 269, Max: 2746 +2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 304 +Avg inp packing efficiency: 0.903 +Avg ctx packing efficiency: 0.090 + +Input IDs length stats: + + Avg: 7399.6, Std: 596.4, Min: 1741, Max: 8187 +Context IDs length stats: + + Avg: 1477.2, Std: 364.4, Min: 77, Max: 2929 +2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 309 +Avg inp packing efficiency: 0.901 +Avg ctx packing efficiency: 0.090 + +Input IDs length stats: + + Avg: 7382.4, Std: 495.5, Min: 6182, Max: 8190 +Context IDs length stats: + + Avg: 1473.0, Std: 358.3, Min: 511, Max: 2732 +2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1654 +# Packed samples: 306 +Avg inp packing efficiency: 0.903 +Avg ctx packing efficiency: 0.090 + +Input IDs length stats: + + Avg: 7399.3, Std: 543.3, Min: 5268, Max: 8190 +Context IDs length stats: + + Avg: 1471.0, Std: 352.9, Min: 761, Max: 2731 +2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 307 +Avg inp packing efficiency: 0.906 +Avg ctx packing efficiency: 0.090 + +Input IDs length stats: + + Avg: 7424.7, Std: 593.1, Min: 2040, Max: 8189 +Context IDs length stats: + + Avg: 1468.3, Std: 377.4, Min: 657, Max: 2476 +2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1655 +# Packed samples: 313 +Avg inp packing efficiency: 0.905 +Avg ctx packing efficiency: 0.088 + +Input IDs length stats: + + Avg: 7412.2, Std: 480.0, Min: 6258, Max: 8184 +Context IDs length stats: + + Avg: 1440.5, Std: 333.8, Min: 713, Max: 2496 +2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp_8ng95dx +2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpn8vn3gl6 +2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmp6b36952_ +2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1654 +# Packed samples: 310 +Avg inp packing efficiency: 0.910 +Avg ctx packing efficiency: 0.087 + +Input IDs length stats: + + Avg: 7452.5, Std: 543.1, Min: 5253, Max: 8192 +Context IDs length stats: + + Avg: 1433.4, Std: 345.5, Min: 697, Max: 2485 +2025-07-03 11:34:29 DEBUG: Packing stats - Original samples: 1654 +# Packed samples: 310 +Avg inp packing efficiency: 0.903 +Avg ctx packing efficiency: 0.088 + +Input IDs length stats: + + Avg: 7399.8, Std: 632.4, Min: 1274, Max: 8192 +Context IDs length stats: + + Avg: 1441.6, Std: 343.1, Min: 637, Max: 2813 +2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpddree1qo +2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpk7c7dq8t +2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpmyru39ll +2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpkzz3opiy +2025-07-03 11:34:29 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-7peiur11/tmpwfptlbzo +2025-07-03 11:34:31 DEBUG: Attempting to acquire lock 23427306306384 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:34:31 DEBUG: Lock 23427306306384 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:34:31 DEBUG: Attempting to release lock 23427306306384 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:34:31 DEBUG: Lock 23427306306384 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:34:31 DEBUG: Attempting to acquire lock 23427306305664 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:34:31 DEBUG: Lock 23427306305664 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:34:31 DEBUG: Attempting to release lock 23427306305664 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:34:31 DEBUG: Lock 23427306305664 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:35:18 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 11:35:18 INFO: Starting packing script... +2025-07-03 11:35:18 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 11:35:18 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 11:35:18 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 11:35:19 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 11:35:20 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 11:35:20 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 11:35:20 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 11:35:20 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 11:35:20 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3 +2025-07-03 11:35:20 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 11:35:20 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 11:35:21 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 11:35:21 DEBUG: Attempting to acquire lock 22909864697312 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:35:21 DEBUG: Lock 22909864697312 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:35:21 DEBUG: Attempting to release lock 22909864697312 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:35:21 DEBUG: Lock 22909864697312 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 11:35:21 DEBUG: Attempting to acquire lock 22909864701008 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:35:21 DEBUG: Lock 22909864701008 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 11:35:21 DEBUG: Attempting to release lock 22909864701008 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:35:21 DEBUG: Lock 22909864701008 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpg6t5m5rz +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmphunzvaq4 +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpd1_f9lw2 +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpr9vqkc81 +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpucggpm9g +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpu_27oix1 +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpgtp6vv1q +2025-07-03 11:35:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpqzov2ae1 +2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpy_8_oizg +2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp9ww_zy_q +2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmppypu3w71 +2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp5eq7w7zo +2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp1rira0hc +2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmph1_ybwe1 +2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp4zpomupi +2025-07-03 11:35:27 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmphpzj44b6 +2025-07-03 11:35:32 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split... +2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpyrol6uvi +2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpsh9k69if +2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpuwd4_eoh +2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpqkf91a81 +2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpxt3tjg2i +2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpoa8b2pbm +2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpj95ctlj6 +2025-07-03 11:35:32 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpb6u_879q +2025-07-03 11:35:43 DEBUG: Tokenizing inputs +2025-07-03 11:36:10 INFO: Tokenizing 1499551 messages... +2025-07-03 11:39:01 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmplkcsgio9 +2025-07-03 11:41:30 INFO: Tokenizing 1499355 messages... +2025-07-03 11:46:10 INFO: Tokenizing 641765 messages... +2025-07-03 11:48:06 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpbe2pu0zt +2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp604ef9nv +2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpnauk7vkk +2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmphnhqefnt +2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpfwassucc +2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpmr2kdbwi +2025-07-03 11:48:09 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpommhop29 +2025-07-03 11:48:10 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp7rlckzvw +2025-07-03 11:50:27 DEBUG: Tokenizing context +2025-07-03 11:50:41 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp7fnpnabo +2025-07-03 11:53:14 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:53:20 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp53rf3qfz +2025-07-03 11:53:20 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp__a4ilv6 +2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmprpdjdszh +2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmppdi34mys +2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpc_p6mol0 +2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp254gcs6r +2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmppdzxvk0g +2025-07-03 11:53:21 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp021gol7o +2025-07-03 11:55:21 DEBUG: Unpacking data +2025-07-03 11:55:21 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpw2g7rzn9 +2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpxgpt4kjo +2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpttv63g6r +2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpdqq8c4an +2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp4x6j4ub5 +2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp13syypyu +2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmputn3kwu7 +2025-07-03 11:55:23 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpio252e_2 +2025-07-03 11:57:35 DEBUG: Split too long QAs with max length 2048 +2025-07-03 11:57:35 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 11:59:35 DEBUG: Longest old qas len: 12226 +2025-07-03 11:59:35 DEBUG: Longest new qas len: 2048 +2025-07-03 11:59:35 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpej90z4gx +2025-07-03 11:59:36 DEBUG: Longest old qas len: 12527 +2025-07-03 11:59:36 DEBUG: Longest new qas len: 2048 +2025-07-03 11:59:36 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpju9zzp11 +2025-07-03 11:59:36 DEBUG: Longest old qas len: 11962 +2025-07-03 11:59:36 DEBUG: Longest new qas len: 2048 +2025-07-03 11:59:36 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp4t1vhbeg +2025-07-03 11:59:36 DEBUG: Longest old qas len: 11859 +2025-07-03 11:59:36 DEBUG: Longest new qas len: 2048 +2025-07-03 11:59:36 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp0dx6i6lv +2025-07-03 11:59:36 DEBUG: Longest old qas len: 12674 +2025-07-03 11:59:36 DEBUG: Longest new qas len: 2048 +2025-07-03 11:59:36 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmpqufjvk04 +2025-07-03 11:59:37 DEBUG: Longest old qas len: 12401 +2025-07-03 11:59:37 DEBUG: Longest new qas len: 2048 +2025-07-03 11:59:37 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmp2qr5hr6p +2025-07-03 11:59:37 DEBUG: Longest old qas len: 12709 +2025-07-03 11:59:37 DEBUG: Longest new qas len: 2048 +2025-07-03 11:59:37 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmphve8947h +2025-07-03 11:59:37 DEBUG: Longest old qas len: 12850 +2025-07-03 11:59:37 DEBUG: Longest new qas len: 2048 +2025-07-03 11:59:37 DEBUG: open file: /home/rujikorn_sakana_ai/.tmp/hf_datasets-ia5efh47/tmps_u40ken +2025-07-03 11:59:40 DEBUG: Attempting to acquire lock 22923963693904 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:59:40 DEBUG: Lock 22923963693904 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:59:40 DEBUG: Attempting to release lock 22923963693904 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:59:40 DEBUG: Lock 22923963693904 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 11:59:40 DEBUG: Attempting to acquire lock 22923963693184 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:59:40 DEBUG: Lock 22923963693184 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:59:41 DEBUG: Attempting to release lock 22923963693184 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 11:59:41 DEBUG: Lock 22923963693184 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 12:03:42 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 12:03:42 INFO: Starting packing script... +2025-07-03 12:03:42 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 12:03:42 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 12:03:42 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 12:03:44 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 12:03:44 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 12:03:45 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 12:03:45 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 12:03:45 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 12:03:45 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3 +2025-07-03 12:03:45 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 12:03:45 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 12:03:45 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 12:03:45 DEBUG: Attempting to acquire lock 22872660751840 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 12:03:45 DEBUG: Lock 22872660751840 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 12:03:45 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 12:03:45 DEBUG: Attempting to release lock 22872660751840 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 12:03:45 DEBUG: Lock 22872660751840 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 12:03:45 DEBUG: Attempting to acquire lock 22872660752560 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 12:03:45 DEBUG: Lock 22872660752560 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 12:03:45 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 12:03:45 DEBUG: Attempting to release lock 22872660752560 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 12:03:45 DEBUG: Lock 22872660752560 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpatxylmqc +2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp0k8k8zr1 +2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpfhjxybgj +2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp7u13ypfm +2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpu434v69u +2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp8jocb6v0 +2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp677bi28y +2025-07-03 12:03:45 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpzu483f9t +2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpvb3_d79c +2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpby_f98i4 +2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp9md5vllh +2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpa2fy0dvk +2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp3n5l2h4e +2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpkk757vk3 +2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpr842lqmn +2025-07-03 12:03:47 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpfs2efbik +2025-07-03 12:03:49 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split... +2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpwfnsrbca +2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp2o2tyv3n +2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp4u32hk3l +2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpre39cklr +2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmptox5kjpe +2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpznp702x3 +2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpdyhtxob0 +2025-07-03 12:03:49 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp6w6ewv2u +2025-07-03 12:03:58 DEBUG: Tokenizing inputs +2025-07-03 12:04:18 INFO: Tokenizing 1499551 messages... +2025-07-03 12:07:09 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp4n0kmslm +2025-07-03 12:09:23 INFO: Tokenizing 1499355 messages... +2025-07-03 12:13:41 INFO: Tokenizing 641765 messages... +2025-07-03 12:15:30 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpm6vpm_hg +2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpr4h3vgk4 +2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpu9m0br2c +2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpy5wb3_m8 +2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpu6a91n1l +2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp2idfwt9i +2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp35oyifub +2025-07-03 12:15:33 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmptc9fiamk +2025-07-03 12:17:38 DEBUG: Tokenizing context +2025-07-03 12:17:51 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpyxcnznhz +2025-07-03 12:18:37 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpdph980y3 +2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpk3n13ft6 +2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp8drlpk5o +2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpa3xpn4ln +2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpvcx577xh +2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpx5ul63xj +2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpfjf8u9rb +2025-07-03 12:18:44 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp6t8ga2po +2025-07-03 12:20:41 DEBUG: Unpacking data +2025-07-03 12:20:41 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpwy16gmju +2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpwqge6832 +2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpdoa3khov +2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp7h5ooggt +2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp2_ieaazs +2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp5h66kvhx +2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpnfbqe_wk +2025-07-03 12:20:43 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpiryfqlj9 +2025-07-03 12:23:17 DEBUG: Split too long QAs with max length 2048 +2025-07-03 12:23:17 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 12:25:30 DEBUG: Longest old qas len: 12401 +2025-07-03 12:25:30 DEBUG: Longest new qas len: 2048 +2025-07-03 12:25:30 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmppodex5po +2025-07-03 12:25:34 DEBUG: Longest old qas len: 11859 +2025-07-03 12:25:34 DEBUG: Longest new qas len: 2048 +2025-07-03 12:25:34 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmp179kperq +2025-07-03 12:25:34 DEBUG: Longest old qas len: 12226 +2025-07-03 12:25:34 DEBUG: Longest new qas len: 2048 +2025-07-03 12:25:34 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpl0_rkrgf +2025-07-03 12:25:35 DEBUG: Longest old qas len: 12674 +2025-07-03 12:25:35 DEBUG: Longest new qas len: 2048 +2025-07-03 12:25:35 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpvzg39rhn +2025-07-03 12:25:35 DEBUG: Longest old qas len: 12527 +2025-07-03 12:25:35 DEBUG: Longest new qas len: 2048 +2025-07-03 12:25:35 DEBUG: open file: /mnt/localssd/hf_datasets-92a9d12r/tmpaqqtl3ge +2025-07-03 12:25:36 DEBUG: Attempting to acquire lock 22886692912000 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 12:25:36 DEBUG: Lock 22886692912000 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 12:25:36 DEBUG: Attempting to release lock 22886692912000 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 12:25:36 DEBUG: Lock 22886692912000 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 12:25:36 DEBUG: Attempting to acquire lock 22886692911280 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 12:25:36 DEBUG: Lock 22886692911280 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 12:25:36 DEBUG: Attempting to release lock 22886692911280 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 12:25:36 DEBUG: Lock 22886692911280 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 14:01:20 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 14:01:20 INFO: Starting packing script... +2025-07-03 14:01:20 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 14:01:20 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 14:01:20 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 14:01:22 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 14:01:22 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 14:01:23 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 14:01:23 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 14:01:23 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 14:01:23 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3 +2025-07-03 14:01:23 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 14:01:23 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 14:01:23 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 14:01:23 DEBUG: Attempting to acquire lock 22684941774000 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 14:01:23 DEBUG: Lock 22684941774000 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 14:01:23 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 14:01:23 DEBUG: Attempting to release lock 22684941774000 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 14:01:23 DEBUG: Lock 22684941774000 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 14:01:23 DEBUG: Attempting to acquire lock 22684941770448 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 14:01:23 DEBUG: Lock 22684941770448 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 14:01:23 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 14:01:23 DEBUG: Attempting to release lock 22684941770448 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 14:01:23 DEBUG: Lock 22684941770448 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp65chd13a +2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmphtvax5kh +2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpnoxtu_ht +2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp1ibijg8u +2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpf4rjzflk +2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp7nlj091y +2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp438oajg8 +2025-07-03 14:01:23 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpirdlu1ac +2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpeg1k3qzr +2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpt8j8odur +2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpjri69jc2 +2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpejubln25 +2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpn901kapf +2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmposb8b4cp +2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpjrkcy79f +2025-07-03 14:01:25 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp699maqif +2025-07-03 14:01:27 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split... +2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmph5cpx91l +2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpff2pcpuv +2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp6x42353v +2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpyjxtjuxz +2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp9i7d7cgm +2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpcugne3h6 +2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpckb_sydx +2025-07-03 14:01:27 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp9lofdv2h +2025-07-03 14:01:36 DEBUG: Tokenizing inputs +2025-07-03 14:01:56 INFO: Tokenizing 1499551 messages... +2025-07-03 14:04:46 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpuh_ybp3y +2025-07-03 14:06:57 INFO: Tokenizing 1499355 messages... +2025-07-03 14:11:12 INFO: Tokenizing 641765 messages... +2025-07-03 14:13:00 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmphes6tbze +2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpdmq6k39r +2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp5garo4er +2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpxktzvsiz +2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmps_hnfb31 +2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpx8ilf4qh +2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpja0vasr6 +2025-07-03 14:13:03 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpqytlu_m5 +2025-07-03 14:15:05 DEBUG: Tokenizing context +2025-07-03 14:15:17 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpnjm53jwr +2025-07-03 14:16:03 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmplgmf33m6 +2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp_8bsks6_ +2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpuo5wr0qy +2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpyx0fk9el +2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpoocxr6if +2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpt4pi0u3g +2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp8xtuyymz +2025-07-03 14:16:09 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpg38766pj +2025-07-03 14:17:59 DEBUG: Unpacking data +2025-07-03 14:17:59 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpcyhhyv31 +2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpjegdtgn6 +2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp8s4q0ngs +2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp0b1ifj7l +2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmppc7cu56f +2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp9m0h47mm +2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp57gx_9_5 +2025-07-03 14:18:01 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp3sj9o07x +2025-07-03 14:20:09 DEBUG: Split too long QAs with max length 2048 +2025-07-03 14:23:24 DEBUG: Longest old qas len: 12850 +2025-07-03 14:23:24 DEBUG: Longest new qas len: 2048 +2025-07-03 14:23:24 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmp1bwpue3l +2025-07-03 14:28:01 DEBUG: Longest old qas len: 12674 +2025-07-03 14:28:01 DEBUG: Longest new qas len: 2048 +2025-07-03 14:30:41 DEBUG: Longest old qas len: 11859 +2025-07-03 14:30:41 DEBUG: Longest new qas len: 2048 +2025-07-03 14:31:16 DEBUG: open file: /mnt/localssd/hf_datasets-pv_knwi0/tmpcuhk8l19 +2025-07-03 14:31:16 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 14:32:46 DEBUG: Attempting to acquire lock 22699041521040 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 14:32:46 DEBUG: Lock 22699041521040 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 14:32:46 DEBUG: Attempting to release lock 22699041521040 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 14:32:46 DEBUG: Lock 22699041521040 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 14:32:46 DEBUG: Attempting to acquire lock 22699041520512 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 14:32:46 DEBUG: Lock 22699041520512 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 14:32:46 DEBUG: Attempting to release lock 22699041520512 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 14:32:46 DEBUG: Lock 22699041520512 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 14:42:46 INFO: Logging to: tmp/packing_debug.log/debug.log +2025-07-03 14:42:46 INFO: Starting packing script... +2025-07-03 14:42:46 DEBUG: Model init kwargs: {'pretrained_model_name_or_path': 'google/gemma-2-2b-it', 'device_map': 'cuda', 'torch_dtype': torch.bfloat16, 'trust_remote_code': True, 'attn_implementation': 'flash_attention_2', 'use_cache': False} +2025-07-03 14:42:46 DEBUG: Starting new HTTPS connection (1): huggingface.co:443 +2025-07-03 14:42:46 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/config.json HTTP/1.1" 200 0 +2025-07-03 14:42:48 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/generation_config.json HTTP/1.1" 200 0 +2025-07-03 14:42:48 DEBUG: https://huggingface.co:443 "HEAD /google/gemma-2-2b-it/resolve/main/tokenizer_config.json HTTP/1.1" 200 0 +2025-07-03 14:42:49 INFO: Using chat template from chat_templates/google/gemma-2-2b-it.jinja +2025-07-03 14:42:49 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 14:42:49 DEBUG: Tokenizing dataset with kwargs: {"ds_name": "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", "split": "train", "add_negative_prompt": false, "add_repeat_prompt": false, "repeat_prob": 0.0, "is_pretrain": false, "streaming": false, "max_qas_len": 2048, "base_model_max_len": 8192, "tokenizer_kwargs": {}, "ctx_model_max_len": 8192, "ctx_tokenizer_kwargs": {}, "add_ctx_to_chat": false, "use_kl_loss": false, "need_ctx_ids": true, "is_paraphrased": false, "set_format": null}google/gemma-2-2b-itgoogle/gemma-2-2b-it +2025-07-03 14:42:49 DEBUG: Dataset hash: bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3 +2025-07-03 14:42:49 INFO: Loading dataset self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with split train... +2025-07-03 14:42:49 DEBUG: Starting new HTTPS connection (1): s3.amazonaws.com:443 +2025-07-03 14:42:49 DEBUG: https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/parquet/parquet.py HTTP/1.1" 404 0 +2025-07-03 14:42:49 DEBUG: Attempting to acquire lock 22852114429216 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 14:42:49 DEBUG: Lock 22852114429216 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 14:42:49 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 14:42:49 DEBUG: Attempting to release lock 22852114429216 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 14:42:49 DEBUG: Lock 22852114429216 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/_home_rujikorn_sakana_ai_.cache_huggingface_datasets_parquet_default-4c64ec6bc2199748_0.0.0_9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d.lock +2025-07-03 14:42:49 DEBUG: Attempting to acquire lock 22852114423552 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 14:42:49 DEBUG: Lock 22852114423552 acquired on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 14:42:49 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/dataset_info.json +2025-07-03 14:42:49 DEBUG: Attempting to release lock 22852114423552 on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 14:42:49 DEBUG: Lock 22852114423552 released on /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d_builder.lock +2025-07-03 14:42:49 INFO: Constructing and tokenizing self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny with train split... +2025-07-03 14:42:50 DEBUG: Tokenizing inputs +2025-07-03 14:42:51 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 14:42:53 DEBUG: Tokenizing context +2025-07-03 14:42:54 WARNING: Setting TOKENIZERS_PARALLELISM=false for forked processes. +2025-07-03 14:42:54 DEBUG: Unpacking data +2025-07-03 14:42:55 DEBUG: Split too long QAs with max length 2048 +2025-07-03 14:45:01 DEBUG: Longest old qas len: 12674 +2025-07-03 14:45:01 DEBUG: Longest new qas len: 2048 +2025-07-03 14:45:01 WARNING: Skipped 0 QA pairs because they were too long (> max_qas_len=2048 tokens) +2025-07-03 14:45:01 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmpdhnf3bxy +2025-07-03 14:45:02 DEBUG: Longest old qas len: 12226 +2025-07-03 14:45:02 DEBUG: Longest new qas len: 2048 +2025-07-03 14:45:02 WARNING: Skipped 0 QA pairs because they were too long (> max_qas_len=2048 tokens) +2025-07-03 14:45:02 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmpd8vu0ypo +2025-07-03 14:45:03 DEBUG: Longest old qas len: 12709 +2025-07-03 14:45:03 DEBUG: Longest new qas len: 2048 +2025-07-03 14:45:03 WARNING: Skipped 0 QA pairs because they were too long (> max_qas_len=2048 tokens) +2025-07-03 14:45:03 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmp946mkwxa +2025-07-03 14:45:04 DEBUG: Longest old qas len: 12850 +2025-07-03 14:45:04 DEBUG: Longest new qas len: 2048 +2025-07-03 14:45:04 WARNING: Skipped 0 QA pairs because they were too long (> max_qas_len=2048 tokens) +2025-07-03 14:45:04 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmpu6o_ygx9 +2025-07-03 14:45:52 DEBUG: open file: /home/rujikorn_sakana_ai/.cache/huggingface/datasets/parquet/default-4c64ec6bc2199748/0.0.0/9c460aabd2aa27d1496e5e38d2060760561f0ac2cd6a110134eefa5b3f153b8d/tmplp6me2_j +2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00000-of-00015.arrow +2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00001-of-00015.arrow +2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00002-of-00015.arrow +2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00003-of-00015.arrow +2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00004-of-00015.arrow +2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00005-of-00015.arrow +2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00006-of-00015.arrow +2025-07-03 14:46:04 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00007-of-00015.arrow +2025-07-03 14:46:31 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00008-of-00015.arrow +2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00009-of-00015.arrow +2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00010-of-00015.arrow +2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00011-of-00015.arrow +2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00012-of-00015.arrow +2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00014-of-00015.arrow +2025-07-03 14:46:32 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/data-00013-of-00015.arrow +2025-07-03 14:46:38 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/state.json +2025-07-03 14:46:38 DEBUG: open file: /home/rujikorn_sakana_ai/research/ctx-to-lora/data/processed_datasets/bcc4824e20d0fdcc01b32703f14ef90aeb9222ce622bfb2d06a2780548a487c3/dataset_info.json +2025-07-03 14:48:19 DEBUG: Packing stats - Original samples: 80046 +# Packed samples: 14967 +Avg inp packing efficiency: 0.903 +Avg ctx packing efficiency: 0.089 + +Input IDs length stats: + + Avg: 7399.1, Std: 508.8, Min: 4053, Max: 8192 +Context IDs length stats: + + Avg: 1455.9, Std: 346.3, Min: 481, Max: 3053 +2025-07-03 14:48:21 DEBUG: Packing stats - Original samples: 80045 +# Packed samples: 14985 +Avg inp packing efficiency: 0.903 +Avg ctx packing efficiency: 0.089 + +Input IDs length stats: + + Avg: 7398.7, Std: 507.8, Min: 6150, Max: 8192 +Context IDs length stats: + + Avg: 1451.4, Std: 338.9, Min: 563, Max: 3196 +2025-07-03 14:48:21 DEBUG: open file: /mnt/localssd/hf_datasets-1s9tvwpf/tmpcbf0v49h +2025-07-03 14:48:21 DEBUG: Packing stats - Original samples: 80046 +# Packed samples: 14984 +Avg inp packing efficiency: 0.903 +Avg ctx packing efficiency: 0.089 + +Input IDs length stats: + + Avg: 7398.8, Std: 511.6, Min: 2849, Max: 8192 +Context IDs length stats: + + Avg: 1454.8, Std: 342.6, Min: 466, Max: 3327 +2025-07-03 14:48:22 DEBUG: Packing stats - Original samples: 80045 +# Packed samples: 14967 +Avg inp packing efficiency: 0.904 +Avg ctx packing efficiency: 0.089 + +Input IDs length stats: + + Avg: 7403.8, Std: 511.3, Min: 6161, Max: 8192 +Context IDs length stats: + + Avg: 1455.5, Std: 343.7, Min: 510, Max: 3459 +2025-07-03 14:48:23 DEBUG: open file: /mnt/localssd/hf_datasets-1s9tvwpf/tmpy0ut6956 +2025-07-03 14:48:23 DEBUG: open file: /mnt/localssd/hf_datasets-1s9tvwpf/tmp5bcdvqml +2025-07-03 14:48:24 DEBUG: open file: /mnt/localssd/hf_datasets-1s9tvwpf/tmpykj2tbpo +2025-07-03 14:48:36 DEBUG: Attempting to acquire lock 22866146311040 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 14:48:36 DEBUG: Lock 22866146311040 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 14:48:36 DEBUG: Attempting to release lock 22866146311040 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 14:48:36 DEBUG: Lock 22866146311040 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_2d_kernel.pickle.lock +2025-07-03 14:48:36 DEBUG: Attempting to acquire lock 22866146310320 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 14:48:36 DEBUG: Lock 22866146310320 acquired on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 14:48:36 DEBUG: Attempting to release lock 22866146310320 on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock +2025-07-03 14:48:36 DEBUG: Lock 22866146310320 released on /home/rujikorn_sakana_ai/.triton/autotune/Fp16Matmul_4d_kernel.pickle.lock diff --git a/tmp/test_chat.py b/tmp/test_chat.py new file mode 100644 index 0000000..d543915 --- /dev/null +++ b/tmp/test_chat.py @@ -0,0 +1,105 @@ +import os + +import torch + +from ctx_to_lora.model_loading import get_model, get_tokenizer + +# tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b-it") +# model = AutoModelForCausalLM.from_pretrained( +# "google/gemma-2-2b-it", +# device_map="auto", +# torch_dtype=torch.bfloat16, +# attn_implementation="sdpa", +# ) + +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" +os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "true" +os.environ["FLASH_ATTENTION_DETERMINISTIC"] = "1" + +# torch.use_deterministic_algorithms(True, warn_only=True) +torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False +torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False +torch.backends.cudnn.benchmark = False +torch.backends.cuda.matmul.allow_tf32 = False +torch.backends.cudnn.allow_tf32 = False + +# model, tokenizer = get_model_and_tokenizer( +# "google/gemma-3-1b-it", +# train=False, +# requires_grad=False, +# use_flash_attn=True, +# peft_config=None, +# model_kwargs={"attn_implementation": "flash_attention_2"}, +# tokenizer_kwargs=None, +# device="cuda", +# dtype=torch.bfloat16, +# ) +device = "cuda" if torch.cuda.is_available() else "cpu" +model_kwargs = {"attn_implementation": "flash_attention_2"} +# model_name_or_path = "Qwen/Qwen3-1.7B" +model_name_or_path = "google/gemma-2-2b-it" +# model_name_or_path = "google/gemma-3-1b-it" +tokenizer = get_tokenizer(model_name_or_path, train=False) +model = get_model( + model_name_or_path, + train=False, + requires_grad=False, + model_kwargs=model_kwargs, + use_flash_attn=True, + dtype=torch.bfloat16, +) + +qa = ( + "Architecturally, the school has a Catholic character. " + "Atop the Main Building's gold dome is a golden statue of the Virgin Mary. " + "Immediately in front of the Main Building and facing it, " + 'is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". ' + "Next to the Main Building is the Basilica of the Sacred Heart. " + "Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. " + "It is a replica of the grotto at Lourdes, " + "France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous " + "in 1858. At the end of the main drive (and in a direct line that connects through 3 " + "statues and the Gold Dome), is a simple, modern stone statue of Mary." + "\n\nAnswer the following question. " + "Output only the answer and do not output any other words." + "\n\nQuestion: To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?" +) +# x = " | ".join([str(random.randint(0, 1000)) for _ in range(800)]) +# qa = ( +# " | ".join([str(random.randint(0, 1000)) for _ in range(10000)]) +# + "\n\nRepeat the text above." +# ) + +messages = [ + [{"role": "system", "content": ""}, {"role": "user", "content": qa}], + [{"role": "system", "content": ""}, {"role": "user", "content": "Hello."}], +] +chat_ids = tokenizer.apply_chat_template( + messages, + tokenize=True, + padding=True, + truncation=False, + add_special_token=False, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, +).to(device) +print(chat_ids) +print(f"{len(chat_ids['input_ids'][0])} tokens in input") +print(tokenizer.decode(chat_ids["input_ids"][0])) + + +print(model(**chat_ids).logits) + +outputs = model.generate( + **chat_ids, + max_new_tokens=2**12, + do_sample=False, + temperature=0.0, +) +for output in outputs: + print(tokenizer.decode(output)) + print("-" * 100) + +print(outputs) +breakpoint() diff --git a/tmp/test_detokenize.py b/tmp/test_detokenize.py new file mode 100644 index 0000000..4562cb4 --- /dev/null +++ b/tmp/test_detokenize.py @@ -0,0 +1,35 @@ +from ctx_to_lora.model_loading import get_tokenizer + +model_name_or_path = "google/gemma-2-2b-it" +tokenizer = get_tokenizer(model_name_or_path, train=True) + + +qa = ( + "Architecturally, the school has a Catholic character. " + "Atop the Main Building's gold dome is a golden statue of the Virgin Mary. " + "Immediately in front of the Main Building and facing it, " + 'is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". ' + "Next to the Main Building is the Basilica of the Sacred Heart. " + "Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. " + "It is a replica of the grotto at Lourdes, " + "France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous " + "in 1858. At the end of the main drive (and in a direct line that connects through 3 " + "statues and the Gold Dome), is a simple, modern stone statue of Mary." + "\n\nAnswer the following question. " + "Output only the answer and do not output any other words." + "\n\nQuestion: To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?" +) + + +messages = [{"role": "system", "content": ""}, {"role": "user", "content": qa}] +chat_ids = tokenizer.apply_chat_template( + messages, + tokenize=True, + add_special_token=False, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, +) +print(chat_ids) +print(tokenizer.batch_decode(chat_ids["input_ids"])) +breakpoint() diff --git a/tmp/test_emb.py b/tmp/test_emb.py new file mode 100644 index 0000000..dd570a8 --- /dev/null +++ b/tmp/test_emb.py @@ -0,0 +1,51 @@ +import os + +import torch + +from ctx_to_lora.model_loading import get_model, get_tokenizer + +# tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b-it") +# model = AutoModelForCausalLM.from_pretrained( +# "google/gemma-2-2b-it", +# device_map="auto", +# torch_dtype=torch.bfloat16, +# attn_implementation="sdpa", +# ) + +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" +os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "true" +os.environ["FLASH_ATTENTION_DETERMINISTIC"] = "1" + +# torch.use_deterministic_algorithms(True, warn_only=True) +torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False +torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False +torch.backends.cudnn.benchmark = False +torch.backends.cuda.matmul.allow_tf32 = False +torch.backends.cudnn.allow_tf32 = False + +# model, tokenizer = get_model_and_tokenizer( +# "google/gemma-3-1b-it", +# train=False, +# requires_grad=False, +# use_flash_attn=True, +# peft_config=None, +# model_kwargs={"attn_implementation": "flash_attention_2"}, +# tokenizer_kwargs=None, +# device="cuda", +# dtype=torch.bfloat16, +# ) +device = "cuda" if torch.cuda.is_available() else "cpu" +model_kwargs = {"attn_implementation": "flash_attention_2"} +# model_name_or_path = "google/gemma-2-2b-it" +model_name_or_path = "google/gemma-2-2b-it" +tokenizer = get_tokenizer(model_name_or_path, train=False) +model = get_model( + model_name_or_path, + train=False, + requires_grad=False, + model_kwargs=model_kwargs, + use_flash_attn=True, + dtype=torch.bfloat16, +) + +breakpoint() diff --git a/tmp/test_logits.py b/tmp/test_logits.py new file mode 100644 index 0000000..f8799e5 --- /dev/null +++ b/tmp/test_logits.py @@ -0,0 +1,121 @@ +import os + +import torch + +from ctx_to_lora.model_loading import get_model, get_tokenizer + +# tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b-it") +# model = AutoModelForCausalLM.from_pretrained( +# "google/gemma-2-2b-it", +# device_map="auto", +# torch_dtype=torch.bfloat16, +# attn_implementation="sdpa", +# ) + +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" +os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "true" +os.environ["FLASH_ATTENTION_DETERMINISTIC"] = "1" + +# torch.use_deterministic_algorithms(True, warn_only=True) +torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False +torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False +torch.backends.cudnn.benchmark = False +torch.backends.cuda.matmul.allow_tf32 = False +torch.backends.cudnn.allow_tf32 = False + +# model, tokenizer = get_model_and_tokenizer( +# "google/gemma-3-1b-it", +# train=False, +# requires_grad=False, +# use_flash_attn=True, +# peft_config=None, +# model_kwargs={"attn_implementation": "flash_attention_2"}, +# tokenizer_kwargs=None, +# device="cuda", +# dtype=torch.bfloat16, +# ) +device = "cuda" if torch.cuda.is_available() else "cpu" +model_kwargs = {"attn_implementation": "flash_attention_2"} +# model_name_or_path = "google/gemma-2-2b-it" +model_name_or_path = "google/gemma-3-1b-it" +tokenizer = get_tokenizer(model_name_or_path, train=False) +model = get_model( + model_name_or_path, + train=False, + requires_grad=False, + model_kwargs=model_kwargs, + use_flash_attn=True, + dtype=torch.bfloat16, +) +# model.config.pad_token_id = tokenizer.pad_token_id +# if getattr(model, "generation_config", None): +# model.generation_config.pad_token_id = tokenizer.pad_token_id +# print(model.name_or_path) + +input_text = [ + "Write me a poem about Machine Learning.", + "Write me a poem about cats.", + "Who is Donal Trump?", + "Who is Mark Zuckerburg?", + "Tell me about Thailand.", + "Is Japan hot in the summer?", + "How computers work?", + "Who sells iPhone?", + "What exclusive games are on PS5?", + "Mechanical vs rubber dome keyboards", +] +input_ids = [tokenizer(t, return_tensors="pt").to(device) for t in input_text] + +padding_kwargs = dict(padding=True, padding_side="left", return_tensors="pt") +input_ids = tokenizer.pad( + {k: [x[k][0] for x in input_ids] for k in input_ids[0].keys()}, + **padding_kwargs, +).to(device) +print(input_ids) +outputs = model.generate(**input_ids, do_sample=False) +for output in outputs: + print(tokenizer.decode(output)) + print("-" * 100) + +print(model(**input_ids).logits) + +# qa = ( +# "Architecturally, the school has a Catholic character. " +# "Atop the Main Building's gold dome is a golden statue of the Virgin Mary. " +# "Immediately in front of the Main Building and facing it, " +# 'is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". ' +# "Next to the Main Building is the Basilica of the Sacred Heart. " +# "Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. " +# "It is a replica of the grotto at Lourdes, " +# "France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous " +# "in 1858. At the end of the main drive (and in a direct line that connects through 3 " +# "statues and the Gold Dome), is a simple, modern stone statue of Mary." +# "\n\nAnswer the following question. " +# "Output only the answer and do not output any other words." +# "\n\nQuestion: To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?" +# ) + + +# messages = [{"role": "system", "content": ""}, {"role": "user", "content": qa}] +# chat_ids = tokenizer.apply_chat_template( +# messages, +# tokenize=True, +# add_special_token=False, +# add_generation_prompt=True, +# return_tensors="pt", +# return_dict=True, +# ).to(device) +# print(chat_ids) +# print(tokenizer.decode(chat_ids["input_ids"][0])) +# outputs = model.generate( +# **chat_ids, +# max_new_tokens=100, +# do_sample=False, +# temperature=0.0, +# ) + +# print(model(**chat_ids).logits) + +# for output in outputs: +# print(tokenizer.decode(output)) +# print("-" * 100) diff --git a/tmp/test_mem_alloc.py b/tmp/test_mem_alloc.py new file mode 100644 index 0000000..36f3087 --- /dev/null +++ b/tmp/test_mem_alloc.py @@ -0,0 +1,151 @@ +import os +import time + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + + +def bytes_to_mib(x: int) -> float: + return x / (1024**2) + + +def bytes_to_gib(x: int) -> float: + return x / (1024**3) + + +def pick_dtype() -> torch.dtype: + if torch.cuda.is_available(): + # Prefer bfloat16 if supported, else float16, else float32 + if torch.cuda.is_bf16_supported(): + return torch.bfloat16 + return torch.float16 + return torch.float32 + + +def main(): + model_name = os.environ.get("MODEL_NAME", "google/gemma-2-2b-it") + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + dtype = pick_dtype() + + if device.type != "cuda": + print( + "CUDA is not available; this script is intended for GPU memory measurements." + ) + return + + torch.cuda.empty_cache() + torch.cuda.synchronize() + + # Load tokenizer on CPU (doesn't affect CUDA mem) + tokenizer = AutoTokenizer.from_pretrained(model_name) + + # Measure peak memory during model load + torch.cuda.reset_peak_memory_stats(device) + load_start = time.time() + + model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=dtype, + device_map=None, # load on CPU first, then move to desired single device + ) + model.to(device) + model.eval() + + torch.cuda.synchronize() + model_load_peak = torch.cuda.max_memory_allocated(device) + load_elapsed = time.time() - load_start + + # Baseline after load + baseline_alloc = torch.cuda.memory_allocated(device) + + # Prepare a chat-style prompt if available; fallback to plain text + doc = """ +Matryoshka dolls (Russian: матрёшка, romanized: matryoshka/ˌmætriˈɒʃkə/), also known as stacking dolls, nesting dolls, Russian tea dolls, or Russian dolls,[1] are a set of wooden dolls of decreasing size placed one inside another. The name Matryoshka is a diminutive form of Matryosha (Матрёша), in turn a hypocorism of the Russian female first name Matryona (Матрёна).[2] + +A set of matryoshkas consists of a wooden figure, which separates at the middle, top from bottom, to reveal a smaller figure of the same sort inside, which has, in turn, another figure inside of it, and so on. + +The first Russian nested doll set was made in 1890 by woodturning craftsman and wood carver Vasily Zvyozdochkin from a design by Sergey Malyutin, who was a folk crafts painter at Abramtsevo. Traditionally the outer layer is a woman, dressed in a Russian sarafan dress. The figures inside may be of any gender; the smallest, innermost doll is typically a baby turned from a single piece of wood. Much of the artistry is in the painting of each doll, which can be very elaborate. The dolls often follow a theme; the themes may vary, from fairy tale characters to Soviet leaders. In some countries, matryoshka dolls are often referred to as babushka dolls, though they are not known by this name in Russian; babushka (бабушка) means 'grandmother; old woman'.[3] +History +The original matryoshka set by Zvyozdochkin and Malyutin, 1892 + +The first Russian nested doll set was carved in 1890 at the Children's Education Workshop by Vasily Zvyozdochkin and designed by Sergey Malyutin, who was a folk crafts painter in the Abramtsevo estate of Savva Mamontov, a Russian industrialist and patron of arts.[4][5] Mamontov's brother, Anatoly Ivanovich Mamontov (1839–1905), created the Children's Education Workshop to make and sell children's toys. The doll set was painted by Malyutin. Malyutin's doll set consisted of eight dolls—the outermost was a mother in a traditional dress holding a red-combed rooster. The inner dolls were her children, girls and a boy, and the innermost a baby. The Children's Education Workshop was closed in the late 1890s, but the tradition of the matryoshka simply relocated to Sergiyev Posad, the Russian city known as a toy-making center since the fourteenth century.[6][4] + +The inspiration for matryoshka dolls is not clear. Matryoshka dolls may have been inspired by a nesting doll imported from Japan.[5][7] The Children's Education workshop where Zvyozdochkin was a lathe operator received a five piece, cylinder-shaped nesting doll featuring Fukuruma (Fukurokuju) in the late 1890s,[8] which is now part of the collection at the Sergiev Posad Museum of Toys.[8] Other east Asian dolls share similarities with matryoshka dolls such as the Kokeshi dolls,[4][9] originating in Northern Honshū, the main island of Japan, although they cannot be placed one inside another, and the round hollow daruma doll depicting a Buddhist monk.[9][10] Another possible source of inspiration is the nesting Easter eggs produced on a lathe by Russian woodworkers during the late 19th Century.[3][11] + +Savva Mamontov's wife presented a set of matryoshka dolls at the Exposition Universelle in Paris in 1900, and the toy earned a bronze medal. Soon after, matryoshka dolls were being made in several places in Russia and shipped around the world. + """ + messages = [ + { + "role": "user", + "content": doc + "\n\nSummarize the article.", + }, + ] + + prompt_text = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + + inputs = tokenizer(prompt_text, return_tensors="pt").to(device) + + # Optional quick warmup to stabilize kernels (doesn't affect peak calc since we reset after) + with torch.no_grad(): + _ = model.generate(**inputs, max_new_tokens=1) + torch.cuda.synchronize() + + # Measure generation peak memory for 512 new tokens + torch.cuda.reset_peak_memory_stats(device) + gen_start = time.time() + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=int(os.environ.get("MAX_NEW_TOKENS", 512)), + do_sample=False, + temperature=None, + top_p=None, + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.eos_token_id, + ) + torch.cuda.synchronize() + gen_elapsed = time.time() - gen_start + + gen_peak_total = torch.cuda.max_memory_allocated( + device + ) # includes baseline present at reset + gen_additional = max(0, gen_peak_total - baseline_alloc) + + decoded = tokenizer.decode(outputs[0], skip_special_tokens=True) + + # Print summary + print("=== Memory allocation summary (CUDA) ===") + print(f"Device: {device} | {torch.cuda.get_device_name(device)}") + print(f"Model: {model_name}") + print(f"DType: {dtype}") + print() + print("Model load:") + print( + f" Peak allocated during load: {bytes_to_mib(model_load_peak):.2f} MiB ({bytes_to_gib(model_load_peak):.2f} GiB)" + ) + print(f" Time to load + move to device: {load_elapsed:.2f}s") + print( + f" Baseline after load (current allocated): {bytes_to_mib(baseline_alloc):.2f} MiB ({bytes_to_gib(baseline_alloc):.2f} GiB)" + ) + print() + print("Generation (max_new_tokens=512):") + print( + f" Peak allocated total during generation: {bytes_to_mib(gen_peak_total):.2f} MiB ({bytes_to_gib(gen_peak_total):.2f} GiB)" + ) + print( + f" Additional peak over baseline during generation: {bytes_to_mib(gen_additional):.2f} MiB ({bytes_to_gib(gen_additional):.2f} GiB)" + ) + print(f" Time to generate: {gen_elapsed:.2f}s") + print() + # Response preview + preview = decoded + if len(preview) > 800: + preview = preview[:800] + "\n... [truncated]" + print("=== Response preview ===") + print(preview) + + +if __name__ == "__main__": + main() diff --git a/tmp/test_pad.py b/tmp/test_pad.py new file mode 100644 index 0000000..4ebea5f --- /dev/null +++ b/tmp/test_pad.py @@ -0,0 +1,8 @@ +import torch +from transformers import AutoTokenizer + +tk = AutoTokenizer.from_pretrained("google/gemma-2-2b-it") +x = [torch.rand(5, 3), torch.rand(10, 3), torch.rand(7, 3)] +padded_x = tk.pad({"input_ids": x}, padding=True) +print(padded_x) +breakpoint() diff --git a/tmp/torch_profiler_test.py b/tmp/torch_profiler_test.py new file mode 100644 index 0000000..73d72d6 --- /dev/null +++ b/tmp/torch_profiler_test.py @@ -0,0 +1,69 @@ +# (c) Meta Platforms, Inc. and affiliates. +import logging +import socket +from datetime import datetime + +import torch +from torch.autograd.profiler import record_function +from torchvision import models + +logging.basicConfig( + format="%(levelname)s:%(asctime)s %(message)s", + level=logging.INFO, + datefmt="%Y-%m-%d %H:%M:%S", +) +logger: logging.Logger = logging.getLogger(__name__) +logger.setLevel(level=logging.INFO) + +TIME_FORMAT_STR: str = "%b_%d_%H_%M_%S" + + +def trace_handler(prof: torch.profiler.profile): + # Prefix for file names. + host_name = socket.gethostname() + timestamp = datetime.now().strftime(TIME_FORMAT_STR) + file_prefix = f"{host_name}_{timestamp}" + + # Construct the trace file. + prof.export_chrome_trace(f"{file_prefix}.json.gz") + + # Construct the memory timeline file. + prof.export_memory_timeline(f"{file_prefix}.html", device="cuda:0") + + +def run_resnet50(num_iters=5, device="cuda:0"): + model = models.resnet50().to(device=device) + inputs = torch.randn(1, 3, 224, 224, device=device) + labels = torch.rand_like(model(inputs)) + optimizer = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.9) + loss_fn = torch.nn.CrossEntropyLoss() + + with torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + schedule=torch.profiler.schedule(wait=0, warmup=0, active=6, repeat=1), + record_shapes=True, + profile_memory=True, + with_stack=True, + on_trace_ready=trace_handler, + ) as prof: + for _ in range(num_iters): + prof.step() + with record_function("## forward ##"): + pred = model(inputs) + + with record_function("## backward ##"): + loss_fn(pred, labels).backward() + + with record_function("## optimizer ##"): + optimizer.step() + optimizer.zero_grad(set_to_none=True) + + +if __name__ == "__main__": + # Warm up + run_resnet50() + # Run the resnet50 model + run_resnet50() diff --git a/tmp/visualize_ds_len.py b/tmp/visualize_ds_len.py new file mode 100644 index 0000000..8d40598 --- /dev/null +++ b/tmp/visualize_ds_len.py @@ -0,0 +1,101 @@ +import os +from functools import partial + +import matplotlib.pyplot as plt +import numpy as np + +from ctx_to_lora.data.processing import get_tokenized_dataset +from ctx_to_lora.model_loading import get_tokenizer +from ctx_to_lora.utils import check_is_iterable + +tokenizer_kwargs = {"max_length": 2**13} +ctx_tokenizer_kwargs = {"max_length": 2**13} # not used for now +tokenizer = get_tokenizer("google/gemma-2-2b-it", train=True) +ctx_tokenizer = get_tokenizer("google/gemma-2-2b-it", train=True) + +_get_tokenized_dataset = partial( + get_tokenized_dataset, + base_model_max_len=2**13, + tokenizer=tokenizer, + tokenizer_kwargs=tokenizer_kwargs, + ctx_model_max_len=2**13, + ctx_tokenizer=ctx_tokenizer, + ctx_tokenizer_kwargs=ctx_tokenizer_kwargs, + add_ctx_to_chat=False, + add_repeat_prompt=False, + add_negative_prompt=False, + use_kl_loss=False, + # streaming=data_args.streaming, +) + +train_ds_names = [ + # "fw_qa", + # "fw_qa_large", + # "ctx_qa", + # "pwc", + # "hotpot_qa", + # "squad", + # "drop", + # "narrativeqa", + # "quoref", + # "ropes", + # "synthetic_convqa", + # "booksum", + # "gov_report", + # "wikitext-2", + # "wikitext-103", + # "fw_qa_2" + # "fw_qa_xl", + # "fw_qa_3_medium", + "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_0_tiny", + "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_1_tiny", + "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", + "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_3_tiny", +] +tokenized_ds = dict() +val_ds_names = [] # ["fw_qa_large", "ctx_qa", "pwc", "hotpot_qa", "squad"] +for split, ds_names in zip(["train", "validation"], [train_ds_names, val_ds_names]): + tokenized_ds[split] = { + os.path.basename(ds_name): _get_tokenized_dataset(ds_name, split) + for ds_name in ds_names + } + for ds_name in tokenized_ds[split]: + print(ds_name) + print(f"Num samples: {len(tokenized_ds[split][ds_name])}") + sample = tokenized_ds[split][ds_name][0] + + # TODO: let's investigate samples that are very long and very short + if check_is_iterable(sample["input_ids"]): + tokens = [] + for sample in tokenized_ds[split][ds_name]: + tokens += [len(ids) for ids in sample["input_ids"]] + else: + tokens = [ + len(sample["input_ids"]) for sample in tokenized_ds[split][ds_name] + ] + + print(f"Num avg. tokens: {sum(tokens) / len(tokens)}") + print(f"Num max. tokens: {max(tokens)}") + print(f"Num min. tokens: {min(tokens)}") + print(f"Num std. tokens: {np.std(tokens)}") + plt.hist(tokens, bins=100) + plt.savefig(f"tmp/n_token_hist_{split}_{ds_name}.png") + plt.close() + + long_indices = [i for i, n_tokens in enumerate(tokens) if n_tokens > 100] + short_indices = [i for i, n_tokens in enumerate(tokens) if n_tokens < 30] + breakpoint() + for i in long_indices: + print(f"Long sample {i}: {tokenized_ds[split][ds_name][i]}") + breakpoint() + for i in short_indices: + print(f"Short sample {i}: {tokenized_ds[split][ds_name][i]}") + + ctx_tokens = [len(sample["ctx_ids"]) for sample in tokenized_ds[split][ds_name]] + print(f"Num avg. ctx tokens: {sum(ctx_tokens) / len(ctx_tokens)}") + print(f"Num max. ctx tokens: {max(ctx_tokens)}") + print(f"Num min. ctx tokens: {min(ctx_tokens)}") + print(f"Num std. ctx tokens: {np.std(ctx_tokens)}") + plt.hist(ctx_tokens, bins=100) + plt.savefig(f"tmp/ctx_n_token_hist_{split}_{ds_name}.png") + plt.close() diff --git a/tmp/visualize_long_samples.py b/tmp/visualize_long_samples.py new file mode 100644 index 0000000..0104a43 --- /dev/null +++ b/tmp/visualize_long_samples.py @@ -0,0 +1,133 @@ +import os +from functools import partial + +import matplotlib.pyplot as plt +import numpy as np + +from ctx_to_lora.data.processing import get_tokenized_dataset, load_and_process_dataset +from ctx_to_lora.model_loading import get_tokenizer + +tokenizer_kwargs = {"max_length": 2**13} +ctx_tokenizer_kwargs = {"max_length": 2**13} # not used for now +tokenizer = get_tokenizer("google/gemma-2-2b-it", train=True) +ctx_tokenizer = get_tokenizer("google/gemma-2-2b-it", train=True) + +_get_tokenized_dataset = partial( + get_tokenized_dataset, + base_model_max_len=2**13, + tokenizer=tokenizer, + tokenizer_kwargs=tokenizer_kwargs, + ctx_model_max_len=2**13, + ctx_tokenizer=ctx_tokenizer, + ctx_tokenizer_kwargs=ctx_tokenizer_kwargs, + add_ctx_to_chat=False, + add_repeat_prompt=False, + add_negative_prompt=False, + use_kl_loss=False, + # streaming=data_args.streaming, +) +_load_and_process_dataset = partial( + load_and_process_dataset, + add_negative_prompt=False, + add_repeat_prompt=False, + repeat_prob=0, + is_pretrain=False, + streaming=False, + num_proc=8, +) + +train_ds_names = [ + # "fw_qa", + # "fw_qa_large", + # "ctx_qa", + # "pwc", + # "hotpot_qa", + # "squad", + # "drop", + # "narrativeqa", + # "quoref", + # "ropes", + # "synthetic_convqa", + # "booksum", + # "gov_report", + # "wikitext-2", + # "wikitext-103", + # "fw_qa_2" + # "fw_qa_xl", + # "fw_qa_3_medium", + "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_0_tiny", + "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_1_tiny", + "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_2_tiny", + "self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_3_tiny", +] +loaded_dataset = dict() +val_ds_names = [] # ["fw_qa_large", "ctx_qa", "pwc", "hotpot_qa", "squad"] +for split, ds_names in zip(["train", "validation"], [train_ds_names, val_ds_names]): + loaded_dataset[split] = { + os.path.basename(ds_name): _load_and_process_dataset(ds_name, split) + for ds_name in ds_names + } + for ds_name in loaded_dataset[split]: + print(ds_name) + print(f"Num samples: {len(loaded_dataset[split][ds_name])}") + ds = loaded_dataset[split][ds_name] + sample = loaded_dataset[split][ds_name][0] + + # TODO: let's investigate samples that are very long and very short + if "responses" in sample: + char_lens = [] + responses = [] + questions = [] + ctxs = [] + for sample in loaded_dataset[split][ds_name]: + char_lens += [len(res) for res in sample["responses"]] + responses += sample["responses"] + questions += sample["prompts"] + ctxs += [sample["context"]] * len(sample["responses"]) + else: + responses = ds["response"] + questions = ds["prompt"] + ctxs = ds["context"] + + char_lens = [ + len(sample["response"]) for sample in loaded_dataset[split][ds_name] + ] + + print(f"Num avg. char_lens: {sum(char_lens) / len(char_lens)}") + print(f"Num max. char_lens: {max(char_lens)}") + print(f"Num min. char_lens: {min(char_lens)}") + print(f"Num std. char_lens: {np.std(char_lens)}") + plt.hist(char_lens, bins=100) + plt.savefig(f"tmp/n_token_hist_{split}_{ds_name}.png") + plt.close() + + long_indices = [i for i, clen in enumerate(char_lens) if clen > 2000] + short_indices = [i for i, clen in enumerate(char_lens) if clen < 100] + breakpoint() + for i in long_indices: + print(f"Long sample {i}") + print(f"Response length: {len(responses[i])}") + print(f"Context: {ctxs[i]}") + print(f"Question: {questions[i]}") + print(f"Response: {responses[i]}") + print("=" * 80) + + breakpoint() + for i in short_indices: + print(f"Short sample {i}") + print(f"Response length: {len(responses[i])}") + print(f"Context: {ctxs[i]}") + print(f"Question: {questions[i]}") + print(f"Response: {responses[i]}") + print("=" * 80) + + ctx_tokens = [ + len(sample["ctx_ids"]) for sample in loaded_dataset[split][ds_name] + ] + print(f"Num avg. ctx char_lens: {sum(ctx_tokens) / len(ctx_tokens)}") + print(f"Num max. ctx char_lens: {max(ctx_tokens)}") + print(f"Num min. ctx char_lens: {min(ctx_tokens)}") + print(f"Num std. ctx char_lens: {np.std(ctx_tokens)}") + plt.hist(ctx_tokens, bins=100) + plt.savefig(f"tmp/ctx_n_token_hist_{split}_{ds_name}.png") + plt.close()