From 2a06affa9a89566c9fba88526f6a3ef7be957c3b Mon Sep 17 00:00:00 2001 From: Evan Chen Date: Tue, 19 Mar 2024 18:14:47 +0800 Subject: [PATCH 1/2] Add sklearn68 dataset --- sub_swebench_dataset/readme.md | 14 ++++++++++++++ sub_swebench_dataset/scikit-learn-68.csv | Bin 0 -> 11428 bytes 2 files changed, 14 insertions(+) create mode 100644 sub_swebench_dataset/scikit-learn-68.csv diff --git a/sub_swebench_dataset/readme.md b/sub_swebench_dataset/readme.md index 77fb23b2a..acd041d1c 100644 --- a/sub_swebench_dataset/readme.md +++ b/sub_swebench_dataset/readme.md @@ -3,6 +3,12 @@ # Dataset Description The index of sub_swebench is a subset of swebench, with two columns in total, each column containing 50 id_instance. The id_instance is a balanced subset of pass and fail samples for CognitionAI on swebench. + +The index of scikit-learn-68 is another subset of CognitionAI in swebench (all tasks of the scikit-learn type), with a total of two columns: + +- pass:12 +- fail:56 + Sampling list:https://github.com/CognitionAI/devin-swebench-results/tree/main/ Original dataset:https://huggingface.co/datasets/princeton-nlp/SWE-bench/ @@ -69,3 +75,11 @@ ### After balanced sampling: - Xarray (pydata): 2 - Matplotlib: 2 - Sphinx: 1 + + + +## scikit-learn-68 dataset Description: + +instance_id_pass:12 + +instance_id_pass:56 diff --git a/sub_swebench_dataset/scikit-learn-68.csv b/sub_swebench_dataset/scikit-learn-68.csv new file mode 100644 index 0000000000000000000000000000000000000000..7d31862d24015bec2773e6752c52db810595b420 GIT binary patch literal 11428 zcmeHtWmH{D(A|Gz26(*i#FUTN8Zx{ZE4f-5A*#Dmd8MIWj4LV3?pk zV1K}F5ZoVmpMindf`fsf{u8EeXUFJfZIv23Bn82O8Fb};g!;LBH~Kx`6Q)piHAkV|5KpE@3- zp~x4UYiDfEp2Fj;8$iKAgd7c70%!HCyUiBC-6Dnae0jWMHvz>K2cOu_*k#SzxH_v| zNe7RR;7jbxtcFN@Xv-(aWg>xe9agk^r|CYQ>hht?6emAwmjG*_wf8-Ys~rkZZf}O)Waz%5@?MD| zU8PLf!vc%E_0k+4#(T7;7=I`_OntG8^5W!Pc?0KnL*u2s8dLxoS`%bww0|1f(AL5D z$#?_ zjf8dR9ko|z2QfID0g&()I)hJ2s`@^SZj~n2<2J z>~7!fP0ve2_|ip9LLb?Yyut}`Zb#`CadNLo%w#9m--xS}?}W;Yh{7e6A6m(Npwv*K zzFdE#it<9gIrkExwBj=WAiswZoZI4mK;C`%yZOCl>rLrE=0^jWALBRkJ3BhrT0c1; zH)ce-lNl3uh*IsE!HU{hmU^wKxm~E-1+`8!uSnbE(eIOHy3X_%BQ+&2 zgp&~WOH);0acj4;I9kGUr9E=REVT<=Utip}x#iB#*H^#SBOv3%FNREMv0E9`B;41H zNj0G2t@7rRlY1#IuqsiGCb?qk7W>#2@M9efCC2h{J*e2NLZe{6`441GtY$@e1tRO zit400pM}g#;-bh4TieE{V0CF%+UrCw%36btk$-niNK}GLOP0}0y-#{Ef_UM(p5U6k zJWuHncdt-9!6S!V%oUq2b+9*a#`ADj9wI+&&%Se7&4lk9yLa74ur`vc>)XIv{9w!_ zRY<15g|v5F;OyF{g*>HLyIG--%Gcg6bYWe%YWdhi@I@!V^G;!&zT4sr{rqSCZn@Rk zM_eRluF3c&wJA$F`MJ>Z&oms90t9em=Qp3ZrLuOcUT7A0%=)*a};$Kp8;Z=bC#AccS1 zC?4vDpw&h|>hp2yA{7B`@ws{)p?1iCg;9GQN``L%zf|^p($UaBMSvh^<=v0>TxQz#! zQZW}#A*eOG7CJhd%VOZlIYszhj%5gEdIN>LVBh{GFtqe*;Y`vHi_M{jb5xsw9ioUD z9$1$({Jk1weGs(@N8451&^MbKXa-duYl-Ptduuq4Z0gA!F$~Q67bOJzYf9})Eb-4b z_veOpX3&9zyY*zRVME%<+3%aIDYG9gBQ>0%{pV~9LhB^wBjGt9`o4SW|8C4jj zIGmehskW~1f!0)clXEKPlwqGxSv%T+5lg!fciLYCD!Hhl@R&qPt-@6%US<)EeytHX z7B?YfyWmxs)#pAIYkellYGug1&pyF$*MAQ%{Z+l4t47t*Z zATAS3HaW>_NpSrM1AqA&q5yqX=WsN9dBMRMid6}?K)n~JUli~29#@a&SK?$lMt`Czs& zl^hT@o+V+rv1}`)U3ZbD>$&R0IizMuck;MbU`V+5FS^Ty*@iJ9u|q{O1zI1?q)xEU z1FzDuP()N-rxo8@-^jMxEA2!v+LVt)*Qn}>mG zBm%@)730V7y@Ip<(IZI3{@UjV;QmVqP#0u46ptZ)znDgBYCOLi@nWxG@Evc1@toP<6C5@ z-2t5X6810AO06!6YA&CBOCCd{T{FZlI2K&zEf1X+3U#iGONY91Fgz=0w^W^*J}!R; z`+dn@Ca9z^!#pSEmm!(V z!4OH_K^{T@R2(m75)+F;FpY`dA|+bjCBUAJ7O#Ri7RqF}c`?8Zp=p?HrFN?8Iv>uI z0u@^ugU0~)4hP;^w$w@oeSS9zff^2pFSUfxKu0RP7!X4$0>kN|aW1YT<4xs%Mw0B* zYj#39B7i^^zlpKh-x}m+U2HL<=gq8V?(a4loAoeChhs8lct)wSzKPmA>>)>PlQ$ew z_cm>pt?2T3MQ({W8KwC)qng1*y1a6_pAK$9jE}D!4%&WJL+W@~TI`}a>L`Ng^I$7G z(&B@vu()Roj8LPusIWugu^Vq0i_BC~*4!=zMKn*#5s1PI?={;|rj@nEjBELd2$00OrG&NO z`R;nv!(Tg|4mLyaaBby4_!d9(;Gn+ARI|SsAC6zhq+-H~%%Ys(R5YNCiq5%(`M>rz z+yD3X|Lgw#_pI{i3m24jI++<;8~^cz8+dz%cnt*x)&hzFPnrMUupi(bx#WTRP&7UV zW-Hl=K*ZM4A^8x{Wy-4N=9{qV{F}JiZ)1`v5%?o+M!bzfNHr4VIpb%< z{A24uH-GY`)4t*N*=(c9qp1#};U6i~3g!qK}^Ub4D3Wzx8|#?iY{Ylyc@;9Pt$7^&bE9!#WyysOa9-#>tcM-0u#lni@l6bxD z!9X+EYHLy{=oCl^BXnsFn7Pj0Tbtlu1MAnaIPXcRK3N44o4~^B7hX)A9qGLC#cT3- z*!aF$brvF!Cg9=pa14_@fW{}f+lB3QGnDV^b)k5CH&L$3|F95^!};*>NZ{cnf8@I$ zbyltzQ;XZ_UKJHn%lC~sTVl@@7y=|Kw{-HikS~wb?ZPQml93}UxrvUa<92}WxzMSh z7m20_ae3SD9YwwCDP00wLqr~rpOp_C^44A&@W*wron3I*zXL{$QLZn6X{kA?qaBWn zhx$73SJ~I0@>~op3Kz!B4bumdNqW(0vbYTLWm3nR`76^yTtv#UT)MeNqf(Utje0-n z8R9OG31OhwzP>eDK{cmwdX3&YuwY7(GZ#8^lA6(1FtW$V?;N9m`raer$naa>30{&U z;9BKUQTX5;F)8j{#QxCDXpeJXpo~1Tv5;wC&~e|w#!7ko`Q9l1cXNTfvo0|Q$vpm4 ziT5n{vnCU&5wU|I2QlommBjC96fmKvWNuf0gQch!$nJu>JxxQ+P2 zTo8CJgnuU6za1i1NiWJDN}rBmqiiWYHuHkF-q`aLY}-NG;op#g9L> zBhceZD1RZNO2S~MW44ZAR_rrrgaAD<)V9k^w1hozA~~4%YcZ@WA1m%F(sd{D-GUL) z83Dz1n2*u*xn`BIc1d%)`Q~=eu9&pK`ply0WCWdlh(vLSYvhwAP-AdfhKLj|-Mukd zjI??=(JTADUGELVmGH z(Z=s1Bg%F9!FNap3a$Sr-@*)4j&H!v{R2QX@&(2?G$E+glOAIE8erzNdzsm zH+1e7OrZpJlt<{eR|bIg_|Al=ATw+-5$!|+5n~+6UG}$M{fl+fs>?0Y?D_IgVe7dA z={g`b-zT9SqDUM&(D?ExTB~~29M_ekXkDlc;;#lZId8q;d*?V)bd=UMi+1wfEsvh${>rZdJ}Ap z@tngXhxICnT{}{_S3Fp@z=>ZqNGJj6zP)Oao@K<=HUQhN?P($a?pse3jT^ zsoo=08}wD?K=o?Bvv@8up7mPOsA*V8uN7)@4l(Ic&>i3 zT;{4gP0AADxOfUFDNA9XIOb+{B#=rRr+Vwko$_hRAR-ko`4Ea5?i; zTvJt7uD;9FBNf5>jcurz4B~qCZ?-PW<V#sVF~{Hu06q(#;MnzthD z5j~6k$}a=rgN@(Dv%(!_1q&uwqRGD(GIZw?s*FI?n5Kp44njWVCek=JjUi2qX`OPv zl42lTM@?USE|Xo|oo5|fsR6;$E-cY5W4Jkf`YL?9MqzCrMT4GU$0-wr4Fl*L2Un>7 zp#gq>c0Q=j-V7!UiWp#aw>3*MsKS>Bj3ZUk0}gTDa+#%>^BmC3LVC39Rl|j?L#GmV z*FXh(WDD&RiRp6b%oEKp>oZPpD?WQR49_^Fnypw|+M8h0eq>>#O&7B~HDdulDeYOc zuzJQuTU_N_udB(4k{u)G-Yois$^2a>p`p+)CkwToBeZ)qW@Z^9VD^2~^Kd;N3RAynKKYfTksB(M z2Y=&bK8fPqyQCE54+WYZUNk?W&a~LZq>!qoOzGdScxMRcfFxC;W?>52O3?%e*9~Jy zrJQ`{B#K~dO@%o_N8PSQ*9TOjDg&t3k2l6BGX`ysgm90iJuX~To&{C8>_v$uR2Nz*!(#yYr8IF z++WK-NNgPJ`7PQ*ZwT$l?Cnn)9-}QrkSv9L5EUhjcKkZEI+4E)JZeXPcu@|>Zdvb; z^&XU{_coMTd0505IYjY&s2V`=!3am=abUT$vhrdQH8TBP4(!Yq2!(=Ws|WMeGaMft z5a5F$%WkC*?JxeiY3(s42(8r=NmnEADyFiX;OxpUAm`Ze zS-lBa-}H4~dPu8}?|I|dScq@r=4%>phy9otxSZ_hVJ3kGZ+f9}7*aIzYhbj%GrMfw zXfey_P&zGcUgv#==3Od!J=ru_k9^4JYi~<>DC$HD!nV5(uU%a5wdtj<$z1~-Jg~~_I9`~pCa{GjO%{M|9akx(pKi%5 zND#TE@~U~HvcS3Z?}MAP;bMjW4xI6HT9Li#%KfuH(Xup-%Gn)Lf?gHMN?RZ(W_T8O zQ3W`S>C?aOQy3)w5^p2fzwnbn8qn}v20HJO)O`%6d5r7RJVfi9Oa82DXL8?1=XhJ+ zWd1v_+32V*hYOb?MOkI12X(zw6wLC^qUHmRONsd4!5`y&Rwk!hk+0}#0z{;ji?FYX zHS_d8)>`D=2b9Ln&IoW-Sco;xxZ$vhEQQswjSfzGp1rM9EvT%^ZoeN&YM5|by*Cot zu!4MmURF~ZgMm)ZU;?wc{K%uYLt$$@e@Fa#ir6L@PmTvVL41o22KID@4%$m_G}Cu5 zHd1zSFt;)NktvqGm5okeMr>A_dwf?v63Y}~1ht>3rn zs0gsqz{28lc~-^3UamXw)+=dhezu0fo6=Z+fmyf2({?~tz&Axj{kqikqfbhz`G7#r z$MVIAx}LjI*ejmpglTI{ouYmeHMWZfcby0XD%&%j3(b11WxCdqyw>YtmU(v72vl|1 zW@lrT!IyP4pmo@G_@U*~(c{Aoo^7MaUP*8Wgj|Nurmo<2v8RQTztm!d~v>(=@KJlxYJ}dV8g>bUjNA z?g&)cV;yo8fS2k8{`nht@XzN>Gr^^WHifg95qY5AkYrlC7J`yuW=9Om%N$QkPWd7g zKe-IX{#6LsLOUFwRRgH>MlO;vNEBi)uwhl8na|vETV?-h6omtL$RI&wZ-WfKNQ*(k zKqtcn3zTA0fQ?I_iOO)!`3#WG^c$wnl~79SfI+#&q_E>#3m=dFXbr+Fa3z)*g^OHYTQ z2DY}AKXxgVwQc8^F}>5PAN6hsQy_PQMGIb&YUC=t0gN4(pQ4Anb>8pg46vB-FmX zyZL9e+nbL{Be1YJq4wt1glGnzW8X}kzZoJ|fzPG0Af41SuoyD4k?OIcc)jATmXoEm zmOwhL&84Dv95M8!wm7kxA=JcQ;gbH|ldZi~Sed37afL zf(4iR94sBLkG>fJp;UknOl?LUHm5=&D_<7vYG0PS<7B=1*f1w>vXH=n3@#8DQ`1l$ zVJnT`%ga~hoW2||GbyH9*0J2?^s!u1l{>0LP)a#YKZH_*-xc`2h^ARGS%$86_z z5a|K;mDwZ*#Rq_5%qz1E88U-?&-NM8nf4{z`M1>*d_BIgiivIk_M`ZDK6CiDhUFj0 z)jF5V(vZbCV6r z)#BOci?zF5_|sKExP1bmL-VtZ^Mm}yAIBB{zMTm={|3`_Ft+;haNLv2rRfp$Y#9`T z5dOw6J|RnEN2I!#QG1R+g*sa;vaVQYvrbGe_GOK{-zCaCj#6U8svop83n|i1eu0Vp zbhDMb>}Ax5zeu9Pk{8wF@2u!AXxr=3qIm4(_IA<#bxHFSgu9Ae2V~>7Q-iy}fGs}2 z93y#*wwKZMreDk zpr#vJ+oHYUl~Z_RopmQhk!(H zhIw-lxR@%MAglRvh`?7qsG6)vyPdwv?cw#oD>gEm{-1=d(#FK@I_0N}m=Gu3Oi4lw z6m}eR*~?gu?LEBRpvA1T^)-)1+ulKj>_6B%2A3K)h&oTip<_uvTWtB{b8K?BB)nM8 zj+5LKF;(wZ*j?H()y$MzcPsAHMd~$4`u6rYEYJYq8~gd%_tsjb0ZJfFy06R5by!f| z95nqBcdrL|2d7Few8_Ei_d>cM?iyIZA*wo#SZ~1o-t-r4l+uXF9Jnps zT*{HX7{FYy*aR6)Y3AKX{c(!KjUMKYiRb%FCau3mEn9b}7mlE)MF@IDpLWwVMQm-H zjBT8BRov~29krh#R}EmmrsK(0@wX^3byzG4g+;XB;rs~PDDH?stw&vg z6sS<+qq5|{G|xjV0qE@9vREiL`qI9)Ob4GKBdUwoCnCIMba<}l+esqO<2gQ5Vv*n@ zm6t=u#Y4yx%<1)np)NpdQzMLmEAO?9%Gfkx`vv>-r+#2#fm%9pIjXOwdi#4;87%IKD+{C5T5fcGGD$Zi}}(@ zy)Y{A*x~nm3V7gqw{l`n>;s`7{R|xQM@I6qs_#!PfgA<)42%SHRipe@pZ#R;?-S@L z82xNd1%j1ize3aRLjMd-Pc%P;PW?^uV>10u!k%X9&-Ntjsr{pH@ppNDmM%Vlf6Al! zC)rbD-_>RR)1W~Vf3>GUTY?e- zP$c@F2!6NHJ^4RE*Pn|18RMP={j?G}#XrdYE8_jh=g-vl7oRu8KluDtBK#+ZKlfLE zabU&z$>Hza*8lUduYu~x{yx$BQ{taTf4?NcfikYY$$lLK{>k9a9PJl_Owu0={%N-V zN$IDmx1a6F!x29{{6zeB4czbV_McU7PqaV1R{vkLziQ(CRR7Nkh+ka5LD>Un{QuD4 m2bUl95dY^~P7(i=%a2ob1!<@sfdd1yOvHh7zI&RKVE+qp0pD-{ literal 0 HcmV?d00001 From 7c345ccc139339607f7bdaf8f2555fcbce256c6f Mon Sep 17 00:00:00 2001 From: Evan Chen Date: Tue, 19 Mar 2024 18:58:00 +0800 Subject: [PATCH 2/2] byte fix --- sub_swebench_dataset/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sub_swebench_dataset/readme.md b/sub_swebench_dataset/readme.md index acd041d1c..9c9f2fea8 100644 --- a/sub_swebench_dataset/readme.md +++ b/sub_swebench_dataset/readme.md @@ -82,4 +82,4 @@ ## scikit-learn-68 dataset Description: instance_id_pass:12 -instance_id_pass:56 +instance_id_fail:56