diff --git a/results/lora_sft/9g-8B/events.out.tfevents.1724591877.ubuntu b/results/lora_sft/9g-8B/events.out.tfevents.1724591877.ubuntu new file mode 100644 index 00000000..4d07ac63 Binary files /dev/null and b/results/lora_sft/9g-8B/events.out.tfevents.1724591877.ubuntu differ diff --git a/results/lora_sft/9g-8B/run-.-tag-Loss_train_1.csv b/results/lora_sft/9g-8B/run-.-tag-Loss_train_1.csv new file mode 100644 index 00000000..859cd16b --- /dev/null +++ b/results/lora_sft/9g-8B/run-.-tag-Loss_train_1.csv @@ -0,0 +1,1001 @@ +Wall time,Step,Value +1724591895.7673328,2,2.0924906730651855 +1724591903.7862196,4,2.1884868144989014 +1724591911.7835774,6,2.089179277420044 +1724591915.8130808,7,2.0493855476379395 +1724591923.8368187,9,2.002346992492676 +1724591931.8776395,11,2.012960910797119 +1724591935.79909,12,2.040778160095215 +1724591939.840039,13,1.9084585905075073 +1724591943.8328624,14,1.980177402496338 +1724591947.8187883,15,1.9899401664733887 +1724591951.8475142,16,1.862898588180542 +1724591955.8360763,17,1.8881759643554688 +1724591959.863927,18,1.8723196983337402 +1724591963.8491657,19,1.8965755701065063 +1724591967.853598,20,1.8681071996688843 +1724591971.822212,21,1.8573286533355713 +1724591979.8300676,23,1.8820374011993408 +1724591987.854274,25,1.823925256729126 +1724591991.8387575,26,1.8618351221084595 +1724591999.875711,28,1.8112305402755737 +1724592011.868203,31,1.7973077297210693 +1724592015.8749664,32,1.6861262321472168 +1724592019.8671262,33,1.8534733057022095 +1724592023.8273726,34,1.7798333168029785 +1724592027.762172,35,1.7968920469284058 +1724592035.697219,37,1.8431357145309448 +1724592047.5892851,40,1.7941187620162964 +1724592051.5849187,41,1.7816107273101807 +1724592059.5945733,43,1.7702220678329468 +1724592063.5747588,44,1.8031026124954224 +1724592067.6001232,45,1.7257362604141235 +1724592071.620778,46,1.7512564659118652 +1724592075.5986588,47,1.7594759464263916 +1724592079.6325896,48,1.6466701030731201 +1724592087.6215284,50,1.7511193752288818 +1724592091.5227664,51,1.7717334032058716 +1724592103.5358627,54,1.7511334419250488 +1724592107.557504,55,1.7263318300247192 +1724592111.503521,56,1.775518536567688 +1724592115.5004547,57,1.7020866870880127 +1724592119.653741,58,1.7273216247558594 +1724592131.5930946,61,1.7568507194519043 +1724592135.6242054,62,1.6650809049606323 +1724592139.6311483,63,1.7246265411376953 +1724592143.6259499,64,1.7642258405685425 +1724592147.6190734,65,1.7747801542282104 +1724592151.6092856,66,1.7013943195343018 +1724592155.6029515,67,1.7020920515060425 +1724592159.6466856,68,1.6936891078948975 +1724592163.5650609,69,1.7581435441970825 +1724592167.5637834,70,1.6886496543884277 +1724592175.5394301,72,1.71756911277771 +1724592179.546017,73,1.6686737537384033 +1724592183.531664,74,1.7287335395812988 +1724592187.5614123,75,1.6750768423080444 +1724592195.2588487,77,1.6948763132095337 +1724592203.1675034,79,1.7345706224441528 +1724592207.1558638,80,1.7281575202941895 +1724592215.0790987,82,1.7174237966537476 +1724592219.0698707,83,1.7096030712127686 +1724592227.0831091,85,1.6864454746246338 +1724592231.1008646,86,1.6455317735671997 +1724592235.1129794,87,1.636845350265503 +1724592239.124649,88,1.672706127166748 +1724592243.0422738,89,1.781375765800476 +1724592247.057729,90,1.7119501829147339 +1724592255.0159981,92,1.7321443557739258 +1724592259.0237758,93,1.7067581415176392 +1724592263.0165985,94,1.6917155981063843 +1724592266.93894,95,1.6568050384521484 +1724592274.9819026,97,1.7193955183029175 +1724592287.0205972,100,1.6833431720733643 +1724592290.9472883,101,1.6715580224990845 +1724592294.923605,102,1.7354767322540283 +1724592298.935973,103,1.675136685371399 +1724592306.959923,105,1.6732913255691528 +1724592310.9599767,106,1.7329044342041016 +1724592314.9755545,107,1.6450945138931274 +1724592327.0043902,110,1.674322247505188 +1724592330.9998517,111,1.6918859481811523 +1724592342.9661105,114,1.6659235954284668 +1724592354.6694667,117,1.5986591577529907 +1724592358.6427186,118,1.6987007856369019 +1724592362.6304786,119,1.6862797737121582 +1724592366.6653185,120,1.659897804260254 +1724592370.6821477,121,1.6688566207885742 +1724592382.6092474,124,1.6560938358306885 +1724592386.618379,125,1.687709093093872 +1724592390.6333547,126,1.708358883857727 +1724592394.6654212,127,1.591117262840271 +1724592398.6609354,128,1.7426371574401855 +1724592406.6591988,130,1.5909861326217651 +1724592410.67122,131,1.6615251302719116 +1724592422.6452827,134,1.7457573413848877 +1724592434.5207329,137,1.6685446500778198 +1724592438.5152159,138,1.705800175666809 +1724592446.483907,140,1.7242212295532227 +1724592454.5152047,142,1.7219339609146118 +1724592458.388004,143,1.6644915342330933 +1724592462.2997324,144,1.692807912826538 +1724592466.3051763,145,1.6614251136779785 +1724592470.2808988,146,1.717132329940796 +1724592474.2542183,147,1.7115483283996582 +1724592482.2362328,149,1.7354657649993896 +1724592486.2322526,150,1.694509744644165 +1724592490.131594,151,1.6406375169754028 +1724592494.1183612,152,1.657658576965332 +1724592501.7827253,154,1.6764616966247559 +1724592505.7194946,155,1.7306265830993652 +1724592509.6982474,156,1.712843656539917 +1724592517.7009306,158,1.722342610359192 +1724592521.7096179,159,1.6728965044021606 +1724592525.725289,160,1.6665853261947632 +1724592533.874415,162,1.656968593597412 +1724592537.8292758,163,1.651583194732666 +1724592545.797959,165,1.7042840719223022 +1724592549.7908978,166,1.6419987678527832 +1724592557.849277,168,1.663612723350525 +1724592561.768128,169,1.680391550064087 +1724592565.768811,170,1.6691410541534424 +1724592573.7745097,172,1.640392780303955 +1724592577.7828119,173,1.6522971391677856 +1724592590.0027785,176,1.7163655757904053 +1724592594.0089204,177,1.6993637084960938 +1724592598.0099542,178,1.626343846321106 +1724592602.017799,179,1.6423274278640747 +1724592610.0240364,181,1.6436882019042969 +1724592614.006249,182,1.6603766679763794 +1724592622.0332181,184,1.6868664026260376 +1724592630.048033,186,1.6719794273376465 +1724592633.756121,187,1.6235815286636353 +1724592637.6917903,188,1.6622204780578613 +1724592641.6868212,189,1.6405177116394043 +1724592649.5647964,191,1.6276273727416992 +1724592653.561267,192,1.6594457626342773 +1724592665.478822,195,1.711427092552185 +1724592681.448392,199,1.6449651718139648 +1724592685.4673026,200,1.6216150522232056 +1724592697.4843075,203,1.6697794198989868 +1724592701.483765,204,1.6801173686981201 +1724592709.4717062,206,1.6686201095581055 +1724592713.380155,207,1.6229263544082642 +1724592717.4130158,208,1.6641591787338257 +1724592721.4295025,209,1.6842747926712036 +1724592725.1223762,210,1.6408222913742065 +1724592737.0867553,213,1.699344277381897 +1724592741.0869913,214,1.630943775177002 +1724592745.1038785,215,1.6736611127853394 +1724592749.170149,216,1.5864779949188232 +1724592761.0776734,219,1.632934808731079 +1724592765.107173,220,1.6631921529769897 +1724592769.109426,221,1.671012043952942 +1724592773.1173482,222,1.6681619882583618 +1724592777.1265402,223,1.6251018047332764 +1724592781.1251152,224,1.6547967195510864 +1724592789.1850176,226,1.7395738363265991 +1724592793.20698,227,1.6144015789031982 +1724592797.2437487,228,1.6483744382858276 +1724592805.2692735,230,1.6624276638031006 +1724592809.262708,231,1.7031261920928955 +1724592813.288053,232,1.6371679306030273 +1724592817.2306206,233,1.6798880100250244 +1724592821.2208228,234,1.657408595085144 +1724592825.2115564,235,1.6718920469284058 +1724592829.2183518,236,1.680406093597412 +1724592837.2319312,238,1.6603364944458008 +1724592841.1505823,239,1.6044965982437134 +1724592849.0757325,241,1.6435601711273193 +1724592864.5618088,245,1.656528115272522 +1724592868.5738158,246,1.6994155645370483 +1724592876.563715,248,1.676395058631897 +1724592880.559988,249,1.6603363752365112 +1724592884.5366688,250,1.6916838884353638 +1724592888.5406754,251,1.6656404733657837 +1724592892.559856,252,1.626572608947754 +1724592896.4570284,253,1.681372046470642 +1724592900.3716512,254,1.619728922843933 +1724592908.3635445,256,1.6832945346832275 +1724592920.3718863,259,1.6463613510131836 +1724592924.3724248,260,1.6481072902679443 +1724592928.3702471,261,1.6230733394622803 +1724592932.3761652,262,1.6568483114242554 +1724592940.372691,264,1.6332180500030518 +1724592948.3005328,266,1.6534239053726196 +1724592956.3727496,268,1.5793068408966064 +1724592964.2763257,270,1.6927387714385986 +1724592972.3172731,272,1.6467803716659546 +1724592976.3550558,273,1.6055824756622314 +1724592980.3717358,274,1.6368635892868042 +1724592984.3882291,275,1.7085376977920532 +1724592988.380189,276,1.7301744222640991 +1724592996.319563,278,1.6638286113739014 +1724593000.3408675,279,1.6236436367034912 +1724593008.3415768,281,1.6727626323699951 +1724593012.3576605,282,1.6347222328186035 +1724593016.348321,283,1.6613423824310303 +1724593024.2418792,285,1.686631441116333 +1724593044.003864,290,1.6305145025253296 +1724593051.9930024,292,1.6325911283493042 +1724593055.9897141,293,1.6256589889526367 +1724593063.969948,295,1.6979279518127441 +1724593067.9812253,296,1.651720643043518 +1724593072.0109863,297,1.6170549392700195 +1724593075.9788516,298,1.6958918571472168 +1724593079.9930897,299,1.693881630897522 +1724593084.0038986,300,1.6779309511184692 +1724593087.9898725,301,1.6943875551223755 +1724593091.9098341,302,1.6429847478866577 +1724593095.9237375,303,1.6420269012451172 +1724593099.906791,304,1.7035083770751953 +1724593115.6082506,308,1.697654128074646 +1724593119.5206094,309,1.6295747756958008 +1724593127.5081851,311,1.6084697246551514 +1724593139.5388815,314,1.6710929870605469 +1724593151.557887,317,1.6267063617706299 +1724593155.5533242,318,1.6186566352844238 +1724593159.5659995,319,1.641950011253357 +1724593163.5410502,320,1.669058918952942 +1724593167.5160465,321,1.691622018814087 +1724593175.5082412,323,1.6472365856170654 +1724593179.5169897,324,1.6725211143493652 +1724593183.5081213,325,1.6404271125793457 +1724593187.5401218,326,1.5998404026031494 +1724593191.2028434,327,1.6780979633331299 +1724593195.1076746,328,1.6328086853027344 +1724593207.1154668,331,1.6702239513397217 +1724593211.1257305,332,1.65586256980896 +1724593219.152687,334,1.648804783821106 +1724593227.1811755,336,1.7019060850143433 +1724593235.1243677,338,1.667287826538086 +1724593243.1833153,340,1.6045408248901367 +1724593251.2294974,342,1.643607258796692 +1724593255.2381866,343,1.6281960010528564 +1724593259.2329223,344,1.6561239957809448 +1724593263.1687214,345,1.6079117059707642 +1724593267.0681806,346,1.6573951244354248 +1724593271.0493135,347,1.7081780433654785 +1724593275.0649452,348,1.6754027605056763 +1724593279.0602028,349,1.6714141368865967 +1724593283.127029,350,1.5705291032791138 +1724593291.1368103,352,1.6349539756774902 +1724593299.1411042,354,1.6191999912261963 +1724593303.1456354,355,1.6756516695022583 +1724593311.0251527,357,1.6125526428222656 +1724593318.9916608,359,1.6516737937927246 +1724593323.027167,360,1.5913676023483276 +1724593327.0712976,361,1.6080219745635986 +1724593331.1137645,362,1.6352267265319824 +1724593335.0153222,363,1.6617048978805542 +1724593339.0139503,364,1.67450749874115 +1724593343.0197537,365,1.726904273033142 +1724593347.0288136,366,1.6130926609039307 +1724593351.0301359,367,1.6253001689910889 +1724593354.7083662,368,1.6273034811019897 +1724593358.6268294,369,1.5890530347824097 +1724593366.638974,371,1.6640105247497559 +1724593370.6347053,372,1.6690783500671387 +1724593378.4389083,374,1.7219219207763672 +1724593386.4370923,376,1.6643303632736206 +1724593390.4155133,377,1.6704201698303223 +1724593394.3945708,378,1.6572843790054321 +1724593398.3946857,379,1.6494579315185547 +1724593402.4250753,380,1.6530559062957764 +1724593410.4026918,382,1.6740442514419556 +1724593414.3925035,383,1.7022160291671753 +1724593422.372594,385,1.6612892150878906 +1724593426.4054008,386,1.6032545566558838 +1724593430.3270085,387,1.6578110456466675 +1724593434.32704,388,1.663527011871338 +1724593438.3529594,389,1.6003470420837402 +1724593446.437341,391,1.6145573854446411 +1724593450.4138873,392,1.6690014600753784 +1724593454.3263898,393,1.6698256731033325 +1724593458.333144,394,1.6588504314422607 +1724593470.2246826,397,1.6768150329589844 +1724593478.14831,399,1.6419646739959717 +1724593482.1428814,400,1.603395938873291 +1724593490.109905,402,1.5870752334594727 +1724593493.799013,403,1.6472591161727905 +1724593497.7602057,404,1.6366127729415894 +1724593505.6865163,406,1.649254322052002 +1724593509.6887264,407,1.6657850742340088 +1724593517.653434,409,1.5946091413497925 +1724593525.6535513,411,1.6037495136260986 +1724593545.4319036,416,1.6362851858139038 +1724593549.4059644,417,1.7176486253738403 +1724593561.4307399,420,1.6239328384399414 +1724593565.4256983,421,1.6481516361236572 +1724593569.4344928,422,1.638138771057129 +1724593573.4391959,423,1.5878921747207642 +1724593577.4272573,424,1.6852614879608154 +1724593581.4245296,425,1.6440485715866089 +1724593589.4303782,427,1.653361439704895 +1724593597.3291976,429,1.6479272842407227 +1724593605.3636599,431,1.7219957113265991 +1724593613.3892953,433,1.6605092287063599 +1724593625.0563846,436,1.6639666557312012 +1724593629.0459404,437,1.5961003303527832 +1724593640.9275117,440,1.667154312133789 +1724593644.9086258,441,1.6481815576553345 +1724593648.9064593,442,1.650304913520813 +1724593652.9078946,443,1.588853359222412 +1724593660.881475,445,1.6840147972106934 +1724593664.879202,446,1.6754071712493896 +1724593672.8636718,448,1.5885560512542725 +1724593676.859381,449,1.6908894777297974 +1724593680.8434682,450,1.6909427642822266 +1724593684.841897,451,1.654309630393982 +1724593688.8323696,452,1.6954340934753418 +1724593692.8167925,453,1.632520079612732 +1724593700.7030706,455,1.691481113433838 +1724593704.701109,456,1.609863519668579 +1724593712.6285903,458,1.6529273986816406 +1724593716.6601307,459,1.584636926651001 +1724593720.6360168,460,1.6853936910629272 +1724593724.6238346,461,1.6067816019058228 +1724593728.5364165,462,1.6568337678909302 +1724593732.5514963,463,1.628319501876831 +1724593736.534518,464,1.6657829284667969 +1724593740.577932,465,1.6313947439193726 +1724593744.4883573,466,1.6815667152404785 +1724593748.4820027,467,1.6323297023773193 +1724593752.4660795,468,1.6379603147506714 +1724593756.4588313,469,1.6123274564743042 +1724593760.159047,470,1.7142006158828735 +1724593764.1667674,471,1.6446115970611572 +1724593768.1618278,472,1.611260175704956 +1724593776.0491865,474,1.6623344421386719 +1724593780.0189118,475,1.6654438972473145 +1724593791.960042,478,1.6803433895111084 +1724593795.9549413,479,1.6742596626281738 +1724593812.0033991,483,1.664589524269104 +1724593823.89094,486,1.6430723667144775 +1724593827.8810506,487,1.6351560354232788 +1724593835.9277942,489,1.5932891368865967 +1724593839.9394362,490,1.6351947784423828 +1724593847.9274378,492,1.651682734489441 +1724593851.8154671,493,1.674103021621704 +1724593855.807743,494,1.6466901302337646 +1724593867.725207,497,1.6434919834136963 +1724593871.740704,498,1.6633150577545166 +1724593879.843009,500,1.684281587600708 +1724593895.9149642,504,1.6306133270263672 +1724593899.911852,505,1.710532784461975 +1724593907.9295805,507,1.6601154804229736 +1724593911.6336465,508,1.5843942165374756 +1724593915.5921137,509,1.6529828310012817 +1724593919.5981255,510,1.6073322296142578 +1724593923.597571,511,1.6321625709533691 +1724593927.6260211,512,1.6442110538482666 +1724593931.5878303,513,1.676708698272705 +1724593939.6227918,515,1.61398184299469 +1724593943.6838543,516,1.6264729499816895 +1724593947.6864586,517,1.6589444875717163 +1724593951.6830482,518,1.6305358409881592 +1724593955.675951,519,1.674702525138855 +1724593959.6473725,520,1.6909335851669312 +1724593963.6187,521,1.667889952659607 +1724593975.604819,524,1.6480381488800049 +1724593979.5107744,525,1.6323785781860352 +1724593983.5167785,526,1.6746914386749268 +1724594007.4477358,532,1.60276198387146 +1724594015.2589617,534,1.6330095529556274 +1724594019.245458,535,1.5920149087905884 +1724594023.2651625,536,1.622429609298706 +1724594027.2788234,537,1.6666884422302246 +1724594031.171994,538,1.6947013139724731 +1724594035.1714737,539,1.6533474922180176 +1724594039.076757,540,1.6567654609680176 +1724594042.7696424,541,1.639661192893982 +1724594046.7464578,542,1.6506903171539307 +1724594050.7503777,543,1.649553656578064 +1724594054.766466,544,1.5894949436187744 +1724594058.7855198,545,1.6412320137023926 +1724594062.7283435,546,1.6520439386367798 +1724594066.743663,547,1.6287124156951904 +1724594070.701404,548,1.6840158700942993 +1724594074.6625125,549,1.7012786865234375 +1724594078.6467812,550,1.6785351037979126 +1724594086.6487207,552,1.6838409900665283 +1724594090.6438777,553,1.6412124633789062 +1724594094.650785,554,1.602677345275879 +1724594102.6436052,556,1.620790958404541 +1724594110.6306179,558,1.569399118423462 +1724594118.6216822,560,1.6374996900558472 +1724594122.616615,561,1.66169273853302 +1724594134.6029255,564,1.6457538604736328 +1724594138.6121483,565,1.6375128030776978 +1724594142.5013802,566,1.6741929054260254 +1724594150.3900392,568,1.6381208896636963 +1724594154.3990006,569,1.6817439794540405 +1724594158.294399,570,1.6507946252822876 +1724594166.2055216,572,1.620227575302124 +1724594170.2051318,573,1.6386134624481201 +1724594174.1906796,574,1.652235984802246 +1724594177.8273659,575,1.6358535289764404 +1724594185.7278693,577,1.6810483932495117 +1724594189.7158685,578,1.628760576248169 +1724594193.6544487,579,1.6164402961730957 +1724594197.6846054,580,1.6739747524261475 +1724594201.6764796,581,1.6322181224822998 +1724594205.6761672,582,1.6797139644622803 +1724594209.6851497,583,1.6134328842163086 +1724594213.6864562,584,1.6270999908447266 +1724594217.7019856,585,1.6709556579589844 +1724594225.6886344,587,1.655944585800171 +1724594229.6896193,588,1.647727131843567 +1724594233.7280018,589,1.6362636089324951 +1724594245.7255454,592,1.5932101011276245 +1724594249.705422,593,1.6804845333099365 +1724594253.844185,594,1.6560276746749878 +1724594261.886095,596,1.6289080381393433 +1724594265.8927746,597,1.6822501420974731 +1724594269.862658,598,1.669326663017273 +1724594273.7619607,599,1.5723774433135986 +1724594281.687851,601,1.5783418416976929 +1724594285.66996,602,1.6436396837234497 +1724594289.6750379,603,1.6489057540893555 +1724594293.7042418,604,1.5779448747634888 +1724594297.3918567,605,1.6824212074279785 +1724594305.602077,607,1.649396300315857 +1724594313.6073205,609,1.6419621706008911 +1724594321.6029541,611,1.6488527059555054 +1724594325.5864985,612,1.6124566793441772 +1724594329.6008184,613,1.6112785339355469 +1724594341.6314585,616,1.5693241357803345 +1724594357.618019,620,1.6563509702682495 +1724594361.5565748,621,1.6289563179016113 +1724594365.5005298,622,1.5988991260528564 +1724594369.4078097,623,1.6642277240753174 +1724594373.4032776,624,1.6425937414169312 +1724594381.3738952,626,1.595876932144165 +1724594385.3751807,627,1.61759614944458 +1724594397.3854368,630,1.6603071689605713 +1724594401.288906,631,1.6656378507614136 +1724594409.3188298,633,1.645189642906189 +1724594417.3296301,635,1.6579138040542603 +1724594421.3287473,636,1.6563223600387573 +1724594433.3654299,639,1.6389105319976807 +1724594441.373341,641,1.646579623222351 +1724594445.3737845,642,1.6825968027114868 +1724594449.4115543,643,1.65375816822052 +1724594453.4051797,644,1.6266741752624512 +1724594457.4054885,645,1.6901628971099854 +1724594461.4096794,646,1.623475193977356 +1724594465.4019,647,1.6566567420959473 +1724594469.4368656,648,1.6259456872940063 +1724594473.4316874,649,1.704770565032959 +1724594477.4449914,650,1.632115364074707 +1724594481.4396353,651,1.6586759090423584 +1724594485.483506,652,1.6076996326446533 +1724594489.4884882,653,1.6536320447921753 +1724594493.480793,654,1.6476144790649414 +1724594505.4161541,657,1.6364078521728516 +1724594509.3968356,658,1.6612385511398315 +1724594517.420088,660,1.598564624786377 +1724594525.4019625,662,1.653748631477356 +1724594529.440306,663,1.5577627420425415 +1724594533.4375153,664,1.6728359460830688 +1724594537.3917356,665,1.6113672256469727 +1724594541.3817294,666,1.6305201053619385 +1724594545.303132,667,1.6178709268569946 +1724594549.30254,668,1.6385592222213745 +1724594553.3017585,669,1.6591699123382568 +1724594557.308716,670,1.6486300230026245 +1724594564.8105311,672,1.6756733655929565 +1724594568.807138,673,1.6123135089874268 +1724594572.7721612,674,1.6318118572235107 +1724594576.7579374,675,1.628105640411377 +1724594580.73953,676,1.6320900917053223 +1724594588.6701272,678,1.6130226850509644 +1724594592.6670983,679,1.646243929862976 +1724594596.658891,680,1.61703360080719 +1724594600.7309227,681,1.5786726474761963 +1724594604.7806618,682,1.6212334632873535 +1724594608.6862693,683,1.6858841180801392 +1724594612.5833797,684,1.5953264236450195 +1724594616.6002898,685,1.6881577968597412 +1724594620.5241554,686,1.5832538604736328 +1724594624.5217314,687,1.653733253479004 +1724594628.5041034,688,1.6823312044143677 +1724594632.3864865,689,1.6477729082107544 +1724594636.5802486,690,1.648080825805664 +1724594648.9260788,693,1.6058913469314575 +1724594656.9358025,695,1.6863245964050293 +1724594664.9468453,697,1.6540290117263794 +1724594676.9609659,700,1.626447081565857 +1724594681.2560055,701,1.5863937139511108 +1724594685.4513624,702,1.6012752056121826 +1724594689.6349163,703,1.573264241218567 +1724594693.3258927,704,1.6704623699188232 +1724594697.3521433,705,1.6248327493667603 +1724594701.3674147,706,1.6592590808868408 +1724594705.3625424,707,1.6413236856460571 +1724594709.3743446,708,1.604191780090332 +1724594713.4123354,709,1.6515394449234009 +1724594717.4326482,710,1.6410752534866333 +1724594721.4212704,711,1.698554277420044 +1724594725.421194,712,1.6945022344589233 +1724594737.4509711,715,1.6453217267990112 +1724594741.5757625,716,1.6143999099731445 +1724594745.567285,717,1.630463719367981 +1724594753.5333252,719,1.6188547611236572 +1724594757.5491915,720,1.5961329936981201 +1724594765.5767753,722,1.6248053312301636 +1724594769.5976968,723,1.6298363208770752 +1724594773.6423423,724,1.6365479230880737 +1724594777.7184505,725,1.5861119031906128 +1724594781.7073648,726,1.673599362373352 +1724594785.6985788,727,1.6919649839401245 +1724594797.8855433,730,1.6492019891738892 +1724594802.1481695,731,1.5658663511276245 +1724594806.3416555,732,1.6696510314941406 +1724594822.5277228,736,1.6868484020233154 +1724594826.2326026,737,1.632452368736267 +1724594834.234296,739,1.6159573793411255 +1724594838.2612507,740,1.6224148273468018 +1724594842.3199055,741,1.6170939207077026 +1724594846.3254168,742,1.6512055397033691 +1724594854.382155,744,1.4980719089508057 +1724594862.4126217,746,1.625914216041565 +1724594866.4282212,747,1.6206632852554321 +1724594870.407984,748,1.6622531414031982 +1724594878.4763937,750,1.607391357421875 +1724594882.4774144,751,1.6462503671646118 +1724594886.4934466,752,1.638118028640747 +1724594890.5089421,753,1.6103962659835815 +1724594894.5064464,754,1.6570580005645752 +1724594898.5160007,755,1.6362388134002686 +1724594902.5050907,756,1.6546720266342163 +1724594906.5080843,757,1.6415318250656128 +1724594918.5161767,760,1.649804949760437 +1724594922.5416071,761,1.6152043342590332 +1724594930.459381,763,1.6056244373321533 +1724594934.4703357,764,1.6656094789505005 +1724594938.4504604,765,1.6323281526565552 +1724594942.3753717,766,1.613464117050171 +1724594946.3922913,767,1.6337839365005493 +1724594950.3770726,768,1.7036200761795044 +1724594958.292612,770,1.6158971786499023 +1724594961.987442,771,1.6687408685684204 +1724594965.9289515,772,1.6347225904464722 +1724594969.8748484,773,1.6553453207015991 +1724594973.9001222,774,1.6997311115264893 +1724594977.8142698,775,1.6249029636383057 +1724594981.8186777,776,1.5948593616485596 +1724594985.748071,777,1.6754095554351807 +1724594997.686142,780,1.6595189571380615 +1724595001.705964,781,1.5879807472229004 +1724595009.681008,783,1.6444337368011475 +1724595013.6130414,784,1.5996390581130981 +1724595017.6396725,785,1.6246557235717773 +1724595021.642593,786,1.6655586957931519 +1724595029.6670392,788,1.660515546798706 +1724595037.6211865,790,1.6265918016433716 +1724595041.6515694,791,1.684316635131836 +1724595045.6639845,792,1.6197624206542969 +1724595049.6801357,793,1.598349928855896 +1724595053.686589,794,1.598034143447876 +1724595057.7161932,795,1.6093639135360718 +1724595061.7113914,796,1.6326438188552856 +1724595065.720799,797,1.6325836181640625 +1724595073.7727914,799,1.6013308763504028 +1724595077.7808728,800,1.6496142148971558 +1724595081.806827,801,1.6269209384918213 +1724595089.758259,803,1.6786900758743286 +1724595097.3624291,805,1.6276605129241943 +1724595101.3284085,806,1.6090761423110962 +1724595105.3526845,807,1.6353713274002075 +1724595109.3426592,808,1.6893097162246704 +1724595113.3671556,809,1.68410062789917 +1724595117.2677786,810,1.6924335956573486 +1724595125.2724657,812,1.6706593036651611 +1724595129.2923334,813,1.6434285640716553 +1724595133.522482,814,1.6557034254074097 +1724595142.1901958,816,1.6502655744552612 +1724595146.4623713,817,1.624997854232788 +1724595155.1284976,819,1.648629069328308 +1724595167.982983,822,1.6551592350006104 +1724595176.7208087,824,1.5970817804336548 +1724595181.0954406,825,1.6422417163848877 +1724595185.4842384,826,1.6131900548934937 +1724595197.8069363,829,1.631683349609375 +1724595201.8061697,830,1.6345610618591309 +1724595209.8100836,832,1.622223138809204 +1724595213.7953148,833,1.6303361654281616 +1724595221.7932613,835,1.6496199369430542 +1724595225.7101483,836,1.6468937397003174 +1724595229.4520407,837,1.6131640672683716 +1724595237.5141838,839,1.6649794578552246 +1724595241.5228121,840,1.6409718990325928 +1724595253.593835,843,1.5928332805633545 +1724595257.589477,844,1.6389602422714233 +1724595265.577536,846,1.6722993850708008 +1724595269.6347077,847,1.6164684295654297 +1724595273.6379786,848,1.6446439027786255 +1724595277.62757,849,1.679344654083252 +1724595285.5439882,851,1.714489459991455 +1724595289.5615404,852,1.632578730583191 +1724595293.5617902,853,1.6382924318313599 +1724595297.4653072,854,1.613431453704834 +1724595305.5024621,856,1.635909080505371 +1724595313.5106752,858,1.6857590675354004 +1724595321.5222652,860,1.6621240377426147 +1724595325.5257695,861,1.6763356924057007 +1724595329.5212967,862,1.6604095697402954 +1724595333.5295043,863,1.6182856559753418 +1724595337.5278132,864,1.5951941013336182 +1724595353.5526593,868,1.593653917312622 +1724595357.5363975,869,1.641008973121643 +1724595361.1503131,870,1.6066300868988037 +1724595365.0521576,871,1.6392405033111572 +1724595369.0805762,872,1.587095022201538 +1724595376.7896936,874,1.602781057357788 +1724595380.7864058,875,1.6163069009780884 +1724595384.7171302,876,1.629913091659546 +1724595388.757772,877,1.5797920227050781 +1724595392.7673283,878,1.6383529901504517 +1724595396.760173,879,1.6148028373718262 +1724595400.7194686,880,1.6691046953201294 +1724595404.7090547,881,1.670861005783081 +1724595408.6148348,882,1.6730868816375732 +1724595412.6165483,883,1.6331655979156494 +1724595420.6873574,885,1.5779566764831543 +1724595424.6859255,886,1.62453031539917 +1724595428.6990426,887,1.5842761993408203 +1724595432.7392545,888,1.6239210367202759 +1724595436.7468975,889,1.640336036682129 +1724595440.7284808,890,1.6601513624191284 +1724595444.7240055,891,1.6224946975708008 +1724595452.6665995,893,1.6318364143371582 +1724595456.635051,894,1.6493552923202515 +1724595460.708312,895,1.6040679216384888 +1724595464.725592,896,1.6113436222076416 +1724595468.7302232,897,1.64014732837677 +1724595472.7639308,898,1.6398757696151733 +1724595480.769415,900,1.6355656385421753 +1724595484.7588637,901,1.6162784099578857 +1724595500.8441598,905,1.6304739713668823 +1724595504.853993,906,1.5931010246276855 +1724595512.8645394,908,1.6682250499725342 +1724595516.8776193,909,1.6359056234359741 +1724595520.881886,910,1.6201233863830566 +1724595524.9255471,911,1.6580114364624023 +1724595528.9304652,912,1.607328176498413 +1724595532.9608083,913,1.62784743309021 +1724595536.9753006,914,1.6358036994934082 +1724595544.9395146,916,1.6505979299545288 +1724595548.9405708,917,1.6517536640167236 +1724595552.9652078,918,1.6158664226531982 +1724595556.9773996,919,1.6628756523132324 +1724595565.0210085,921,1.6526951789855957 +1724595569.0434,922,1.5957448482513428 +1724595573.0351775,923,1.6412488222122192 +1724595577.0670462,924,1.571773648262024 +1724595580.977261,925,1.6279140710830688 +1724595585.007511,926,1.5959867238998413 +1724595604.8594067,931,1.5979797840118408 +1724595608.8495753,932,1.6254420280456543 +1724595612.8330398,933,1.657322645187378 +1724595616.8377616,934,1.662083625793457 +1724595620.8481143,935,1.6225652694702148 +1724595624.8513894,936,1.661851167678833 +1724595632.8919737,938,1.6184592247009277 +1724595640.5168304,940,1.6599979400634766 +1724595652.5404313,943,1.6532552242279053 +1724595656.5374372,944,1.6322330236434937 +1724595660.5749292,945,1.6176331043243408 +1724595664.2490144,946,1.5348970890045166 +1724595676.1562274,949,1.555751085281372 +1724595680.1633406,950,1.638298749923706 +1724595688.0444129,952,1.6627318859100342 +1724595692.0179644,953,1.637871265411377 +1724595695.9251676,954,1.6668626070022583 +1724595699.9177244,955,1.6667022705078125 +1724595707.945492,957,1.64393949508667 +1724595711.9727504,958,1.6731659173965454 +1724595715.891651,959,1.6164259910583496 +1724595719.8924325,960,1.640811562538147 +1724595723.8189259,961,1.5944880247116089 +1724595727.8285072,962,1.613485336303711 +1724595731.8481064,963,1.5861973762512207 +1724595735.8668604,964,1.6331359148025513 +1724595739.8569965,965,1.6151044368743896 +1724595747.9202726,967,1.611311912536621 +1724595751.9134223,968,1.632597804069519 +1724595759.9889264,970,1.5805387496948242 +1724595768.0662642,972,1.6269910335540771 +1724595772.0207024,973,1.5632432699203491 +1724595780.0925632,975,1.6434667110443115 +1724595788.1052403,977,1.662858009338379 +1724595792.1671925,978,1.5614322423934937 +1724595796.1924868,979,1.5928683280944824 +1724595800.1860323,980,1.6912808418273926 +1724595812.1507843,983,1.6379824876785278 +1724595816.1479208,984,1.641485571861267 +1724595820.1397512,985,1.6243258714675903 +1724595824.146208,986,1.571020245552063 +1724595832.1544273,988,1.613083839416504 +1724595836.6242657,989,1.6395342350006104 +1724595841.1457143,990,1.651950716972351 +1724595845.6903884,991,1.6093226671218872 +1724595849.6587396,992,1.6333720684051514 +1724595861.6902277,995,1.6706610918045044 +1724595869.6486294,997,1.672074556350708 +1724595873.5983553,998,1.6311790943145752 +1724595881.7150016,1000,1.6664949655532837 +1724595885.7246175,1001,1.5774742364883423 +1724595893.7696862,1003,1.5688732862472534 +1724595897.7598684,1004,1.627173900604248 +1724595901.8686602,1005,1.5309216976165771 +1724595905.8734615,1006,1.6119725704193115 +1724595913.8945484,1008,1.6074180603027344 +1724595917.9255204,1009,1.5714199542999268 +1724595921.9302514,1010,1.5535799264907837 +1724595929.5423117,1012,1.6422137022018433 +1724595933.5285301,1013,1.6334582567214966 +1724595937.5181456,1014,1.6697564125061035 +1724595941.5016313,1015,1.6483052968978882 +1724595949.4901261,1017,1.6571868658065796 +1724595953.4178698,1018,1.5788861513137817 +1724595961.403212,1020,1.6669809818267822 +1724595973.29473,1023,1.6193833351135254 +1724595981.2174544,1025,1.6130716800689697 +1724595985.2320795,1026,1.6696724891662598 +1724595989.23509,1027,1.618309497833252 +1724595993.3022175,1028,1.6330125331878662 +1724596001.2446117,1030,1.6277228593826294 +1724596009.27138,1032,1.5885354280471802 +1724596013.272641,1033,1.6376346349716187 +1724596017.2478487,1034,1.6649689674377441 +1724596021.4187746,1035,1.6325229406356812 +1724596025.4245198,1036,1.6631476879119873 +1724596029.4433122,1037,1.6205347776412964 +1724596033.4569464,1038,1.6232020854949951 +1724596037.462428,1039,1.647625207901001 +1724596041.4523714,1040,1.6594955921173096 +1724596053.5242798,1043,1.5960817337036133 +1724596057.2500618,1044,1.6078557968139648 +1724596061.18782,1045,1.620850682258606 +1724596077.7334018,1049,1.616768479347229 +1724596081.8919337,1050,1.6685476303100586 +1724596086.2100086,1051,1.5507731437683105 +1724596090.3819551,1052,1.6297305822372437 +1724596094.4016283,1053,1.7026491165161133 +1724596102.471127,1055,1.614331841468811 +1724596106.5033605,1056,1.6130486726760864 +1724596110.5435755,1057,1.5976022481918335 +1724596114.5804152,1058,1.648413062095642 +1724596118.6099515,1059,1.6295591592788696 +1724596138.5751324,1064,1.6665284633636475 +1724596142.5962787,1065,1.6174640655517578 +1724596146.635858,1066,1.662560224533081 +1724596150.679355,1067,1.5500142574310303 +1724596154.6821456,1068,1.646782398223877 +1724596159.1438162,1069,1.653539776802063 +1724596168.1387973,1071,1.6312311887741089 +1724596172.625883,1072,1.6392295360565186 +1724596177.0879612,1073,1.558337688446045 +1724596181.0999289,1074,1.6146913766860962 +1724596185.107547,1075,1.6841588020324707 +1724596189.0242038,1076,1.5948644876480103 +1724596192.6954162,1077,1.6500824689865112 +1724596196.5984192,1078,1.6738598346710205 +1724596200.4906383,1079,1.6582374572753906 +1724596208.3925524,1081,1.6107219457626343 +1724596212.061335,1082,1.6530249118804932 +1724596216.0431015,1083,1.6008049249649048 +1724596219.9613826,1084,1.5876096487045288 +1724596223.968095,1085,1.6461634635925293 +1724596227.9688957,1086,1.6369413137435913 +1724596231.9756935,1087,1.5501283407211304 +1724596240.0424027,1089,1.554017186164856 +1724596248.0298932,1091,1.617236852645874 +1724596251.9306688,1092,1.6443212032318115 +1724596255.9209082,1093,1.6549057960510254 +1724596259.9074013,1094,1.60243821144104 +1724596263.8189166,1095,1.6253039836883545 +1724596271.8356035,1097,1.6620067358016968 +1724596279.8233018,1099,1.6475448608398438 +1724596283.7222009,1100,1.6178905963897705 +1724596291.6146548,1102,1.6186888217926025 +1724596299.5914829,1104,1.63981032371521 +1724596303.6179435,1105,1.6137821674346924 +1724596307.6336694,1106,1.6554847955703735 +1724596315.62002,1108,1.6410298347473145 +1724596327.7198257,1111,1.6107420921325684 +1724596331.7519538,1112,1.6427828073501587 +1724596343.881464,1115,1.615569829940796 +1724596347.549467,1116,1.655094861984253 +1724596351.4976583,1117,1.5960372686386108 +1724596363.6787977,1120,1.6498881578445435 +1724596367.8022609,1121,1.5958176851272583 +1724596379.648714,1123,1.6803193092346191 +1724596385.812804,1124,1.664862036705017 +1724596391.9698153,1125,1.5743579864501953 +1724596398.112674,1126,1.6459842920303345 +1724596404.3186214,1127,1.6359018087387085 +1724596410.6438887,1128,1.6662495136260986 +1724596416.9721212,1129,1.6240102052688599 +1724596423.2936525,1130,1.6728330850601196 +1724596429.4066668,1131,1.6252390146255493 +1724596435.5587363,1132,1.6332392692565918 +1724596447.90079,1134,1.600312352180481 +1724596454.005718,1135,1.679064154624939 +1724596460.273938,1136,1.691753625869751 +1724596478.9611712,1139,1.7173012495040894 +1724596485.2245548,1140,1.5892283916473389 +1724596497.5153854,1142,1.685258388519287 +1724596503.6281004,1143,1.6498172283172607 +1724596509.7692564,1144,1.6070634126663208 +1724596515.865658,1145,1.6368026733398438 +1724596528.1651158,1147,1.5721629858016968 +1724596534.228979,1148,1.6269662380218506 +1724596546.5594234,1150,1.5650761127471924 +1724596552.6973205,1151,1.6223411560058594 +1724596558.8319967,1152,1.545701265335083 +1724596564.9784284,1153,1.6251206398010254 +1724596571.1274407,1154,1.6495940685272217 +1724596577.2316942,1155,1.5906049013137817 +1724596583.3269272,1156,1.6245079040527344 +1724596589.404562,1157,1.6859034299850464 +1724596595.6077814,1158,1.5586402416229248 +1724596601.7425432,1159,1.6733531951904297 +1724596607.8787117,1160,1.6171554327011108 +1724596614.0638976,1161,1.623660922050476 +1724596620.197362,1162,1.6390693187713623 +1724596626.330251,1163,1.6502761840820312 +1724596638.6410828,1165,1.6573089361190796 +1724596644.8156302,1166,1.6310112476348877 +1724596651.0005965,1167,1.6572296619415283 +1724596657.1544883,1168,1.679098129272461 +1724596669.4306445,1170,1.5997979640960693 +1724596675.5693908,1171,1.6309797763824463 +1724596700.3272407,1175,1.5738433599472046 +1724596718.8950937,1178,1.5717339515686035 +1724596731.4299858,1180,1.6105846166610718 +1724596738.4554155,1181,1.6869081258773804 +1724596745.7904592,1182,1.6059627532958984 +1724596753.1731067,1183,1.632718563079834 +1724596760.5396812,1184,1.607689619064331 +1724596775.2684658,1186,1.5714552402496338 +1724596782.6186347,1187,1.6358915567398071 +1724596789.990157,1188,1.6756608486175537 +1724596797.3447328,1189,1.6041203737258911 +1724596804.6878498,1190,1.6320841312408447 +1724596819.4543211,1192,1.6454815864562988 +1724596826.9933825,1193,1.6653683185577393 +1724596834.6366372,1194,1.606353998184204 +1724596842.2320228,1195,1.560014009475708 +1724596849.7398415,1196,1.6637204885482788 +1724596857.3150687,1197,1.6509037017822266 +1724596864.8475459,1198,1.662248134613037 +1724596872.274506,1199,1.6365344524383545 +1724596879.7032895,1200,1.5884162187576294 +1724596887.043473,1201,1.6793408393859863 +1724596894.4808173,1202,1.6138989925384521 +1724596909.2531154,1204,1.6807196140289307 +1724596916.5660093,1205,1.658231496810913 +1724596931.39193,1207,1.6416857242584229 +1724596938.7243783,1208,1.5824191570281982 +1724596946.0341954,1209,1.6695090532302856 +1724596961.0922108,1211,1.651387095451355 +1724596968.6298463,1212,1.6769760847091675 +1724596976.2299995,1213,1.6024072170257568 +1724596983.5165288,1214,1.6524096727371216 +1724596991.0831401,1215,1.6337223052978516 +1724596998.7055533,1216,1.6373271942138672 +1724597028.982115,1220,1.6615045070648193 +1724597044.1290853,1222,1.6743690967559814 +1724597067.4880142,1225,1.6777210235595703 +1724597075.7156796,1226,1.620629072189331 +1724597083.9231222,1227,1.617063045501709 +1724597092.2888625,1228,1.649963140487671 +1724597100.4354439,1229,1.6462651491165161 +1724597108.658315,1230,1.5989800691604614 +1724597125.291166,1232,1.64650297164917 +1724597133.5377138,1233,1.604837417602539 +1724597141.8986323,1234,1.588707685470581 +1724597150.1606128,1235,1.6344839334487915 +1724597158.6172845,1236,1.6305713653564453 +1724597167.4337788,1237,1.6123712062835693 +1724597176.3668706,1238,1.6329470872879028 +1724597185.0477026,1239,1.5891978740692139 +1724597194.3025298,1240,1.6222245693206787 +1724597204.8398378,1241,1.6221110820770264 +1724597226.018584,1243,1.6542186737060547 +1724597236.3608925,1244,1.567331075668335 +1724597246.8118937,1245,1.6156898736953735 +1724597268.5090847,1247,1.599122405052185 +1724597279.5826137,1248,1.6336579322814941 +1724597290.9662213,1249,1.6102914810180664 +1724597302.3364277,1250,1.645733118057251 +1724597313.7063947,1251,1.6656056642532349 +1724597325.0992978,1252,1.6497478485107422 +1724597347.5382895,1254,1.644105076789856 +1724597358.707233,1255,1.5785504579544067 +1724597370.0268059,1256,1.6490209102630615 +1724597381.1999896,1257,1.5931639671325684 +1724597392.4032543,1258,1.6141234636306763 +1724597403.7268944,1259,1.6771050691604614 +1724597414.793562,1260,1.659956932067871 +1724597437.0614405,1262,1.6649973392486572 +1724597448.1113331,1263,1.6026077270507812 +1724597459.3542829,1264,1.6297717094421387 +1724597470.5375664,1265,1.5960613489151 +1724597481.6615915,1266,1.6136659383773804 +1724597492.927759,1267,1.621364712715149 +1724597504.2619584,1268,1.6729367971420288 +1724597515.2572951,1269,1.625281810760498 +1724597526.5053873,1270,1.6640424728393555 +1724597548.6959422,1272,1.6247841119766235 +1724597559.881047,1273,1.6065276861190796 +1724597582.3867648,1275,1.5719432830810547 +1724597626.81783,1279,1.5756245851516724 +1724597637.9286454,1280,1.674991250038147 +1724597660.3170104,1282,1.6249139308929443 +1724597671.5699637,1283,1.5649566650390625 +1724597682.663689,1284,1.6448215246200562 +1724597705.000186,1286,1.5698758363723755 +1724597727.3364973,1288,1.6255364418029785 +1724597738.545603,1289,1.637969732284546 +1724597749.8125582,1290,1.66135573387146 +1724597772.53697,1292,1.6578171253204346 +1724597783.921252,1293,1.654626727104187 +1724597795.3338156,1294,1.6530821323394775 +1724597806.560859,1295,1.620492696762085 +1724597829.342197,1297,1.5947380065917969 +1724597840.6212451,1298,1.621121883392334 +1724597851.6558774,1299,1.600787878036499 +1724597862.8649073,1300,1.566412091255188 +1724597873.748322,1301,1.6534464359283447 +1724597896.0613475,1303,1.7217464447021484 +1724597907.2558799,1304,1.6730904579162598 +1724597918.5632682,1305,1.6456869840621948 +1724597940.9596405,1307,1.6145915985107422 +1724597951.732777,1308,1.6213791370391846 +1724597983.7775161,1311,1.6040383577346802 +1724597994.3691,1312,1.659391164779663 +1724598005.0782955,1313,1.6198093891143799 +1724598015.9159527,1314,1.6468684673309326 +1724598037.7216706,1316,1.6192693710327148 +1724598048.5503073,1317,1.6554871797561646 +1724598059.3873506,1318,1.6295629739761353 +1724598070.2964568,1319,1.6110410690307617 +1724598081.0836298,1320,1.6211018562316895 +1724598091.9236484,1321,1.5834333896636963 +1724598102.6084585,1322,1.6071540117263794 +1724598112.9728148,1323,1.6491771936416626 +1724598134.3794425,1325,1.6259533166885376 +1724598145.054056,1326,1.6916069984436035 +1724598155.8841846,1327,1.693203330039978 +1724598166.450285,1328,1.605634093284607 +1724598177.079627,1329,1.6222869157791138 +1724598187.981964,1330,1.6040701866149902 +1724598198.565311,1331,1.6474764347076416 +1724598220.0073695,1333,1.6051751375198364 +1724598230.7216883,1334,1.606697678565979 +1724598241.4182098,1335,1.6377084255218506 +1724598252.088654,1336,1.6958249807357788 +1724598262.9457984,1337,1.626418113708496 +1724598273.5613632,1338,1.6108812093734741 +1724598305.6463304,1341,1.6223511695861816 +1724598326.6880877,1343,1.650027871131897 +1724598337.5152473,1344,1.6026207208633423 +1724598348.0225165,1345,1.5822603702545166 +1724598358.9256415,1346,1.6160839796066284 +1724598369.713465,1347,1.6663644313812256 +1724598380.5010152,1348,1.6945399045944214 +1724598391.2754958,1349,1.6534466743469238 +1724598401.9060426,1350,1.647518277168274 +1724598412.6053817,1351,1.6447678804397583 +1724598423.2830393,1352,1.6200326681137085 +1724598456.159543,1355,1.6046369075775146 +1724598466.8867586,1356,1.6366561651229858 +1724598499.3201582,1359,1.6204979419708252 +1724598509.8584757,1360,1.6338157653808594 +1724598520.570811,1361,1.62336003780365 +1724598531.339796,1362,1.6235897541046143 +1724598542.1272736,1363,1.5933055877685547 +1724598553.0695684,1364,1.611230731010437 +1724598574.3910477,1366,1.587636947631836 +1724598585.0286226,1367,1.5996074676513672 +1724598595.8404872,1368,1.6538680791854858 +1724598606.5613065,1369,1.6328442096710205 +1724598617.2709935,1370,1.6682194471359253 +1724598628.114787,1371,1.657191276550293 +1724598638.8829713,1372,1.595916509628296 +1724598660.3988664,1374,1.6138721704483032 +1724598671.143464,1375,1.6901963949203491 +1724598681.9701734,1376,1.6256623268127441 +1724598703.2896628,1378,1.6439322233200073 +1724598713.8481607,1379,1.6205520629882812 +1724598724.4296267,1380,1.619495153427124 +1724598745.8422155,1382,1.6729583740234375 +1724598756.7575321,1383,1.5815800428390503 +1724598777.9491696,1385,1.6558609008789062 +1724598788.5465024,1386,1.589239239692688 +1724598799.0864666,1387,1.7017796039581299 +1724598809.6558807,1388,1.6007835865020752 diff --git a/sort_result.py b/sort_result.py new file mode 100644 index 00000000..2bccd71c --- /dev/null +++ b/sort_result.py @@ -0,0 +1,146 @@ +import os +import time +import json + +AVG_ONLY = True +GPU_STATUS_FILE = "gpu_status_0824.json" +RESULTS_PATH = r"C:\Users\wengq\Desktop\QY项目\repo\LLaMA-Factory-Mirror\results" +TRAIN_TYPE = "lora_sft" +MODEL_FOLDER = ["Qwen-7B", "Llama2-7B", "ChatGLM2-6B", "Baichuan2-7B"] +MODEL_PREFIX = ["Qwen", "llama2", "ChatGLM2", "Baichuan2"] + +def get_train_result(step500): + all_total_sec = [] + all_step_sec = [] + all_token_per_sec = [] + for model_idx in range(4): + for single in range(2): + total_sec = [] + step_sec = [] + token_per_sec = [] + for run_no in range(1,4): + train_result_path = os.path.join(get_detail_folder_path(MODEL_FOLDER[model_idx], MODEL_PREFIX[model_idx], run_no, single, step500), "train_results.json") + # print(train_result_path) + with open(train_result_path, 'r', encoding='utf-8') as file: + train_results = json.load(file) + # print(train_results) + total_sec.append(train_results['train_runtime']) + step_sec.append(train_results['train_runtime'] / (500 if step500 else 1000)) + token_per_sec.append(train_results['train_tokens_per_second']) + + if AVG_ONLY: + total_sec = [sum(total_sec) / 3] + step_sec = [sum(step_sec) / 3] + token_per_sec = [sum(token_per_sec) / 3] + else: + total_sec.append(sum(total_sec) / 3) + step_sec.append(sum(step_sec) / 3) + token_per_sec.append(sum(token_per_sec) / 3) + + all_total_sec.extend(total_sec) + all_step_sec.extend(step_sec) + all_token_per_sec.extend(token_per_sec) + + result = ",".join(map(str, all_total_sec)) + "\n" + ",".join(map(str, all_step_sec)) + "\n" + ",".join(map(str, all_token_per_sec)) + "\n" + print(result) + return result + +def get_detail_folder_path(model_folder, model_prefix, run_no, single, step500): + detail_folder = model_prefix + "_" + TRAIN_TYPE + "_" + str(run_no) + ("_single" if single else "") + ("_step500" if step500 else "") + return os.path.join(RESULTS_PATH, TRAIN_TYPE, model_folder, detail_folder) + +def get_train_start_end_time(model_folder, model_prefix, run_no, single, step500): + trainer_log_path = os.path.join(get_detail_folder_path(model_folder, model_prefix, run_no, single, step500), "trainer_log.jsonl") + + start_time = time.strptime(get_first_json(trainer_log_path)['cur_time'], "%Y-%m-%d %H:%M:%S") + end_time = time.strptime(get_last_json(trainer_log_path)['cur_time'], "%Y-%m-%d %H:%M:%S") + + return start_time, end_time + + +def get_gpu_result(step500): + all_mem = [] + all_power_consumption = [] + for model_idx in range(4): + for single in range(2): + all_run_max_mem = [] + all_run_avg_power_consumption = [] + for run_no in range(1,4): + start_time, end_time = get_train_start_end_time(MODEL_FOLDER[model_idx], MODEL_PREFIX[model_idx], run_no, single, step500) + max_mems = [] + power_consumptions = [] + gpu_result_path = os.path.join(RESULTS_PATH, "gpu_status", GPU_STATUS_FILE) + with open(gpu_result_path, 'r', encoding='utf-8') as file: + for line in file: + gpu_results = json.loads(line) + cur_time = time.strptime(gpu_results["cur_time"], "%Y-%m-%d %H:%M:%S") + if cur_time >= start_time and cur_time <= end_time: + max_mems.append(get_max_mem_in_multi_gpu(gpu_results)) + power_consumptions.append(get_sum_power_consumption(gpu_results, single)) + + elif cur_time > end_time: + break + all_run_max_mem.append(max(max_mems)) + all_run_avg_power_consumption.append(sum(power_consumptions) / len(power_consumptions)) + + # print(all_run_max_mem) + + if AVG_ONLY: + all_run_max_mem = [sum(all_run_max_mem) / 3] + all_run_avg_power_consumption = [sum(all_run_avg_power_consumption) / 3] + + else: + all_run_max_mem.append(sum(all_run_max_mem) / 3) + all_run_avg_power_consumption.append(sum(all_run_avg_power_consumption) / 3) + + all_mem.extend(all_run_max_mem) + all_power_consumption.extend(all_run_avg_power_consumption) + + + result = ",".join(map(str, all_mem)) + "\n" + ",".join(map(str, all_power_consumption)) + print(result) + return result + +def get_sum_power_consumption(gpu_results, single, card_no = 0): + if single: + return gpu_results["all_gpu_status"][card_no]["powerusage_W"] + else: + sum = 0 + for idx in range(7): + sum += gpu_results["all_gpu_status"][idx]["powerusage_W"] + + return sum + +def get_max_mem_in_multi_gpu(gpu_results): + max_mem = 0 + for gpu in gpu_results["all_gpu_status"]: + if gpu["used_mem_GB"] > max_mem: + max_mem = gpu["used_mem_GB"] + return max_mem + + + +def get_first_json(jsonl_file_path): + first_json = None + with open(jsonl_file_path, 'r', encoding='utf-8') as file: + first_json = json.loads(file.readline()) + return first_json + +def get_last_json(jsonl_file_path): + last_json = None + with open(jsonl_file_path, 'r', encoding='utf-8') as file: + for line in file: + last_json = json.loads(line) + return last_json + + +def main(): + # get_train_result(False) + # get_gpu_result(False) + + get_train_result(True) + get_gpu_result(True) + + +if __name__ == "__main__": + main()