From 546609a5b2e21e0775b9121dacff867af7755f2d Mon Sep 17 00:00:00 2001 From: Zixiao Wang Date: Sat, 22 Mar 2025 20:54:10 +0800 Subject: [PATCH] updated celery cpu gpu optimization --- .gitignore | 4 +- __pycache__/__init__.cpython-311.pyc | Bin 0 -> 144 bytes __pycache__/celery_worker.cpython-310.pyc | Bin 0 -> 395 bytes __pycache__/celery_worker.cpython-311.pyc | Bin 0 -> 725 bytes __pycache__/celery_worker.cpython-312.pyc | Bin 0 -> 546 bytes __pycache__/main.cpython-311.pyc | Bin 0 -> 2107 bytes __pycache__/main.cpython-312.pyc | Bin 2973 -> 3393 bytes __pycache__/pipeline_setup.cpython-311.pyc | Bin 0 -> 719 bytes __pycache__/tasks.cpython-310.pyc | Bin 0 -> 1733 bytes __pycache__/tasks.cpython-311.pyc | Bin 0 -> 11186 bytes __pycache__/tasks.cpython-312.pyc | Bin 1783 -> 2600 bytes celery_app.py | 25 - celery_debug.py | 214 +++ celery_worker.py | 89 + connect_rabbitmq.py | 10 + endpoints/image.py | 89 +- endpoints/text.py | 44 +- endpoints/video.py | 339 ++-- main.py | 160 +- monitor.py | 33 + nohup.out | 38 + pipeline_setup.py | 6 +- requirements.txt | 125 +- resource_usage.log | 1623 +++++++++++++++++ tasks.py | 343 +++- test_audio.py | 92 + .../audio_transcription.cpython-311.pyc | Bin 0 -> 2917 bytes .../image_processing.cpython-311.pyc | Bin 0 -> 837 bytes .../video_processing.cpython-311.pyc | Bin 0 -> 6315 bytes utils/audio_transcription.py | 87 +- utils/video_processing.py | 4 +- 31 files changed, 2937 insertions(+), 388 deletions(-) create mode 100644 __pycache__/__init__.cpython-311.pyc create mode 100644 __pycache__/celery_worker.cpython-310.pyc create mode 100644 __pycache__/celery_worker.cpython-311.pyc create mode 100644 __pycache__/celery_worker.cpython-312.pyc create mode 100644 __pycache__/main.cpython-311.pyc create mode 100644 __pycache__/pipeline_setup.cpython-311.pyc create mode 100644 __pycache__/tasks.cpython-310.pyc create mode 100644 __pycache__/tasks.cpython-311.pyc delete mode 100644 celery_app.py create mode 100644 celery_debug.py create mode 100644 celery_worker.py create mode 100644 connect_rabbitmq.py create mode 100644 monitor.py create mode 100644 nohup.out create mode 100644 resource_usage.log create mode 100644 test_audio.py create mode 100644 utils/__pycache__/audio_transcription.cpython-311.pyc create mode 100644 utils/__pycache__/image_processing.cpython-311.pyc create mode 100644 utils/__pycache__/video_processing.cpython-311.pyc diff --git a/.gitignore b/.gitignore index 0dab052..f260bf4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,6 @@ __pycache__/ endpoints/__pycache__/ -flagged/ \ No newline at end of file +flagged/ + +*.dat \ No newline at end of file diff --git a/__pycache__/__init__.cpython-311.pyc b/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c160804f11f7e9c2a0dbb1695d3c6e1a10f97a42 GIT binary patch literal 144 zcmZ3^%ge<81hO}mri19mAOZ#$p^VRLK*n^26oz01O-8?!3`I;p{%4TnFJ=9V{M=Oi z{QS&3{o)e+lFaOq{Oou~PyP7#%)HE!_;|g7%3mBdx%nxjIjMFJ5ViMDekDSjfTf6$kU&HT@u_HNpxS1&*_k9}y>YO;LdgN>I0z-zaZ8me zASCc^8fG+aG^2U%jT{bp#KoKcInQH4zS`k`L~gitrEeVo0V+sMGfD|k@W6NJ%_#)$ zK}H~i2x3k-Me^+BJ|tZaV6W>T^d3pt-#Q(VQXAc&K()4^DRiYb2MU4f31dQ*g;+>! zCTFMT7wjbUZ8TM~LS^G=QC7GF8}nr;8o*85k;lh2d}**j8w1XYMwl&iCQlD(WTUdP zQJ!05K`EEBtlyGyB^!e}JA9SOY3D12Gc@HQuN|GQ1V@#Px4g1YWg2>3s)-f#-R&&45Nf$td|vf&u*l D!i{ZO literal 0 HcmV?d00001 diff --git a/__pycache__/celery_worker.cpython-311.pyc b/__pycache__/celery_worker.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13f716f7003bb2b96e775beb63d5e8b5089de4cc GIT binary patch literal 725 zcmZ8ezi-n(6n^KQY15=gDJqE&LKG=Yhj>Q_fsl|`SYXOxxxP!{#?CqCvrsZ+;1B55 zj!Z?U6@LYPK#($6w=#7zDpMz(V^R^Gp5J}`?tAy%yZchBZ3EgQ{609V0Q@X|)@J?% zH{TGP0R>7zh)7Q(paIiP^tF&`RKGGVG=NL|N+X5d42{U_nJc|XEo#%s)z$^(UEbyP zEV@mrbSEGIVb+Pg&Udk!(r9gi-@4=V4SqF8u}8^!=+v(Z_)ZypE=C_ZM41SDhQNBw zLMEoF=C4-wBMO+9wc@E4P2y*cGe}t?UxwWG!XZzj+J{B!~BAJASx8$tdMNNWMje?5N;c5U9zj7C;%XFGejZP?l`!0@{W rb3Jc?SwFSUn~U9pT-P2T^{KLiy}SbL!xA54tw$x=UqInvADjCFyIjaU literal 0 HcmV?d00001 diff --git a/__pycache__/celery_worker.cpython-312.pyc b/__pycache__/celery_worker.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1a86e52ef602947dad1f16490492ca80ca5586d GIT binary patch literal 546 zcmZ`!zb^zq6rS0=-MzbYL`ZNbM2I7?5(yz95tT+I3e9HiymM>kZqDp%j-^=&o&10!lM5@m9hXdC_=q&|VnZM1)jcTqDK?GA& z$9otf4FM(^(bdro@O|zs*X8aZVi1#90S+*r!{2FUn7*!_A(H=x@uwn%e;7vQ(JEH# z_Y_pHG&g|)Z%IRVQ5c;Jb09&q>Nu479<8xRtS&7suQ&@8U7Guxg}|k?@A)A#h_r%+ zPh$cnR!2NPD~+QVVvrVrsuxr7j#bFccEyyY--+SrKV;5ljjFU$k;_>uAd=-{#+A3* z{8dJ{{95Qsa&}@*B|XzgY4CFEvtSgpQ?osWjNU8j lX5^`FvaM^QYII?r+0SUGokz2C8JbSUC%XL$D)WA2{R6CFh#>#~ literal 0 HcmV?d00001 diff --git a/__pycache__/main.cpython-311.pyc b/__pycache__/main.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8a40201a668f8b5bc2e2038f93958aff1ee2e16d GIT binary patch literal 2107 zcmcIkO>7%Q6rS0cwH-TlvW-dY2B~{Ms1Z@+mv zKP8iK1nrj(e_wkch0ve;WzeF1;p9h*(0ybe16Prai`bTml1+*PbDdPBwp^5jOsbJ$ zL_#LsgV2Kk4k|{?Xft;DIwBZ!53mnl_8P=WhCHrY#YV)OT9u6Gj(i3;elBio9OE3^ zDU%3nYQ!h76X#+l{w4PG1opI5L%QfPJB)wqRo$|&XjM%Ohtuz^yuY+;`kw3f z=2CqsSN5!2V15yFi&2CLBMEKeZFES+ zX}F6F`Zf6k-6!AE6{N}acj78ENJ=#w!*eYsPy<)>m|HP@U*-F&mgCg|^-C*QSA%s^ zt@$Rq;;WDo+tNie;~c>!#JPJ(7&OdyVJ>M>m>K~4I=8ba?kowjkbNGls!uQH)?M4o zxsXH74|0LE5x5(r!tI=0wjABt3g?FKr2!_JhG4#v{}wENL2saL9LZN(R2v-~WnuSH z^V621w-xfkhTg7?BRc`rMvNZ@%`PYBN*?fVjg`vIMDKQrLT*M%3G1*=AePD0`HQi3XA33!I- z5xgPKb711ZCk0rBJYR3oYonv1ytup2eCa6DQu1vjf2`zNiHSk;ED0!mrZ+9abpIEQ zZ#|o1RqU+tBYrNPg?o=@ra=C@xJL*UW;PFoF@m|`48bEnLxMBlmxamFeBu0ru~LcG zeyLQC@n^p5dG+Wky6D#1ml66XjMO%(uIp(ur0cF9gw%6c5Js5EZ+iVzJg^LI7 z6UGmk@e*eIq6(G1UT6J#$7d0)kCy!{r((Gwfh{>EAQL}e;%{TT6c|sJC+L)Ivs4PB zwN0zyGAAUg=Ir|F6C_;(@2~1quKF literal 0 HcmV?d00001 diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc index 67850100bac471c046e3ec7a17622377af2c9a42..3221fdf8c88dbfdaadc9c9cfc3b0e16cc09bd5bc 100644 GIT binary patch delta 1220 zcmY*ZO>7%Q6rS0gwbyGWP15yGjT^QNN>~%ww7rDz+cZs-78NN;l`5Cj$}@Ggb-eEE zI>jatDI8jHLRtw0p^9FR5+xNy6;e)>dTWIQ0a|{Bo+t;7hDeA52WHl>+RkdfdHdd* z@6CH}X0NC3W{1A%>e2uoU)vkyQWpS^#YbmE_k`UWUkzDsg6O#S;)?*#90pjfB6i2wJLcEQ2z{%2uS~up&Lh5?z*pBT+r_gevswhQ}hFj|rtz6oAF;TCLKsY_Cx*G0DL+!nLMMMH^|K@IZpaleuUF$K_X? z43}QzuXl;VpWxqB@CSLChUAwTOnGaC@G3tIKR3qD&ZEZ(8ac|(e{QdkxboNm^ z`*VEo`}pAcx%E@q3lHLxtK;F~+}iBtbDz$wP3>rd>oZ@@Z_GbLBf(wyxzrdKlVDRT z+BK)>EZSv46FltgFCuTl;7z4BW<1`8EM1ft*E)YRCJJ=CT{z|g}gy0xK3PT55VA# zHZ+_Al5#b3C3EeiA7Sd(!qHpDHl4qK3JdG<-(uR8fs}6Xe?g(82fgtRj8f(x+xB(W7e zwMSt<6e)TUTB@M-u80>w4;BmRQZIV&;H9-!5#QS;A`a~LzW2j>^Szx{$+<-4ZCjg0 z;H-atRQe6<8#=9{Hp#|?XPE+Jb=hne@8})rIT4oB6Y=C`UTg^)XQoJrK9&~gW^9^z zO-6_%PtnV*Uaj6>?~}Q|^^|y#R?1>nFVbqa`b54)O+{2?BeOt#R^%nOp`R@z<+g$2DgbFbyaSjY4_X$k)@)uc*cGcf z2{QN|NDVD@gnrS}381FVlVErkLJDCALN~%rgkUtT@W0VK4Y7Ymit$L&%s_Aii+d1y z0U8~FJlt6BdUL7MAnRe|>sUYJ{ZJL|=2S|s>9&h@87`aUwp%NE+{CoP?V?rRlso_d zQ|2)+L6T#@#n%#u7R#EU>;rF9{Ls!yW?anZ`_({7QoPXj=J7`b3*dN9UoP1m$NB}b z0Z2C27CtsHY1PX1ah``5J~sCg5&39lAVwr*rt?Q2+n{ diff --git a/__pycache__/pipeline_setup.cpython-311.pyc b/__pycache__/pipeline_setup.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..951a1a0d36c3a423abf9baab03279e9edabda420 GIT binary patch literal 719 zcmZ`#Pj3=I6rcUW-vw#{jkU4HHZkd{h<{?@p;0MCLZropipd(YS(quT?9Oa=ms&V+ z(1V}AZ%|{}Z{Zi9x6PbLJaMCGPdzyUV!Zg?%x~U%c`x((z3<6n957q?a@HD)0Q};~ z+>}4T#UTs60SKUA0gb<+CD@`S+L9*OvL*`vN?HiY2z(6Bhhb=gg)po|M1VxZQGyW| zg)tb13XGj5j#=-?-~Q9&G5i07-I-}oDAa(O8uN5Vm1xv49b}mp(Z#*4+aR`yVG*}j zp+NAy*`n7<2qV|J_7rD`OwIxL$dzKg~Yd%x=G_X7lfMsbqXQ;2Q!acnQE*U{UMI)PBBD z!!%(Nh%9}dxVGU_(RZlgAu~Ci?6X!fX=bIg$bo6{VH{UfQ;dh+}PVXwZ!REUbdhRZ> zNiN^}%3vZ1f)Ie%VCn&QHT?e>rB(*Y=+51N_&GjQR?ooNBmrXg2B9FbFgOb2Sdd;H wrq_eTY{GB<+jOTn$R;O6RNA)X1J0|u8ICZSYRJO>OeISgd(?h<438&XfuT>t<8 literal 0 HcmV?d00001 diff --git a/__pycache__/tasks.cpython-310.pyc b/__pycache__/tasks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c39dcfcdb212f176a9876802594304a02eccb6a GIT binary patch literal 1733 zcmZ`(&2A($5Vqa!p8ucCCLth%yn8_-p&2A35D`M46$dWSiX2w-W$28ZOz-rJ2iw`~ zv}O=e_PS31NO0UY;J{mOL4D=KD{z5w%}+XEajUD{Rd%`RtFNqEtvZ46`?KFJetn0K zKmAX6RN&_`chM=AhWL0MI`U#Nd5 zQT6}ouf3j0E<<>tmqAPwecF%}S$#=mO*PM%tjop=eof;RTAFBSsU}*vY|9Q>+i30H zvEGw=Xzif&K3ezX!3zTZx-vK=y~DpCJJO>?G*X!|3qN~=vy&opiJjQ|$S9fGyAJC;G>TR@Ap_7of$n*rl_1Hqlucy`~dg#oqFzu-(G^lp;v%cK1-&msDjSA zRL@4#pSu|hz5%?C;I$7yND2@G+$(^`F&Qk$59BB2!CKNK1P_7p6XRWg=lacI`N3nO zN6OlCb|JP$9gAKNw;{N>PG`bedeK5*$VCJk8W_p1ASI=e`{w zO#JbP!#6Mi2Y`joTLKE6bSoeo=ztJ_1zQ3h*K8FmgD2#aaN>f+fRYubByTOL<4ag4 z!ynlb5|OWgxGVl0Kizz<=oRTjC;5XCWa&)YjkVwJZ-H)X)#}OSR?}D^F%fak=Vs-G&XvsFKxEyGT3h3C0NM>^DOj(h-^`>1pOWB(aya5K|5jFQg+^P$jJkI98 z3(wqgSxWA)EO5yZ+Xca*=IxRkfimx`d&R;1fXhdqyfu8gV`p%fNPh>glYDQ3U24UZ zHWTRk2coPcQQx1|%91QGwk6A!V_8;gE4FOOms8@%&NaHE49#6hl=-3T zu561X4b?Cj`D);F*K6(I96&kb61ma!T%bVRL$64H+$9D2$L?W^BNlMsqVOO2k46$8 z@Lzp1%O!V7*+C1mtI^xp*_qjyH*eniy*K=*s>+Mtn)|!|m~Y*K(0}0v^Kut5U)^^g zbPMr_r{ZW9@w9*zDYFvq7=tW%M<*P!425VUFpJLOdxm#zxjv*3`WXI=i^BEYaaY1U z>n3?B&d#z>#>PDf@2ofBoAo99v;IWYY*pbs{x1H0^=!3Jy%;ci+Nd#e);o-^R3m78 zW9Dr&YRyvXU8wsI=J;`Op3izBbFR1b`TzA*H}B!Sym!m@A*|8I@NZnR4ZL4ySZp+( z!%b$6ui~p?6wm#7mFdo*KU>ZjjtEl z=4ig*I#bq1`&0X9tmvajXe0eFd~-!V9Z&72MlDJ?}VC(nf_sF)I?0%Tk#gt#ECgczMoUP{ELk}+AY#krHhrC3xrEyyQQ z%Sl;pwUtdql6+ivab+sW3s>|G+q?5}EFMcoOuoFEFxs9+R?=qHWjBLkMImtwq-deUH&ml}}bXi-hUckqV7|6e@^HtKY zva19NWl)AnLp@+RJ)N!|=lNK2p2Hs*7#P5eT@qtS8Ky*C(Ww;)e~06qaDxk}gfN&& z#gc=PJSfL5%BhRt@u|VWs8@7Hgy+RpsQniFOE|NN9x)!p+p02c8q=mQZ8@KR?d=-} zS6})H-lBA2X0mWa##|6r!niXS0+Wm+1f7L42`@s_G9Sc1OET9v$b8pad75{`sRW&+ zC_7h}4Ei~Gk1pp84ZO)9dnxeAIh}5rN-U*BnXsxjUSODr`2$1tmS&M|c*R(2T-lt% zC>L^wSkdX#ARSAWbHpwv72sbw1Ej#z}c$uS|to*v*8*t5|O1d__^)tJ2svp45!SQ}kmP<@@6uTx<<#Wv_JT@b-oA~H|p zvalk$XD${O!b=f(!6tUtdXlC14P&G&x_mM0499jVLt`R(ENl1H={GDTrc93^@%U2$Y0?^e6Vf_;t_3m8h_UMP7GE zGOoN6iVe{q66C^Gs8;wle93ovuHz&yVJPg#)Stix`|Gc-?Q!~Ju_gC+zDdJl}4 z%dYJ}x8mLkSBB!9ylcyC$K6@R)-x+(Wn#7{)1KRzNn2}=WnA9(92KLn?yEAJkaa$_ zC*L+bS#`YA{++DrsXh5CdjhUhU58zMvdm-1xcKTjU}Hjo>2#gNA{1nZ6S2Scgc^K8 zk8)pPDf|*E5?FC>B67tPrWz?WpNNDLpr>OXjDncn85ffC@&cEd<3Q+OTVtWa2E&xi zQ7+BS7;?&`dq+7_94jOh>`gKzEeQNTx~EjFTxu<~+DEx^`7OP*i6It&I{5zztAJZ* z9t82tV3xA76$~D}s6A9gD{#8H@hWbaz1^ znwxT9j38<0C}hk^*ng8S%2`E9<8dS1JMr&L&ut!1n+CO}!D1=FEU@SO1vV0ZqI)g- z+Ol`~Fh)`Z^D7>J{&fc|i#QB9@gR_pyVMaMkt5!_KJxW!ym9}6GILJxomYM5HQ)KwlcfFVucUj!cUQJn z6z?b`l{um@M-=7=c{#QI!;Roi-c-C(MpBtG8goWr&O8c+HutE(Lt60A>KRgX(yThE zcyUshDUF#@n5hR%eVbmj>44UBKw;`h-Q#B6!=fCa z=fYKv`6VZ)2=E>5&@;#iElUBT@)mEv7qH6C(0|^Lp*JiL)%*tdVr8n8e8pM#3P^%^ z@s%naE1trYfuPhDYlp@^@>};Q@44asPPOfoEy14~e*4Y_glZ4w$8zf2e?x!V|7`$GG03;Kn}{JjzTy58Bb|s&uY!r-J#9# z5C_V8)q67qjuhNW@E4#r`LnMig_6E*G=wDPGHp0o6nY&|@MoJty?wk8kZJ(a?5!dsGfabBcLE~8G&j74<1ob4 zggqK&Agn@MJ3`V6SctGldJNYFK3x?sP zX<+zekwl!*-a?54pAH(ICZHAo1Gp4vZxJ;Ne2KMMx}_4RNZjQZ;ElA8yPOgsDkT|_ z9Un$1g%X?X$NQ7m{v>4nn!bf%*!m9N!8B{K$aGa!SBa$su))Wo z1+dXm9!ayHjbg2TgHK5KbKim+gB)GeCGwy3z3V4#x91%w*m5iI)4(qS_x60^_}$DG z&7Z#c*_+DHj9PO}t2w8<_Ilo7YiPt$S|+f+Z+dN5o(hBJ#24`NXfi3W5Ax|C0gHei zz-!?6R8r`&Ee{wzV|j2;tpLr~EXBiyxJ3Qhm7xHLc>uLiq&I`3G~u4O8(LW5n0Ok0 z?^&!V0g`1}5Yhr62)=lxs9Z;{vhTD~ zH;I#f@*5Xw?$VleZ=O+`4rxt?Zur+c>q9wTz2a-%nEGSS$RBz}?l-GFFKRt6LJ?^_ zt<=4YlmF#MI6SmdWxF-DTVcB&)a}`v{i0i`n^x=uwOk>9sb_^ik4o+cPA2hbS*L{28-Mx4A{;GDX{qByf z9rwG`&=Iw9Olurl^O5hrxVc-Ye$hxOdqQJRDC~(xRh^sPQ>sSPsu8VfB#(lg#`}|b z+P3dXTlZ0D|Gg14G^T~d@=mDp81L+_F;V;-c|YQMZqNNv+8lX5`$1OSF}(KjgVtS2 z--z0JSZh6;N1m!iLe^cm#=cFL);PHC$kXm%V{Z4*uV47v7w#V0I=24u#tF5)U#ai^ z_CXVuN6sL$G0B7a7NvFKUK9wV-_Cpz{HNFc`8DH7tv{vJpHk{iJ?!6iuWqaAL08Yl z+4mbiXoQjOfj)qg`iAa{iNw#G@Ch78^WZTCfeVBcggnMf`5Kc_*>*YHe4AR|>>`5K zrLsFUcBjJbd`LF^y<@QR)s8nbcw$>&A=`R$&0KEhFrfDuFQ9iqJl=1=@gsjjDIiF4 z|I?@-jwxL~ILV;j><9Ac-WU7f_F2QY=XA5(MU);O}kJ?9Xe=5gb_P2m+S>BUz^WM@7 zu)mODFxE_i{>!qwAL>-yq30>S8lySum&<#{A+4;(>SghPjAyIHju+-Bz`l4z3RgMK z&H6H~?Y*vyZ?VV)`%EZw^P>#)lva)m1L2_Hor-|g&Bqxyuml8ZIn)Pi-j?+P61R9q znKGX$MiKm3m-S!u1O6oU5{3@dr;n0IZ~@IhfGEOUh{#bAHYynp6JAPODoNtSr(Zd5 zM1#|zSC^AX3^a;Ig}9ET7h;?-7A%=TV))W^Q-#w=3+&bHo1qvU;$Q&^7{q5_5S|<- zf(X#>IRJ^kNc%_J$r(cdmvL_L=|QPK7JqpSrRwknL9KvqZ*$c>K-FNye!5I zhBV;-(5$ieg3KbEhw4sPjd{U9cML}8;`1;_ z-4$6{g3~gx5>_my*IRc**u-YFk@u-Z?5ZL9&w%kApm4@EzfO^X)PmbXL3#bo2^>Y@Y*Egl+X)W)6uob zNB*EvJG|)y0!j5B)cgk(|3NU}J-ZY?muu?=7*KVDkahRNzR*XPe{nfSHp_39!Odw> z&c3Nl{i!+;)+WO0vA48iZ^0&0>myoyy1Yd@;A9b0D~v~_N$eR!f^vROdO*y_)H5aUUx;w?_fl1>~zii6$?q=QkqhuE8}Q56aG0ELhmBq8 zfQ_g#u&?LzRwIfKhJ7_BfR3`9cIZC*BaX0`TE@SeVekX_RR}qCBtKse@yWC}gh>>C zxDk(38YsAq#d#aP-T_<<2kG?|qnZ(IHxD()VL6H4>(%&BE*?t?;AzUsONR0&^lj+h zf-ee2P8`8K*1}w(%Od;?M;FZv_*R7(ShiIo_PdQY#N^7WY{q)ALg8Eg|uc3a( zfHYIB3Tm;Yhp1|m{N>Q1@?9zKMU=NN1ms;Me-Dx4I{QxGT40_2nH$G5Hs}xB9|e99 zxW}s8VU0U%Y#J3E(a;eE9myf*s&nBk7j*O;t2@~x<^e-#Fxq3--X#PuotqsopW zxgEXWb}1c)Ve2>3pc9&*F0NFcKM7~}&g;#2nx=wz1mv*;(O$EFg_rL%Uu#wZyH=Z3 zs!yZ(@-#znx1+F;DR9S3f?M^QZ(*2bvp(DmUi9Z_4>i2u0Dq4f-jv9#L~!enSt;MR zU2C7w8g95X+)AX>8W)csDyX0uBjwQB%F|O0J+IhP4s|N_v|XKJ3fd;+nIHwnEJd7%Q6rR~1d)GhNByFPpiB3pCY-!`BwLlFBsVX2Lp$O4abHH-ETidncH8Y#0 zwj7jHmC|D(4z8d`>8XuUiHZ{kB*dAEk&V!I19IRn+%%uqK_u{R*`_tVIxAMwp6X^n;o6to%%}5UFvA&ORjFj>Y)=CobBj(T z*8_lvngW!EeQOE`fEWXC0mO>{VpkOsKrFd|zC>S1LFhSD8n}Ui{W)YR(-f+Jzb3Zr zKz-yR>f~(3ij_PGuAd^N>T-9&Q3X? zkz#JjEtXuTlsWT3%71dT<{8_!if2<^B?;}Pq7ungtiYJf zNYV9``tS+Gh3pA-Pi36J$NI1on%_|ZWiFTfyV-iHAJDXznK{{k2! zB5E44|DTi2eZ4Zc2Ys%m8-7!8p?8WKb0GN45G{b`u8C^cl&0&$ zz0emKS#W3xdf(UvqA!C49ppp+TJIaAHioq+ziwgzlrjj zJjr2DLN(?oyeoN;Y_$p%=S8g|6}>*1on_)g%Vb0iXZ*TU_+v!SMSrBch$!KNJMo;4 zPqnw_9BaO8zeA71DQ|9U04S6R#t#vBi1t55`Y$N<2+1qjvbJI@8>>_a-xGKf7OlE zXp2m>-u2zc*09;ZN4t`Y$@Sb51k;v`q}K(b8$(hu5Ifqzz}0bj wUG5^7ZnGzR;?lk literal 1783 zcmd^9&2JM&6rY(L$96XH(gbK5A=~+Yu&NXb2~?>sy zd-Hqa-zO)>35AOs$Qd67fP5~cyx3SM=u{Qr&uh$ES4^L4N>wST*+4@ z<*X@h%_@IYQWbyAUuw1j<8_J_8f6Y#F;o9j5Ld}lZua%5o<7^rXZQ5^t6%oH-s3YJ zKC^SO%Zm?nqo>bx^ts!g{kqki|9np`|G~>ca3w+ZqEi=Ev!X6l@jl{_1|=l#0VoOb zSdzRpv9;Zi*D12CDf9|Q?SJTcMJrF>!ef*u7Fn7*4p$gkE6#P zRR+%nzSvI5IWkX95T9;B=I058uSK@B^JJ5nsaGwaJB`Y!hY!aJ-kO8`6RC zx*I~IoPsO@?lpxgu~L~ufwf5NLNg@~ppFcMPn`w^J& zkl`{2r1q13cJjKjbFiDu-&*J!hkM4+j&XF)SiH{r8KalUcQW}M)y*tCG^TsTv5s-< z_T_v0{#JMK?4I%U-)S;l=wGzrP4aXKj1v9h5oU@a*-vI3wr4hJT3u6%YfBQu>JG9+TN8RR5vyePO^j{etdPUl5#!8etzq krojYZ6A>aszoECZ0m1nf|3x^DkZ2m@_D4Qvhzt|_7YNTxTmS$7 diff --git a/celery_app.py b/celery_app.py deleted file mode 100644 index 20bff9f..0000000 --- a/celery_app.py +++ /dev/null @@ -1,25 +0,0 @@ -# from celery import Celery - -# celery_app = Celery( -# "tasks", -# broker="redis://localhost:6379/0", # Redis as broker -# backend="redis://localhost:6379/0", # Redis for storing results -# ) - -# celery_app.conf.task_routes = { -# "tasks.*": {"queue": "default"}, -# } - -from celery import Celery - -celery_app = Celery( - "tasks", - broker="redis://localhost:6379/0", - backend="redis://localhost:6379/0", - include=["tasks"] # ✅ Prevents import issues -) - -celery_app.conf.task_routes = { - "tasks.*": {"queue": "default"}, -} - diff --git a/celery_debug.py b/celery_debug.py new file mode 100644 index 0000000..7eecf8e --- /dev/null +++ b/celery_debug.py @@ -0,0 +1,214 @@ +import os +import time +import random +from tasks import app + +print(app.control.ping()) + +print(app.conf.result_backend) +print(app.conf.broker_url) + +# task = celery.send_task("tasks.text_query_task", args=["What is string?"]) + +# from celery.result import AsyncResult +# result = AsyncResult(task.id) +# print(result.get(timeout=1000)) # Should return "Celery is working!" + +# print(celery.tasks) + +# print(result.state) # Possible states: PENDING, STARTED, SUCCESS, FAILURE +# print(result.get()) # Get result if completed + +# # Check status +# result = AsyncResult(task.id) +# print(result.status) +# print(result.result) # If it failed, this will contain an error message + +from celery import chain +from tasks import text_query_task, image_query_task, preprocess_video, inference_video +from tasks import text_query_task, image_query_task, video_query_task +from celery.result import AsyncResult +from concurrent.futures import ThreadPoolExecutor +import time + +# Check Celery connectivity +print("Celery Ping Response:", app.control.ping()) + +# Print Celery configuration details +print("Backend:", app.conf.result_backend) +print("Broker:", app.conf.broker_url) + +# Define the number of concurrent tasks +NUM_TASKS = 4 +delay_seconds = 0 +file_paths = [f"../video/1.mp4" for _ in range(NUM_TASKS)] + +# video_folder = "../video" +# video_files = [f for f in os.listdir(video_folder) if f.endswith(('.mp4', '.avi', '.mov', '.mkv'))] +# video_files = ['1.2.mp4', '1.mp4', '3.mp4', '4.mp4', '5.mp4'] +# print(video_files) +# file_paths = [os.path.join(video_folder, random.choice(video_files)) for _ in range(NUM_TASKS)] +# print(file_paths) + +# profile_folder = "../profile" +# image_files = [f for f in os.listdir(profile_folder) if f.endswith(('.png', '.jpg', '.jpeg'))] +# file_paths = [os.path.join(profile_folder, random.choice(image_files)) for _ in range(NUM_TASKS)] + +# questions = [f"What is AI? {i}" for i in range(NUM_TASKS)] + +# questions = [ +# f"Extract the following information from this image and return the result in JSON format:\n" +# f"- Name: \n" +# f"- ID: \n" +# f"- Profile Picture: \n" +# f"- Follower Count: \n" +# f"- Likes Count: \n" +# f"- Bio: \n" +# f"- Following Count: \n" +# f"- External Links: \n" +# f"Do not include any disclaimers or comments like 'I'm sorry' or 'I can't assist.' Task {i}" +# for i in range(NUM_TASKS) +# ] + +questions = [ + f"Based on the given images and audio script, extract detailed information about the products recommended in the video and format the output as JSON with the following fields:\n" + f"- Product Name: \n" + f"- Category: \n" + f"- Styles or Variants: \n" + f"- Highlights: \n" + f"- Promotional Details: \n" + f"Do not include any disclaimers or comments like 'I'm sorry' or 'I can't assist.' Task {i}" + for i in range(NUM_TASKS) +] + +# questions = [ +# "Generate a screenplay based on the uploaded video, incorporating detailed elements such as dialogue, scene descriptions, and character actions. The screenplay should follow a structured format with the following components:\n" +# "1. **Scene Descriptions**: Provide a detailed visual setting for each scene, describing the background, lighting, and overall atmosphere.\n" +# "2. **Character Introductions**: Identify key characters, their appearance, and any notable traits or expressions.\n" +# "3. **Dialogue**: Transcribe or adapt spoken lines from the video into screenplay format, ensuring natural flow and emphasis on key moments.\n" +# "4. **Actions & Expressions**: Capture non-verbal cues, body language, and interactions between characters or with objects in the scene.\n" +# "5. **Product Integrations**: If the video features product recommendations, weave them naturally into the script, highlighting their name, category, features, and promotional details as part of the dialogue or narration.\n" +# "6. **Narrative Flow**: Ensure the screenplay has a coherent progression, with clear transitions between scenes, maintaining engagement and pacing similar to the video’s tone and style.\n\n" +# "Format the output as a properly structured screenplay:\n" +# "- Scene headers (INT./EXT. - LOCATION - TIME OF DAY)\n" +# "- Character names in uppercase\n" +# "- Dialogue centered\n" +# "- Actions and descriptions formatted accordingly" +# ] + +# def submit_task(question, index): # sends tasks to Celery asynchronously, queues the tasks in Celery broker. If multiple Celery workers, they process tasks in parallel. +# """ Submits a Celery task with increasing delay """ +# countdown_time = index * delay_seconds # Dynamic delay +# task = text_query_task.apply_async(args=[question], countdown=countdown_time) +# print("Running celery_debug...") +# # task = text_query_task.delay(question) +# print(f"Task {index} scheduled with {countdown_time}s delay.") +# return task.id + +# def submit_task(file_path, question, index): +# """ Submits a Celery task with increasing delay """ +# countdown_time = index * delay_seconds # Dynamic delay +# task = image_query_task.apply_async(args=[file_path, question], countdown=countdown_time) +# print(f"Task {index} scheduled with {countdown_time}s delay.") +# return task.id + +# def submit_task(file_path, question, index): +# """ Submits a video query task with increasing delay """ +# countdown_time = index * delay_seconds # Dynamic delay +# task = video_query_task.apply_async(args=[file_path, question], countdown=countdown_time) +# print(f"Task {index} scheduled with {countdown_time}s delay for file {file_path}.") +# return task.id + +# def submit_task(file_path, question, index): +# # countdown_time = index * delay_seconds +# countdown_time = delay_seconds + +# # Chain preprocessing and inference tasks, inference only after preprocess done +# task_chain = chain( +# preprocess_video.s(file_path, question), +# inference_video.s() +# ).apply_async(countdown=countdown_time) + +# print(f"Task {index} scheduled with {countdown_time}s delay for file {file_path}.") +# return task_chain.id + +from celery import chord, group +from tasks import preprocess_video, inference_video + +def submit_task(file_paths, questions, batch_size=4): + task_ids = [] + for i in range(0, len(file_paths), batch_size): + # Get the current batch of file paths and questions + batch_file_paths = file_paths[i:i + batch_size] + batch_questions = questions[i:i + batch_size] + + # Create preprocessing tasks for the current batch + preprocessing_tasks = [ + preprocess_video.s(file_path, question) + for file_path, question in zip(batch_file_paths, batch_questions) + ] + + # Submit the batch as a chord + chord_task = chord(preprocessing_tasks)(inference_video.s()) + task_ids.append(chord_task.id) + print(f"Batch {i // batch_size + 1} submitted with task ID: {chord_task.id}") + + return task_ids + +# def submit_task(file_path, question, index): +# preprocess_task = preprocess_video.apply_async( +# args=[file_path, question], +# queue="preprocess_queue" +# ) +# print(f"Task {index} preprocessing scheduled for file {file_path}.") + +# # Add a callback to submit inference task after preprocessing is done +# preprocess_task.then( +# inference_video.s().set(queue="inference_queue") +# ) +# print(f"Task {index} inference will be scheduled after preprocessing.") +# return preprocess_task.id + +start_time = time.time() + +print(f"\nSubmitting {NUM_TASKS} tasks concurrently...") +task_ids = [] + +# from tasks import add +# result = add.delay(questions) +# print(result) +# print(f"Task ID: {result.id}") +# try: +# task_result = result.get(timeout=5) +# print(f"Task Result: {task_result}") +# except Exception as e: +# print(f"Task not ready or failed: {e}") +# task_ids.append(result.id) + +# with ThreadPoolExecutor(max_workers=10) as executor: +# # resultID = executor.map(submit_task, questions, range(NUM_TASKS)) +# resultID = executor.map(submit_task, file_paths, questions, range(NUM_TASKS)) +# task_ids.extend(resultID) + +task_ids = submit_task(file_paths, questions) + +print("\nAll tasks submitted!") +print(task_ids) + +# Monitor Task Statuses +print("\nChecking Task Results...") +for task_id in task_ids: + async_result = AsyncResult(task_id, app=app) + while async_result.status not in ["SUCCESS", "FAILURE"]: + print(f"Task {task_id} status: {async_result.status}") + time.sleep(1) + + # Fetch final result + print(f"Task {task_id} final status: {async_result.status}") + if async_result.status == "SUCCESS": + print(f"Result: {async_result.get()}") + +print("\nAll tasks completed.") + +end_time = time.time() +print(f"Total time taken: {end_time - start_time:.2f} seconds.") \ No newline at end of file diff --git a/celery_worker.py b/celery_worker.py new file mode 100644 index 0000000..f49f440 --- /dev/null +++ b/celery_worker.py @@ -0,0 +1,89 @@ +import torch.multiprocessing as mp +mp.set_start_method("fork", force=True) + +# from celery import Celery + +# celery_app = Celery( +# "tasks", +# broker="redis://localhost:6379/0", # Redis as broker +# backend="redis://localhost:6379/0", # Redis for storing results +# ) + +# celery_app.conf.task_routes = { +# "tasks.*": {"queue": "default"}, +# } + +from celery import Celery + +# app = Celery( +# "tasks", +# broker="redis://localhost:6379/0", +# backend="redis://localhost:6379/0", +# include=["tasks"] +# ) + +# app = Celery( +# "tasks", +# broker="pyamqp://guest@localhost//", +# backend="rpc://", +# include=["tasks"] +# ) + +app = Celery( + "celery_worker", + broker="pyamqp://guest@localhost//", + backend="rpc://", +) + +# celery = Celery( +# "tasks", +# broker="pyamqp://guest@localhost//", +# backend="rpc://" +# ) + +app.conf.task_routes = { + "tasks.*": {"queue": "default"}, +} + +app.conf.worker_prefetch_multiplier = 1 +app.conf.task_acks_late = True + + +# from celery import Celery +# from kombu import Queue + +# celery = Celery( +# "tasks", +# broker="redis://localhost:6379/0", +# backend="redis://localhost:6379/0", +# include=["tasks"] +# ) + +# # Define task queues properly +# celery.conf.task_queues = ( +# Queue("high_priority"), +# Queue("default"), +# Queue("low_priority"), +# ) + +# # Define task routing +# celery.conf.task_routes = { +# "tasks.text_query_task": {"queue": "high_priority"}, +# "tasks.image_query_task": {"queue": "default"}, +# "tasks.video_query_task": {"queue": "low_priority"}, +# } + +# # Define task rate limits +# celery.conf.task_annotations = { +# "tasks.text_query_task": {"rate_limit": "10/m"}, +# "tasks.image_query_task": {"rate_limit": "5/m"}, +# "tasks.video_query_task": {"rate_limit": "3/m"}, +# } + +# # Define task retries +# celery.conf.task_acks_late = True # Ensure task is only removed from queue when fully processed +# celery.conf.worker_prefetch_multiplier = 1 # Avoid one worker taking too many tasks at once + +# # Define task time limits +# celery.conf.task_time_limit = 60 # 60 seconds max execution time +# celery.conf.task_soft_time_limit = 50 # Warn at 50 seconds diff --git a/connect_rabbitmq.py b/connect_rabbitmq.py new file mode 100644 index 0000000..562af5d --- /dev/null +++ b/connect_rabbitmq.py @@ -0,0 +1,10 @@ +from kombu import Connection + +rabbitmq_url = "amqp://guest:guest@localhost//" + +with Connection(rabbitmq_url) as conn: + try: + conn.connect() + print("RabbitMQ connection successful!") + except Exception as e: + print(f"Failed to connect: {e}") diff --git a/endpoints/image.py b/endpoints/image.py index fa09f51..1f03968 100644 --- a/endpoints/image.py +++ b/endpoints/image.py @@ -8,43 +8,64 @@ from PIL import Image from pipeline_setup import pipe, IMAGE_TOKEN from utils.image_processing import encode_image_base64 -# async def image_query(file: UploadFile, question: str = Form(...)): -# """ -# API endpoint to process an image with the user's query. -# """ -# try: -# if file.content_type not in ["image/jpeg", "image/png"]: -# return JSONResponse({"query": question, "error": "Unsupported file type."}) - -# image_data = await file.read() -# image = Image.open(io.BytesIO(image_data)).convert("RGB").resize((512, 512)) -# encoded_image_base64 = encode_image_base64(image) - -# question_with_image_token = f"{question}\n{IMAGE_TOKEN}" -# response = await asyncio.to_thread(pipe, (question, image)) -# return JSONResponse({"query": question, "response": response.text}) -# except Exception as e: -# return JSONResponse({"query": question, "error": str(e)}) - - -async def image_query(image: np.ndarray, question: str): +async def image_query(file: UploadFile, question: str = Form(...)): """ - API endpoint to process an image (as numpy array) with the user's query. + API endpoint to process an image with the user's query. """ try: - # Convert the numpy array to a PIL Image - image = Image.fromarray(image).convert("RGB").resize((512, 512)) + if file.content_type not in ["image/jpeg", "image/png"]: + return JSONResponse({"query": question, "error": "Unsupported file type."}) + + image_data = await file.read() + image = Image.open(io.BytesIO(image_data)).convert("RGB").resize((512, 512)) + encoded_image_base64 = encode_image_base64(image) - # Encode the image to base64 (optional, if needed by your pipeline) - buffered = io.BytesIO() - image.save(buffered, format="JPEG") - encoded_image_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8") - - # Prepare the query with the image token question_with_image_token = f"{question}\n{IMAGE_TOKEN}" - - # Query the model - response = await asyncio.to_thread(pipe, (question, image)) - return {"query": question, "response": response.text} + response = await asyncio.to_thread(pipe, (question, image)) + return JSONResponse({"query": question, "response": response.text}) except Exception as e: - return {"query": question, "error": str(e)} + return JSONResponse({"query": question, "error": str(e)}) + + +# async def image_query(image: np.ndarray, question: str): +# """ +# API endpoint to process an image (as numpy array) with the user's query. +# """ +# try: +# # Convert the numpy array to a PIL Image +# image = Image.fromarray(image).convert("RGB").resize((512, 512)) + +# # Encode the image to base64 (optional, if needed by your pipeline) +# buffered = io.BytesIO() +# image.save(buffered, format="JPEG") +# encoded_image_base64 = base64.b64encode(buffered.getvalue()).decode("utf-8") + +# # Prepare the query with the image token +# question_with_image_token = f"{question}\n{IMAGE_TOKEN}" + +# # Query the model +# response = await asyncio.to_thread(pipe, (question, image)) +# return {"query": question, "response": response.text} +# except Exception as e: +# return {"query": question, "error": str(e)} + + +# def image_query(image_path: str, question: str): +# try: +# print("image_path in image_query...") +# with open(image_path, "rb") as file: +# image_data = file.read() + +# image = Image.open(io.BytesIO(image_data)).convert("RGB").resize((512, 512)) +# encoded_image_base64 = encode_image_base64(image) + +# # Prepare the question with the image token +# question_with_image_token = f"{question}\n{IMAGE_TOKEN}" + +# # Run model inference (blocking call, but can be async) +# response = pipe((question_with_image_token, image)) + +# return {"query": question, "response": response.text} + +# except Exception as e: +# return {"query": question, "error": str(e)} diff --git a/endpoints/text.py b/endpoints/text.py index 446b4a8..60e4d52 100644 --- a/endpoints/text.py +++ b/endpoints/text.py @@ -4,23 +4,39 @@ from fastapi.responses import JSONResponse from asyncio import to_thread from pipeline_setup import pipe -# async def text_query(question: str = Form(...)): -# """ -# API endpoint to process text input with the user's query. -# """ -# try: -# response = await to_thread(pipe, question) -# return JSONResponse({"query": question, "response": response.text}) -# except Exception as e: -# return JSONResponse({"query": question, "error": str(e)}) - - +# api async def text_query(question: str = Form(...)): """ API endpoint to process text input with the user's query. """ try: - response = await to_thread(pipe, question) - return {"query": question, "response": response.text} + print("starting text querying...") + response = await to_thread(pipe, question) + return JSONResponse({"query": question, "response": response.text}) except Exception as e: - return {"query": question, "error": str(e)} + return JSONResponse({"query": question, "error": str(e)}) + +# gradio +# async def text_query(question: str = Form(...)): +# """ +# API endpoint to process text input with the user's query. +# """ +# try: +# print("Processing in text.py...") +# response = await to_thread(pipe, question) +# return {"query": question, "response": response.text} +# except Exception as e: +# return {"query": question, "error": str(e)} + +# celery +# def text_query(question: str = Form(...)): +# """ +# API endpoint to process text input with the user's query. +# """ +# print("Testing....") +# try: +# print("Processing in text.py...") +# response = pipe(question) # Call pipe synchronously +# return {"query": question, "response": response.text} +# except Exception as e: +# return {"query": question, "error": str(e)} diff --git a/endpoints/video.py b/endpoints/video.py index 3cd4ea3..7188f8f 100644 --- a/endpoints/video.py +++ b/endpoints/video.py @@ -4,39 +4,155 @@ from pipeline_setup import pipe from utils.image_processing import encode_image_base64 from utils.video_processing import split_video_into_segments, extract_motion_key_frames, extract_audio_from_video from utils.audio_transcription import transcribe_audio +import os +import torch +import json import time import asyncio import mimetypes from concurrent.futures import ThreadPoolExecutor +def save_checkpoint(video_id, checkpoint_data): + checkpoint_path = f"/tmp/{video_id}_progress.json" + with open(checkpoint_path, "w") as f: + json.dump(checkpoint_data, f) + +def load_checkpoint(video_id): + checkpoint_path = f"/tmp/{video_id}_progress.json" + if os.path.exists(checkpoint_path): + with open(checkpoint_path, "r") as f: + return json.load(f) + return None + # async def video_query(file: UploadFile, question: str = Form(...)): +# try: +# print("Processing video...") + +# if file.content_type not in ["video/mp4", "video/avi", "video/mkv"]: +# return JSONResponse({"query": question, "error": "Unsupported video file type."}) + +# overall_start_time = time.time() + +# video_data = await file.read() +# video_id = str(hash(video_data)) # Unique ID for checkpointing +# temp_video_path = f"/tmp/{video_id}.mp4" +# with open(temp_video_path, "wb") as temp_video_file: +# temp_video_file.write(video_data) + +# video_reading_time = time.time() +# segments = split_video_into_segments(temp_video_path, segment_duration=30) + +# checkpoint = load_checkpoint(video_id) or {} +# aggregated_responses = checkpoint.get("responses", []) +# segment_timings = checkpoint.get("timings", []) +# completed_segments = set(checkpoint.get("completed_segments", [])) +# preprocessed_segments = set(checkpoint.get("preprocessed_segments", [])) +# inference_completed_segments = set(checkpoint.get("inference_completed_segments", [])) + +# for i, segment_path in enumerate(segments): +# if i in completed_segments: +# print(f"Skipping already processed segment {i+1}") +# continue + +# segment_start_time = time.time() + +# if i not in preprocessed_segments: +# frame_start_time = time.time() +# imgs = extract_motion_key_frames(segment_path, max_frames=50, sigma_multiplier=2) +# frame_time = time.time() + +# audio_start_time = time.time() +# audio_path = extract_audio_from_video(segment_path) +# transcribed_text = transcribe_audio(audio_path) +# audio_time = time.time() + +# preprocessed_segments.add(i) +# save_checkpoint(video_id, { +# "responses": aggregated_responses, +# "timings": segment_timings, +# "completed_segments": list(completed_segments), +# "preprocessed_segments": list(preprocessed_segments), +# "inference_completed_segments": list(inference_completed_segments) +# }) + +# if i not in inference_completed_segments: +# combined_query = f"Audio Transcript: {transcribed_text}\n{question}" +# question_with_frames = "".join([f"Frame{j+1}: {{IMAGE_TOKEN}}\n" for j, _ in enumerate(imgs)]) +# question_with_frames += combined_query + +# content = [{"type": "text", "text": question_with_frames}] + [ +# {"type": "image_url", "image_url": {"max_dynamic_patch": 1, "url": f"data:image/jpeg;base64,{encode_image_base64(img)}"}} +# for img in imgs +# ] + +# inference_start_time = time.time() +# messages = [dict(role="user", content=content)] +# response = await asyncio.to_thread(pipe, messages) +# inference_time = time.time() + +# aggregated_responses.append(response.text) + +# inference_completed_segments.add(i) +# save_checkpoint(video_id, { +# "responses": aggregated_responses, +# "timings": segment_timings, +# "completed_segments": list(completed_segments), +# "preprocessed_segments": list(preprocessed_segments), +# "inference_completed_segments": list(inference_completed_segments) +# }) + +# segment_timings.append({ +# "segment_index": i + 1, +# "segment_processing_time": inference_time - segment_start_time, +# "frame_extraction_time": frame_time - frame_start_time, +# "audio_extraction_time": audio_time - audio_start_time, +# "model_inference_time": inference_time - inference_start_time +# }) + +# completed_segments.add(i) +# save_checkpoint(video_id, { +# "responses": aggregated_responses, +# "timings": segment_timings, +# "completed_segments": list(completed_segments), +# "preprocessed_segments": list(preprocessed_segments), +# "inference_completed_segments": list(inference_completed_segments) +# }) + +# return JSONResponse({ +# "question": question, +# "responses": aggregated_responses, +# "timings": { +# "video_reading_time": video_reading_time - overall_start_time, +# "total_segments": len(segments), +# "total_processing_time": time.time() - overall_start_time, +# "segment_details": segment_timings +# } +# }) +# except Exception as e: +# return JSONResponse({"query": question, "error": str(e)}) + + +# async def video_query(video_path: str, question: str): # """ # API endpoint to process a video file with the user's query. # """ # try: # print("Processing video...") -# # Validate file type -# if file.content_type not in ["video/mp4", "video/avi", "video/mkv"]: -# return JSONResponse({"query": question, "error": "Unsupported video file type."}) +# if not video_path or not isinstance(video_path, str): +# return {"query": question, "error": "No video file provided or invalid file input."} -# # Start overall timer -# overall_start_time = time.time() +# # Determine the file type using the file extension +# file_type, _ = mimetypes.guess_type(video_path) +# if file_type is None or not file_type.startswith("video/"): +# return {"query": question, "error": "Unsupported video file type."} -# # Save the uploaded video to a temporary file -# print("Reading video...") -# video_data = await file.read() -# temp_video_path = "/tmp/temp_video.mp4" -# with open(temp_video_path, "wb") as temp_video_file: -# temp_video_file.write(video_data) -# print(f"Temp video saved to: {temp_video_path}") - -# # Record the time after reading the video -# video_reading_time = time.time() +# # Log the video path +# print(f"Video path: {video_path}") # # Split the video into segments # print("Splitting video...") -# segments = split_video_into_segments(temp_video_path, segment_duration=30) +# segments = split_video_into_segments(video_path, segment_duration=30) # print(f"Video split into {len(segments)} segments.") # aggregated_responses = [] @@ -45,19 +161,12 @@ from concurrent.futures import ThreadPoolExecutor # for i, segment_path in enumerate(segments): # print(f"Processing segment {i+1}/{len(segments)}: {segment_path}") -# # Start timing for the segment -# segment_start_time = time.time() - # # Extract key frames -# frame_start_time = time.time() # imgs = extract_motion_key_frames(segment_path, max_frames=50, sigma_multiplier=2) -# frame_time = time.time() # # Extract audio and transcribe -# audio_start_time = time.time() # audio_path = extract_audio_from_video(segment_path) # transcribed_text = transcribe_audio(audio_path) -# audio_time = time.time() # # Combine transcribed text with the query # combined_query = f"Audio Transcript: {transcribed_text}\n{question}" @@ -79,110 +188,130 @@ from concurrent.futures import ThreadPoolExecutor # }) # # Query the model -# inference_start_time = time.time() # messages = [dict(role="user", content=content)] # response = await asyncio.to_thread(pipe, messages) -# inference_time = time.time() # # Aggregate response # aggregated_responses.append(response.text) -# # Calculate timing for the segment -# segment_timings.append({ -# "segment_index": i + 1, -# "segment_processing_time": inference_time - segment_start_time, -# "frame_extraction_time": frame_time - frame_start_time, -# "audio_extraction_time": audio_time - audio_start_time, -# "model_inference_time": inference_time - inference_start_time -# }) - -# print(f"transcription: {transcribed_text}") -# # print(f"content: {content}") - -# overall_end_time = time.time() - -# # Aggregate total timings -# total_timings = { -# "video_reading_time": video_reading_time - overall_start_time, -# "total_segments": len(segments), -# "total_processing_time": overall_end_time - overall_start_time, -# "segment_details": segment_timings -# } - -# return JSONResponse({ +# return { # "question": question, # "responses": aggregated_responses, -# "timings": total_timings, -# }) +# } # except Exception as e: -# return JSONResponse({"query": question, "error": str(e)}) +# return {"query": question, "error": str(e)} -async def video_query(video_path: str, question: str): - """ - API endpoint to process a video file with the user's query. - """ - try: - print("Processing video...") +# def video_query(video_path: str, question: str): +# """ +# Processes a video file using the model. +# Reads the video from disk, extracts key frames, transcribes audio, and queries the model. +# """ +# try: +# print("Processing video...") - if not video_path or not isinstance(video_path, str): - return {"query": question, "error": "No video file provided or invalid file input."} +# if not os.path.exists(video_path): +# return {"query": question, "error": "Video file not found."} - # Determine the file type using the file extension - file_type, _ = mimetypes.guess_type(video_path) - if file_type is None or not file_type.startswith("video/"): - return {"query": question, "error": "Unsupported video file type."} +# # Determine the file type +# file_type, _ = mimetypes.guess_type(video_path) +# if file_type is None or not file_type.startswith("video/"): +# return {"query": question, "error": "Unsupported video file type."} - # Log the video path - print(f"Video path: {video_path}") +# # Split video into segments +# print("Splitting video...") +# segments = split_video_into_segments(video_path, segment_duration=30) +# print(f"Video split into {len(segments)} segments.") - # Split the video into segments - print("Splitting video...") - segments = split_video_into_segments(video_path, segment_duration=30) - print(f"Video split into {len(segments)} segments.") +# aggregated_responses = [] +# segment_timings = [] - aggregated_responses = [] - segment_timings = [] +# for i, segment_path in enumerate(segments): +# print(f"Processing segment {i+1}/{len(segments)}: {segment_path}") - for i, segment_path in enumerate(segments): - print(f"Processing segment {i+1}/{len(segments)}: {segment_path}") +# # Extract key frames +# imgs = extract_motion_key_frames(segment_path, max_frames=50, sigma_multiplier=2) - # Extract key frames - imgs = extract_motion_key_frames(segment_path, max_frames=50, sigma_multiplier=2) +# # Extract audio and transcribe +# audio_path = extract_audio_from_video(segment_path) +# transcribed_text = transcribe_audio(audio_path) - # Extract audio and transcribe - audio_path = extract_audio_from_video(segment_path) - transcribed_text = transcribe_audio(audio_path) +# # Combine transcribed text with the query +# combined_query = f"Audio Transcript: {transcribed_text}\n{question}" - # Combine transcribed text with the query - combined_query = f"Audio Transcript: {transcribed_text}\n{question}" +# # Prepare content for the pipeline +# question_with_frames = "".join([f"Frame{j+1}: {{IMAGE_TOKEN}}\n" for j in range(len(imgs))]) +# question_with_frames += combined_query - # Prepare content for the pipeline - question_with_frames = "" - for j, img in enumerate(imgs): - question_with_frames += f"Frame{j+1}: {{IMAGE_TOKEN}}\n" - question_with_frames += combined_query +# content = [{"type": "text", "text": question_with_frames}] + [ +# {"type": "image_url", "image_url": {"max_dynamic_patch": 1, "url": f"data:image/jpeg;base64,{encode_image_base64(img)}"}} +# for img in imgs +# ] - content = [{"type": "text", "text": question_with_frames}] - for img in imgs: - content.append({ - "type": "image_url", - "image_url": { - "max_dynamic_patch": 1, - "url": f"data:image/jpeg;base64,{encode_image_base64(img)}" - } - }) +# # Query the model +# messages = [dict(role="user", content=content)] +# response = pipe(messages) - # Query the model - messages = [dict(role="user", content=content)] - response = await asyncio.to_thread(pipe, messages) +# # Aggregate response +# aggregated_responses.append(response.text) - # Aggregate response - aggregated_responses.append(response.text) +# return { +# "question": question, +# "responses": aggregated_responses, +# } + +# except Exception as e: +# return {"query": question, "error": str(e)} - return { - "question": question, - "responses": aggregated_responses, - } - except Exception as e: - return {"query": question, "error": str(e)} \ No newline at end of file + +# def run_video_inference(preprocessed_data): +# """ +# **Inference Step (Runs on GPU)** +# - Takes preprocessed data (key frames + transcribed audio). +# - Constructs a query for the model. +# - Runs inference on the GPU. +# - Returns the aggregated responses. +# """ +# import torch +# torch.cuda.empty_cache() # Free up GPU memory before inference + +# try: +# print("Starting video inference...") + +# question = preprocessed_data["question"] +# segments = preprocessed_data["segments"] +# aggregated_responses = [] + +# for segment in segments: +# segment_index = segment["segment_index"] +# transcribed_text = segment["transcription"] +# encoded_imgs = segment["encoded_images"] + +# print(f"Running inference on segment {segment_index + 1}...") + +# # Prepare query content +# question_with_frames = "".join( +# [f"Frame{j+1}: {{IMAGE_TOKEN}}\n" for j in range(len(encoded_imgs))] +# ) +# combined_query = f"Audio Transcript: {transcribed_text}\n{question}" +# question_with_frames += combined_query + +# content = [{"type": "text", "text": question_with_frames}] + [ +# {"type": "image_url", "image_url": {"max_dynamic_patch": 1, "url": f"data:image/jpeg;base64,{img}"}} +# for img in encoded_imgs +# ] + +# # Query the model (GPU-heavy operation) +# messages = [dict(role="user", content=content)] +# response = pipe(messages) + +# # Collect responses +# aggregated_responses.append(response.text) + +# return { +# "question": question, +# "responses": aggregated_responses, +# } + +# except Exception as e: +# return {"query": question, "error": str(e)} diff --git a/main.py b/main.py index 832bf12..16df0b7 100644 --- a/main.py +++ b/main.py @@ -1,83 +1,105 @@ -# from fastapi import FastAPI, Form, UploadFile -# from fastapi.responses import JSONResponse -# import asyncio - -# app = FastAPI() - -# @app.post("/api/text") -# async def text_query_endpoint(question: str = Form(...)): -# """ -# API endpoint to process text input with the user's query. -# """ -# from endpoints.text import text_query -# return await text_query(question=question) - - -# @app.post("/api/image") -# async def image_query_endpoint(file: UploadFile, question: str = Form(...)): -# """ -# API endpoint to process an image with the user's query. -# """ -# from endpoints.image import image_query -# return await image_query(file=file, question=question) - - -# @app.post("/api/video") -# async def video_query_endpoint(file: UploadFile, question: str = Form(...)): -# """ -# API endpoint to process a video file with the user's query. -# """ -# from endpoints.video import video_query -# return await video_query(file=file, question=question) - - -# if __name__ == "__main__": -# import uvicorn -# uvicorn.run("main:app", host="0.0.0.0", port=8002, reload=True, loop="uvloop") - - from fastapi import FastAPI, Form, UploadFile from fastapi.responses import JSONResponse -import shutil -import uuid -from tasks import text_query_task, image_query_task, video_query_task +import asyncio app = FastAPI() -# @app.post("/api/text") -# async def text_query_endpoint(question: str = Form(...)): -# task = text_query_task.apply_async(args=[question]) -# return JSONResponse({"task_id": task.id}) - @app.post("/api/text") async def text_query_endpoint(question: str = Form(...)): - print(f"Received request: {question}") - task = text_query_task.apply_async(args=[question]) - print(f"Task sent: {task.id}") - return JSONResponse({"task_id": task.id}) + """ + API endpoint to process text input with the user's query. + """ + from endpoints.text import text_query + return await text_query(question=question) + @app.post("/api/image") async def image_query_endpoint(file: UploadFile, question: str = Form(...)): - file_path = f"/tmp/{uuid.uuid4()}_{file.filename}" - with open(file_path, "wb") as buffer: - shutil.copyfileobj(file.file, buffer) - - task = image_query_task.apply_async(args=[file_path, question]) - return JSONResponse({"task_id": task.id}) + """ + API endpoint to process an image with the user's query. + """ + from endpoints.image import image_query + return await image_query(file=file, question=question) + @app.post("/api/video") async def video_query_endpoint(file: UploadFile, question: str = Form(...)): - file_path = f"/tmp/{uuid.uuid4()}_{file.filename}" - with open(file_path, "wb") as buffer: - shutil.copyfileobj(file.file, buffer) - - task = video_query_task.apply_async(args=[file_path, question]) - return JSONResponse({"task_id": task.id}) + """ + API endpoint to process a video file with the user's query. + """ + from endpoints.video import video_query + return await video_query(file=file, question=question) -@app.get("/api/task/{task_id}") -async def get_task_result(task_id: str): - from celery.result import AsyncResult - result = AsyncResult(task_id) - if result.ready(): - return JSONResponse({"status": "completed", "result": result.result}) - return JSONResponse({"status": "pending"}) + +if __name__ == "__main__": + import uvicorn + uvicorn.run("main:app", host="0.0.0.0", port=8002, reload=True, loop="uvloop") + + + + +# from fastapi import FastAPI, Form, UploadFile +# from fastapi.responses import JSONResponse +# import shutil +# import uuid +# from celery import chain +# from tasks import text_query_task, image_query_task, video_query_task +# from tasks import text_query_task, image_query_task, preprocess_video, inference_video + +# app = FastAPI() + +# @app.get("/") +# def read_root(): +# return {"message": "FastAPI with Celery & RabbitMQ"} + +# @app.post("/api/text") +# async def text_query_endpoint(question: str = Form(...)): +# print(f"Received request: {question}") +# task = text_query_task.apply_async(args=[question]) +# print(f"Task sent: {task.id}") +# return JSONResponse({"task_id": task.id}) + +# @app.post("/api/image") +# async def image_query_endpoint(file: UploadFile, question: str = Form(...)): +# file_path = f"/tmp/{uuid.uuid4()}_{file.filename}" +# with open(file_path, "wb") as buffer: +# shutil.copyfileobj(file.file, buffer) + +# task = image_query_task.apply_async(args=[file_path, question]) +# return JSONResponse({"task_id": task.id}) + +# # @app.post("/api/video") +# # async def video_query_endpoint(file: UploadFile, question: str = Form(...)): +# # file_path = f"/tmp/{uuid.uuid4()}_{file.filename}" +# # with open(file_path, "wb") as buffer: +# # shutil.copyfileobj(file.file, buffer) + +# # task = video_query_task.apply_async(args=[file_path, question]) +# # return JSONResponse({"task_id": task.id}) + +# @app.post("/api/video") +# async def video_query_endpoint(file: UploadFile, question: str = Form(...)): +# # Save the uploaded file to a temporary location +# file_path = f"/tmp/{uuid.uuid4()}_{file.filename}" +# with open(file_path, "wb") as buffer: +# shutil.copyfileobj(file.file, buffer) + +# # Chain the preprocessing and inference tasks +# task_chain = chain( +# preprocess_video.s(file_path, question), # Preprocessing task +# inference_video.s() # Inference task +# ).apply_async() + +# return JSONResponse({"task_id": task_chain.id}) + +# @app.get("/api/task/{task_id}") +# async def get_task_result(task_id: str): +# from celery.result import AsyncResult +# result = AsyncResult(task_id) +# if result.ready(): +# return JSONResponse({"status": "completed", "result": result.result}) +# return JSONResponse({"status": "pending"}) + +# if __name__ == "__main__": +# import uvicorn +# uvicorn.run("main:app", host="0.0.0.0", port=8002, reload=True, loop="uvloop") \ No newline at end of file diff --git a/monitor.py b/monitor.py new file mode 100644 index 0000000..f613fe2 --- /dev/null +++ b/monitor.py @@ -0,0 +1,33 @@ +import psutil +import GPUtil +import time +import logging + +# Specify the full path for the log file +log_file_path = "resource_usage.log" + +# Configure logging +logging.basicConfig(filename=log_file_path, level=logging.INFO) +logging.info("Logging started") # Add this line to confirm logging is working + +try: + while True: + # Monitor CPU + cpu_usage = psutil.cpu_percent(interval=1) + memory_usage = psutil.virtual_memory().percent + cpu_message = f"CPU Usage: {cpu_usage}% | Memory Usage: {memory_usage}%" + print(cpu_message) # Print to console + logging.info(cpu_message) # Log to file + + # Monitor GPU + GPUs = GPUtil.getGPUs() + for gpu in GPUs: + gpu_message = f"GPU {gpu.id} | Usage: {gpu.load * 100}% | Memory: {gpu.memoryUsed}MB / {gpu.memoryTotal}MB" + print(gpu_message) # Print to console + logging.info(gpu_message) # Log to file + + time.sleep(1) +except Exception as e: + error_message = f"An error occurred: {e}" + print(error_message) # Print to console + logging.error(error_message) # Log to file \ No newline at end of file diff --git a/nohup.out b/nohup.out new file mode 100644 index 0000000..b625a81 --- /dev/null +++ b/nohup.out @@ -0,0 +1,38 @@ +INFO: Will watch for changes in these directories: ['/home/ooin/st/tiktok_AI'] +INFO: Uvicorn running on http://0.0.0.0:8002 (Press CTRL+C to quit) +INFO: Started reloader process [2653752] using StatReload +INFO: Started server process [2653764] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Will watch for changes in these directories: ['/home/ooin/st/tiktok_AI'] +INFO: Uvicorn running on http://0.0.0.0:8002 (Press CTRL+C to quit) +INFO: Started reloader process [2662681] using StatReload +INFO: Started server process [2662696] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: 127.0.0.1:40090 - "GET / HTTP/1.1" 404 Not Found +INFO: 127.0.0.1:53220 - "GET /docs HTTP/1.1" 200 OK +WARNING: StatReload detected changes in 'pipeline_setup.py'. Reloading... +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [2662696] +INFO: Started server process [2876691] +INFO: Waiting for application startup. +INFO: Application startup complete. +WARNING: StatReload detected changes in 'pipeline_setup.py'. Reloading... +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [2876691] +INFO: Started server process [2876710] +INFO: Waiting for application startup. +INFO: Application startup complete. +WARNING: StatReload detected changes in 'model_loader.py'. Reloading... +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [2876710] +INFO: Started server process [2877032] +INFO: Waiting for application startup. +INFO: Application startup complete. diff --git a/pipeline_setup.py b/pipeline_setup.py index 5fc6f85..97e1e3b 100644 --- a/pipeline_setup.py +++ b/pipeline_setup.py @@ -7,7 +7,9 @@ os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" # os.environ["CUDA_VISIBLE_DEVICES"] = "2,3" # Model initialization -model = "OpenGVLab/InternVL2-26B-AWQ" +# model = "OpenGVLab/InternVL2-26B-AWQ" +# model = "OpenGVLab/InternVL2_5-4B-AWQ" +model = "OpenGVLab/InternVL2_5-8B-MPO-AWQ" pipe = pipeline( model, backend_config=TurbomindEngineConfig( @@ -16,7 +18,7 @@ pipe = pipeline( # tp=4, session_len=16384, # 4096, 8192, 16384, 32768 max_batch_size=1, - cache_max_entry_count=0.2, # 0.05 + cache_max_entry_count=0.15, # 0.05 cache_block_seq_len=16384, # 8192, 16384, 32768 # quant_policy=8, # precision="fp16", diff --git a/requirements.txt b/requirements.txt index 8806a97..cb8a209 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,117 +1,14 @@ -accelerate==1.2.1 -addict==2.4.0 -aiohappyeyeballs==2.4.4 -aiohttp==3.11.11 -aiosignal==1.3.2 -annotated-types==0.7.0 -anyio==4.7.0 -attrs==24.3.0 -bitsandbytes==0.45.0 -certifi==2024.12.14 -charset-normalizer==3.4.0 -click==8.1.8 -cloudpickle==3.1.0 -datasets==3.2.0 +celery==5.4.0 decord==0.6.0 -dill==0.3.8 -diskcache==5.6.3 -distro==1.9.0 -einops==0.8.0 -fastapi==0.115.6 -filelock==3.16.1 -fire==0.7.0 -# flash-attn==2.7.2.post1 -frozenlist==1.5.0 -fsspec==2024.9.0 -h11==0.14.0 -httpcore==1.0.7 +fastapi==0.115.8 +gradio==5.17.0 +gradio_image_prompter==0.1.0 httpx==0.28.1 -huggingface-hub==0.27.0 -idna==3.10 -interegular==0.3.3 -Jinja2==3.1.5 -jiter==0.8.2 -jsonschema==4.23.0 -jsonschema-specifications==2024.10.1 -lark==1.2.2 -llvmlite==0.43.0 -lmdeploy==0.6.4 -markdown-it-py==3.0.0 -MarkupSafe==3.0.2 -mdurl==0.1.2 -mmengine-lite==0.10.5 -modelscope==1.21.0 -mpmath==1.3.0 -multidict==6.1.0 -multiprocess==0.70.16 -nest-asyncio==1.6.0 -networkx==3.4.2 -ninja==1.11.1.3 -numba==0.60.0 -numpy==1.26.4 -nvidia-cublas-cu12==12.1.3.1 -nvidia-cuda-cupti-cu12==12.1.105 -nvidia-cuda-nvrtc-cu12==12.1.105 -nvidia-cuda-runtime-cu12==12.1.105 -nvidia-cudnn-cu12==9.1.0.70 -nvidia-cufft-cu12==11.0.2.54 -nvidia-curand-cu12==10.3.2.106 -nvidia-cusolver-cu12==11.4.5.107 -nvidia-cusparse-cu12==12.1.0.106 -nvidia-ml-py==12.560.30 -nvidia-nccl-cu12==2.20.5 -nvidia-nvjitlink-cu12==12.6.85 -nvidia-nvtx-cu12==12.1.105 -openai==1.58.1 -opencv-python==4.10.0.84 -outlines==0.0.46 -packaging==24.2 -pandas==2.2.3 -peft==0.11.1 -pillow==11.0.0 -platformdirs==4.3.6 -propcache==0.2.1 -protobuf==5.29.2 -psutil==6.1.1 -pyairports==2.1.1 -pyarrow==18.1.0 -pycountry==24.6.1 -pydantic==2.10.4 -pydantic_core==2.27.2 -Pygments==2.18.0 -pynvml==12.0.0 -python-dateutil==2.9.0.post0 -python-multipart==0.0.20 -pytz==2024.2 -PyYAML==6.0.2 -referencing==0.35.1 -regex==2024.11.6 -requests==2.32.3 -rich==13.9.4 -rpds-py==0.22.3 -safetensors==0.4.5 -sentencepiece==0.2.0 -setuptools==75.6.0 -shortuuid==1.0.13 -six==1.17.0 -sniffio==1.3.1 -starlette==0.41.3 -sympy==1.13.3 -termcolor==2.5.0 -tiktoken==0.8.0 -timm==1.0.12 -tokenizers==0.21.0 -# torch==2.4.0 -# torchaudio==2.4.0 -# torchvision==0.19.0 -tqdm==4.67.1 -transformers==4.47.1 -triton==3.0.0 -typing_extensions==4.12.2 -tzdata==2024.2 -urllib3==2.3.0 +lmdeploy==0.7.0.post3 +locustio==0.999 +numpy==2.2.3 +Pillow==11.1.0 +pydub==0.25.1 +torch==2.6.0 uvicorn==0.34.0 -wheel==0.45.1 -xxhash==3.5.0 -yapf==0.43.0 -yarl==1.18.3 +whisper==1.1.10 diff --git a/resource_usage.log b/resource_usage.log new file mode 100644 index 0000000..d632041 --- /dev/null +++ b/resource_usage.log @@ -0,0 +1,1623 @@ +INFO:root:CPU Usage: 3.6% | Memory Usage: 3.6% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.9% | Memory Usage: 3.6% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 3.5% | Memory Usage: 3.6% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 3.7% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 3.7% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 3.7% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 1.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 3.7% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 3.7% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.2% | Memory Usage: 3.7% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.2% | Memory Usage: 3.8% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 15.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.1% | Memory Usage: 3.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 278.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 278.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 11.9% | Memory Usage: 4.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 278.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 278.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 3.6% | Memory Usage: 4.2% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 620.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 560.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 3.2% | Memory Usage: 4.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 3290.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 3294.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.0% | Memory Usage: 4.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 3290.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 3830.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 6.8% | Memory Usage: 4.7% +INFO:root:GPU 0 | Usage: 3.0% | Memory: 3290.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 71.0% | Memory: 3958.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 4.0% | Memory Usage: 4.6% +INFO:root:GPU 0 | Usage: 87.0% | Memory: 5970.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 90.0% | Memory: 6510.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 13.9% | Memory Usage: 4.9% +INFO:root:GPU 0 | Usage: 95.0% | Memory: 5970.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 95.0% | Memory: 6510.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 3.9% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 100.0% | Memory: 6344.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 99.0% | Memory: 6824.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 4.1% | Memory Usage: 5.0% +INFO:root:GPU 0 | Usage: 100.0% | Memory: 6680.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 100.0% | Memory: 7208.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 7.6% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 9334.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 28.000000000000004% | Memory: 10414.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.7% | Memory Usage: 5.7% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 9334.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 10414.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.7% | Memory Usage: 5.3% +INFO:root:CPU Usage: 2.8% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 95.0% | Memory: 10730.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 97.0% | Memory: 11810.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 98.0% | Memory: 10954.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 100.0% | Memory: 12034.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.0% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.2% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 10.8% | Memory Usage: 5.4% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11082.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.2% | Memory Usage: 5.4% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11097.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12162.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 4.3% | Memory Usage: 5.6% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11341.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12421.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11683.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12703.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.7% | Memory Usage: 6.1% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 11683.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 12767.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 7.8% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 7.000000000000001% | Memory: 14353.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 31.0% | Memory: 15799.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.8% | Memory Usage: 6.7% +INFO:root:GPU 0 | Usage: 1.0% | Memory: 14353.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 14.000000000000002% | Memory: 15973.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.6% | Memory Usage: 6.3% +INFO:root:GPU 0 | Usage: 93.0% | Memory: 15749.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 93.0% | Memory: 17369.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.6% | Memory Usage: 6.3% +INFO:root:GPU 0 | Usage: 93.0% | Memory: 15749.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 92.0% | Memory: 17369.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.9% | Memory Usage: 6.3% +INFO:root:GPU 0 | Usage: 99.0% | Memory: 15749.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 97.0% | Memory: 17369.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.9% | Memory Usage: 6.3% +INFO:root:GPU 0 | Usage: 100.0% | Memory: 15909.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 99.0% | Memory: 17529.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.5% | Memory Usage: 6.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.1% | Memory Usage: 6.3% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 39.3% | Memory Usage: 7.4% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 7.9% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 98.2% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 50.4% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:Logging started +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.3% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 16101.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 1.0% | Memory: 17721.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.1% | Memory Usage: 5.8% +INFO:root:GPU 0 | Usage: 25.0% | Memory: 10058.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 100.0% | Memory: 11138.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:Logging started +INFO:root:CPU Usage: 1.8% | Memory Usage: 6.2% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10659.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11743.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 8.0% | Memory Usage: 6.8% +INFO:root:GPU 0 | Usage: 6.0% | Memory: 13329.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 32.0% | Memory: 14949.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.6% | Memory Usage: 7.2% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 13329.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 14949.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.8% | Memory Usage: 6.7% +INFO:root:GPU 0 | Usage: 88.0% | Memory: 14725.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 88.0% | Memory: 16345.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.6% | Memory Usage: 6.7% +INFO:root:GPU 0 | Usage: 98.0% | Memory: 14725.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 98.0% | Memory: 16345.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.8% | Memory Usage: 6.7% +INFO:root:GPU 0 | Usage: 98.0% | Memory: 14725.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 98.0% | Memory: 16345.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.7% | Memory Usage: 6.7% +INFO:root:GPU 0 | Usage: 100.0% | Memory: 14885.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 100.0% | Memory: 16505.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.9% | Memory Usage: 6.7% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15077.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 16697.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 4.5% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 99.0% | Memory: 15135.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 90.0% | Memory: 16713.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 2.9% | Memory Usage: 6.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 15135.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 16713.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.1% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.4% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.0% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.1% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.1% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.4% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.0% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 1.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 1.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 1.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.4% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.2% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.4% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 1.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 5.9% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.5% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.9% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.6% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.1% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 1.3% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.7% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:CPU Usage: 0.8% | Memory Usage: 6.0% +INFO:root:GPU 0 | Usage: 0.0% | Memory: 10116.0MB / 24576.0MB +INFO:root:GPU 1 | Usage: 0.0% | Memory: 11154.0MB / 24576.0MB +INFO:root:GPU 2 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB +INFO:root:GPU 3 | Usage: 0.0% | Memory: 18.0MB / 24576.0MB diff --git a/tasks.py b/tasks.py index f9c756a..a23c34e 100644 --- a/tasks.py +++ b/tasks.py @@ -1,25 +1,330 @@ -import asyncio -from celery_app import celery_app +import os +os.environ["TOKENIZERS_PARALLELISM"] = "false" -@celery_app.task +import torch.multiprocessing as mp +mp.set_start_method("spawn", force=True) +# mp.set_start_method("fork", force=True) + +from celery import Celery + +import psutil +from pynvml import nvmlInit, nvmlDeviceGetCount, nvmlDeviceGetHandleByIndex, nvmlDeviceGetUtilizationRates + +# from endpoints.video import run_video_inference +# from endpoints.video2 import preprocess_video + +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +print(os.path.dirname(os.path.abspath(__file__))) + +# celery_app.conf.task_routes = { +# "tasks.*": {"queue": "default"}, +# } + +app = Celery( + "tasks", + broker="redis://localhost:6379/0", + backend="redis://localhost:6379/0", + # include=["tasks"] +) + +# app = Celery( +# "tasks", +# broker="pyamqp://guest@localhost//", +# # backend="rpc://", +# backend="redis://localhost:6379/0", +# ) +app.conf.worker_prefetch_multiplier = 1 + +# app.conf.task_routes = { +# "tasks.*": {"queue": "default"}, +# } + +# app.conf.task_acks_late = True + +# from celery import Celery +# from kombu import Queue + +# celery = Celery( +# "tasks", +# broker="redis://localhost:6379/0", +# backend="redis://localhost:6379/0", +# include=["tasks"] +# ) + +# app.conf.task_routes = { +# 'tasks.preprocess_video': {'queue': 'preprocess_queue'}, +# 'tasks.inference_video': {'queue': 'inference_queue'}, +# } + +app.conf.task_routes = { + 'tasks.preprocess_video': {'queue': 'preprocess_queue'}, + 'tasks.inference_video': {'queue': 'inference_queue'}, +} + +# # Define task queues properly +# celery.conf.task_queues = ( +# Queue("high_priority"), +# Queue("default"), +# Queue("low_priority"), +# ) + +# # Define task routing +# celery.conf.task_routes = { +# "tasks.text_query_task": {"queue": "high_priority"}, +# "tasks.image_query_task": {"queue": "default"}, +# "tasks.video_query_task": {"queue": "low_priority"}, +# } + +# # Define task rate limits +# celery.conf.task_annotations = { +# "tasks.text_query_task": {"rate_limit": "10/m"}, +# "tasks.image_query_task": {"rate_limit": "5/m"}, +# "tasks.video_query_task": {"rate_limit": "3/m"}, +# } + +# # Define task retries +# celery.conf.task_acks_late = True # Ensure task is only removed from queue when fully processed +# celery.conf.worker_prefetch_multiplier = 1 # Avoid one worker taking too many tasks at once + +# # Define task time limits +# celery.conf.task_time_limit = 60 # 60 seconds max execution time +# celery.conf.task_soft_time_limit = 50 # Warn at 50 seconds + +@app.task +def add(x, y): + print("Adding task...") + return x + y + +@app.task(name="tasks.text_query_task") def text_query_task(question: str): - from endpoints.text import text_query # Import inside the function to avoid circular import - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - return loop.run_until_complete(text_query(question=question)) # ✅ Correct way to call async functions in Celery + print("Importing text_query...") + from endpoints.text import text_query + print(f"Processing question: {question}") + return text_query(question) -@celery_app.task +@app.task(name="tasks.image_query_task") def image_query_task(file_path: str, question: str): - from endpoints.image import image_query # Import inside the function - with open(file_path, "rb") as file: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - return loop.run_until_complete(image_query(file=file, question=question)) # ✅ Use event loop + try: + print("Processing in image_query_task...") + from endpoints.image import image_query + print("file_path in image_query_task...") + result = image_query(file_path, question) + return result + + except Exception as e: + return {"query": question, "error": str(e)} -@celery_app.task +@app.task(name="tasks.video_query_task") def video_query_task(file_path: str, question: str): - from endpoints.video import video_query # Import inside the function - with open(file_path, "rb") as file: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - return loop.run_until_complete(video_query(file=file, question=question)) # ✅ Use event loop + """ + Celery task to process a video query asynchronously. + Reads the video file from disk and processes it. + """ + try: + from endpoints.video import video_query + result = video_query(file_path, question) + return result + + except Exception as e: + return {"query": question, "error": str(e)} + +# @celery.task(name="tasks.video_preprocessing_task", priority=5, queue="preprocessing") +# def video_preprocessing_task(file_path: str, question: str): +# return preprocess_video(file_path, question) + +# @celery.task(name="tasks.video_query_task", priority=10, queue="inference") +# def video_query_task(preprocessed_data): +# return run_video_inference(preprocessed_data) + + +# @celery.task(name="tasks.test_task") +# def test_task(): +# return "Celery is working!" + + + +import mimetypes +from utils.video_processing import split_video_into_segments, extract_motion_key_frames, extract_audio_from_video +from utils.audio_transcription import transcribe_audio +from pipeline_setup import pipe +from utils.image_processing import encode_image_base64 +from concurrent.futures import ThreadPoolExecutor, as_completed + +def process_segment(segment_data): + segment_path, segment_idx, total_segments = segment_data + print(f"Processing segment {segment_idx+1}/{total_segments}: {segment_path}") + + imgs = extract_motion_key_frames(segment_path, max_frames=20, sigma_multiplier=4) + print(f"length of key frames in segments: {len(imgs)}") + print(f"Segment {segment_idx+1}: extract_motion_key_frames finished.") + + audio_path = extract_audio_from_video(segment_path) + print(f"Segment {segment_idx+1}: extract_audio_from_video finished.") + transcribed_text = transcribe_audio(audio_path) + print(f"Segment {segment_idx+1}: transcribe_audio finished.") + + return { + "segment_path": segment_path, + "key_frames": [encode_image_base64(img) for img in imgs], + "transcribed_text": transcribed_text + } + +@app.task(name="tasks.preprocess_video") +def preprocess_video(video_path, question): + try: + # Monitor CPU usage + # cpu_usage = psutil.cpu_percent(interval=1) + # print(f"CPU Usage during preprocessing: {cpu_usage}%") + + print(f"Preprocessing video: {video_path}") + + if not os.path.exists(video_path): + return {"query": question, "error": "Video file not found."} + + # Determine the file type + file_type, _ = mimetypes.guess_type(video_path) + if file_type is None or not file_type.startswith("video/"): + return {"query": question, "error": "Unsupported video file type."} + + print("Splitting video...") + segments = split_video_into_segments(video_path, segment_duration=100) + print(f"segments: {segments}") + print(f"Video split into {len(segments)} segments.") + + # Process segments in parallel + processed_segments = [] + max_workers = min(len(segments), os.cpu_count() * 2) + + print(f"Processing segments with {max_workers} workers...") + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_segment = { + executor.submit(process_segment, (segment_path, idx, len(segments))): idx + for idx, segment_path in enumerate(segments) + } + + # Collect results as they complete + segment_results = [None] * len(segments) + for future in as_completed(future_to_segment): + idx = future_to_segment[future] + try: + segment_results[idx] = future.result() + except Exception as e: + print(f"Error processing segment {idx}: {str(e)}") + segment_results[idx] = { + "segment_path": segments[idx], + "error": str(e) + } + + print("multithread done!") + + processed_segments = [result for result in segment_results if "error" not in result] + + return { + "video_path": video_path, + "question": question, + "processed_segments": processed_segments + } + + except Exception as e: + return {"query": question, "error": str(e)} + +# @app.task(name="tasks.inference_video") +# def inference_video(preprocessed_data): +# try: +# # Monitor GPU usage +# # nvmlInit() +# # device_count = nvmlDeviceGetCount() +# # for i in range(device_count): +# # handle = nvmlDeviceGetHandleByIndex(i) +# # utilization = nvmlDeviceGetUtilizationRates(handle) +# # print(f"GPU {i} Usage during inference: {utilization.gpu}%") + +# # print(preprocessed_data) +# video_path = preprocessed_data["video_path"] +# question = preprocessed_data["question"] +# # print(f"question: {question}") +# segments = preprocessed_data["processed_segments"] + +# print(f"Running inference on: {video_path}") + +# aggregated_responses = [] +# for i, segment in enumerate(segments): +# print(f"Inferencing segment {i+1}/{len(segments)}") + +# # Prepare input content +# question_with_frames = "".join( +# [f"Frame{j+1}: {{IMAGE_TOKEN}}\n" for j in range(len(segment["key_frames"]))] +# ) +# question_with_frames += f"Audio Transcript: {segment['transcribed_text']}\n{question}" + +# content = [{"type": "text", "text": question_with_frames}] + [ +# {"type": "image_url", "image_url": {"max_dynamic_patch": 1, "url": f"data:image/jpeg;base64,{img}"}} +# for img in segment["key_frames"] +# ] + +# # Query model +# messages = [dict(role="user", content=content)] +# response = pipe(messages) + +# # Aggregate response +# aggregated_responses.append(response.text) + +# return { +# "question": question, +# "responses": aggregated_responses, +# } + +# except Exception as e: +# return {"query": question, "error": str(e)} + + +@app.task(name="tasks.inference_video") +def inference_video(preprocessed_results): + """ + Processes a batch of preprocessed videos on the GPU. + """ + try: + print("Running inference on a batch of videos...") + aggregated_results = [] + for preprocessed_data in preprocessed_results: + video_path = preprocessed_data["video_path"] + question = preprocessed_data["question"] + segments = preprocessed_data["processed_segments"] + + print(f"Inferencing video: {video_path}") + + # Run inference on the GPU + aggregated_responses = [] + for segment in segments: + # Prepare input for inference + question_with_frames = "".join( + [f"Frame{j+1}: {{IMAGE_TOKEN}}\n" for j in range(len(segment["key_frames"]))] + ) + question_with_frames += f"Audio Transcript: {segment['transcribed_text']}\n{question}" + + content = [{"type": "text", "text": question_with_frames}] + [ + {"type": "image_url", "image_url": {"max_dynamic_patch": 1, "url": f"data:image/jpeg;base64,{img}"}} + for img in segment["key_frames"] + ] + + # Query model + messages = [dict(role="user", content=content)] + response = pipe(messages) + + # Aggregate response + aggregated_responses.append(response.text) + + aggregated_results.append({ + "video_path": video_path, + "question": question, + "responses": aggregated_responses + }) + + return aggregated_results + + except Exception as e: + return {"error": str(e)} \ No newline at end of file diff --git a/test_audio.py b/test_audio.py new file mode 100644 index 0000000..9ca5572 --- /dev/null +++ b/test_audio.py @@ -0,0 +1,92 @@ +import torch +import os +from whisper import load_model +from pydub import AudioSegment + +def extract_audio_from_video(video_path: str) -> str: + audio = AudioSegment.from_file(video_path) + audio_path = "/tmp/temp_audio_test.wav" + audio.export(audio_path, format="wav") + print("video extracted!") + return audio_path + +# def transcribe_audio(audio_path: str) -> str: +# print("Loading model in transcribe_audio...") +# from transformers import WhisperProcessor, WhisperForConditionalGeneration +# import torch + +# # Load processor and model from transformers +# processor = WhisperProcessor.from_pretrained("openai/whisper-base") +# model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-base") + +# if torch.cuda.device_count() > 1: +# print(f"Using {torch.cuda.device_count()} GPUs!") +# model = torch.nn.DataParallel(model) + +# device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +# model.to(device) +# print("Model loaded successfully.") +# print(audio_path) + +# # Load and process the audio file +# import librosa +# audio_input, sr = librosa.load(audio_path, sr=16000) +# input_features = processor(audio_input, sampling_rate=sr, return_tensors="pt").input_features.to(device) + +# # Generate transcription +# with torch.no_grad(): +# if isinstance(model, torch.nn.DataParallel): +# generated_ids = model.module.generate(input_features) +# else: +# generated_ids = model.generate(input_features) + +# # Decode the generated tokens to text +# transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] +# return transcription + +def transcribe_audio(audio_path: str) -> str: + print("Loading model in transcribe_audio...") + os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" + + from transformers import WhisperProcessor, WhisperForConditionalGeneration + import torch + + # Load processor and model from transformers + processor = WhisperProcessor.from_pretrained("openai/whisper-tiny") + model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny") + + if torch.cuda.device_count() > 1: + print(f"Using {torch.cuda.device_count()} GPUs!") + model = torch.nn.DataParallel(model) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model.to(device) + print("Model loaded successfully.") + print(audio_path) + + # Load and process the audio file + import librosa + audio_input, sr = librosa.load(audio_path, sr=16000) + input_features = processor(audio_input, sampling_rate=sr, return_tensors="pt").input_features.to(device) + + # Generate transcription + with torch.no_grad(): + if isinstance(model, torch.nn.DataParallel): + generated_ids = model.module.generate(input_features) + else: + generated_ids = model.generate(input_features) + + # Decode the generated tokens to text + transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + return transcription + +if __name__ == "__main__": + extract_audio_from_video("../video/1.mp4") + + audio_file = "/tmp/temp_audio_test.wav" + + for i in range(3): + print(f"\nTranscription attempt {i + 1}:") + transcription = transcribe_audio(audio_file) + print("Transcription:") + print(transcription) \ No newline at end of file diff --git a/utils/__pycache__/audio_transcription.cpython-311.pyc b/utils/__pycache__/audio_transcription.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bfa52b66659479918c81079e537cb53a4e5fdb1b GIT binary patch literal 2917 zcmbVOU2NOP5nht3f67r}$;qXTqvV_fcCV6s>7hw{3E;MgldG+!`MEa60V=JfZQ2wm zEU7qgDL_3OP#4I9akwHudxz@bE=`d<L-ddlw(0!mE_U8?ba1Gw#T0987)GVIx zkA(Zcd)Vi5$l?JdE^Q7M&1$V^%GFv~Vn7QU(oIi*4xmSuby}6o34t<5USgT*)$+2U z%3f5yRnsZsFu-2K7xj|<;2UQ!iF`i)^|y2qe7LMspwjvcH283--5M@*~3)!5B5Drhd^%xe#S}Yn&^pmz!i_S&NyPh6${Pt z-Mz<~?{snE7xVAWZy#)(Z_Yb-(#4ZDp6sS_HqJc-F+}0|QONI6Sn&P+H|WEDCtTRu z5CJ?9RIbW5xT?_L!IxYsh##iC-v|y%HUx`*#huvnW3{JhjiK{fD;7|L<9(Ps>@SeW zx{rpI2BU{W|9_3YK_vEx5VnOwA${@S_*NpI21jeRC$O1%!C&j!8q)c~>Vds4q{3EH zp2RhdB*G{g5ev82Wxv#{NO&jM)Bm0jBFX>7-TnRf5rNcnBWgukEI?ms79y!fV)z?; z55C4mtP$stC0H>lPSTHt?nCrFWO3-XwLJm#YBn?yA+Kfv>9D8XLSXCb0S_T79BciX zT38XT?N_mI?ba1*b%7qTrp;Zr%JrCvwV$iN%?E#U^*;5c*x zEP=02Xk`b_4rnV$_B_gjwGjTbfQ=d{D+M`lZ)tOUUYCfXt>mjZk=4AS<%5K@BtOrJ zg#u;CbAtCWKVDUgnoKWKy&@ZiPQAm=S1#!Ed%8vxmc%4=PS#{9F?w@YugRLE6gPU3 z-!c{L_E#+0Hlx=KcJ5|0KX>`Mam2%wI*~k~QmbzsT=2oD08J)&qh4XBE!S1`c0p;L z11Oxw8`R@#CV)yCQnjXnsxllYdqcg-QZ{7`02(F} zk||x5sHCd0>P35ePgIPubW>7PX$eXPZ|Yu5RhFo3NFHY9c|(d;tDEIzS%P{f8(vJ) z%PSNZNd??vn37h3k^`4h2l@C)&#|&6E=gu(wM^s+cxpoQlllPmM6G{A5&vT8QD)aM z20hiQJN~(Jge{+7%jrNjDJ#VA(m@#bb;)3P1WH=J(KM6E1e=0bp>M)6`xzMs87NUW zVJD{?JniCX8&7v}$^K7oHkXPoh~?)X{z;Kh5# z?;Zc+TTc3to4(Y%zzXZd_b+a}>)@=5vo_B9#vOO?2^XKR@d=o2O*%N|VpeN+q9`)l zEU~9Ma_Fx5q1rn0_}I4UjGS^uPW}9c%^z&<>*mI`Z`}LYAC7);&CXqMa#vjTN}U{O z=ZGz@J2}AqA0oN&Y@o(zH@j7;mC~NJxS%Zx9Jgv3mD z^vK8aozdy`==9@)Gdk;z&TcJiE&OHl$Zt!(Ei_BS!|L~lvYOyW-q-QBd!NKP z`W;wf`NX&d-HyOAcY1(qm(qzL&1xz?r1krH_Xh2Cbb}? z&=zv)B{}rq$+pdK09Vocd<18V`N5`+m$f^S*g+=1V-jgg~-epQ`sU zLcdfn74pv+XTdo~4svh}6($xJFvOu+sIC<>j2w+>2ikO_Lnx^bC;W~S!VZ3pQu=QN z?WM2`=jwKq%CH+4ol)VlMj4oAqF7a?@efevNZh~55 zcFI_3W1R-cWwc7gv0d{4bf}_Bg_x>QMoWIBLK$_qYK3WDKeS$Yb!vH@+pxH>gnKBw z!{XzdB7^N6afz&ege&PUPx@q&UGQBE`;i$_zO@Y4P!h+ ju~smKDBcRj5Z&o5ej}petqcG2>k$IyD-X_x?T{W)Y<2o8M%98Aj+YxZ)&Fqwu&0h1*C#0+mDa z_M4igNZm8s(4fici3sM zS=Ky)=TrI)jJn&LPa)Ooj6`S=uS1fC?!dge&AAm?4LW@YX>_e+)=s<1?35RMPT9SZ zyH;WT;dT9g*ScQKt1;yeLKPoS8*t{dzVATgZnJ?x8C3DB@n!=O+_d@Jlxx%-kk;L% z#Ll5NuU7)<&UCa$v=KMdfR3jV?1xti{00fW4|nZKB`ElXfDqJ>z^Z$9W@-U94MND( zOYQ3Ags>LOget7Me`k%wh*Q>B(1*xwebD^1HfKcZTMN2Zr_ejA>cO26LYwA)dM(iX z_VNsyfONT-%c@F(Pb_0aPh<b zz9?pf$TH>lu$nIplSPuw{3~EA0r7fPQIpb=jJ4YeH2RWdW+-2L$?~Psc#*Q_=qmiP zW%{V1KtY9n%3@MdaTQxUg|X!5rPnWBvHY@_6M03nyfRA)Pi_Sv^Zz&bT%hi{*;u@ z^94b{>WsIvk-mFjctOgG!xC^Xtf<3kc1e|%k|UGDB{iE<8hh2?2=;TRxN7ZHusS4d zKCmt{_fmciR>!8JoC5b#McV-s>#t62F}@qq>ul|c&OB={&zj7$Q25q|LJ^bQw{g;7 z51Z`a>gX05UQ4~3c{}q#%IrF9MEcE0zrhZe>;Uw|yVjN4EAOv-sF=^ZVsxJ{yH6PL zSIzjV)tP$d!P=NFGZ?|o5M&QS0;Kw?7{>lW`hBr5s$DlPqlbx zguNp6UynbB{H^#~9T+`sAn?gezYo41sd!8y2_Cl>*o`L@@Vm7px;V4&TW$Jn1la2G z`d;^%8twM1?%wZ59=9*A_ypz#(tPKTyZ!CB3UPsx7Mm2^j0>I}H>+YqZKTB}a=3ll z+Z}81wT?7+@EZXaXTDa>ZpPiTyVuxFUBwRw6lg^tH}x;0N>J@|D1)5|{s68pyFIa+ ziV6$p0zWr36e@)J&DT(E_k%KEeO%mVI;cw^yQC zyB1y|)*xSP-7_uXo~cnwgamn9?c1ScG3J!NU!%0BON*LO#owZ_64N3zY_3lwE#{^P zuy;X=0Ja9pV+0h(uB$Si0(4yxR};2}hnJL3IB0tf@J2`gpynlltKf0Pf|6BNZBR|M z7(59mF~N`j3$okb0Eo!`Mp)?zdYZ=XW)%X+mT$BKnv_Nhl;9C)fq}mlZ|WWJ{&$g=mI4>8(YkX?S@)Q8N zl~R7tXR%~clSLjNJy?{*fU!t zt6liwn0ai>=pHw_$Je;6wwT#=WOMSK_Tb7_$Mo5s>X%>FfA({I{ulb2|Egb08Et~u zCaf`A(fF;d4W>r@d(epXn$g}hU%jn!UDMn8^gy3{6h2183*j1_Qm?yI7kg`>x>zM| zLqxBkH(#z$F4b$Qp+;%cVpClOw1WTwRfcv~Z;PL?d($4bg1_f^=Qt=GEkBP5#9MDu zN!$DHlSL(4hL}%T;EN)h_dG+-TVC5#F%6Av`HnrEgD4gPt>T&T(Z6Fuc7~sVRg~8t zt0Fy|ZalNE|6A&v#f{Ed=STZDfBI3+y%&swFWnPAUwUxx^TJn@aePuwOc{}BGcw(r zL`3WoE7oly){?Tad=r&7Q$~5cgm-s zu3)#DNK~st6Xo+z`v-YFi&yD|>NIAh`cifje4c{j_Cr@BX=xjbC z&)_zqjX0!&t=yend(C!~dw(_iV7A(VHWm z_&)XN+_1q7o7`}9tRCNU`|SH?KYn>L`q}6`?-X zu7|`RHva|e!00$_cATzW08ZnbxB54Z8nFR0Hc*|mQ6g+a51Y}$)v0>>e!XYt-aez} z6o{ZY2|UF*;LzC955m82|M-K04|*T;{_&(SFm4Wv8$Ap#ys8z{h>H#orBoG<)tbZ@0qPw_pRZN`#ySZv-CyEJU(Ico;Q2X z+frN`t#e)9a0kBT4s4_huGi#xb*}fx!%(NrbSgMl{B!KkIS>7%hdJl-e;K48f65(U z`+NKVvfHQ?v4masI&f!)M>f(9f<1L-=fb_+CpU=e=?XaSTFY9_nt**Cf%(+yMmYFx z6FcDy=hHmDK_Biq4^l7_P|#nVYmG}2JihC+BU0i88uvQIRiSh(0XGT>g@@3K$G1bO zm@i5)FRwb2;O3!m^Ru5^nI;0|=144*}0j2pI-Op;3xpLiYURRQ=DxW`M@w^;Im zcuie!L4~!)@h0{F>}>5z5V;7`Ux&Z42w4?<8$e!GXZsCiz+?t=W?+lk16ox(X>da( zH&h+lVuG#Rbz%lg@U59&&8+XMja6rwwHfVUsPTI2Z31Ui;g%;3?L44haOo!4X zd}F2%Y$7*3#EBB-3C`MZjU&|W0;ltmP|AsCci8Mq#bsw1{aesvVAl3yJ~ zb?2$0nC?7v)Tz6lI_lBgPaXB?-}ltf3%dK+W`0C str: audio = AudioSegment.from_file(video_path) @@ -7,7 +8,85 @@ def extract_audio_from_video(video_path: str) -> str: audio.export(audio_path, format="wav") return audio_path +# def transcribe_audio(audio_path: str) -> str: +# print("Loading model in transcribe_audio...") +# from whisper import load_model +# model = load_model("base", device="cpu") +# # model = load_model("base") +# print("Model loaded successfully.") + +# print(f"Model is running on: {next(model.parameters()).device}") +# print("Model loaded successfully on CPU.") + +# result = model.transcribe(audio_path) +# print(result) +# return result["text"] + +# def transcribe_audio(audio_path: str) -> str: +# print("Loading model in transcribe_audio...") +# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" + +# from whisper import load_model +# model = load_model("base") + +# if torch.cuda.device_count() > 1: +# print(f"Using {torch.cuda.device_count()} GPUs!") +# model = torch.nn.DataParallel(model) + +# device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +# model.to(device) +# print("Model loaded successfully.") +# print(audio_path) + +# # Access the underlying model if using DataParallel +# if isinstance(model, torch.nn.DataParallel): +# result = model.module.transcribe(audio_path) +# else: +# result = model.transcribe(audio_path) + +# print(result) +# return result["text"] + def transcribe_audio(audio_path: str) -> str: - model = load_model("base") - result = model.transcribe(audio_path) - return result["text"] + print("Loading model in transcribe_audio...") + from transformers import WhisperProcessor, WhisperForConditionalGeneration + processor = WhisperProcessor.from_pretrained("openai/whisper-tiny") + model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny") + + if torch.cuda.device_count() > 1: + print(f"Using {torch.cuda.device_count()} GPUs!") + model = torch.nn.DataParallel(model) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model.to(device) + print("Model loaded successfully.") + print(audio_path) + + # Load and process the audio file + import librosa + audio_input, sr = librosa.load(audio_path, sr=16000) + input_features = processor(audio_input, sampling_rate=sr, return_tensors="pt").input_features.to(device) + + # Generate transcription + with torch.no_grad(): + if isinstance(model, torch.nn.DataParallel): + generated_ids = model.module.generate(input_features) + else: + generated_ids = model.generate(input_features) + + # Decode the generated tokens to text + transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + return transcription + +# audio_path = "/tmp/temp_audio.wav" +# num_iterations = 5 + +# import time +# start_time = time.time() +# for i in range(num_iterations): +# print(f"Processing iteration {i+1}...") +# transcription = transcribe_audio(audio_path) +# print(f"Transcription (iteration {i+1}): {transcription}") +# end_time = time.time() +# elapsed_time = end_time - start_time +# print(f"Time taken for iteration {i+1}: {elapsed_time:.2f} seconds\n") \ No newline at end of file diff --git a/utils/video_processing.py b/utils/video_processing.py index 28c18fb..dcc19fc 100644 --- a/utils/video_processing.py +++ b/utils/video_processing.py @@ -25,7 +25,8 @@ def split_video_into_segments(video_path, segment_duration=30): for start_time in range(0, int(total_duration), segment_duration): segment_file = os.path.join(output_dir, f"segment_{start_time}.mp4") command = [ - "ffmpeg", "-i", video_path, + "ffmpeg", "-y", + "-i", video_path, "-ss", str(start_time), "-t", str(segment_duration), "-c", "copy", segment_file @@ -33,6 +34,7 @@ def split_video_into_segments(video_path, segment_duration=30): subprocess.run(command, check=True) segments.append(segment_file) + print(f"segments: \n", segments) return segments def extract_motion_key_frames(video_path, max_frames=20, sigma_multiplier=2, frame_interval=1):