From 6b1e7a0fb740c6a293226bd1d23cbc43522ca168 Mon Sep 17 00:00:00 2001
From: Ubuntu <ubuntu@ip-172-31-9-178.us-west-2.compute.internal>
Date: Thu, 7 Nov 2024 11:56:45 +0000
Subject: [PATCH] update to fix bbox error

---
 .gitignore                                    |    1 +
 __pycache__/trainer.cpython-37.pyc            |  Bin 35819 -> 35819 bytes
 bert/__pycache__/activations.cpython-37.pyc   |  Bin 1964 -> 1964 bytes
 .../configuration_bert.cpython-37.pyc         |  Bin 7881 -> 7881 bytes
 .../configuration_utils.cpython-37.pyc        |  Bin 16243 -> 16243 bytes
 bert/__pycache__/file_utils.cpython-37.pyc    |  Bin 24440 -> 24440 bytes
 .../generation_utils.cpython-37.pyc           |  Bin 27906 -> 27906 bytes
 bert/__pycache__/modeling_bert.cpython-37.pyc |  Bin 56399 -> 56399 bytes
 .../__pycache__/modeling_utils.cpython-37.pyc |  Bin 48040 -> 48040 bytes
 .../tokenization_bert.cpython-37.pyc          |  Bin 19136 -> 19136 bytes
 .../tokenization_utils.cpython-37.pyc         |  Bin 24769 -> 24769 bytes
 .../tokenization_utils_base.cpython-37.pyc    |  Bin 81740 -> 81740 bytes
 .../__pycache__/__init__.cpython-37.pyc       |  Bin 252 -> 252 bytes
 ...abel_smoothed_cross_entropy.cpython-37.pyc |  Bin 11402 -> 11402 bytes
 data/__pycache__/__init__.cpython-37.pyc      |  Bin 278 -> 278 bytes
 data/__pycache__/base_dataset.cpython-37.pyc  |  Bin 2107 -> 2107 bytes
 data/__pycache__/data_utils.cpython-37.pyc    |  Bin 18330 -> 18330 bytes
 data/__pycache__/file_dataset.cpython-37.pyc  |  Bin 4264 -> 4264 bytes
 data/__pycache__/poly_utils.cpython-37.pyc    |  Bin 9668 -> 9668 bytes
 .../refcoco_dataset.cpython-37.pyc            |  Bin 11164 -> 11164 bytes
 .../refcoco_pretrain_dataset.cpython-37.pyc   |  Bin 8979 -> 8979 bytes
 data/create_aihub_data.py                     |   65 +-
 demo.py                                       |    9 +-
 models/__pycache__/__init__.cpython-37.pyc    |  Bin 277 -> 277 bytes
 .../__pycache__/__init__.cpython-37.pyc       |  Bin 288 -> 288 bytes
 .../__pycache__/polyformer.cpython-37.pyc     |  Bin 5939 -> 5939 bytes
 .../__pycache__/swin.cpython-37.pyc           |  Bin 23524 -> 23524 bytes
 .../unify_multihead_attention.cpython-37.pyc  |  Bin 10596 -> 10596 bytes
 .../unify_transformer.cpython-37.pyc          |  Bin 42595 -> 42595 bytes
 .../unify_transformer_layer.cpython-37.pyc    |  Bin 14994 -> 14994 bytes
 .../__pycache__/__init__.cpython-37.pyc       |  Bin 256 -> 256 bytes
 .../evaluate_polyformer_b_aihub_indoor.sh     |    3 +-
 .../100_5e-5_512.log                          | 8424 ++++++-----------
 .../finetune/train_polyformer_b_aihub.sh      |    4 +-
 tasks/__pycache__/__init__.cpython-37.pyc     |  Bin 257 -> 257 bytes
 tasks/__pycache__/base_task.cpython-37.pyc    |  Bin 9158 -> 9158 bytes
 tasks/__pycache__/refcoco.cpython-37.pyc      |  Bin 9360 -> 9360 bytes
 .../refcoco_pretrain.cpython-37.pyc           |  Bin 8847 -> 8847 bytes
 utils/__pycache__/__init__.cpython-37.pyc     |  Bin 154 -> 154 bytes
 .../checkpoint_utils.cpython-37.pyc           |  Bin 20159 -> 20159 bytes
 utils/__pycache__/eval_utils.cpython-37.pyc   |  Bin 8179 -> 8179 bytes
 utils/__pycache__/transforms.cpython-37.pyc   |  Bin 14274 -> 14274 bytes
 utils/__pycache__/vis_utils.cpython-37.pyc    |  Bin 1817 -> 1817 bytes
 43 files changed, 2726 insertions(+), 5780 deletions(-)

diff --git a/.gitignore b/.gitignore
index 64c3763..57ef660 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 datasets/*
 refer/data/*
 pretrained_weights/*
+data/aihub_csv_error_csv
 run_scripts/finetune/polyformer_b_aihub_indoor_checkpoints
 run_scripts/finetune/polyformer_b_aihub_indoor_logs
 run_scripts/finetune/polyformer_b_aihub_manufact_checkpoints
diff --git a/__pycache__/trainer.cpython-37.pyc b/__pycache__/trainer.cpython-37.pyc
index 09f0e66403a25ab7fd1b80fd1de5c47d125c52eb..e724578a13290ea2e3f19422cc311ea0678b73ee 100644
GIT binary patch
delta 29
jcmaDoo$2*-CT=HQUM>b8_{yoZk$b%WWBKOY0tVFpgoy~g

delta 29
jcmaDoo$2*-CT=HQUM>b8xOr7&Blmg%#_-L%1q`YIj8zFp

diff --git a/bert/__pycache__/activations.cpython-37.pyc b/bert/__pycache__/activations.cpython-37.pyc
index 5a3683be39b522690af031d05e05809c9a7b2ffd..9646da8bab7109b25d63e2861eca8dbeee295be1 100644
GIT binary patch
delta 27
hcmZ3(zlNXNiI<m)0SLZuYHj4^W@IeiEY3KU4FFG#1+@SG

delta 27
hcmZ3(zlNXNiI<m)0SInfRoTeR&Bz$OS)6ey8vs{X20s7*

diff --git a/bert/__pycache__/configuration_bert.cpython-37.pyc b/bert/__pycache__/configuration_bert.cpython-37.pyc
index fe94b1d6fb1b696d93282d90cd6e258f96d7968a..158b6a71ef2a1a30dd874eec67e6b26195f6a962 100644
GIT binary patch
delta 27
hcmX?Ud(xKMiI<m)0SLZuYHj46C&O62d6kSj698r%2Q2^q

delta 27
hcmX?Ud(xKMiI<m)0SInfRoTcrPlhpk^C}s6CID^e2e$wK

diff --git a/bert/__pycache__/configuration_utils.cpython-37.pyc b/bert/__pycache__/configuration_utils.cpython-37.pyc
index 40e4e3ece284950a68ee3bfa6635c15f16affdd7..24ef7886d0baafaf5c4672b13e00ad1194b81604 100644
GIT binary patch
delta 27
hcmexd_qmSSiI<m)0SLZuYHj2`!^2p<`3BEyBLIFS2<rd<

delta 27
hcmexd_qmSSiI<m)0SInfRoTdWhKDhH^9`QaMgWEJ33UJf

diff --git a/bert/__pycache__/file_utils.cpython-37.pyc b/bert/__pycache__/file_utils.cpython-37.pyc
index 03486742bfa581ef7d6c642f4c1409e60e3732be..e80d87f566968ed0a5f226d0aab698be56d61104 100644
GIT binary patch
delta 29
jcmeydkMYMoMs6owUM>b8_`<2Rkz0(Dv3#=<XIK~jhdT%4

delta 29
jcmeydkMYMoMs6owUM>b8xN%iwBexhQWB6tz&af~5j|T{D

diff --git a/bert/__pycache__/generation_utils.cpython-37.pyc b/bert/__pycache__/generation_utils.cpython-37.pyc
index 226610ef3488cbc00186cfb34dae4998d950021e..131d5b716a9cfa0c96f0643f504a8c8a7fab97c5 100644
GIT binary patch
delta 29
jcmZp=#n^O<k=u!vmx}=izHn-7<d$P*EZ?lfd?OP8bm|Am

delta 29
jcmZp=#n^O<k=u!vmx}=iZd_H_$Sudr7`|DH`9>xHe6|Qv

diff --git a/bert/__pycache__/modeling_bert.cpython-37.pyc b/bert/__pycache__/modeling_bert.cpython-37.pyc
index ec6a62c221dde9e28041292037a5791f4dc60669..abc1ce72fafc0052b5bc19b51864bd873eb1a5f6 100644
GIT binary patch
delta 29
jcmX@VgZcapW^N~5UM>b8_`<2Rk^34qWBKMs+_@J3h!P2(

delta 29
kcmX@VgZcapW^N~5UM>b8xN%iwBlk6K#_-LLxN|Q60FT}ZDF6Tf

diff --git a/bert/__pycache__/modeling_utils.cpython-37.pyc b/bert/__pycache__/modeling_utils.cpython-37.pyc
index f48a757c8ca919733a4497a8d9519b47eef41cfc..8f7213db623e37168628ef897bbc0b5a7ba58b7b 100644
GIT binary patch
delta 29
jcmZ4SooU5)CT=HQUM>b8_`<2Rkz0?2v3#=?OWt+>gPjOh

delta 29
jcmZ4SooU5)CT=HQUM>b8xN%iwBexz4WB6t(mb~o%i)jeq

diff --git a/bert/__pycache__/tokenization_bert.cpython-37.pyc b/bert/__pycache__/tokenization_bert.cpython-37.pyc
index 2895d11b7e3a914fc77032ed223b40080dc6404c..52e0378a379304134a4f0e90f7b67c0457137400 100644
GIT binary patch
delta 29
jcmX>wmGQt-Ms6owUM>b8_`<2Rk=vbzv3zqdPp%yRb!rER

delta 29
jcmX>wmGQt-Ms6owUM>b8xN%iwBey#bWBBG^o?JTseKrUa

diff --git a/bert/__pycache__/tokenization_utils.cpython-37.pyc b/bert/__pycache__/tokenization_utils.cpython-37.pyc
index 863744d97aed9865801d2bd964bba03622b7d4fa..ec155b1be9b32af4d7f8f51e7bfa3257bafa4b10 100644
GIT binary patch
delta 29
jcmX?jkn!L_Ms6owUM>b8_`<2Rk$XEcWBKO8%>QBlg-Quv

delta 29
kcmX?jkn!L_Ms6owUM>b8xN%iwBlmV@#_-LDng7KA0F4$2>Hq)$

diff --git a/bert/__pycache__/tokenization_utils_base.cpython-37.pyc b/bert/__pycache__/tokenization_utils_base.cpython-37.pyc
index d645fddd84d373a1eb7b3936e755ed9debd05795..cb03aa9a3b75cfc68496e8ae3d7ef50e42b722ff 100644
GIT binary patch
delta 33
ncmX@}kLAoi7H%hAUM>b8_`<2Rk^3$yV|nvS*6lA@8Ivsmz<LV)

delta 33
ocmX@}kLAoi7H%hAUM>b8xN%iwBllfa#_;BstlM9*GA3IB0LpF)hyVZp

diff --git a/criterions/__pycache__/__init__.cpython-37.pyc b/criterions/__pycache__/__init__.cpython-37.pyc
index 9f3dcd65b75dc59d4114bd0126f324aef29bc29d..99c333205620eecc66d8ad59fa1761030b1a9b20 100644
GIT binary patch
delta 24
ecmeyv_=l0(iI<m)0SLZuYE9%`%ve5gy)OVy>ju{V

delta 24
ecmeyv_=l0(iI<m)0SInfRhh`Wm@$0fdS3ul`3GSD

diff --git a/criterions/__pycache__/label_smoothed_cross_entropy.cpython-37.pyc b/criterions/__pycache__/label_smoothed_cross_entropy.cpython-37.pyc
index 292de8028b00caaa0769db919ae19105d6cb0540..20c9aaf5c3d35c96f15c957fb27b876ca943f0ed 100644
GIT binary patch
delta 27
hcmeB*?26=e;^pOH0D>=^S{u39*%-?=3$vB00RUG31_=NF

delta 27
hcmeB*?26=e;^pOH0D>D=RW@?7voVHm7G^6~0{~vk29p2)

diff --git a/data/__pycache__/__init__.cpython-37.pyc b/data/__pycache__/__init__.cpython-37.pyc
index 96fad4fe4379b6aad3640d848a821d3a5af26621..2263a1f9eb4ad58c38c8520c4f2f96fb3955e460 100644
GIT binary patch
delta 24
ecmbQnG>wVdiI<m)0SLZuYE9%m$XGt{tS10IuLeZ`

delta 24
ecmbQnG>wVdiI<m)0SInfRhh_rkTHDXSx*2&y#~(!

diff --git a/data/__pycache__/base_dataset.cpython-37.pyc b/data/__pycache__/base_dataset.cpython-37.pyc
index 419351a5451ee44c3a5dccc707ba3fdc94cc054d..308c8f29784d3a17e63f3cefd3245846ea0cdcf0 100644
GIT binary patch
delta 27
hcmdljuv>uJiI<m)0SLZuYHj3x&&XK5`8T5j8vtFk2S@+_

delta 27
hcmdljuv>uJiI<m)0SInfRoTe>o{=$p^KV86HUMRT2hsol

diff --git a/data/__pycache__/data_utils.cpython-37.pyc b/data/__pycache__/data_utils.cpython-37.pyc
index 99f9b4344145eca4fabd3ec8f7b3533bc209965a..f4da43fbc24adf7baab7837dcee087bfc7be51db 100644
GIT binary patch
delta 59
zcmbQ$&p4}}k=u!vmx}=izHn-7<mP8)EZ;26%&snRi@UfqsXV_ZB|a~=<QKDpgTgN+
N2Zy2+o2|4GWC7Y95vl+H

delta 59
zcmbQ$&p4}}k=u!vmx}=iZd_H_$j#5p7`|DWnO$At7n6g-FJ=b^g<IUkrAg)aMJe%l
Oxg|xbHd|>W$N~W9{t`9-

diff --git a/data/__pycache__/file_dataset.cpython-37.pyc b/data/__pycache__/file_dataset.cpython-37.pyc
index 450cd2532a72afe8c7e1c7c474e94548d94ec16f..c55bd329f2628d0099be94720dba7bd2f81648f9 100644
GIT binary patch
delta 27
hcmZ3XxI&TJiI<m)0SLZuYHj4+&BR!~`8d-<UI1I$2ZjIu

delta 27
hcmZ3XxI&TJiI<m)0SInfRoTeBn~5=e^Kqt!yZ~dT2oL}O

diff --git a/data/__pycache__/poly_utils.cpython-37.pyc b/data/__pycache__/poly_utils.cpython-37.pyc
index ec465debf1237748951885c70755d3f9cb74ab57..007c3d5fbc6613319fa53a8fc0c60b64f351fc2d 100644
GIT binary patch
delta 1787
zcmaJ>TWDNm5YBA&eu=p^n>L%vrp1$rfrKVVpd{K&({7q>N`rJErYD<|G$ea=GiNt(
zo7xbmRY?%zQ0P^9r6_`yJO&ZkqE9|4BInIVK_7~J6U3Q+H5<ja%lZCu=AX+y-^@Qx
z<4@y(S3@Cx8ULR9IvxdHalq1M%<%^}Y@jOF>vYrU1$*;>GX;<&pL+=o(5m|<=%NoD
zVY=Ws3>~!PY2>{hJU!4uA@3dNrO&<No@6=B8`w)VzEf~SX|MQB?C<74ng=_0!A=I-
zgNAiZ&l)Iaz<z-~dghD4G4>Uvg#Ux#m<SsMfI;(>tZ`7J+m^Ol$C>o3X$MjnGo|MY
zlOOC8ougvu#v$qm=#ZfGKqk8FR&(1f)Xf>A7`|EbiwC4gun`7mpuUENgAo{`x!^~K
zWy^M*H!x!r4csh92D6#@!c4~OO6%4fj)@P+4-X26OS|04r%|K6&@7D8gV2*cbpT08
zC8w3|2tUeF&5uK32PO`S(h2!+>S*XVK=&#VFh$QRRud&Jid*ll>|k8Hl-O`7_jp#M
zy1=ABN@e@w3on5|8#Q5yRCe*(Xk`|r>2c+!%`>c&3}y1UY(6(JnV+6E(DqFha>m4z
zp3`kws~Ug>`nKvDh|!hm@yaQ2dQ2c8V34Dxy-KQ{!6`O%rRFo1k)+<5GqA`3v(UZi
zs0DB@v-Zzn?zJ$r)%7c`cJ|G`D!(JGa%^K$+cRZNpp<_^CG<sI6fV;Dbt9pRysJ5}
zRZjC3b=JpWg_i4&LqC08Kj>Z+ag3_>JXt9bt9nnd8`zy)DVgm|>$!8b(^|lsD*BFd
z`HI!_<uZINbTefny{54=A#H;L67{S=sSuE&v4%KYraKMG+9lo-=H=JVouZDZ8uc<u
zG<VV^ek{C1v2Y6vsgw)hPFSN)!bkavJ`0b+#^!5{1%Ph)a_=EP`fcwjV}4(>Pql;`
zADsd&0p;5g-xanaYw=^0x-jW@(?~iB!tHnaqJCsG8w$ya0zHZ}QB%`NxJv1!tuy~*
zQ2QV9R5*2ntKuw}#k^@SH?wBW+QFe))t1AqM#6kAZ$u2ZPSAWiBE>-o$J{74{UQ*r
zJT6A)tL8n!|HW}iwB8X2Z9|2;U1Rv2AFGm~4E#_FLH;*@7peiTQCrI}N6xgIf?M>s
zWqm}2_V3;t!X$bGN@V{vl2o?_EIpl_H~#L}o2?1BOTV^Gww4Ngm#uk%wThGX1>O+2
zP1&~NaG$o?W}HhbAaCs<2c~H}x)3Ocg1E=?sv&-f9z6MwKeh7Hrw28_3-Z3bs1zLd
zuIRs4+__O)J}xfzWO*6uc(Gk(KAkOK2Tiv}UVNJ+rPfJ$thw2=&eK%pBCZRl%;)H{
z_U4rX^4%rSBXC5ZG#=F492c3I8-~113oHmM3M>oACB_v2MUY$qToc$3kRHP80#Zl3
SB_Q<S{W9)wd`Khv7ykqgV2>97

delta 1787
zcmaJ>TWnNi5bm7r{Zf0UTj*}vEy1Nk3>2klAW&OcyK65{18mZi)9u;q+V0t9&aP!C
zx>Q4~+SJS7liU$RF;UXT#+U%nC!f58oHrj$^nvITCdPmM)$U5HyE)(gpZVvX%Qti8
ze)xXa-Rt$bD#Y)>4;^>h-3lCC?N|N;<q=Sg^(bAiIN;Ih4NDBbB&Tg2w$Y;PC)iEz
zDn2@6-w&O%W^Wc}KiZ#xz2tRVg>L%XF=00=aZbZ-s&gKO0jXVZzOc1RI11UZQ!Mzj
z0Mm||F{5TQ<QTA*VIMto_CrM2^3kYkB{ImuW(E)ta+Yr6^DJGzY1&jwq>{Snj-~aO
zn$>ji;6BzF;+r-M({{HCM`+od4z8OG*-R^{dO|C@53qhQK=ON<p^pwW)=|XchhsGB
zd4E42nby-9rj6nSn-D}jne<#fk=A#mRAUy8u?@}-cQCL?v(m_=a69#QlW>A=c<=Sd
z05~O?T&S|l{kV{ZTo~qipyLQDjq-~{MNP#%x?VL33HrTiakOMba_iWT9UNvWB{r<m
zJjPf$&7d*FWwsBV*a0*2UY(Ep)w{%QtU3cpx?BBe;3c6H^`>*#OfEY)m5awUj5??C
zS#5G!&8ks)yXGJueOvPl4AKX+6V(Ye8eurXFhfdRM@>{zJ&n^En%3A{+R#jTw(dC0
z3x^r#T2<-+Y-fb_&py#>K5B0`Ah{~<o5w0&mR2RUi>s!6+L%Nx|CmhZi-sT+==+9Y
zZ$X@fEZD};yh=M8!*G@s8i$~lzHU5ZdyU0|RJ-Y3p+v0gJ;|<nV|KY@rX{6jXH1K6
z60@@CWpO!+)pTal;<Hfom}WY3ZGMv52K(6SV}*b?z1S3nCA!+Q5IQFmm`VN}5S=2!
zl#RM1B;3Gk$G~&c?`wsyOgZn{2`lu8Z$Nya4}D{Be)YNLJU|zHxp@y5^xNh|fw?Wg
z9@!EcK3W*84AOR%ZC9B}#t@HI)`iL`%~bNt?RQ&(E<>o9kdId7>5jjJT3SZnBBff^
zj{lp%(EpGp-N_IxvN2BvbGjzDN$Odnj6<8Ot#G^O_lfWFL%#->2?AIAT^uvJ6WmBM
za*(i9F)qgFtH7qne{me=Yi}@k*P)7cmjwJF9-~@71-PIdJmTL34yXmZMD48+;W^!U
z7_QOX*5zU8+O=_WFq3F!D3Sg5NaAjdx(zjznbZF1*b8l=aFc#*n`$c+`liqm6HNB)
zGFws^KB7$f5Zt1*_Jrk>5RjvOkAfMR2%dDGW(78n3$h`833iU$7O$bo@$o|;z*l&`
zzQ`05e3SLxDxO>^F4v38$Gkj+4Pr5^dM=g8V<*Ks{7=3vB&F6d?Z#{}rHX0Fa}nQR
zkeTcBSx2C-jo)`O>}42WD2)d>H&3uE!7#&+WI%>_1|AGpU^vSl30mUa3d4B@?jgL)
Wz;(oH3``&1st_HHw`q9m%YOlT-jrGZ

diff --git a/data/__pycache__/refcoco_dataset.cpython-37.pyc b/data/__pycache__/refcoco_dataset.cpython-37.pyc
index 91e2b94769f5bfa1b2612b89d0397f28594918d6..b094f22762f2b9996b18b5503cb177291fbf14c7 100644
GIT binary patch
delta 27
hcmbOeJ|~>piI<m)0SLZuYHj4UWo0bi?8zFe1^`^I25$fW

delta 27
hcmbOeJ|~>piI<m)0SJ`2)Hib5vNDEm_GArK0{~EJ1$+Pi

diff --git a/data/__pycache__/refcoco_pretrain_dataset.cpython-37.pyc b/data/__pycache__/refcoco_pretrain_dataset.cpython-37.pyc
index 2f6ddbb6e13fcd2aed3119bee810e8927fd7f4d7..875c1f5b1a01d39d8d6865dc909b4284fa2fac32 100644
GIT binary patch
delta 27
hcmbR2Hrb8aiI<m)0SLZuYHj4c$--E^`6-K@8~|S!2Sfk>

delta 27
hcmbR2Hrb8aiI<m)0SInfRoTdWlZ7#S^HUZ*IRIwl2hIQh

diff --git a/data/create_aihub_data.py b/data/create_aihub_data.py
index fc851b6..9557d82 100644
--- a/data/create_aihub_data.py
+++ b/data/create_aihub_data.py
@@ -5,6 +5,8 @@
 from PIL import Image
 import random
 import os
+import pandas as pd
+import glob
 from tqdm import tqdm
 
 import pickle
@@ -16,8 +18,8 @@
 
 data_root = './refer/data'
 # datasets = ['refcoco', 'refcoco+', 'refcocog']
-# datasets = ['aihub_indoor']
-datasets = ['aihub_manufact']
+datasets = ['aihub_indoor']
+# datasets = ['aihub_manufact']
 
 if datasets[0] == 'aihub_indoor':
     image_dir = './refer/data/aihub_refcoco_format/indoor_80/images'
@@ -28,6 +30,28 @@
 val_test_files = pickle.load(open("data/val_test_files.p", "rb"))
 
 
+# Define the directory containing your CSV files
+if datasets[0] == 'aihub_indoor':
+    csv_dir = 'data/aihub_csv_error_csv/indoor'  # Replace with the actual directory path
+elif datasets[0] == 'aihub_manufact':
+    csv_dir = 'data/aihub_csv_error_csv/manufact'  # Replace with the actual directory path
+csv_files = glob.glob(f'{csv_dir}/*.csv')
+
+# Initialize an empty dictionary to store bounding box values from all CSV files
+bbox_dict = {}
+
+# Load and combine data from all CSV files
+for csv_file in csv_files:
+    bbox_data = pd.read_csv(csv_file)
+    
+    # Determine prefix based on the file name
+    prefix = "real_" if "real_" in csv_file else "syn_"
+    
+    # Convert filenames to the appropriate format and store in bbox_dict
+    bbox_data['파일명'] = bbox_data['파일명'].apply(lambda x: f'{prefix}{x}')
+    # Update bbox_dict with bbox data from this file
+    bbox_dict.update(dict(zip(bbox_data['파일명'], bbox_data['bbox'])))  # Replace 'bbox_column_name' with actual column name
+
 
 combined_train_data = []
 
@@ -50,7 +74,7 @@
         splits = ['train', 'val', 'test']
         splitBy = None
 
-    save_dir = f'datasets/finetune/{dataset}'
+    save_dir = f'datasets/finetune/{dataset}_bbox_fix'
     os.makedirs(save_dir, exist_ok=True)
     for split in splits:
         num_pts = []
@@ -78,6 +102,10 @@
             fn = this_img['file_name']
             img_id = fn.split(".")[0].split("_")[-1]
 
+            # Determine the appropriate prefix for file_name_key
+            prefix = fn.split(".")[0].split("_")[0] + "_"
+            file_name_key = f"{prefix}{img_id}"
+
             # load image
             img = Image.open(os.path.join(image_dir, this_img['file_name'])).convert("RGB")
 
@@ -114,9 +142,34 @@
             pts_string_interpolated = polygons_to_string(polygons_interpolated)
 
             # load box
-            box = refer.getRefBox(this_ref_id)  # x,y,w,h
-            x, y, w, h = box
-            box_string = f'{x},{y},{x + w},{y + h}'
+            if file_name_key in bbox_dict:
+                print('bbox dict')
+                # Update bbox value based on CSV data
+                x1, y1, x2, y2 = map(int, bbox_dict[file_name_key].split(','))
+                box_string = f'{x1},{y1},{x2},{y2}'
+            else:
+                box = refer.getRefBox(this_ref_id)  # x,y,w,h
+                # Fallback to the default logic if not in combined CSV data
+                if prefix == "real_":
+                    x, y, w, h = box
+                    box_string = f'{x},{y},{x + w},{y + h}'
+                elif prefix == "syn_":
+                    x1, y1, x2, y2 = box
+                    box_string = f'{x1},{y1},{x2},{y2}'
+                else:
+                    print("Image must be either real or syn")
+                    exit()
+            # box = refer.getRefBox(this_ref_id)  # x,y,w,h
+            # print(fn.split(".")[0].split("_")[0])
+            # if fn.split(".")[0].split("_")[0] == "real": 
+            #     x, y, w, h = box
+            #     box_string = f'{x},{y},{x + w},{y + h}'
+            # elif fn.split(".")[0].split("_")[0] == "syn":
+            #     x1, y1, x2, y2 = box
+            #     box_string = f'{x1},{y1},{x2},{y2}'
+            # else:
+            #     print("Image must be either real or syn")
+            #     exit()
 
             max_num_pts = max(max_num_pts, check_length(polygons))
 
diff --git a/demo.py b/demo.py
index 3b89d95..19de338 100644
--- a/demo.py
+++ b/demo.py
@@ -21,7 +21,7 @@
 # Load pretrained ckpt & config
 overrides={"bpe_dir":"utils/BPE"}
 models, cfg, task = load_model_ensemble_and_task(
-        utils.split_paths('weights/polyformer_l_refcocog.pt'),
+        utils.split_paths('run_scripts/finetune/polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt'),
         arg_overrides=overrides
     )
 
@@ -33,8 +33,11 @@
 cfg.generation.no_repeat_ngram_size = 3
 cfg.task.patch_image_size = 512
 
-from bert.tokenization_bert import BertTokenizer
-tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+# from bert.tokenization_bert import BertTokenizer
+# tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+from transformers import AutoTokenizer, AutoModelForMaskedLM
+tokenizer = AutoTokenizer.from_pretrained('xlm-roberta-base')
+
 
 # Fix seed for stochastic decoding
 if cfg.common.seed is not None and not cfg.generation.no_seed_provided:
diff --git a/models/__pycache__/__init__.cpython-37.pyc b/models/__pycache__/__init__.cpython-37.pyc
index 2b822c2985c2c8b3e786bfd305905d85290d981a..5580d40956c88444e4d1ad1de63a33e5ab0c2feb 100644
GIT binary patch
delta 24
ecmbQrG?j_liI<m)0SLZuYE9%mz*s)<j3)p-c?LlM

delta 24
ecmbQrG?j_liI<m)0SInfRhh_rfH8dH8BYL2hX%_4

diff --git a/models/polyformer/__pycache__/__init__.cpython-37.pyc b/models/polyformer/__pycache__/__init__.cpython-37.pyc
index c9826fc050d2e225862ee9ae2ad05b94d6690872..cba4b180f6522c2631d91e67b654df5a1b4d81c0 100644
GIT binary patch
delta 24
ecmZ3$w1A1*iI<m)0SLZuYE9%mz*s)<Ob7r#eg<Ly

delta 24
ecmZ3$w1A1*iI<m)0SInfRhh_rfH8dHnGgU*j0Wrg

diff --git a/models/polyformer/__pycache__/polyformer.cpython-37.pyc b/models/polyformer/__pycache__/polyformer.cpython-37.pyc
index 92b3350dff304866c8463a00fa104fffb40393a0..cee46465e79ccf8381749ca6e73eff8a4fc08438 100644
GIT binary patch
delta 27
hcmdn2w^@(diI<m)0SLZuYHj2;VrDGgY{&dn7yw!22EYIS

delta 27
hcmdn2w^@(diI<m)0SInfRoTdG#LO7J*^c?EFaTk(2TA|{

diff --git a/models/polyformer/__pycache__/swin.cpython-37.pyc b/models/polyformer/__pycache__/swin.cpython-37.pyc
index 505e46d46990421e6ce421637ba26390da7dd54d..b4e2ba847793c8fab967dd0739c96588f033b959 100644
GIT binary patch
delta 29
jcmaE|o$<+bMs6owUM>b8_`<2Rk=vDtv3zqN)8t41gbxTl

delta 29
jcmaE|o$<+bMs6owUM>b8xN%iwBeyFPWBBGkrpb{2i`xju

diff --git a/models/polyformer/__pycache__/unify_multihead_attention.cpython-37.pyc b/models/polyformer/__pycache__/unify_multihead_attention.cpython-37.pyc
index 422736031635b41e118e47f39e139f2b3fd88998..041a54cf2057be731895e2ce539b47f73917d88c 100644
GIT binary patch
delta 27
hcmaD7^dyMeiI<m)0SLZuYHj4MWoInk+{u1h5ddzs2mAm4

delta 27
hcmaD7^dyMeiI<m)0SInfRoTd0%gz|Sxs(02A^>)R2!;Rv

diff --git a/models/polyformer/__pycache__/unify_transformer.cpython-37.pyc b/models/polyformer/__pycache__/unify_transformer.cpython-37.pyc
index 0696918df79a03a1d587ef3191102974df5e7dbb..f642dbfa5a7a574d871859f56b37be0da057e016 100644
GIT binary patch
delta 29
jcmaEShUxJcCT=HQUM>b8_`<2Rk-L_Kv3zqU%dLd~j3^0f

delta 29
jcmaEShUxJcCT=HQUM>b8xN%iwBX=zeWBBGymRkz}lk^Go

diff --git a/models/polyformer/__pycache__/unify_transformer_layer.cpython-37.pyc b/models/polyformer/__pycache__/unify_transformer_layer.cpython-37.pyc
index 29d34c71c4e669b630ba6a9a2a9185cb56471f0b..ccd84124d21081863c90d82dd4b83f2ba078aecd 100644
GIT binary patch
delta 27
hcmbPKI;oV~iI<m)0SLZuYHj4cz`|I*`3}oAQvhhh2rU2r

delta 27
hcmbPKI;oV~iI<m)0SInfRoTdWfrT-A^BtCLrT}n|2)6(L

diff --git a/polyformer_module/__pycache__/__init__.cpython-37.pyc b/polyformer_module/__pycache__/__init__.cpython-37.pyc
index a54762b1586d61f7f8b405f35b5cd763cb6b4383..46bb3ff2537067b5b72f1e7fa7e4f83cd8667e46 100644
GIT binary patch
delta 24
ecmZo*YGC4a;^pOH0D`ZaS`)cvFqThT5(5A^kp<fT

delta 24
ecmZo*YGC4a;^pOH0D_xWRVH%JU<{wQBnALKp9W<B

diff --git a/run_scripts/evaluation/evaluate_polyformer_b_aihub_indoor.sh b/run_scripts/evaluation/evaluate_polyformer_b_aihub_indoor.sh
index ddbf641..fdbf3bc 100644
--- a/run_scripts/evaluation/evaluate_polyformer_b_aihub_indoor.sh
+++ b/run_scripts/evaluation/evaluate_polyformer_b_aihub_indoor.sh
@@ -18,7 +18,7 @@ num_bins=64
 batch_size=16
 
 dataset='aihub_indoor'
-ckpt_path=../finetune/polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt
+ckpt_path=../finetune/polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
 # dataset='refcocog'
 # ckpt_path=../../weights/polyformer_b_refcocog.pt
 
@@ -44,6 +44,7 @@ python3 -m torch.distributed.launch --nproc_per_node=${GPUS_PER_NODE} --master_p
     --fp16 \
     --num-workers=0 \
     --num-bins=${num_bins} \
+    --vis \
     --vis_dir=${vis_dir} \
     --result_dir=${result_dir} \
     --model-overrides="{\"data\":\"${data}\",\"bpe_dir\":\"${bpe_dir}\",\"selected_cols\":\"${selected_cols}\"}"
diff --git a/run_scripts/finetune/polyformer_b_aihub_indoor_80_logs/100_5e-5_512.log b/run_scripts/finetune/polyformer_b_aihub_indoor_80_logs/100_5e-5_512.log
index c25e7d7..e0fcb7c 100644
--- a/run_scripts/finetune/polyformer_b_aihub_indoor_80_logs/100_5e-5_512.log
+++ b/run_scripts/finetune/polyformer_b_aihub_indoor_80_logs/100_5e-5_512.log
@@ -11,65 +11,65 @@ WARNING:torch.distributed.run:
 *****************************************
 Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
 *****************************************
-2024-11-03 05:29:39 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
-2024-11-03 05:29:39 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
-2024-11-03 05:29:39 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
-2024-11-03 05:29:39 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
-2024-11-03 05:29:39 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
-2024-11-03 05:29:39 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
-2024-11-03 05:29:39 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
-2024-11-03 05:29:39 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
-2024-11-03 05:29:41 - utils.py[line:258] - INFO: distributed init (rank 4): env://
-2024-11-03 05:29:41 - utils.py[line:261] - INFO: Start init
-2024-11-03 05:29:41 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 4
-2024-11-03 05:29:41 - utils.py[line:258] - INFO: distributed init (rank 5): env://
-2024-11-03 05:29:41 - utils.py[line:261] - INFO: Start init
-2024-11-03 05:29:41 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 5
-2024-11-03 05:29:41 - utils.py[line:258] - INFO: distributed init (rank 0): env://
-2024-11-03 05:29:41 - utils.py[line:261] - INFO: Start init
-2024-11-03 05:29:41 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 0
-2024-11-03 05:29:41 - utils.py[line:258] - INFO: distributed init (rank 1): env://
-2024-11-03 05:29:41 - utils.py[line:261] - INFO: Start init
-2024-11-03 05:29:41 - utils.py[line:258] - INFO: distributed init (rank 7): env://
-2024-11-03 05:29:41 - utils.py[line:261] - INFO: Start init
-2024-11-03 05:29:41 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 1
-2024-11-03 05:29:41 - utils.py[line:258] - INFO: distributed init (rank 2): env://
-2024-11-03 05:29:41 - utils.py[line:261] - INFO: Start init
-2024-11-03 05:29:41 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 7
-2024-11-03 05:29:41 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 2
-2024-11-03 05:29:41 - utils.py[line:258] - INFO: distributed init (rank 6): env://
-2024-11-03 05:29:41 - utils.py[line:261] - INFO: Start init
-2024-11-03 05:29:41 - utils.py[line:258] - INFO: distributed init (rank 3): env://
-2024-11-03 05:29:41 - utils.py[line:261] - INFO: Start init
-2024-11-03 05:29:41 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 6
-2024-11-03 05:29:41 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 3
-2024-11-03 05:29:41 - distributed_c10d.py[line:354] - INFO: Rank 3: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
-2024-11-03 05:29:41 - utils.py[line:274] - INFO: initialized host ip-172-31-38-178 as rank 3
+2024-11-06 13:40:37 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
+2024-11-06 13:40:37 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
+2024-11-06 13:40:37 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
+2024-11-06 13:40:37 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
+2024-11-06 13:40:37 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
+2024-11-06 13:40:37 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
+2024-11-06 13:40:37 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
+2024-11-06 13:40:37 - file_utils.py[line:39] - INFO: PyTorch version 1.13.1+cu116 available.
+2024-11-06 13:40:39 - utils.py[line:258] - INFO: distributed init (rank 6): env://
+2024-11-06 13:40:39 - utils.py[line:261] - INFO: Start init
+2024-11-06 13:40:39 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 6
+2024-11-06 13:40:39 - utils.py[line:258] - INFO: distributed init (rank 3): env://
+2024-11-06 13:40:39 - utils.py[line:261] - INFO: Start init
+2024-11-06 13:40:39 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 3
+2024-11-06 13:40:39 - utils.py[line:258] - INFO: distributed init (rank 2): env://
+2024-11-06 13:40:39 - utils.py[line:261] - INFO: Start init
+2024-11-06 13:40:39 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 2
+2024-11-06 13:40:39 - utils.py[line:258] - INFO: distributed init (rank 1): env://
+2024-11-06 13:40:39 - utils.py[line:261] - INFO: Start init
+2024-11-06 13:40:39 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 1
+2024-11-06 13:40:39 - utils.py[line:258] - INFO: distributed init (rank 7): env://
+2024-11-06 13:40:39 - utils.py[line:261] - INFO: Start init
+2024-11-06 13:40:39 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 7
+2024-11-06 13:40:39 - utils.py[line:258] - INFO: distributed init (rank 5): env://
+2024-11-06 13:40:39 - utils.py[line:261] - INFO: Start init
+2024-11-06 13:40:39 - utils.py[line:258] - INFO: distributed init (rank 0): env://
+2024-11-06 13:40:39 - utils.py[line:261] - INFO: Start init
+2024-11-06 13:40:39 - utils.py[line:258] - INFO: distributed init (rank 4): env://
+2024-11-06 13:40:39 - utils.py[line:261] - INFO: Start init
+2024-11-06 13:40:39 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 5
+2024-11-06 13:40:39 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 0
+2024-11-06 13:40:39 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:1 to store for rank: 4
+2024-11-06 13:40:39 - distributed_c10d.py[line:354] - INFO: Rank 4: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
+2024-11-06 13:40:39 - utils.py[line:274] - INFO: initialized host ip-172-31-9-178 as rank 4
 single-machine distributed training is initialized.
-2024-11-03 05:29:41 - distributed_c10d.py[line:354] - INFO: Rank 1: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
-2024-11-03 05:29:41 - utils.py[line:274] - INFO: initialized host ip-172-31-38-178 as rank 1
+2024-11-06 13:40:39 - distributed_c10d.py[line:354] - INFO: Rank 1: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
+2024-11-06 13:40:39 - utils.py[line:274] - INFO: initialized host ip-172-31-9-178 as rank 1
 single-machine distributed training is initialized.
-2024-11-03 05:29:41 - distributed_c10d.py[line:354] - INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
-2024-11-03 05:29:41 - utils.py[line:274] - INFO: initialized host ip-172-31-38-178 as rank 0
+2024-11-06 13:40:39 - distributed_c10d.py[line:354] - INFO: Rank 7: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
+2024-11-06 13:40:39 - utils.py[line:274] - INFO: initialized host ip-172-31-9-178 as rank 7
 single-machine distributed training is initialized.
-2024-11-03 05:29:41 - distributed_c10d.py[line:354] - INFO: Rank 7: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
-2024-11-03 05:29:41 - distributed_c10d.py[line:354] - INFO: Rank 4: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
-2024-11-03 05:29:41 - utils.py[line:274] - INFO: initialized host ip-172-31-38-178 as rank 7
+2024-11-06 13:40:39 - distributed_c10d.py[line:354] - INFO: Rank 3: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
+2024-11-06 13:40:39 - distributed_c10d.py[line:354] - INFO: Rank 6: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
+2024-11-06 13:40:39 - utils.py[line:274] - INFO: initialized host ip-172-31-9-178 as rank 3
+2024-11-06 13:40:39 - utils.py[line:274] - INFO: initialized host ip-172-31-9-178 as rank 6
 single-machine distributed training is initialized.
-2024-11-03 05:29:41 - utils.py[line:274] - INFO: initialized host ip-172-31-38-178 as rank 4
 single-machine distributed training is initialized.
-2024-11-03 05:29:41 - distributed_c10d.py[line:354] - INFO: Rank 5: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
-2024-11-03 05:29:41 - utils.py[line:274] - INFO: initialized host ip-172-31-38-178 as rank 5
+2024-11-06 13:40:39 - distributed_c10d.py[line:354] - INFO: Rank 5: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
+2024-11-06 13:40:39 - distributed_c10d.py[line:354] - INFO: Rank 2: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
+2024-11-06 13:40:39 - utils.py[line:274] - INFO: initialized host ip-172-31-9-178 as rank 5
 single-machine distributed training is initialized.
-2024-11-03 05:29:41 - distributed_c10d.py[line:354] - INFO: Rank 2: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
-2024-11-03 05:29:41 - utils.py[line:274] - INFO: initialized host ip-172-31-38-178 as rank 2
+2024-11-06 13:40:39 - distributed_c10d.py[line:354] - INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
+2024-11-06 13:40:39 - utils.py[line:274] - INFO: initialized host ip-172-31-9-178 as rank 2
 single-machine distributed training is initialized.
-2024-11-03 05:29:41 - distributed_c10d.py[line:354] - INFO: Rank 6: Completed store-based barrier for key:store_based_barrier_key:1 with 8 nodes.
-2024-11-03 05:29:41 - utils.py[line:274] - INFO: initialized host ip-172-31-38-178 as rank 6
+2024-11-06 13:40:39 - utils.py[line:274] - INFO: initialized host ip-172-31-9-178 as rank 0
 single-machine distributed training is initialized.
-2024-11-03 05:29:45 - train.py[line:77] - INFO: {'_name': None, 'common': {'_name': None, 'no_progress_bar': False, 'log_interval': 10, 'log_format': 'simple', 'log_file': None, 'tensorboard_logdir': None, 'wandb_project': None, 'azureml_logging': False, 'seed': 1, 'cpu': False, 'tpu': False, 'bf16': False, 'memory_efficient_bf16': False, 'fp16': True, 'memory_efficient_fp16': False, 'fp16_no_flatten_grads': False, 'fp16_init_scale': 128, 'fp16_scale_window': 512, 'fp16_scale_tolerance': 0.0, 'on_cpu_convert_precision': False, 'min_loss_scale': 0.0001, 'threshold_loss_scale': None, 'amp': False, 'amp_batch_retries': 2, 'amp_init_scale': 128, 'amp_scale_window': None, 'user_dir': '../../polyformer_module', 'empty_cache_freq': 0, 'all_gather_list_size': 16384, 'model_parallel_size': 1, 'quantization_config_path': None, 'profile': False, 'reset_logging': False, 'suppress_crashes': False, 'use_plasma_view': False, 'plasma_path': '/tmp/plasma'}, 'common_eval': {'_name': None, 'path': None, 'post_process': None, 'quiet': False, 'model_overrides': '{}', 'results_path': None}, 'distributed_training': {'_name': None, 'distributed_world_size': 8, 'distributed_num_procs': 8, 'distributed_rank': 0, 'distributed_backend': 'nccl', 'distributed_init_method': 'env://', 'distributed_port': -1, 'device_id': 0, 'distributed_no_spawn': True, 'ddp_backend': 'pytorch_ddp', 'ddp_comm_hook': 'none', 'bucket_cap_mb': 25, 'fix_batches_to_gpus': False, 'find_unused_parameters': True, 'gradient_as_bucket_view': False, 'fast_stat_sync': False, 'heartbeat_timeout': -1, 'broadcast_buffers': False, 'slowmo_momentum': None, 'slowmo_algorithm': 'LocalSGD', 'localsgd_frequency': 3, 'nprocs_per_node': 8, 'pipeline_model_parallel': False, 'pipeline_balance': None, 'pipeline_devices': None, 'pipeline_chunks': 0, 'pipeline_encoder_balance': None, 'pipeline_encoder_devices': None, 'pipeline_decoder_balance': None, 'pipeline_decoder_devices': None, 'pipeline_checkpoint': 'never', 'zero_sharding': 'none', 'fp16': True, 'memory_efficient_fp16': False, 'tpu': False, 'no_reshard_after_forward': False, 'fp32_reduce_scatter': False, 'cpu_offload': False, 'use_sharded_state': False}, 'dataset': {'_name': None, 'num_workers': 0, 'skip_invalid_size_inputs_valid_test': False, 'max_tokens': None, 'batch_size': 16, 'required_batch_size_multiple': 8, 'required_seq_len_multiple': 1, 'dataset_impl': None, 'data_buffer_size': 10, 'train_subset': 'train', 'valid_subset': 'valid', 'combine_valid_subsets': None, 'ignore_unused_valid_subsets': False, 'validate_interval': 1, 'validate_interval_updates': 500, 'validate_after_updates': 0, 'fixed_validation_seed': 7, 'disable_validation': False, 'max_tokens_valid': None, 'batch_size_valid': 16, 'max_valid_steps': None, 'curriculum': 0, 'gen_subset': 'test', 'num_shards': 1, 'shard_id': 0}, 'optimization': {'_name': None, 'max_epoch': 100, 'max_update': 0, 'stop_time_hours': 0.0, 'clip_norm': 1.0, 'sentence_avg': False, 'update_freq': [8], 'lr': [5e-05], 'stop_min_lr': -1.0, 'use_bmuf': False}, 'checkpoint': {'_name': None, 'save_dir': './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512', 'restore_file': '../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume_2/20_5e-5_512/checkpoint.best_score_0.5220.pt', 'finetune_from_model': None, 'reset_dataloader': True, 'reset_lr_scheduler': False, 'reset_meters': True, 'reset_optimizer': True, 'optimizer_overrides': '{}', 'save_interval': 1, 'save_interval_updates': 500, 'keep_interval_updates': -1, 'keep_interval_updates_pattern': -1, 'keep_last_epochs': -1, 'keep_best_checkpoints': 1, 'no_save': False, 'no_epoch_checkpoints': True, 'no_last_checkpoints': False, 'no_save_optimizer_state': False, 'best_checkpoint_metric': 'score', 'maximize_best_checkpoint_metric': True, 'patience': -1, 'checkpoint_suffix': '', 'checkpoint_shard_count': 1, 'load_checkpoint_on_all_dp_ranks': False, 'write_checkpoints_asynchronously': False, 'model_parallel_size': 1, 'use_ema_weights_to_init_param': False, 'use_latest_weights_to_init_ema': False}, 'bmuf': {'_name': None, 'block_lr': 1.0, 'block_momentum': 0.875, 'global_sync_iter': 50, 'warmup_iterations': 500, 'use_nbm': False, 'average_sync': False, 'distributed_world_size': 8}, 'generation': {'_name': None, 'beam': 5, 'nbest': 1, 'max_len_a': 0.0, 'max_len_b': 200, 'min_len': 1, 'match_source_len': False, 'unnormalized': False, 'no_early_stop': False, 'no_beamable_mm': False, 'lenpen': 1.0, 'unkpen': 0.0, 'replace_unk': None, 'sacrebleu': False, 'score_reference': False, 'prefix_size': 0, 'no_repeat_ngram_size': 0, 'sampling': False, 'sampling_topk': -1, 'sampling_topp': -1.0, 'constraints': None, 'temperature': 1.0, 'diverse_beam_groups': -1, 'diverse_beam_strength': 0.5, 'diversity_rate': -1.0, 'print_alignment': None, 'print_step': False, 'lm_path': None, 'lm_weight': 0.0, 'iter_decode_eos_penalty': 0.0, 'iter_decode_max_iter': 10, 'iter_decode_force_max_iter': False, 'iter_decode_with_beam': 1, 'iter_decode_with_external_reranker': False, 'retain_iter_history': False, 'retain_dropout': False, 'retain_dropout_modules': None, 'decoding_format': None, 'no_seed_provided': False}, 'eval_lm': {'_name': None, 'output_word_probs': False, 'output_word_stats': False, 'context_window': 0, 'softmax_batch': 9223372036854775807}, 'interactive': {'_name': None, 'buffer_size': 0, 'input': '-'}, 'model': Namespace(_name='polyformer_b', activation_fn='gelu', adam_betas='(0.9,0.999)', adam_eps=1e-08, adaptive_softmax_cutoff=None, adaptive_softmax_dropout=0, add_type_embedding=True, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, arch='polyformer_b', attention_dropout=0.0, attn_scale_factor=2, azureml_logging=False, batch_size=16, batch_size_valid=16, best_checkpoint_metric='score', bf16=False, bpe=None, bpe_dir='../../utils/BPE', broadcast_buffers=False, bucket_cap_mb=25, checkpoint_shard_count=1, checkpoint_suffix='', clip_norm=1.0, cls_weight=0.0005, code_dict_size=8192, code_image_size=128, code_layernorm_embedding=True, combine_valid_subsets=None, constraint_range=None, cpu=False, cpu_offload=False, criterion='adjust_label_smoothed_cross_entropy', cross_self_attention=False, curriculum=0, data='../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv,../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv', data_buffer_size=10, dataset_impl=None, ddp_backend='pytorch_ddp', ddp_comm_hook='none', decoder_attention_heads=12, decoder_drop_path_rate=0.1, decoder_embed_dim=768, decoder_embed_path=None, decoder_ffn_embed_dim=3072, decoder_input_dim=768, decoder_layerdrop=0, decoder_layers=6, decoder_layers_to_keep=None, decoder_learned_pos=True, decoder_normalize_before=True, decoder_output_dim=768, det_weight=0.1, device_id=0, disable_entangle=True, disable_validation=False, distributed_backend='nccl', distributed_init_method=None, distributed_no_spawn=False, distributed_num_procs=8, distributed_port=-1, distributed_rank=0, distributed_world_size=8, drop_worst_after=0, drop_worst_ratio=0.0, dropout=0.1, ema_decay=0.9999, ema_fp32=False, ema_seed_model=None, ema_start_update=0, ema_update_freq=1, empty_cache_freq=0, encoder_attention_heads=12, encoder_drop_path_rate=0.1, encoder_embed_dim=768, encoder_embed_path=None, encoder_ffn_embed_dim=3072, encoder_layerdrop=0, encoder_layers=6, encoder_layers_to_keep=None, encoder_learned_pos=True, encoder_normalize_before=True, end_learning_rate=0.0, entangle_position_embedding=False, eos=2, eval_acc=True, eval_args='{"beam":5,"min_len":2,"max_len_a":0,"max_len_b":2}', eval_print_samples=False, fast_stat_sync=False, find_unused_parameters=True, finetune_from_model=None, fix_batches_to_gpus=False, fixed_validation_seed=7, force_anneal=None, fp16=True, fp16_adam_stats=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=512, fp32_reduce_scatter=False, freeze_decoder_embedding=False, freeze_encoder_embedding=False, gen_subset='test', gradient_as_bucket_view=False, heartbeat_timeout=-1, ignore_eos=False, ignore_prefix_size=0, ignore_unused_valid_subsets=False, image_bucket_size=42, imagenet_default_mean_and_std=False, keep_best_checkpoints=1, keep_interval_updates=-1, keep_interval_updates_pattern=-1, keep_last_epochs=-1, label_smoothing=0.1, layernorm_embedding=True, load_checkpoint_on_all_dp_ranks=False, localsgd_frequency=3, log_file=None, log_format='simple', log_interval=10, lr=[5e-05], lr_scheduler='polynomial_decay', max_epoch=100, max_image_size=512, max_source_positions=1024, max_src_length=80, max_target_positions=1024, max_tgt_length=420, max_tokens=None, max_tokens_valid=None, max_update=0, max_valid_steps=None, maximize_best_checkpoint_metric=True, memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, min_params_to_wrap=100000000, model_parallel_size=1, no_cross_attention=False, no_epoch_checkpoints=True, no_last_checkpoints=False, no_progress_bar=False, no_reshard_after_forward=False, no_save=False, no_save_optimizer_state=False, no_scale_embedding=True, no_seed_provided=False, no_token_positional_embeddings=False, nprocs_per_node=8, num_bins=64, num_shards=1, num_workers=0, on_cpu_convert_precision=False, optimizer='adam', optimizer_overrides='{}', out_index=3, pad=1, patch_image_size=512, patch_layernorm_embedding=True, patience=-1, pipeline_balance=None, pipeline_checkpoint='never', pipeline_chunks=0, pipeline_decoder_balance=None, pipeline_decoder_devices=None, pipeline_devices=None, pipeline_encoder_balance=None, pipeline_encoder_devices=None, pipeline_model_parallel=False, plasma_path='/tmp/plasma', pooler_activation_fn='tanh', pooler_classifier='mlp', pooler_dropout=0.0, power=1.0, profile=False, quant_noise_pq=0, quant_noise_pq_block_size=8, quant_noise_scalar=0, quantization_config_path=None, reg_alpha=1.0, relu_dropout=0.0, report_accuracy=False, required_batch_size_multiple=8, required_seq_len_multiple=1, reset_dataloader=True, reset_logging=False, reset_lr_scheduler=False, reset_meters=True, reset_optimizer=True, resnet_drop_path_rate=0.0, restore_file='../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume_2/20_5e-5_512/checkpoint.best_score_0.5220.pt', sample_patch_num=196, save_dir='./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512', save_interval=1, save_interval_updates=500, scale_attn=True, scale_fc=True, scale_heads=True, scale_resids=False, scoring='bleu', scst=False, scst_args='{}', seed=1, selected_cols='0,5,6,2,4,3,7', sentence_avg=False, shard_id=0, share_all_embeddings=True, share_decoder_input_output_embed=True, simul_type=None, skip_invalid_size_inputs_valid_test=False, slowmo_algorithm='LocalSGD', slowmo_momentum=None, stop_min_lr=-1.0, stop_time_hours=0, store_ema=False, suppress_crashes=False, sync_bn=False, task='refcoco', tensorboard_logdir=None, threshold_loss_scale=None, token_bucket_size=256, tokenizer=None, total_num_update=1000000, tpu=False, train_subset='train', unk=3, update_freq=[8], use_bmuf=False, use_ema_weights_to_init_param=False, use_latest_weights_to_init_ema=False, use_old_adam=False, use_plasma_view=False, use_rdrop=False, use_sharded_state=False, user_dir='../../polyformer_module', uses_ema=False, valid_subset='valid', validate_after_updates=0, validate_interval=1, validate_interval_updates=500, vis_encoder_type='swin-base', wandb_project=None, warmup_ratio=0.06, warmup_updates=0, weight_decay=0.01, write_checkpoints_asynchronously=False, zero_sharding='none'), 'task': {'_name': 'refcoco', 'data': '../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv,../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv', 'selected_cols': '0,5,6,2,4,3,7', 'bpe_dir': '../../utils/BPE', 'max_source_positions': 1024, 'max_target_positions': 1024, 'max_src_length': 80, 'max_tgt_length': 420, 'code_dict_size': 8192, 'patch_image_size': 512, 'num_bins': 64, 'imagenet_default_mean_and_std': False, 'constraint_range': None, 'eval_acc': True, 'eval_args': '{"beam":5,"min_len":2,"max_len_a":0,"max_len_b":2}', 'uses_ema': False, 'eval_print_samples': False, 'max_image_size': 512, 'scst': False, 'scst_args': '{}'}, 'criterion': {'_name': 'adjust_label_smoothed_cross_entropy', 'label_smoothing': 0.1, 'report_accuracy': False, 'det_weight': 0.1, 'cls_weight': 0.0005, 'ignore_prefix_size': 0, 'ignore_eos': False, 'sentence_avg': False, 'drop_worst_ratio': 0.0, 'drop_worst_after': 0, 'use_rdrop': False, 'reg_alpha': 1.0, 'sample_patch_num': 196, 'constraint_range': None}, 'optimizer': {'_name': 'adam', 'adam_betas': '(0.9,0.999)', 'adam_eps': 1e-08, 'weight_decay': 0.01, 'use_old_adam': False, 'fp16_adam_stats': False, 'tpu': False, 'lr': [5e-05]}, 'lr_scheduler': {'_name': 'polynomial_decay', 'warmup_updates': 0, 'warmup_ratio': 0.06, 'force_anneal': None, 'end_learning_rate': 0.0, 'power': 1.0, 'total_num_update': 1000000.0, 'lr': [5e-05]}, 'scoring': {'_name': 'bleu', 'pad': 1, 'eos': 2, 'unk': 3}, 'bpe': None, 'tokenizer': None, 'ema': {'_name': None, 'store_ema': False, 'ema_decay': 0.9999, 'ema_start_update': 0, 'ema_seed_model': None, 'ema_update_freq': 1, 'ema_fp32': False}, 'simul_type': None}
-2024-11-03 05:29:45 - base_task.py[line:187] - INFO: source dictionary: 4100 types
-2024-11-03 05:29:45 - base_task.py[line:188] - INFO: target dictionary: 4100 types
+2024-11-06 13:40:43 - train.py[line:77] - INFO: {'_name': None, 'common': {'_name': None, 'no_progress_bar': False, 'log_interval': 10, 'log_format': 'simple', 'log_file': None, 'tensorboard_logdir': None, 'wandb_project': None, 'azureml_logging': False, 'seed': 1, 'cpu': False, 'tpu': False, 'bf16': False, 'memory_efficient_bf16': False, 'fp16': True, 'memory_efficient_fp16': False, 'fp16_no_flatten_grads': False, 'fp16_init_scale': 128, 'fp16_scale_window': 512, 'fp16_scale_tolerance': 0.0, 'on_cpu_convert_precision': False, 'min_loss_scale': 0.0001, 'threshold_loss_scale': None, 'amp': False, 'amp_batch_retries': 2, 'amp_init_scale': 128, 'amp_scale_window': None, 'user_dir': '../../polyformer_module', 'empty_cache_freq': 0, 'all_gather_list_size': 16384, 'model_parallel_size': 1, 'quantization_config_path': None, 'profile': False, 'reset_logging': False, 'suppress_crashes': False, 'use_plasma_view': False, 'plasma_path': '/tmp/plasma'}, 'common_eval': {'_name': None, 'path': None, 'post_process': None, 'quiet': False, 'model_overrides': '{}', 'results_path': None}, 'distributed_training': {'_name': None, 'distributed_world_size': 8, 'distributed_num_procs': 8, 'distributed_rank': 0, 'distributed_backend': 'nccl', 'distributed_init_method': 'env://', 'distributed_port': -1, 'device_id': 0, 'distributed_no_spawn': True, 'ddp_backend': 'pytorch_ddp', 'ddp_comm_hook': 'none', 'bucket_cap_mb': 25, 'fix_batches_to_gpus': False, 'find_unused_parameters': True, 'gradient_as_bucket_view': False, 'fast_stat_sync': False, 'heartbeat_timeout': -1, 'broadcast_buffers': False, 'slowmo_momentum': None, 'slowmo_algorithm': 'LocalSGD', 'localsgd_frequency': 3, 'nprocs_per_node': 8, 'pipeline_model_parallel': False, 'pipeline_balance': None, 'pipeline_devices': None, 'pipeline_chunks': 0, 'pipeline_encoder_balance': None, 'pipeline_encoder_devices': None, 'pipeline_decoder_balance': None, 'pipeline_decoder_devices': None, 'pipeline_checkpoint': 'never', 'zero_sharding': 'none', 'fp16': True, 'memory_efficient_fp16': False, 'tpu': False, 'no_reshard_after_forward': False, 'fp32_reduce_scatter': False, 'cpu_offload': False, 'use_sharded_state': False}, 'dataset': {'_name': None, 'num_workers': 0, 'skip_invalid_size_inputs_valid_test': False, 'max_tokens': None, 'batch_size': 16, 'required_batch_size_multiple': 8, 'required_seq_len_multiple': 1, 'dataset_impl': None, 'data_buffer_size': 10, 'train_subset': 'train', 'valid_subset': 'valid', 'combine_valid_subsets': None, 'ignore_unused_valid_subsets': False, 'validate_interval': 1, 'validate_interval_updates': 500, 'validate_after_updates': 0, 'fixed_validation_seed': 7, 'disable_validation': False, 'max_tokens_valid': None, 'batch_size_valid': 16, 'max_valid_steps': None, 'curriculum': 0, 'gen_subset': 'test', 'num_shards': 1, 'shard_id': 0}, 'optimization': {'_name': None, 'max_epoch': 100, 'max_update': 0, 'stop_time_hours': 0.0, 'clip_norm': 1.0, 'sentence_avg': False, 'update_freq': [8], 'lr': [5e-05], 'stop_min_lr': -1.0, 'use_bmuf': False}, 'checkpoint': {'_name': None, 'save_dir': './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512', 'restore_file': '../finetune/polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_73.pt', 'finetune_from_model': None, 'reset_dataloader': True, 'reset_lr_scheduler': False, 'reset_meters': True, 'reset_optimizer': True, 'optimizer_overrides': '{}', 'save_interval': 1, 'save_interval_updates': 500, 'keep_interval_updates': -1, 'keep_interval_updates_pattern': -1, 'keep_last_epochs': -1, 'keep_best_checkpoints': 1, 'no_save': False, 'no_epoch_checkpoints': True, 'no_last_checkpoints': False, 'no_save_optimizer_state': False, 'best_checkpoint_metric': 'score', 'maximize_best_checkpoint_metric': True, 'patience': -1, 'checkpoint_suffix': '', 'checkpoint_shard_count': 1, 'load_checkpoint_on_all_dp_ranks': False, 'write_checkpoints_asynchronously': False, 'model_parallel_size': 1, 'use_ema_weights_to_init_param': False, 'use_latest_weights_to_init_ema': False}, 'bmuf': {'_name': None, 'block_lr': 1.0, 'block_momentum': 0.875, 'global_sync_iter': 50, 'warmup_iterations': 500, 'use_nbm': False, 'average_sync': False, 'distributed_world_size': 8}, 'generation': {'_name': None, 'beam': 5, 'nbest': 1, 'max_len_a': 0.0, 'max_len_b': 200, 'min_len': 1, 'match_source_len': False, 'unnormalized': False, 'no_early_stop': False, 'no_beamable_mm': False, 'lenpen': 1.0, 'unkpen': 0.0, 'replace_unk': None, 'sacrebleu': False, 'score_reference': False, 'prefix_size': 0, 'no_repeat_ngram_size': 0, 'sampling': False, 'sampling_topk': -1, 'sampling_topp': -1.0, 'constraints': None, 'temperature': 1.0, 'diverse_beam_groups': -1, 'diverse_beam_strength': 0.5, 'diversity_rate': -1.0, 'print_alignment': None, 'print_step': False, 'lm_path': None, 'lm_weight': 0.0, 'iter_decode_eos_penalty': 0.0, 'iter_decode_max_iter': 10, 'iter_decode_force_max_iter': False, 'iter_decode_with_beam': 1, 'iter_decode_with_external_reranker': False, 'retain_iter_history': False, 'retain_dropout': False, 'retain_dropout_modules': None, 'decoding_format': None, 'no_seed_provided': False}, 'eval_lm': {'_name': None, 'output_word_probs': False, 'output_word_stats': False, 'context_window': 0, 'softmax_batch': 9223372036854775807}, 'interactive': {'_name': None, 'buffer_size': 0, 'input': '-'}, 'model': Namespace(_name='polyformer_b', activation_fn='gelu', adam_betas='(0.9,0.999)', adam_eps=1e-08, adaptive_softmax_cutoff=None, adaptive_softmax_dropout=0, add_type_embedding=True, all_gather_list_size=16384, amp=False, amp_batch_retries=2, amp_init_scale=128, amp_scale_window=None, arch='polyformer_b', attention_dropout=0.0, attn_scale_factor=2, azureml_logging=False, batch_size=16, batch_size_valid=16, best_checkpoint_metric='score', bf16=False, bpe=None, bpe_dir='../../utils/BPE', broadcast_buffers=False, bucket_cap_mb=25, checkpoint_shard_count=1, checkpoint_suffix='', clip_norm=1.0, cls_weight=0.0005, code_dict_size=8192, code_image_size=128, code_layernorm_embedding=True, combine_valid_subsets=None, constraint_range=None, cpu=False, cpu_offload=False, criterion='adjust_label_smoothed_cross_entropy', cross_self_attention=False, curriculum=0, data='../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv,../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv', data_buffer_size=10, dataset_impl=None, ddp_backend='pytorch_ddp', ddp_comm_hook='none', decoder_attention_heads=12, decoder_drop_path_rate=0.1, decoder_embed_dim=768, decoder_embed_path=None, decoder_ffn_embed_dim=3072, decoder_input_dim=768, decoder_layerdrop=0, decoder_layers=6, decoder_layers_to_keep=None, decoder_learned_pos=True, decoder_normalize_before=True, decoder_output_dim=768, det_weight=0.1, device_id=0, disable_entangle=True, disable_validation=False, distributed_backend='nccl', distributed_init_method=None, distributed_no_spawn=False, distributed_num_procs=8, distributed_port=-1, distributed_rank=0, distributed_world_size=8, drop_worst_after=0, drop_worst_ratio=0.0, dropout=0.1, ema_decay=0.9999, ema_fp32=False, ema_seed_model=None, ema_start_update=0, ema_update_freq=1, empty_cache_freq=0, encoder_attention_heads=12, encoder_drop_path_rate=0.1, encoder_embed_dim=768, encoder_embed_path=None, encoder_ffn_embed_dim=3072, encoder_layerdrop=0, encoder_layers=6, encoder_layers_to_keep=None, encoder_learned_pos=True, encoder_normalize_before=True, end_learning_rate=0.0, entangle_position_embedding=False, eos=2, eval_acc=True, eval_args='{"beam":5,"min_len":2,"max_len_a":0,"max_len_b":2}', eval_print_samples=False, fast_stat_sync=False, find_unused_parameters=True, finetune_from_model=None, fix_batches_to_gpus=False, fixed_validation_seed=7, force_anneal=None, fp16=True, fp16_adam_stats=False, fp16_init_scale=128, fp16_no_flatten_grads=False, fp16_scale_tolerance=0.0, fp16_scale_window=512, fp32_reduce_scatter=False, freeze_decoder_embedding=False, freeze_encoder_embedding=False, gen_subset='test', gradient_as_bucket_view=False, heartbeat_timeout=-1, ignore_eos=False, ignore_prefix_size=0, ignore_unused_valid_subsets=False, image_bucket_size=42, imagenet_default_mean_and_std=False, keep_best_checkpoints=1, keep_interval_updates=-1, keep_interval_updates_pattern=-1, keep_last_epochs=-1, label_smoothing=0.1, layernorm_embedding=True, load_checkpoint_on_all_dp_ranks=False, localsgd_frequency=3, log_file=None, log_format='simple', log_interval=10, lr=[5e-05], lr_scheduler='polynomial_decay', max_epoch=100, max_image_size=512, max_source_positions=1024, max_src_length=80, max_target_positions=1024, max_tgt_length=420, max_tokens=None, max_tokens_valid=None, max_update=0, max_valid_steps=None, maximize_best_checkpoint_metric=True, memory_efficient_bf16=False, memory_efficient_fp16=False, min_loss_scale=0.0001, min_params_to_wrap=100000000, model_parallel_size=1, no_cross_attention=False, no_epoch_checkpoints=True, no_last_checkpoints=False, no_progress_bar=False, no_reshard_after_forward=False, no_save=False, no_save_optimizer_state=False, no_scale_embedding=True, no_seed_provided=False, no_token_positional_embeddings=False, nprocs_per_node=8, num_bins=64, num_shards=1, num_workers=0, on_cpu_convert_precision=False, optimizer='adam', optimizer_overrides='{}', out_index=3, pad=1, patch_image_size=512, patch_layernorm_embedding=True, patience=-1, pipeline_balance=None, pipeline_checkpoint='never', pipeline_chunks=0, pipeline_decoder_balance=None, pipeline_decoder_devices=None, pipeline_devices=None, pipeline_encoder_balance=None, pipeline_encoder_devices=None, pipeline_model_parallel=False, plasma_path='/tmp/plasma', pooler_activation_fn='tanh', pooler_classifier='mlp', pooler_dropout=0.0, power=1.0, profile=False, quant_noise_pq=0, quant_noise_pq_block_size=8, quant_noise_scalar=0, quantization_config_path=None, reg_alpha=1.0, relu_dropout=0.0, report_accuracy=False, required_batch_size_multiple=8, required_seq_len_multiple=1, reset_dataloader=True, reset_logging=False, reset_lr_scheduler=False, reset_meters=True, reset_optimizer=True, resnet_drop_path_rate=0.0, restore_file='../finetune/polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_73.pt', sample_patch_num=196, save_dir='./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512', save_interval=1, save_interval_updates=500, scale_attn=True, scale_fc=True, scale_heads=True, scale_resids=False, scoring='bleu', scst=False, scst_args='{}', seed=1, selected_cols='0,5,6,2,4,3,7', sentence_avg=False, shard_id=0, share_all_embeddings=True, share_decoder_input_output_embed=True, simul_type=None, skip_invalid_size_inputs_valid_test=False, slowmo_algorithm='LocalSGD', slowmo_momentum=None, stop_min_lr=-1.0, stop_time_hours=0, store_ema=False, suppress_crashes=False, sync_bn=False, task='refcoco', tensorboard_logdir=None, threshold_loss_scale=None, token_bucket_size=256, tokenizer=None, total_num_update=1000000, tpu=False, train_subset='train', unk=3, update_freq=[8], use_bmuf=False, use_ema_weights_to_init_param=False, use_latest_weights_to_init_ema=False, use_old_adam=False, use_plasma_view=False, use_rdrop=False, use_sharded_state=False, user_dir='../../polyformer_module', uses_ema=False, valid_subset='valid', validate_after_updates=0, validate_interval=1, validate_interval_updates=500, vis_encoder_type='swin-base', wandb_project=None, warmup_ratio=0.06, warmup_updates=0, weight_decay=0.01, write_checkpoints_asynchronously=False, zero_sharding='none'), 'task': {'_name': 'refcoco', 'data': '../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv,../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv', 'selected_cols': '0,5,6,2,4,3,7', 'bpe_dir': '../../utils/BPE', 'max_source_positions': 1024, 'max_target_positions': 1024, 'max_src_length': 80, 'max_tgt_length': 420, 'code_dict_size': 8192, 'patch_image_size': 512, 'num_bins': 64, 'imagenet_default_mean_and_std': False, 'constraint_range': None, 'eval_acc': True, 'eval_args': '{"beam":5,"min_len":2,"max_len_a":0,"max_len_b":2}', 'uses_ema': False, 'eval_print_samples': False, 'max_image_size': 512, 'scst': False, 'scst_args': '{}'}, 'criterion': {'_name': 'adjust_label_smoothed_cross_entropy', 'label_smoothing': 0.1, 'report_accuracy': False, 'det_weight': 0.1, 'cls_weight': 0.0005, 'ignore_prefix_size': 0, 'ignore_eos': False, 'sentence_avg': False, 'drop_worst_ratio': 0.0, 'drop_worst_after': 0, 'use_rdrop': False, 'reg_alpha': 1.0, 'sample_patch_num': 196, 'constraint_range': None}, 'optimizer': {'_name': 'adam', 'adam_betas': '(0.9,0.999)', 'adam_eps': 1e-08, 'weight_decay': 0.01, 'use_old_adam': False, 'fp16_adam_stats': False, 'tpu': False, 'lr': [5e-05]}, 'lr_scheduler': {'_name': 'polynomial_decay', 'warmup_updates': 0, 'warmup_ratio': 0.06, 'force_anneal': None, 'end_learning_rate': 0.0, 'power': 1.0, 'total_num_update': 1000000.0, 'lr': [5e-05]}, 'scoring': {'_name': 'bleu', 'pad': 1, 'eos': 2, 'unk': 3}, 'bpe': None, 'tokenizer': None, 'ema': {'_name': None, 'store_ema': False, 'ema_decay': 0.9999, 'ema_start_update': 0, 'ema_seed_model': None, 'ema_update_freq': 1, 'ema_fp32': False}, 'simul_type': None}
+2024-11-06 13:40:43 - base_task.py[line:187] - INFO: source dictionary: 4100 types
+2024-11-06 13:40:43 - base_task.py[line:188] - INFO: target dictionary: 4100 types
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3190.)
   return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3190.)
@@ -86,7 +86,14 @@ single-machine distributed training is initialized.
   return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3190.)
   return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
-2024-11-03 05:29:57 - train.py[line:101] - INFO: PolyFormerModel(
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 13:40:56 - train.py[line:101] - INFO: PolyFormerModel(
   (encoder): TransformerEncoder(
     (dropout_module): FairseqDropout()
     (embed_tokens): Embedding(4100, 768, padding_idx=1)
@@ -1211,94 +1218,88 @@ single-machine distributed training is initialized.
   )
   (classification_heads): ModuleDict()
 )
-2024-11-03 05:29:57 - train.py[line:102] - INFO: task: RefcocoTask
-2024-11-03 05:29:57 - train.py[line:103] - INFO: model: PolyFormerModel
-2024-11-03 05:29:57 - train.py[line:104] - INFO: criterion: AdjustLabelSmoothedCrossEntropyCriterion
-2024-11-03 05:29:57 - train.py[line:108] - INFO: num. shared model params: 478,547,732 (num. trained: 478,547,732)
-2024-11-03 05:29:57 - train.py[line:115] - INFO: num. expert model params: 0 (num. trained: 0)
+2024-11-06 13:40:56 - train.py[line:102] - INFO: task: RefcocoTask
+2024-11-06 13:40:56 - train.py[line:103] - INFO: model: PolyFormerModel
+2024-11-06 13:40:56 - train.py[line:104] - INFO: criterion: AdjustLabelSmoothedCrossEntropyCriterion
+2024-11-06 13:40:56 - train.py[line:108] - INFO: num. shared model params: 478,547,732 (num. trained: 478,547,732)
+2024-11-06 13:40:56 - train.py[line:115] - INFO: num. expert model params: 0 (num. trained: 0)
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 4 row count 6839 total row count 54715
+
Downloading tokenizer_config.json:   0%|          | 0.00/25.0 [00:00<?, ?B/s]
Downloading tokenizer_config.json: 100%|██████████| 25.0/25.0 [00:00<00:00, 5.00kB/s]
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 5 row count 6839 total row count 54715
+
Downloading (…)tencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]
Downloading (…)tencepiece.bpe.model: 100%|██████████| 5.07M/5.07M [00:00<00:00, 21.1MB/s]
Downloading (…)tencepiece.bpe.model: 100%|██████████| 5.07M/5.07M [00:00<00:00, 20.8MB/s]
+
Downloading tokenizer.json:   0%|          | 0.00/9.10M [00:00<?, ?B/s]local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 2 row count 6840 total row count 54715
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 0 row count 6840 total row count 54715
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 6 row count 6839 total row count 54715
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 1 row count 6840 total row count 54715
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
 
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 7 row count 6839 total row count 54715file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 5 row count 6839 total row count 54715
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 1 row count 6840 total row count 54715
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 3 row count 6839 total row count 54715
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 7 row count 6839 total row count 54715
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 6 row count 6839 total row count 54715
+
Downloading tokenizer.json: 100%|██████████| 9.10M/9.10M [00:00<00:00, 19.5MB/s]
Downloading tokenizer.json: 100%|██████████| 9.10M/9.10M [00:00<00:00, 19.2MB/s]
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 0 row count 6840 total row count 54715
+2024-11-06 13:41:19 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:2 to store for rank: 0
+2024-11-06 13:41:19 - distributed_c10d.py[line:354] - INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:2 with 8 nodes.
+2024-11-06 13:41:19 - trainer.py[line:124] - INFO: detected shared parameter: encoder.embed_tokens.weight <- decoder.embed_tokens.weight
+2024-11-06 13:41:19 - trainer.py[line:124] - INFO: detected shared parameter: encoder.embed_images.layers.0.downsample.reduction.bias <- encoder.embed_images.layers.1.downsample.reduction.bias
+2024-11-06 13:41:19 - trainer.py[line:124] - INFO: detected shared parameter: encoder.embed_images.layers.0.downsample.reduction.bias <- encoder.embed_images.layers.2.downsample.reduction.bias
+2024-11-06 13:41:19 - trainer.py[line:124] - INFO: detected shared parameter: encoder.embed_images.layers.0.downsample.reduction.bias <- decoder.cls_head.bias
+2024-11-06 13:41:19 - trainer.py[line:124] - INFO: detected shared parameter: encoder.bert.roberta.embeddings.word_embeddings.weight <- encoder.bert.lm_head.decoder.weight
+2024-11-06 13:41:19 - trainer.py[line:124] - INFO: detected shared parameter: encoder.bert.lm_head.bias <- encoder.bert.lm_head.decoder.bias
+2024-11-06 13:41:21 - utils.py[line:759] - INFO: ***********************CUDA enviroments for all 8 workers***********************
+2024-11-06 13:41:21 - utils.py[line:765] - INFO: rank   0: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
+2024-11-06 13:41:21 - utils.py[line:765] - INFO: rank   1: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
+2024-11-06 13:41:21 - utils.py[line:765] - INFO: rank   2: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
+2024-11-06 13:41:21 - utils.py[line:765] - INFO: rank   3: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
+2024-11-06 13:41:21 - utils.py[line:765] - INFO: rank   4: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
+2024-11-06 13:41:21 - utils.py[line:765] - INFO: rank   5: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
+2024-11-06 13:41:21 - utils.py[line:765] - INFO: rank   6: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
+2024-11-06 13:41:21 - utils.py[line:765] - INFO: rank   7: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
+2024-11-06 13:41:21 - utils.py[line:767] - INFO: ***********************CUDA enviroments for all 8 workers***********************
+2024-11-06 13:41:21 - train.py[line:145] - INFO: training on 8 devices (GPUs/TPUs)
+2024-11-06 13:41:21 - train.py[line:151] - INFO: max tokens per device = None and max sentences per device = 16
+2024-11-06 13:41:21 - trainer.py[line:458] - INFO: Preparing to load checkpoint ../finetune/polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_73.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_val.tsv slice_id 4 row count 6839 total row count 54715
-2024-11-03 05:30:21 - distributed_c10d.py[line:319] - INFO: Added key: store_based_barrier_key:2 to store for rank: 0
-2024-11-03 05:30:21 - distributed_c10d.py[line:354] - INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:2 with 8 nodes.
-2024-11-03 05:30:21 - trainer.py[line:124] - INFO: detected shared parameter: encoder.embed_tokens.weight <- decoder.embed_tokens.weight
-2024-11-03 05:30:21 - trainer.py[line:124] - INFO: detected shared parameter: encoder.embed_images.layers.0.downsample.reduction.bias <- encoder.embed_images.layers.1.downsample.reduction.bias
-2024-11-03 05:30:21 - trainer.py[line:124] - INFO: detected shared parameter: encoder.embed_images.layers.0.downsample.reduction.bias <- encoder.embed_images.layers.2.downsample.reduction.bias
-2024-11-03 05:30:21 - trainer.py[line:124] - INFO: detected shared parameter: encoder.embed_images.layers.0.downsample.reduction.bias <- decoder.cls_head.bias
-2024-11-03 05:30:21 - trainer.py[line:124] - INFO: detected shared parameter: encoder.bert.roberta.embeddings.word_embeddings.weight <- encoder.bert.lm_head.decoder.weight
-2024-11-03 05:30:21 - trainer.py[line:124] - INFO: detected shared parameter: encoder.bert.lm_head.bias <- encoder.bert.lm_head.decoder.bias
-2024-11-03 05:30:23 - utils.py[line:759] - INFO: ***********************CUDA enviroments for all 8 workers***********************
-2024-11-03 05:30:23 - utils.py[line:765] - INFO: rank   0: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
-2024-11-03 05:30:23 - utils.py[line:765] - INFO: rank   1: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
-2024-11-03 05:30:23 - utils.py[line:765] - INFO: rank   2: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
-2024-11-03 05:30:23 - utils.py[line:765] - INFO: rank   3: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
-2024-11-03 05:30:23 - utils.py[line:765] - INFO: rank   4: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
-2024-11-03 05:30:23 - utils.py[line:765] - INFO: rank   5: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
-2024-11-03 05:30:23 - utils.py[line:765] - INFO: rank   6: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
-2024-11-03 05:30:23 - utils.py[line:765] - INFO: rank   7: capabilities =  8.0  ; total memory = 39.564 GB ; name = NVIDIA A100-SXM4-40GB                   
-2024-11-03 05:30:23 - utils.py[line:767] - INFO: ***********************CUDA enviroments for all 8 workers***********************
-2024-11-03 05:30:23 - train.py[line:145] - INFO: training on 8 devices (GPUs/TPUs)
-2024-11-03 05:30:23 - train.py[line:151] - INFO: max tokens per device = None and max sentences per device = 16
-2024-11-03 05:30:23 - trainer.py[line:458] - INFO: Preparing to load checkpoint ../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume_2/20_5e-5_512/checkpoint.best_score_0.5220.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 05:30:34 - trainer.py[line:619] - INFO: Loaded checkpoint ../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume_2/20_5e-5_512/checkpoint.best_score_0.5220.pt (epoch 16 @ 0 updates)
-2024-11-03 05:30:34 - trainer.py[line:639] - INFO: loading train data for epoch 1
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 13:41:32 - trainer.py[line:619] - INFO: Loaded checkpoint ../finetune/polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_73.pt (epoch 74 @ 0 updates)
+2024-11-06 13:41:32 - trainer.py[line:639] - INFO: loading train data for epoch 1
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-/home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torchvision/transforms/functional.py:443: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.
-  "Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. "
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
-/home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torchvision/transforms/functional.py:443: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.
-  "Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. "
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torchvision/transforms/functional.py:443: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.
   "Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. "
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
 Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
-Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+/home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torch/nn/functional.py:1967: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
+  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
 
 
 
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+/home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torchvision/transforms/functional.py:443: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.
+  "Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. "
+/home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torchvision/transforms/functional.py:443: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.
+  "Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. "
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torchvision/transforms/functional.py:443: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.
   "Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. "
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torchvision/transforms/functional.py:443: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.
@@ -1310,19 +1311,21 @@ file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torchvision/transforms/functional.py:443: UserWarning: Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. Please use InterpolationMode enum.
   "Argument 'interpolation' of type int is deprecated since 0.13 and will be removed in 0.15. "
 slice_id 5 seek offset 250000
-slice_id 4 seek offset 200000
-slice_id 3 seek offset 150000
 Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
+slice_id 0 seek offset 0
+slice_id 4 seek offset 200000
 Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
 Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
+2024-11-06 13:44:25 - trainer.py[line:703] - INFO: begin training epoch 1
+2024-11-06 13:44:25 - train.py[line:297] - INFO: Start iterating over samples
+slice_id 6 seek offset 300000
 slice_id 7 seek offset 350000
-slice_id 0 seek offset 0
+slice_id 3 seek offset 150000
+Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
+slice_id 2 seek offset 100000
+Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
 Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
 Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
-2024-11-03 05:33:24 - trainer.py[line:703] - INFO: begin training epoch 1
-2024-11-03 05:33:24 - train.py[line:297] - INFO: Start iterating over samples
-/home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torch/nn/functional.py:1967: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
-  warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torch/nn/functional.py:1967: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
   warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torch/nn/functional.py:1967: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
@@ -1337,2386 +1340,2413 @@ Total steps 39100, warmup steps 2346, warmup_factor 0.00042625745950554135
   warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
 /home/ubuntu/anaconda3/envs/polyformer/lib/python3.7/site-packages/torch/nn/functional.py:1967: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.
   warnings.warn("nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.")
-loss_reg: 0.3372189658910365 loss_cls: 0.00998353399336338
-loss_reg: 0.38415348452752085 loss_cls: 0.01047496497631073
-loss_reg: 0.09417515691753478 loss_cls: 0.00729618314653635
-loss_reg: 0.34215670488777106 loss_cls: 0.008932111784815788
-loss_reg: 0.3249204756645938 loss_cls: 0.01011343952268362
-loss_reg: 0.43155686362630563 loss_cls: 0.010162674821913242
-loss_reg: 0.36384949735094985 loss_cls: 0.011974421329796314
-loss_reg: 0.3730860141606566 loss_cls: 0.011719017289578915
-loss_reg: 0.3976116598231898 loss_cls: 0.009354079142212868
-loss_reg: 0.3263368617637766 loss_cls: 0.010335861705243587
-loss_reg: 0.08878352672604281 loss_cls: 0.007757115177810192
-loss_reg: 0.37081073323840824 loss_cls: 0.014285520650446415
-loss_reg: 0.3834619344181994 loss_cls: 0.009991586208343506
-loss_reg: 0.37959803008982446 loss_cls: 0.0163688063621521
-loss_reg: 0.3547841177285383 loss_cls: 0.017778735607862473
-loss_reg: 0.4111460676254898 loss_cls: 0.012158003635704517
-loss_reg: 0.39376572598870435 loss_cls: 0.008445999585092068
-loss_reg: 0.37395459870246966 loss_cls: 0.010649893432855606
-loss_reg: 0.07695943917098358 loss_cls: 0.008405568078160286
-loss_reg: 0.41325731326957177 loss_cls: 0.013923757709562778
-loss_reg: 0.3119105353754643 loss_cls: 0.009916731156408787
-loss_reg: 0.3648215199946308 loss_cls: 0.009635234251618385
-loss_reg: 0.3776925819122094 loss_cls: 0.009465004317462444
-loss_reg: 0.3751377142175711 loss_cls: 0.006813083775341511
-loss_reg: 0.28439589427794854 loss_cls: 0.010893076658248901
-loss_reg: 0.07347764417462452 loss_cls: 0.01100878044962883
-loss_reg: 0.4236512362022082 loss_cls: 0.014109434559941292
-loss_reg: 0.3466435463401501 loss_cls: 0.009289976209402084
-loss_reg: 0.3643196163911359 loss_cls: 0.012857717461884022
-loss_reg: 0.40479941330496283 loss_cls: 0.012652215547859669
-loss_reg: 0.39792850315335143 loss_cls: 0.012009701691567898
-loss_reg: 0.3878230083405658 loss_cls: 0.010022107511758804
-loss_reg: 0.42099895846036284 loss_cls: 0.022499626502394676
-loss_reg: 0.07477745687138912 loss_cls: 0.008130799047648907
-loss_reg: 0.29819489395608656 loss_cls: 0.015109546482563019
-loss_reg: 0.417082450653003 loss_cls: 0.013627692125737667
-loss_reg: 0.3627492048010609 loss_cls: 0.015795230865478516
-loss_reg: 0.41537928616639813 loss_cls: 0.014123686589300632
-loss_reg: 0.4057936082276256 loss_cls: 0.01775825396180153
-loss_reg: 0.3873659493423867 loss_cls: 0.019567450508475304
-loss_reg: 0.37130198571124307 loss_cls: 0.009334290400147438
-loss_reg: 0.08479443801125489 loss_cls: 0.007689611054956913
-loss_reg: 0.4245217722200109 loss_cls: 0.012746463529765606
-loss_reg: 0.3997808382874347 loss_cls: 0.010439220815896988
-loss_reg: 0.37241994554208513 loss_cls: 0.010562262497842312
-loss_reg: 0.41015018742977516 loss_cls: 0.011540415696799755
-loss_reg: 0.4018556351122749 loss_cls: 0.009903889149427414
-loss_reg: 0.4047100995386298 loss_cls: 0.011474339291453362
-loss_reg: 0.43796232982556943 loss_cls: 0.014752591028809547
-loss_reg: 0.09310703141692081 loss_cls: 0.009078622795641422
-loss_reg: 0.3907301182470793 loss_cls: 0.019073083996772766
-loss_reg: 0.4207889607261076 loss_cls: 0.012326940894126892
-loss_reg: 0.39334749788085116 loss_cls: 0.009903103113174438
-loss_reg: 0.3657388709219386 loss_cls: 0.01236018817871809
-loss_reg: 0.3455863222012787 loss_cls: 0.008577457629144192
-loss_reg: 0.378038613536809 loss_cls: 0.010501011274755001
-loss_reg: 0.4408388368074221 loss_cls: 0.012007343582808971
-loss_reg: 0.11242024271021256 loss_cls: 0.011279945261776447
-loss_reg: 0.3798947902683884 loss_cls: 0.01294621266424656
-loss_reg: 0.3934003792367832 loss_cls: 0.008703178726136684
-loss_reg: 0.40439907453264556 loss_cls: 0.00927309226244688
-loss_reg: 0.3864652417603391 loss_cls: 0.009415273554623127
-loss_reg: 0.37692916924141456 loss_cls: 0.01532721146941185
-loss_reg: 0.3823656814572685 loss_cls: 0.012167513370513916
-2024-11-03 05:34:22 - progress_bar.py[line:274] - INFO: epoch 001:     10 / 391 loss=0.022, loss_v1=0, loss_v2=0, nll_loss=24.815, ntokens=17535.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=2.95159e+07, wps=3352.8, ups=0.19, wpb=17535.3, bsz=1024, num_updates=10, lr=2.13129e-07, gnorm=0.061, clip=0, loss_scale=128, train_wall=44, gb_free=13.5, wall=239
-2024-11-03 05:35:15 - progress_bar.py[line:274] - INFO: epoch 001:     20 / 391 loss=0.022, loss_v1=0, loss_v2=0, nll_loss=24.903, ntokens=17733.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=3.13621e+07, wps=3339.9, ups=0.19, wpb=17733.4, bsz=1024, num_updates=20, lr=4.26257e-07, gnorm=0.064, clip=0, loss_scale=128, train_wall=40, gb_free=13.5, wall=292
-2024-11-03 05:36:08 - progress_bar.py[line:274] - INFO: epoch 001:     30 / 391 loss=0.021, loss_v1=0, loss_v2=0, nll_loss=23.101, ntokens=17420.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.00002e+06, wps=3288.8, ups=0.19, wpb=17420.7, bsz=1024, num_updates=30, lr=6.39386e-07, gnorm=0.113, clip=0, loss_scale=128, train_wall=35, gb_free=13.5, wall=345
-2024-11-03 05:37:02 - progress_bar.py[line:274] - INFO: epoch 001:     40 / 391 loss=0.015, loss_v1=0, loss_v2=0, nll_loss=22.078, ntokens=17831.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=4.42833e+06, wps=3315.9, ups=0.19, wpb=17831.8, bsz=1024, num_updates=40, lr=8.52515e-07, gnorm=0.211, clip=0, loss_scale=128, train_wall=31, gb_free=13.6, wall=399
-2024-11-03 05:37:55 - progress_bar.py[line:274] - INFO: epoch 001:     50 / 391 loss=0.008, loss_v1=0, loss_v2=0, nll_loss=19.641, ntokens=17628.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=817561, wps=3303.9, ups=0.19, wpb=17628.1, bsz=1024, num_updates=50, lr=1.06564e-06, gnorm=0.068, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=452
-2024-11-03 05:38:49 - progress_bar.py[line:274] - INFO: epoch 001:     60 / 391 loss=0.007, loss_v1=0, loss_v2=0, nll_loss=18.417, ntokens=17924.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=349922, wps=3357.7, ups=0.19, wpb=17924.3, bsz=1024, num_updates=60, lr=1.27877e-06, gnorm=0.026, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=506
-2024-11-03 05:39:42 - progress_bar.py[line:274] - INFO: epoch 001:     70 / 391 loss=0.006, loss_v1=0, loss_v2=0, nll_loss=16.319, ntokens=17717.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=81739.5, wps=3296.8, ups=0.19, wpb=17717.1, bsz=1024, num_updates=70, lr=1.4919e-06, gnorm=0.019, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=559
-2024-11-03 05:40:36 - progress_bar.py[line:274] - INFO: epoch 001:     80 / 391 loss=0.006, loss_v1=0, loss_v2=0, nll_loss=13.933, ntokens=17601.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15639.6, wps=3297.5, ups=0.19, wpb=17601.8, bsz=1024, num_updates=80, lr=1.70503e-06, gnorm=0.019, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=613
-2024-11-03 05:41:30 - progress_bar.py[line:274] - INFO: epoch 001:     90 / 391 loss=0.005, loss_v1=0, loss_v2=0, nll_loss=12.387, ntokens=17624.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5358.07, wps=3269.9, ups=0.19, wpb=17624.4, bsz=1024, num_updates=90, lr=1.91816e-06, gnorm=0.018, clip=0, loss_scale=128, train_wall=31, gb_free=13.6, wall=667
-2024-11-03 05:42:23 - progress_bar.py[line:274] - INFO: epoch 001:    100 / 391 loss=0.005, loss_v1=0, loss_v2=0, nll_loss=10.979, ntokens=17657, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=2018.85, wps=3282, ups=0.19, wpb=17657, bsz=1024, num_updates=100, lr=2.13129e-06, gnorm=0.013, clip=0, loss_scale=128, train_wall=31, gb_free=13.6, wall=721
-2024-11-03 05:43:17 - progress_bar.py[line:274] - INFO: epoch 001:    110 / 391 loss=0.005, loss_v1=0, loss_v2=0, nll_loss=9.938, ntokens=17488, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=980.92, wps=3278.1, ups=0.19, wpb=17488, bsz=1024, num_updates=110, lr=2.34442e-06, gnorm=0.011, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=774
-2024-11-03 05:44:10 - progress_bar.py[line:274] - INFO: epoch 001:    120 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=8.909, ntokens=17657, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=480.75, wps=3319.6, ups=0.19, wpb=17657, bsz=1024, num_updates=120, lr=2.55754e-06, gnorm=0.01, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=827
-2024-11-03 05:45:03 - progress_bar.py[line:274] - INFO: epoch 001:    130 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=8.181, ntokens=17663.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=290.12, wps=3333.9, ups=0.19, wpb=17663.1, bsz=1024, num_updates=130, lr=2.77067e-06, gnorm=0.008, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=880
-2024-11-03 05:45:55 - progress_bar.py[line:274] - INFO: epoch 001:    140 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=7.389, ntokens=17542.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=167.57, wps=3345.8, ups=0.19, wpb=17542.9, bsz=1024, num_updates=140, lr=2.9838e-06, gnorm=0.006, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=933
-2024-11-03 05:46:48 - progress_bar.py[line:274] - INFO: epoch 001:    150 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=6.913, ntokens=17394.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=120.52, wps=3298.5, ups=0.19, wpb=17394.6, bsz=1024, num_updates=150, lr=3.19693e-06, gnorm=0.006, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=985
-2024-11-03 05:47:41 - progress_bar.py[line:274] - INFO: epoch 001:    160 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=6.667, ntokens=17608.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=101.6, wps=3335.1, ups=0.19, wpb=17608.3, bsz=1024, num_updates=160, lr=3.41006e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=1038
-2024-11-03 05:48:34 - progress_bar.py[line:274] - INFO: epoch 001:    170 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=6.497, ntokens=18153.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=90.34, wps=3446.3, ups=0.19, wpb=18153.7, bsz=1024, num_updates=170, lr=3.62319e-06, gnorm=0.006, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=1091
-2024-11-03 05:49:26 - progress_bar.py[line:274] - INFO: epoch 001:    180 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=6.132, ntokens=17369.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=70.11, wps=3290.8, ups=0.19, wpb=17369.1, bsz=1024, num_updates=180, lr=3.83632e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=1144
-2024-11-03 05:50:19 - progress_bar.py[line:274] - INFO: epoch 001:    190 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=6.091, ntokens=17980.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=68.16, wps=3425.7, ups=0.19, wpb=17980.9, bsz=1024, num_updates=190, lr=4.04945e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=1196
-2024-11-03 05:51:11 - progress_bar.py[line:274] - INFO: epoch 001:    200 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=5.952, ntokens=17957.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=61.89, wps=3413.7, ups=0.19, wpb=17957.7, bsz=1024, num_updates=200, lr=4.26257e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.3, wall=1249
-2024-11-03 05:52:04 - progress_bar.py[line:274] - INFO: epoch 001:    210 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=5.811, ntokens=17907.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=56.15, wps=3415.4, ups=0.19, wpb=17907.3, bsz=1024, num_updates=210, lr=4.4757e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=31, gb_free=13.3, wall=1301
-2024-11-03 05:52:56 - progress_bar.py[line:274] - INFO: epoch 001:    220 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=5.761, ntokens=17882.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=54.22, wps=3404.5, ups=0.19, wpb=17882.3, bsz=1024, num_updates=220, lr=4.68883e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=1354
-2024-11-03 05:53:50 - progress_bar.py[line:274] - INFO: epoch 001:    230 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=5.669, ntokens=17841, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=50.88, wps=3360.1, ups=0.19, wpb=17841, bsz=1024, num_updates=230, lr=4.90196e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=1407
-2024-11-03 05:54:43 - progress_bar.py[line:274] - INFO: epoch 001:    240 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=5.644, ntokens=18014.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=50.02, wps=3361.5, ups=0.19, wpb=18014.1, bsz=1024, num_updates=240, lr=5.11509e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=1460
-2024-11-03 05:55:36 - progress_bar.py[line:274] - INFO: epoch 001:    250 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=5.528, ntokens=17552.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=46.13, wps=3336.8, ups=0.19, wpb=17552.9, bsz=1024, num_updates=250, lr=5.32822e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=1513
-2024-11-03 05:56:28 - progress_bar.py[line:274] - INFO: epoch 001:    260 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.509, ntokens=17776.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=45.54, wps=3392.7, ups=0.19, wpb=17776.1, bsz=1024, num_updates=260, lr=5.54135e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=1565
-2024-11-03 05:57:21 - progress_bar.py[line:274] - INFO: epoch 001:    270 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.451, ntokens=17647.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=43.75, wps=3368.7, ups=0.19, wpb=17647.3, bsz=1024, num_updates=270, lr=5.75448e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=1618
-2024-11-03 05:58:13 - progress_bar.py[line:274] - INFO: epoch 001:    280 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.416, ntokens=17675.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=42.7, wps=3344.6, ups=0.19, wpb=17675.8, bsz=1024, num_updates=280, lr=5.9676e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=1670
-2024-11-03 05:59:06 - progress_bar.py[line:274] - INFO: epoch 001:    290 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.346, ntokens=17688.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=40.68, wps=3333.5, ups=0.19, wpb=17688.6, bsz=1024, num_updates=290, lr=6.18073e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=1724
-2024-11-03 05:59:58 - progress_bar.py[line:274] - INFO: epoch 001:    300 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.313, ntokens=17589.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=39.77, wps=3379.5, ups=0.19, wpb=17589.2, bsz=1024, num_updates=300, lr=6.39386e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=1776
-2024-11-03 06:00:50 - progress_bar.py[line:274] - INFO: epoch 001:    310 / 391 loss=0.004, loss_v1=0, loss_v2=0, nll_loss=5.504, ntokens=18811.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=45.39, wps=3643.5, ups=0.19, wpb=18811.6, bsz=1024, num_updates=310, lr=6.60699e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=1827
-2024-11-03 06:01:43 - progress_bar.py[line:274] - INFO: epoch 001:    320 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.398, ntokens=18484.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=42.16, wps=3528.2, ups=0.19, wpb=18484.5, bsz=1024, num_updates=320, lr=6.82012e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=1880
-2024-11-03 06:02:34 - progress_bar.py[line:274] - INFO: epoch 001:    330 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.3, ntokens=18256.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=39.4, wps=3544.8, ups=0.19, wpb=18256.4, bsz=1024, num_updates=330, lr=7.03325e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=1931
-2024-11-03 06:03:26 - progress_bar.py[line:274] - INFO: epoch 001:    340 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.298, ntokens=18254.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=39.34, wps=3479.8, ups=0.19, wpb=18254.4, bsz=1024, num_updates=340, lr=7.24638e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=31, gb_free=13.3, wall=1984
-2024-11-03 06:04:19 - progress_bar.py[line:274] - INFO: epoch 001:    350 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.335, ntokens=18619.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=40.37, wps=3553.6, ups=0.19, wpb=18619.8, bsz=1024, num_updates=350, lr=7.45951e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=32, gb_free=13.3, wall=2036
-2024-11-03 06:05:12 - progress_bar.py[line:274] - INFO: epoch 001:    360 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.316, ntokens=18682, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=39.85, wps=3520.6, ups=0.19, wpb=18682, bsz=1024, num_updates=360, lr=7.67263e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=45, gb_free=13.3, wall=2089
-2024-11-03 06:06:05 - progress_bar.py[line:274] - INFO: epoch 001:    370 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.215, ntokens=18394.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=37.13, wps=3491.6, ups=0.19, wpb=18394.6, bsz=1024, num_updates=370, lr=7.88576e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=53, gb_free=13.3, wall=2142
-2024-11-03 06:06:57 - progress_bar.py[line:274] - INFO: epoch 001:    380 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.121, ntokens=17954.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=34.81, wps=3418.4, ups=0.19, wpb=17954.2, bsz=1024, num_updates=380, lr=8.09889e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=52, gb_free=13.4, wall=2194
-reach the end of datafile, start a new reader
+loss_reg: 0.02297143856220666 loss_cls: 0.0032473441679030657
+loss_reg: 0.01135831071139285 loss_cls: 0.0030096080154180527
+loss_reg: 0.015267837781967726 loss_cls: 0.00426668394356966
+loss_reg: 0.016800305808969386 loss_cls: 0.002847817027941346
+loss_reg: 0.0046933634395530605 loss_cls: 0.0027561127208173275
+loss_reg: 0.01810007709464248 loss_cls: 0.0036316849291324615
+loss_reg: 0.016607880958196312 loss_cls: 0.003142159665003419
+loss_reg: 0.0200729431093098 loss_cls: 0.0034592438023537397
+loss_reg: 0.008691143454579133 loss_cls: 0.004253067076206207
+loss_reg: 0.010352737209302448 loss_cls: 0.0032175793312489986
+loss_reg: 0.017078402007872723 loss_cls: 0.0031507546082139015
+loss_reg: 0.012029270207420104 loss_cls: 0.0034205110277980566
+loss_reg: 0.0046958169723910845 loss_cls: 0.0027743675746023655
+loss_reg: 0.012815560433010457 loss_cls: 0.003555023344233632
+loss_reg: 0.013148609837097477 loss_cls: 0.003827559994533658
+loss_reg: 0.01955852016509199 loss_cls: 0.0028929756954312325
+loss_reg: 0.017421121488708457 loss_cls: 0.0031427741050720215
+loss_reg: 0.01814756441935296 loss_cls: 0.004545845091342926
+loss_reg: 0.009650886750427932 loss_cls: 0.0045293159782886505
+loss_reg: 0.01207214689012066 loss_cls: 0.0027653828728944063
+loss_reg: 0.004453710429665267 loss_cls: 0.0027635330334305763
+loss_reg: 0.02025564252110163 loss_cls: 0.0036491109058260918
+loss_reg: 0.013934677254264814 loss_cls: 0.0024831329938024282
+loss_reg: 0.009546260798303136 loss_cls: 0.004379210062325001
+loss_reg: 0.012900140583534506 loss_cls: 0.003889333223924041
+loss_reg: 0.018291824205732696 loss_cls: 0.0032235169783234596
+loss_reg: 0.013833876751743617 loss_cls: 0.0027637926395982504
+loss_reg: 0.019095095230619627 loss_cls: 0.003779342398047447
+loss_reg: 0.01652060399816062 loss_cls: 0.0038687698543071747
+loss_reg: 0.00502547242863737 loss_cls: 0.0035307134967297316
+loss_reg: 0.01893817901238808 loss_cls: 0.0028468933887779713
+loss_reg: 0.012948630414719161 loss_cls: 0.00463188998401165
+loss_reg: 0.014358982297745824 loss_cls: 0.00331475050188601
+loss_reg: 0.0149840942823279 loss_cls: 0.0035055342596024275
+loss_reg: 0.013464229872063013 loss_cls: 0.003587101586163044
+loss_reg: 0.018096910413256784 loss_cls: 0.004202715121209621
+loss_reg: 0.010608566167937494 loss_cls: 0.006255564279854298
+loss_reg: 0.005809843509050404 loss_cls: 0.002468743361532688
+loss_reg: 0.016533973110168874 loss_cls: 0.0052381278946995735
+loss_reg: 0.01759285274776229 loss_cls: 0.0034208165016025305
+loss_reg: 0.010690071515280663 loss_cls: 0.0036722554359585047
+loss_reg: 0.010076800564421555 loss_cls: 0.003900547744706273
+loss_reg: 0.010574025011432932 loss_cls: 0.004665974993258715
+loss_reg: 0.018186121256652418 loss_cls: 0.0032005419488996267
+loss_reg: 0.012680793502672996 loss_cls: 0.0031559907365590334
+loss_reg: 0.007933183296575342 loss_cls: 0.0026460494846105576
+loss_reg: 0.02226356007328792 loss_cls: 0.0029250120278447866
+loss_reg: 0.016781901812098116 loss_cls: 0.005312762223184109
+loss_reg: 0.009590464384801323 loss_cls: 0.0035091645549982786
+loss_reg: 0.013568339989412884 loss_cls: 0.0024786086287349463
+loss_reg: 0.015775202380151062 loss_cls: 0.002998781856149435
+loss_reg: 0.012094462922784273 loss_cls: 0.0035288543440401554
+loss_reg: 0.013174907943231555 loss_cls: 0.003168631112203002
+loss_reg: 0.007991684268505158 loss_cls: 0.002687267493456602
+loss_reg: 0.011028426779056844 loss_cls: 0.0038372662384063005
+loss_reg: 0.012713617603672796 loss_cls: 0.004056405276060104
+loss_reg: 0.018438797454097287 loss_cls: 0.0030762795358896255
+loss_reg: 0.016204074614914292 loss_cls: 0.00368868769146502
+loss_reg: 0.01650741563730948 loss_cls: 0.002504548756405711
+loss_reg: 0.017962555503810658 loss_cls: 0.004098428413271904
+loss_reg: 0.004590075030426458 loss_cls: 0.0033059155102819204
+loss_reg: 0.009751376406582364 loss_cls: 0.0038041556254029274
+loss_reg: 0.016853120120576364 loss_cls: 0.004635250195860863
+loss_reg: 0.013542388517793921 loss_cls: 0.0032604229636490345
+2024-11-06 13:45:24 - progress_bar.py[line:274] - INFO: epoch 001:     10 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.599, ntokens=17535.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3313.9, ups=0.19, wpb=17535.3, bsz=1024, num_updates=10, lr=2.13129e-07, gnorm=0.008, clip=0, loss_scale=128, train_wall=41, gb_free=13.5, wall=243
+2024-11-06 13:46:16 - progress_bar.py[line:274] - INFO: epoch 001:     20 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.582, ntokens=17733.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3394.1, ups=0.19, wpb=17733.4, bsz=1024, num_updates=20, lr=4.26257e-07, gnorm=0.007, clip=0, loss_scale=128, train_wall=33, gb_free=13.5, wall=295
+2024-11-06 13:47:11 - progress_bar.py[line:274] - INFO: epoch 001:     30 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.553, ntokens=17420.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3192.4, ups=0.18, wpb=17420.7, bsz=1024, num_updates=30, lr=6.39386e-07, gnorm=0.004, clip=0, loss_scale=128, train_wall=33, gb_free=13.5, wall=350
+2024-11-06 13:48:04 - progress_bar.py[line:274] - INFO: epoch 001:     40 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.592, ntokens=17831.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3344.8, ups=0.19, wpb=17831.8, bsz=1024, num_updates=40, lr=8.52515e-07, gnorm=0.003, clip=0, loss_scale=128, train_wall=48, gb_free=13.5, wall=403
+2024-11-06 13:48:56 - progress_bar.py[line:274] - INFO: epoch 001:     50 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.588, ntokens=17628.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3375.2, ups=0.19, wpb=17628.1, bsz=1024, num_updates=50, lr=1.06564e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=50, gb_free=13.5, wall=455
+2024-11-06 13:49:48 - progress_bar.py[line:274] - INFO: epoch 001:     60 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.613, ntokens=17924.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3434.1, ups=0.19, wpb=17924.3, bsz=1024, num_updates=60, lr=1.27877e-06, gnorm=0.002, clip=0, loss_scale=128, train_wall=45, gb_free=13.5, wall=507
+2024-11-06 13:50:41 - progress_bar.py[line:274] - INFO: epoch 001:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=17717.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3351.4, ups=0.19, wpb=17717.1, bsz=1024, num_updates=70, lr=1.4919e-06, gnorm=0.002, clip=0, loss_scale=128, train_wall=39, gb_free=13.5, wall=560
+2024-11-06 13:51:34 - progress_bar.py[line:274] - INFO: epoch 001:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.574, ntokens=17601.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3345.9, ups=0.19, wpb=17601.8, bsz=1024, num_updates=80, lr=1.70503e-06, gnorm=0.002, clip=0, loss_scale=128, train_wall=39, gb_free=13.5, wall=613
+2024-11-06 13:52:27 - progress_bar.py[line:274] - INFO: epoch 001:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.591, ntokens=17624.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3343.7, ups=0.19, wpb=17624.4, bsz=1024, num_updates=90, lr=1.91816e-06, gnorm=0.002, clip=0, loss_scale=128, train_wall=35, gb_free=13.5, wall=666
+2024-11-06 13:53:20 - progress_bar.py[line:274] - INFO: epoch 001:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.575, ntokens=17657, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3312.4, ups=0.19, wpb=17657, bsz=1024, num_updates=100, lr=2.13129e-06, gnorm=0.002, clip=0, loss_scale=128, train_wall=40, gb_free=13.5, wall=719
+2024-11-06 13:54:13 - progress_bar.py[line:274] - INFO: epoch 001:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.57, ntokens=17488, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3316.5, ups=0.19, wpb=17488, bsz=1024, num_updates=110, lr=2.34442e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=39, gb_free=13.6, wall=772
+2024-11-06 13:55:06 - progress_bar.py[line:274] - INFO: epoch 001:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.595, ntokens=17657, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3313.7, ups=0.19, wpb=17657, bsz=1024, num_updates=120, lr=2.55754e-06, gnorm=0.002, clip=0, loss_scale=128, train_wall=36, gb_free=13.6, wall=825
+2024-11-06 13:55:59 - progress_bar.py[line:274] - INFO: epoch 001:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.59, ntokens=17663.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3300.2, ups=0.19, wpb=17663.1, bsz=1024, num_updates=130, lr=2.77067e-06, gnorm=0.002, clip=0, loss_scale=128, train_wall=44, gb_free=13.5, wall=878
+2024-11-06 13:56:52 - progress_bar.py[line:274] - INFO: epoch 001:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=17542.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3342.6, ups=0.19, wpb=17542.9, bsz=1024, num_updates=140, lr=2.9838e-06, gnorm=0.002, clip=0, loss_scale=128, train_wall=51, gb_free=13.6, wall=931
+2024-11-06 13:57:45 - progress_bar.py[line:274] - INFO: epoch 001:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.557, ntokens=17394.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3307.3, ups=0.19, wpb=17394.6, bsz=1024, num_updates=150, lr=3.19693e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=50, gb_free=13.4, wall=983
+2024-11-06 13:58:37 - progress_bar.py[line:274] - INFO: epoch 001:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.57, ntokens=17608.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3352.6, ups=0.19, wpb=17608.3, bsz=1024, num_updates=160, lr=3.41006e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=39, gb_free=13.5, wall=1036
+2024-11-06 13:59:30 - progress_bar.py[line:274] - INFO: epoch 001:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.624, ntokens=18153.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.17, wps=3415.5, ups=0.19, wpb=18153.7, bsz=1024, num_updates=170, lr=3.62319e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=1089
+2024-11-06 14:00:23 - progress_bar.py[line:274] - INFO: epoch 001:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.558, ntokens=17369.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3310.2, ups=0.19, wpb=17369.1, bsz=1024, num_updates=180, lr=3.83632e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=1142
+2024-11-06 14:01:16 - progress_bar.py[line:274] - INFO: epoch 001:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.615, ntokens=17980.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3392.1, ups=0.19, wpb=17980.9, bsz=1024, num_updates=190, lr=4.04945e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=1195
+2024-11-06 14:02:08 - progress_bar.py[line:274] - INFO: epoch 001:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.636, ntokens=17957.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.22, wps=3413, ups=0.19, wpb=17957.7, bsz=1024, num_updates=200, lr=4.26257e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.3, wall=1247
+2024-11-06 14:03:01 - progress_bar.py[line:274] - INFO: epoch 001:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.59, ntokens=17907.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3398.6, ups=0.19, wpb=17907.3, bsz=1024, num_updates=210, lr=4.4757e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.3, wall=1300
+2024-11-06 14:03:54 - progress_bar.py[line:274] - INFO: epoch 001:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.605, ntokens=17882.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3369.4, ups=0.19, wpb=17882.3, bsz=1024, num_updates=220, lr=4.68883e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=1353
+2024-11-06 14:04:47 - progress_bar.py[line:274] - INFO: epoch 001:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.596, ntokens=17841, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3389.1, ups=0.19, wpb=17841, bsz=1024, num_updates=230, lr=4.90196e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=1406
+2024-11-06 14:05:40 - progress_bar.py[line:274] - INFO: epoch 001:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.635, ntokens=18014.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3380.9, ups=0.19, wpb=18014.1, bsz=1024, num_updates=240, lr=5.11509e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=1459
+2024-11-06 14:06:34 - progress_bar.py[line:274] - INFO: epoch 001:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.581, ntokens=17552.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3258.9, ups=0.19, wpb=17552.9, bsz=1024, num_updates=250, lr=5.32822e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=33, gb_free=13.6, wall=1513
+2024-11-06 14:07:28 - progress_bar.py[line:274] - INFO: epoch 001:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.604, ntokens=17776.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3301, ups=0.19, wpb=17776.1, bsz=1024, num_updates=260, lr=5.54135e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=52, gb_free=13.6, wall=1567
+2024-11-06 14:08:20 - progress_bar.py[line:274] - INFO: epoch 001:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.573, ntokens=17647.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3365.8, ups=0.19, wpb=17647.3, bsz=1024, num_updates=270, lr=5.75448e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=52, gb_free=13.6, wall=1619
+2024-11-06 14:09:13 - progress_bar.py[line:274] - INFO: epoch 001:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.59, ntokens=17675.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3352.7, ups=0.19, wpb=17675.8, bsz=1024, num_updates=280, lr=5.9676e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=53, gb_free=13.5, wall=1672
+2024-11-06 14:10:05 - progress_bar.py[line:274] - INFO: epoch 001:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.588, ntokens=17688.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3373.2, ups=0.19, wpb=17688.6, bsz=1024, num_updates=290, lr=6.18073e-06, gnorm=0.006, clip=0, loss_scale=128, train_wall=47, gb_free=13.6, wall=1724
+2024-11-06 14:10:59 - progress_bar.py[line:274] - INFO: epoch 001:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.604, ntokens=17589.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3305.7, ups=0.19, wpb=17589.2, bsz=1024, num_updates=300, lr=6.39386e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=43, gb_free=13.4, wall=1777
+2024-11-06 14:11:51 - progress_bar.py[line:274] - INFO: epoch 001:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.782, ntokens=18811.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.88, wps=3556.4, ups=0.19, wpb=18811.6, bsz=1024, num_updates=310, lr=6.60699e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=48, gb_free=13.4, wall=1830
+2024-11-06 14:12:44 - progress_bar.py[line:274] - INFO: epoch 001:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.732, ntokens=18484.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.64, wps=3515.4, ups=0.19, wpb=18484.5, bsz=1024, num_updates=320, lr=6.82012e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=46, gb_free=13.4, wall=1883
+2024-11-06 14:13:36 - progress_bar.py[line:274] - INFO: epoch 001:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.707, ntokens=18256.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=3499.1, ups=0.19, wpb=18256.4, bsz=1024, num_updates=330, lr=7.03325e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=46, gb_free=13.4, wall=1935
+2024-11-06 14:14:29 - progress_bar.py[line:274] - INFO: epoch 001:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.689, ntokens=18254.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3484, ups=0.19, wpb=18254.4, bsz=1024, num_updates=340, lr=7.24638e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=50, gb_free=13.3, wall=1987
+2024-11-06 14:15:21 - progress_bar.py[line:274] - INFO: epoch 001:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.742, ntokens=18619.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3548.2, ups=0.19, wpb=18619.8, bsz=1024, num_updates=350, lr=7.45951e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=52, gb_free=13.3, wall=2040
+2024-11-06 14:16:14 - progress_bar.py[line:274] - INFO: epoch 001:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.768, ntokens=18682, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.81, wps=3546.4, ups=0.19, wpb=18682, bsz=1024, num_updates=360, lr=7.67263e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=53, gb_free=13.3, wall=2093
+2024-11-06 14:17:06 - progress_bar.py[line:274] - INFO: epoch 001:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=18394.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3494.2, ups=0.19, wpb=18394.6, bsz=1024, num_updates=370, lr=7.88576e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=53, gb_free=13.3, wall=2145
+2024-11-06 14:17:59 - progress_bar.py[line:274] - INFO: epoch 001:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.655, ntokens=17954.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3423.4, ups=0.19, wpb=17954.2, bsz=1024, num_updates=380, lr=8.09889e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=52, gb_free=13.4, wall=2198
+reach the end of datafile, start a new reader
+2024-11-06 14:18:51 - progress_bar.py[line:274] - INFO: epoch 001:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.677, ntokens=17998.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=3424.5, ups=0.19, wpb=17998.3, bsz=1024, num_updates=390, lr=8.31202e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=52, gb_free=13.5, wall=2250
 reach the end of datafile, start a new reader
-2024-11-03 06:07:49 - progress_bar.py[line:274] - INFO: epoch 001:    390 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.115, ntokens=17998.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=34.65, wps=3447.6, ups=0.19, wpb=17998.3, bsz=1024, num_updates=390, lr=8.31202e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=52, gb_free=13.5, wall=2246
-2024-11-03 06:07:51 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 1 @ 391 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
-2024-11-03 06:07:51 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt'cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt': No such file or directory: No such file or directory
+2024-11-06 14:18:53 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 1 @ 391 updates
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
+2024-11-06 14:18:53 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cp: cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt'cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt'cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt'cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt': No such file or directory: No such file or directory
 
-cp: cp: cp: cp: cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt'cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt'cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt'cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt'cp: : No such file or directory: No such file or directorycannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt': No such file or directory
-: No such file or directory
+: No such file or directory: No such file or directory
 
+cp: cp: cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt'cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt': No such file or directory: No such file or directory
 
-: No such file or directory
+cp: cannot stat './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt': No such file or directory
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 06:08:02 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-03 06:08:13 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 1 @ 391 updates, score 0) (writing took 22.016057339991676 seconds)
+2024-11-06 14:19:04 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+2024-11-06 14:19:15 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 1 @ 391 updates, score 0) (writing took 21.988907869003015 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_1.pt
-2024-11-03 06:08:18 - train.py[line:336] - INFO: end of epoch 1 (average epoch stats below)
-2024-11-03 06:08:18 - progress_bar.py[line:282] - INFO: epoch 001 | loss 0.006 | loss_v1 0 | loss_v2 0 | nll_loss 9.239 | ntokens 17842.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 604.16 | wps 3341.2 | ups 0.19 | wpb 17842.9 | bsz 1023 | num_updates 391 | lr 8.33333e-06 | gnorm 0.019 | clip 0 | loss_scale 128 | train_wall 1296 | gb_free 13.4 | wall 2275
-2024-11-03 06:08:18 - trainer.py[line:639] - INFO: loading train data for epoch 2
+2024-11-06 14:19:20 - train.py[line:336] - INFO: end of epoch 1 (average epoch stats below)
+2024-11-06 14:19:20 - progress_bar.py[line:282] - INFO: epoch 001 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.621 | ntokens 17842.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.15 | wps 3340.1 | ups 0.19 | wpb 17842.9 | bsz 1023 | num_updates 391 | lr 8.33333e-06 | gnorm 0.003 | clip 0 | loss_scale 128 | train_wall 1640 | gb_free 13.4 | wall 2279
+2024-11-06 14:19:20 - trainer.py[line:639] - INFO: loading train data for epoch 2
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
 
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 
-slice_id 7 seek offset 350000
-slice_id 3 seek offset 150000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
 
 
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 slice_id 2 seek offset 100000
 slice_id 6 seek offset 300000
+slice_id 3 seek offset 150000
+slice_id 7 seek offset 350000
 slice_id 5 seek offset 250000
+slice_id 4 seek offset 200000
+slice_id 1 seek offset 50000
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 06:11:03 - trainer.py[line:703] - INFO: begin training epoch 2
-2024-11-03 06:11:03 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 06:11:52 - progress_bar.py[line:274] - INFO: epoch 002:      9 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.059, ntokens=16920.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=33.34, wps=697.5, ups=0.04, wpb=16920.7, bsz=985.6, num_updates=400, lr=8.52515e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=29, gb_free=13.5, wall=2489
-2024-11-03 06:12:44 - progress_bar.py[line:274] - INFO: epoch 002:     19 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.072, ntokens=17781.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=33.64, wps=3411.8, ups=0.19, wpb=17781.7, bsz=1024, num_updates=410, lr=8.73828e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=34, gb_free=13.5, wall=2541
-2024-11-03 06:13:36 - progress_bar.py[line:274] - INFO: epoch 002:     29 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.005, ntokens=17664.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=32.11, wps=3378.2, ups=0.19, wpb=17664.5, bsz=1024, num_updates=420, lr=8.95141e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=31, gb_free=13.6, wall=2593
-2024-11-03 06:14:28 - progress_bar.py[line:274] - INFO: epoch 002:     39 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=5.029, ntokens=17781.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=32.65, wps=3413.8, ups=0.19, wpb=17781.4, bsz=1024, num_updates=430, lr=9.16454e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.3, wall=2646
-2024-11-03 06:15:21 - progress_bar.py[line:274] - INFO: epoch 002:     49 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.991, ntokens=17766.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=31.79, wps=3400.3, ups=0.19, wpb=17766.1, bsz=1024, num_updates=440, lr=9.37766e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=2698
-2024-11-03 06:16:13 - progress_bar.py[line:274] - INFO: epoch 002:     59 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.954, ntokens=17745.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=30.99, wps=3392.8, ups=0.19, wpb=17745.1, bsz=1024, num_updates=450, lr=9.59079e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=2750
-2024-11-03 06:17:05 - progress_bar.py[line:274] - INFO: epoch 002:     69 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.983, ntokens=17994.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=31.62, wps=3457.3, ups=0.19, wpb=17994.6, bsz=1024, num_updates=460, lr=9.80392e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=2802
-2024-11-03 06:17:57 - progress_bar.py[line:274] - INFO: epoch 002:     79 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.918, ntokens=17623, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=30.23, wps=3389, ups=0.19, wpb=17623, bsz=1024, num_updates=470, lr=1.00171e-05, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=2854
-2024-11-03 06:18:49 - progress_bar.py[line:274] - INFO: epoch 002:     89 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.949, ntokens=17736, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=30.88, wps=3405.2, ups=0.19, wpb=17736, bsz=1024, num_updates=480, lr=1.02302e-05, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=2906
-2024-11-03 06:19:41 - progress_bar.py[line:274] - INFO: epoch 002:     99 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.918, ntokens=17772.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=30.23, wps=3400.3, ups=0.19, wpb=17772.9, bsz=1024, num_updates=490, lr=1.04433e-05, gnorm=0.006, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=2959
-2024-11-03 06:20:34 - progress_bar.py[line:274] - INFO: epoch 002:    109 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.89, ntokens=17619.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=29.66, wps=3361.6, ups=0.19, wpb=17619.3, bsz=1024, num_updates=500, lr=1.06564e-05, gnorm=0.006, clip=0, loss_scale=128, train_wall=30, gb_free=13.3, wall=3011
-2024-11-03 06:21:26 - progress_bar.py[line:274] - INFO: epoch 002:    119 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.898, ntokens=17921.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=29.82, wps=3446, ups=0.19, wpb=17921.9, bsz=1024, num_updates=510, lr=1.08696e-05, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=3063
-2024-11-03 06:22:19 - progress_bar.py[line:274] - INFO: epoch 002:    129 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.825, ntokens=17451.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=28.34, wps=3261.1, ups=0.19, wpb=17451.5, bsz=1024, num_updates=520, lr=1.10827e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=36, gb_free=13.5, wall=3116
-2024-11-03 06:23:13 - progress_bar.py[line:274] - INFO: epoch 002:    139 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.865, ntokens=17567.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=29.15, wps=3290.9, ups=0.19, wpb=17567.8, bsz=1024, num_updates=530, lr=1.12958e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=53, gb_free=13.5, wall=3170
-2024-11-03 06:24:05 - progress_bar.py[line:274] - INFO: epoch 002:    149 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.838, ntokens=17618.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=28.61, wps=3377.1, ups=0.19, wpb=17618.6, bsz=1024, num_updates=540, lr=1.1509e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=52, gb_free=13.6, wall=3222
-2024-11-03 06:24:56 - progress_bar.py[line:274] - INFO: epoch 002:    159 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.842, ntokens=17768.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=28.68, wps=3447.7, ups=0.19, wpb=17768.3, bsz=1024, num_updates=550, lr=1.17221e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=50, gb_free=13.5, wall=3274
-2024-11-03 06:25:49 - progress_bar.py[line:274] - INFO: epoch 002:    169 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.851, ntokens=17867.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=28.86, wps=3426.4, ups=0.19, wpb=17867.5, bsz=1024, num_updates=560, lr=1.19352e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=49, gb_free=13.5, wall=3326
-2024-11-03 06:26:41 - progress_bar.py[line:274] - INFO: epoch 002:    179 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.78, ntokens=17460.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=27.47, wps=3326.4, ups=0.19, wpb=17460.5, bsz=1024, num_updates=570, lr=1.21483e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=52, gb_free=13.6, wall=3378
-2024-11-03 06:27:33 - progress_bar.py[line:274] - INFO: epoch 002:    189 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.803, ntokens=17606.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=27.92, wps=3391, ups=0.19, wpb=17606.1, bsz=1024, num_updates=580, lr=1.23615e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=52, gb_free=13.6, wall=3430
-2024-11-03 06:28:25 - progress_bar.py[line:274] - INFO: epoch 002:    199 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.895, ntokens=18257.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=29.74, wps=3523, ups=0.19, wpb=18257.9, bsz=1024, num_updates=590, lr=1.25746e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=52, gb_free=13.5, wall=3482
-2024-11-03 06:29:17 - progress_bar.py[line:274] - INFO: epoch 002:    209 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.777, ntokens=17891.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=27.42, wps=3429.3, ups=0.19, wpb=17891.7, bsz=1024, num_updates=600, lr=1.27877e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=52, gb_free=13.4, wall=3534
-2024-11-03 06:30:09 - progress_bar.py[line:274] - INFO: epoch 002:    219 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.791, ntokens=17928.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=27.69, wps=3417.5, ups=0.19, wpb=17928.4, bsz=1024, num_updates=610, lr=1.30009e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=52, gb_free=13.5, wall=3587
-2024-11-03 06:31:01 - progress_bar.py[line:274] - INFO: epoch 002:    229 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.737, ntokens=17493.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=26.67, wps=3364, ups=0.19, wpb=17493.7, bsz=1024, num_updates=620, lr=1.3214e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=50, gb_free=13.5, wall=3639
-2024-11-03 06:31:53 - progress_bar.py[line:274] - INFO: epoch 002:    239 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.735, ntokens=17502.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=26.63, wps=3380, ups=0.19, wpb=17502.6, bsz=1024, num_updates=630, lr=1.34271e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=44, gb_free=13.4, wall=3690
-2024-11-03 06:32:45 - progress_bar.py[line:274] - INFO: epoch 002:    249 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.741, ntokens=17517.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=26.74, wps=3368.4, ups=0.19, wpb=17517.1, bsz=1024, num_updates=640, lr=1.36402e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=40, gb_free=13.5, wall=3742
-2024-11-03 06:33:37 - progress_bar.py[line:274] - INFO: epoch 002:    259 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.695, ntokens=17444, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=25.91, wps=3361.9, ups=0.19, wpb=17444, bsz=1024, num_updates=650, lr=1.38534e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=32, gb_free=13.5, wall=3794
-2024-11-03 06:34:29 - progress_bar.py[line:274] - INFO: epoch 002:    269 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.724, ntokens=17774.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=26.44, wps=3403.8, ups=0.19, wpb=17774.7, bsz=1024, num_updates=660, lr=1.40665e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.2, wall=3846
-2024-11-03 06:35:21 - progress_bar.py[line:274] - INFO: epoch 002:    279 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.706, ntokens=17653.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=26.1, wps=3387.3, ups=0.19, wpb=17653.5, bsz=1024, num_updates=670, lr=1.42796e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.4, wall=3899
-2024-11-03 06:36:13 - progress_bar.py[line:274] - INFO: epoch 002:    289 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.676, ntokens=17454.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=25.56, wps=3358.7, ups=0.19, wpb=17454.1, bsz=1024, num_updates=680, lr=1.44928e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=3951
-2024-11-03 06:37:05 - progress_bar.py[line:274] - INFO: epoch 002:    299 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.716, ntokens=17843.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=26.29, wps=3460.4, ups=0.19, wpb=17843.2, bsz=1024, num_updates=690, lr=1.47059e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=4002
-2024-11-03 06:37:56 - progress_bar.py[line:274] - INFO: epoch 002:    309 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.785, ntokens=18559.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=27.57, wps=3613.8, ups=0.19, wpb=18559.6, bsz=1024, num_updates=700, lr=1.4919e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.2, wall=4053
-2024-11-03 06:38:48 - progress_bar.py[line:274] - INFO: epoch 002:    319 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.76, ntokens=18438.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=27.1, wps=3564.3, ups=0.19, wpb=18438.7, bsz=1024, num_updates=710, lr=1.51321e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=31, gb_free=13.5, wall=4105
-2024-11-03 06:39:40 - progress_bar.py[line:274] - INFO: epoch 002:    329 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.698, ntokens=18044.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=25.95, wps=3463.2, ups=0.19, wpb=18044.4, bsz=1024, num_updates=720, lr=1.53453e-05, gnorm=0.007, clip=0, loss_scale=256, train_wall=33, gb_free=13.4, wall=4157
-2024-11-03 06:40:32 - progress_bar.py[line:274] - INFO: epoch 002:    339 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.709, ntokens=18149.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=26.16, wps=3487.7, ups=0.19, wpb=18149.8, bsz=1024, num_updates=730, lr=1.55584e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=42, gb_free=13.4, wall=4209
-2024-11-03 06:41:25 - progress_bar.py[line:274] - INFO: epoch 002:    349 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.729, ntokens=18508.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=26.53, wps=3533.3, ups=0.19, wpb=18508.1, bsz=1024, num_updates=740, lr=1.57715e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=50, gb_free=13.4, wall=4262
-2024-11-03 06:42:17 - progress_bar.py[line:274] - INFO: epoch 002:    359 / 391 loss=0.003, loss_v1=0, loss_v2=0, nll_loss=4.67, ntokens=18280.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=25.46, wps=3504.8, ups=0.19, wpb=18280.9, bsz=1024, num_updates=750, lr=1.59847e-05, gnorm=0.007, clip=0, loss_scale=256, train_wall=52, gb_free=13.4, wall=4314
-2024-11-03 06:43:09 - progress_bar.py[line:274] - INFO: epoch 002:    369 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.67, ntokens=18018.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=25.46, wps=3438.5, ups=0.19, wpb=18018.9, bsz=1024, num_updates=760, lr=1.61978e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=52, gb_free=13.5, wall=4366
-2024-11-03 06:44:04 - progress_bar.py[line:274] - INFO: epoch 002:    379 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.626, ntokens=17930.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=24.68, wps=3267.3, ups=0.18, wpb=17930.3, bsz=1024, num_updates=770, lr=1.64109e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=55, gb_free=13.5, wall=4421
-2024-11-03 06:44:57 - progress_bar.py[line:274] - INFO: epoch 002:    389 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.586, ntokens=17700.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=24.02, wps=3321.9, ups=0.19, wpb=17700.9, bsz=1024, num_updates=780, lr=1.6624e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=53, gb_free=13.5, wall=4474
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
-2024-11-03 06:45:06 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 2 @ 782 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
-2024-11-03 06:45:06 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
+2024-11-06 14:21:58 - trainer.py[line:703] - INFO: begin training epoch 2
+2024-11-06 14:21:58 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 14:22:47 - progress_bar.py[line:274] - INFO: epoch 002:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.589, ntokens=16920.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=718.3, ups=0.04, wpb=16920.7, bsz=985.6, num_updates=400, lr=8.52515e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=29, gb_free=13.5, wall=2486
+2024-11-06 14:23:39 - progress_bar.py[line:274] - INFO: epoch 002:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.58, ntokens=17781.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3409, ups=0.19, wpb=17781.7, bsz=1024, num_updates=410, lr=8.73828e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=31, gb_free=13.5, wall=2538
+2024-11-06 14:24:31 - progress_bar.py[line:274] - INFO: epoch 002:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.571, ntokens=17664.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3398.3, ups=0.19, wpb=17664.5, bsz=1024, num_updates=420, lr=8.95141e-06, gnorm=0.003, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=2590
+2024-11-06 14:25:23 - progress_bar.py[line:274] - INFO: epoch 002:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.58, ntokens=17781.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3399.1, ups=0.19, wpb=17781.4, bsz=1024, num_updates=430, lr=9.16454e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.3, wall=2642
+2024-11-06 14:26:16 - progress_bar.py[line:274] - INFO: epoch 002:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.604, ntokens=17766.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3384.9, ups=0.19, wpb=17766.1, bsz=1024, num_updates=440, lr=9.37766e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=2695
+2024-11-06 14:27:08 - progress_bar.py[line:274] - INFO: epoch 002:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.583, ntokens=17745.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3422.6, ups=0.19, wpb=17745.1, bsz=1024, num_updates=450, lr=9.59079e-06, gnorm=0.004, clip=0, loss_scale=128, train_wall=30, gb_free=13.6, wall=2747
+2024-11-06 14:28:00 - progress_bar.py[line:274] - INFO: epoch 002:     69 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.648, ntokens=17994.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3428.4, ups=0.19, wpb=17994.6, bsz=1024, num_updates=460, lr=9.80392e-06, gnorm=0.005, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=2799
+2024-11-06 14:28:52 - progress_bar.py[line:274] - INFO: epoch 002:     79 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.577, ntokens=17623, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3381.6, ups=0.19, wpb=17623, bsz=1024, num_updates=470, lr=1.00171e-05, gnorm=0.006, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=2851
+2024-11-06 14:29:44 - progress_bar.py[line:274] - INFO: epoch 002:     89 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.601, ntokens=17736, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3411.6, ups=0.19, wpb=17736, bsz=1024, num_updates=480, lr=1.02302e-05, gnorm=0.008, clip=0, loss_scale=128, train_wall=30, gb_free=13.5, wall=2903
+2024-11-06 14:30:37 - progress_bar.py[line:274] - INFO: epoch 002:     99 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.591, ntokens=17772.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3405.1, ups=0.19, wpb=17772.9, bsz=1024, num_updates=490, lr=1.04433e-05, gnorm=0.007, clip=0, loss_scale=128, train_wall=31, gb_free=13.4, wall=2955
+2024-11-06 14:31:29 - progress_bar.py[line:274] - INFO: epoch 002:    109 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.614, ntokens=17619.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3343.4, ups=0.19, wpb=17619.3, bsz=1024, num_updates=500, lr=1.06564e-05, gnorm=0.007, clip=0, loss_scale=128, train_wall=30, gb_free=13.3, wall=3008
+2024-11-06 14:32:22 - progress_bar.py[line:274] - INFO: epoch 002:    119 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.631, ntokens=17921.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3376.6, ups=0.19, wpb=17921.9, bsz=1024, num_updates=510, lr=1.08696e-05, gnorm=0.007, clip=0, loss_scale=128, train_wall=30, gb_free=13.4, wall=3061
+2024-11-06 14:33:14 - progress_bar.py[line:274] - INFO: epoch 002:    129 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.57, ntokens=17451.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3354, ups=0.19, wpb=17451.5, bsz=1024, num_updates=520, lr=1.10827e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=3113
+2024-11-06 14:34:06 - progress_bar.py[line:274] - INFO: epoch 002:    139 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.585, ntokens=17567.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3400, ups=0.19, wpb=17567.8, bsz=1024, num_updates=530, lr=1.12958e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=3165
+2024-11-06 14:35:02 - progress_bar.py[line:274] - INFO: epoch 002:    149 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.591, ntokens=17618.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3136, ups=0.18, wpb=17618.6, bsz=1024, num_updates=540, lr=1.1509e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=31, gb_free=13.5, wall=3221
+2024-11-06 14:35:58 - progress_bar.py[line:274] - INFO: epoch 002:    159 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.579, ntokens=17768.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3165.9, ups=0.18, wpb=17768.3, bsz=1024, num_updates=550, lr=1.17221e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=31, gb_free=13.5, wall=3277
+2024-11-06 14:36:51 - progress_bar.py[line:274] - INFO: epoch 002:    169 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.603, ntokens=17867.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3370.3, ups=0.19, wpb=17867.5, bsz=1024, num_updates=560, lr=1.19352e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=3330
+2024-11-06 14:37:44 - progress_bar.py[line:274] - INFO: epoch 002:    179 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.569, ntokens=17460.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3300.3, ups=0.19, wpb=17460.5, bsz=1024, num_updates=570, lr=1.21483e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.6, wall=3383
+2024-11-06 14:38:36 - progress_bar.py[line:274] - INFO: epoch 002:    189 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.57, ntokens=17606.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3391.3, ups=0.19, wpb=17606.1, bsz=1024, num_updates=580, lr=1.23615e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.6, wall=3435
+2024-11-06 14:39:28 - progress_bar.py[line:274] - INFO: epoch 002:    199 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.669, ntokens=18257.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3522.3, ups=0.19, wpb=18257.9, bsz=1024, num_updates=590, lr=1.25746e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=3487
+2024-11-06 14:40:20 - progress_bar.py[line:274] - INFO: epoch 002:    209 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.566, ntokens=17891.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3414.9, ups=0.19, wpb=17891.7, bsz=1024, num_updates=600, lr=1.27877e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=32, gb_free=13.4, wall=3539
+2024-11-06 14:41:14 - progress_bar.py[line:274] - INFO: epoch 002:    219 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.623, ntokens=17928.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3360.8, ups=0.19, wpb=17928.4, bsz=1024, num_updates=610, lr=1.30009e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=42, gb_free=13.4, wall=3593
+2024-11-06 14:42:05 - progress_bar.py[line:274] - INFO: epoch 002:    229 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.549, ntokens=17493.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=3381.4, ups=0.19, wpb=17493.7, bsz=1024, num_updates=620, lr=1.3214e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=47, gb_free=13.5, wall=3644
+2024-11-06 14:42:57 - progress_bar.py[line:274] - INFO: epoch 002:    239 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.57, ntokens=17502.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3393.4, ups=0.19, wpb=17502.6, bsz=1024, num_updates=630, lr=1.34271e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=35, gb_free=13.4, wall=3696
+2024-11-06 14:43:50 - progress_bar.py[line:274] - INFO: epoch 002:    249 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.583, ntokens=17517.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3323.3, ups=0.19, wpb=17517.1, bsz=1024, num_updates=640, lr=1.36402e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=34, gb_free=13.5, wall=3749
+2024-11-06 14:44:45 - progress_bar.py[line:274] - INFO: epoch 002:    259 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.569, ntokens=17444, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3155.7, ups=0.18, wpb=17444, bsz=1024, num_updates=650, lr=1.38534e-05, gnorm=0.007, clip=0, loss_scale=256, train_wall=50, gb_free=13.5, wall=3804
+2024-11-06 14:45:39 - progress_bar.py[line:274] - INFO: epoch 002:    269 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.599, ntokens=17774.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3288.6, ups=0.19, wpb=17774.7, bsz=1024, num_updates=660, lr=1.40665e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=54, gb_free=13.2, wall=3858
+2024-11-06 14:46:32 - progress_bar.py[line:274] - INFO: epoch 002:    279 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.585, ntokens=17653.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3352.5, ups=0.19, wpb=17653.5, bsz=1024, num_updates=670, lr=1.42796e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=53, gb_free=13.4, wall=3911
+2024-11-06 14:47:24 - progress_bar.py[line:274] - INFO: epoch 002:    289 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.551, ntokens=17454.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.86, wps=3340.2, ups=0.19, wpb=17454.1, bsz=1024, num_updates=680, lr=1.44928e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=52, gb_free=13.4, wall=3963
+2024-11-06 14:48:16 - progress_bar.py[line:274] - INFO: epoch 002:    299 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.644, ntokens=17843.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3411.2, ups=0.19, wpb=17843.2, bsz=1024, num_updates=690, lr=1.47059e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=52, gb_free=13.5, wall=4015
+2024-11-06 14:49:08 - progress_bar.py[line:274] - INFO: epoch 002:    309 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.756, ntokens=18559.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.76, wps=3574.3, ups=0.19, wpb=18559.6, bsz=1024, num_updates=700, lr=1.4919e-05, gnorm=0.003, clip=0, loss_scale=256, train_wall=52, gb_free=13.2, wall=4067
+2024-11-06 14:50:00 - progress_bar.py[line:274] - INFO: epoch 002:    319 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.725, ntokens=18438.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.61, wps=3570.9, ups=0.19, wpb=18438.7, bsz=1024, num_updates=710, lr=1.51321e-05, gnorm=0.003, clip=0, loss_scale=256, train_wall=52, gb_free=13.5, wall=4119
+2024-11-06 14:50:52 - progress_bar.py[line:274] - INFO: epoch 002:    329 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.677, ntokens=18044.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.4, wps=3482.1, ups=0.19, wpb=18044.4, bsz=1024, num_updates=720, lr=1.53453e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=52, gb_free=13.4, wall=4171
+2024-11-06 14:51:44 - progress_bar.py[line:274] - INFO: epoch 002:    339 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.675, ntokens=18149.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=3473.4, ups=0.19, wpb=18149.8, bsz=1024, num_updates=730, lr=1.55584e-05, gnorm=0.003, clip=0, loss_scale=256, train_wall=52, gb_free=13.4, wall=4223
+2024-11-06 14:52:37 - progress_bar.py[line:274] - INFO: epoch 002:    349 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.713, ntokens=18508.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.56, wps=3502.2, ups=0.19, wpb=18508.1, bsz=1024, num_updates=740, lr=1.57715e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=53, gb_free=13.3, wall=4276
+2024-11-06 14:53:29 - progress_bar.py[line:274] - INFO: epoch 002:    359 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=18280.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3503.5, ups=0.19, wpb=18280.9, bsz=1024, num_updates=750, lr=1.59847e-05, gnorm=0.003, clip=0, loss_scale=256, train_wall=52, gb_free=13.4, wall=4328
+2024-11-06 14:54:21 - progress_bar.py[line:274] - INFO: epoch 002:    369 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.686, ntokens=18018.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3483.5, ups=0.19, wpb=18018.9, bsz=1024, num_updates=760, lr=1.61978e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=52, gb_free=13.5, wall=4380
+2024-11-06 14:55:12 - progress_bar.py[line:274] - INFO: epoch 002:    379 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.64, ntokens=17930.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.23, wps=3469.3, ups=0.19, wpb=17930.3, bsz=1024, num_updates=770, lr=1.64109e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=52, gb_free=13.5, wall=4431
+2024-11-06 14:56:04 - progress_bar.py[line:274] - INFO: epoch 002:    389 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=17700.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3420.7, ups=0.19, wpb=17700.9, bsz=1024, num_updates=780, lr=1.6624e-05, gnorm=0.003, clip=0, loss_scale=256, train_wall=52, gb_free=13.5, wall=4483
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 14:56:11 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 2 @ 782 updates
+2024-11-06 14:56:11 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt': File exists: File exists
+: File exists
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 06:46:58 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-03 06:48:49 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 2 @ 782 updates, score 0) (writing took 223.95740796698374 seconds)
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 14:58:01 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+
+
+slice_id 2 seek offset 100000
+slice_id 3 seek offset 150000
+slice_id 6 seek offset 300000
+2024-11-06 14:59:52 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 2 @ 782 updates, score 0) (writing took 221.05887915899802 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_2.pt
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 slice_id 5 seek offset 250000
-slice_id 6 seek offset 300000
-slice_id 2 seek offset 100000
-slice_id 3 seek offset 150000
-2024-11-03 06:49:44 - train.py[line:336] - INFO: end of epoch 2 (average epoch stats below)
-2024-11-03 06:49:44 - progress_bar.py[line:282] - INFO: epoch 002 | loss 0.003 | loss_v1 0 | loss_v2 0 | nll_loss 4.816 | ntokens 17795.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 28.17 | wps 2798.5 | ups 0.16 | wpb 17795.9 | bsz 1023 | num_updates 782 | lr 1.66667e-05 | gnorm 0.005 | clip 0 | loss_scale 256 | train_wall 1557 | gb_free 13.4 | wall 4761
-2024-11-03 06:49:44 - trainer.py[line:639] - INFO: loading train data for epoch 3
+slice_id 4 seek offset 200000
+slice_id 1 seek offset 50000
+2024-11-06 15:00:47 - train.py[line:336] - INFO: end of epoch 2 (average epoch stats below)
+2024-11-06 15:00:47 - progress_bar.py[line:282] - INFO: epoch 002 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.615 | ntokens 17795.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.13 | wps 2798.1 | ups 0.16 | wpb 17795.9 | bsz 1023 | num_updates 782 | lr 1.66667e-05 | gnorm 0.005 | clip 0 | loss_scale 256 | train_wall 1529 | gb_free 13.4 | wall 4765
+2024-11-06 15:00:47 - trainer.py[line:639] - INFO: loading train data for epoch 3
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 06:52:29 - trainer.py[line:703] - INFO: begin training epoch 3
-2024-11-03 06:52:29 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 06:53:15 - progress_bar.py[line:274] - INFO: epoch 003:      8 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.552, ntokens=16694.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=23.46, wps=335.2, ups=0.02, wpb=16694.7, bsz=985.6, num_updates=790, lr=1.68372e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=42, gb_free=13.4, wall=4972
-2024-11-03 06:54:09 - progress_bar.py[line:274] - INFO: epoch 003:     18 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.636, ntokens=17901.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=24.87, wps=3318.4, ups=0.19, wpb=17901.5, bsz=1024, num_updates=800, lr=1.70503e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=43, gb_free=13.5, wall=5026
-2024-11-03 06:55:03 - progress_bar.py[line:274] - INFO: epoch 003:     28 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.57, ntokens=17881.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=23.75, wps=3302.1, ups=0.18, wpb=17881.7, bsz=1024, num_updates=810, lr=1.72634e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=43, gb_free=13.4, wall=5081
-2024-11-03 06:55:58 - progress_bar.py[line:274] - INFO: epoch 003:     38 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.597, ntokens=17912.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=24.2, wps=3303.8, ups=0.18, wpb=17912.9, bsz=1024, num_updates=820, lr=1.74766e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=42, gb_free=13.4, wall=5135
-2024-11-03 06:56:51 - progress_bar.py[line:274] - INFO: epoch 003:     48 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.583, ntokens=17721, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=23.97, wps=3306.1, ups=0.19, wpb=17721, bsz=1024, num_updates=830, lr=1.76897e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=43, gb_free=13.5, wall=5188
-2024-11-03 06:57:45 - progress_bar.py[line:274] - INFO: epoch 003:     58 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.567, ntokens=17823.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=23.7, wps=3330.6, ups=0.19, wpb=17823.4, bsz=1024, num_updates=840, lr=1.79028e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=42, gb_free=13.5, wall=5242
-2024-11-03 06:58:39 - progress_bar.py[line:274] - INFO: epoch 003:     68 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.495, ntokens=17643.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=22.55, wps=3248.6, ups=0.18, wpb=17643.4, bsz=1024, num_updates=850, lr=1.81159e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=43, gb_free=13.5, wall=5296
-2024-11-03 06:59:33 - progress_bar.py[line:274] - INFO: epoch 003:     78 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.563, ntokens=17884.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=23.63, wps=3327.3, ups=0.19, wpb=17884.5, bsz=1024, num_updates=860, lr=1.83291e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=43, gb_free=13.4, wall=5350
-2024-11-03 07:00:27 - progress_bar.py[line:274] - INFO: epoch 003:     88 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.442, ntokens=17202.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.74, wps=3204.7, ups=0.19, wpb=17202.1, bsz=1024, num_updates=870, lr=1.85422e-05, gnorm=0.008, clip=0, loss_scale=256, train_wall=43, gb_free=13.6, wall=5404
-2024-11-03 07:01:20 - progress_bar.py[line:274] - INFO: epoch 003:     98 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.498, ntokens=17716, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=22.59, wps=3299.7, ups=0.19, wpb=17716, bsz=1024, num_updates=880, lr=1.87553e-05, gnorm=0.009, clip=0, loss_scale=256, train_wall=41, gb_free=13.5, wall=5457
-2024-11-03 07:02:17 - progress_bar.py[line:274] - INFO: epoch 003:    108 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.444, ntokens=17544.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.76, wps=3092, ups=0.18, wpb=17544.8, bsz=1024, num_updates=890, lr=1.89685e-05, gnorm=0.007, clip=0, loss_scale=256, train_wall=52, gb_free=13.5, wall=5514
-2024-11-03 07:03:14 - progress_bar.py[line:274] - INFO: epoch 003:    118 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.537, ntokens=17929.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=23.22, wps=3143.6, ups=0.18, wpb=17929.2, bsz=1024, num_updates=900, lr=1.91816e-05, gnorm=0.009, clip=0, loss_scale=256, train_wall=57, gb_free=13.5, wall=5571
-2024-11-03 07:04:11 - progress_bar.py[line:274] - INFO: epoch 003:    128 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.419, ntokens=17466, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.39, wps=3084.4, ups=0.18, wpb=17466, bsz=1024, num_updates=910, lr=1.93947e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=57, gb_free=13.5, wall=5628
-2024-11-03 07:05:07 - progress_bar.py[line:274] - INFO: epoch 003:    138 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.401, ntokens=17273.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.12, wps=3042.9, ups=0.18, wpb=17273.5, bsz=1024, num_updates=920, lr=1.96078e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=57, gb_free=13.5, wall=5685
-2024-11-03 07:06:04 - progress_bar.py[line:274] - INFO: epoch 003:    148 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.428, ntokens=17656.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.53, wps=3120.1, ups=0.18, wpb=17656.9, bsz=1024, num_updates=930, lr=1.9821e-05, gnorm=0.007, clip=0, loss_scale=256, train_wall=56, gb_free=13.4, wall=5741
-2024-11-03 07:07:00 - progress_bar.py[line:274] - INFO: epoch 003:    158 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.465, ntokens=17831.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=22.08, wps=3208.7, ups=0.18, wpb=17831.6, bsz=1024, num_updates=940, lr=2.00341e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=55, gb_free=13.5, wall=5797
-2024-11-03 07:07:55 - progress_bar.py[line:274] - INFO: epoch 003:    168 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.456, ntokens=17982.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.94, wps=3241.4, ups=0.18, wpb=17982.9, bsz=1024, num_updates=950, lr=2.02472e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=55, gb_free=13.4, wall=5852
-2024-11-03 07:08:52 - progress_bar.py[line:274] - INFO: epoch 003:    178 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.372, ntokens=17181.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.71, wps=3034.4, ups=0.18, wpb=17181.7, bsz=1024, num_updates=960, lr=2.04604e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=57, gb_free=13.6, wall=5909
-2024-11-03 07:09:48 - progress_bar.py[line:274] - INFO: epoch 003:    188 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.416, ntokens=17835.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.35, wps=3140.9, ups=0.18, wpb=17835.7, bsz=1024, num_updates=970, lr=2.06735e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=57, gb_free=13.4, wall=5966
-2024-11-03 07:10:43 - progress_bar.py[line:274] - INFO: epoch 003:    198 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.378, ntokens=17554.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.8, wps=3192.1, ups=0.18, wpb=17554.7, bsz=1024, num_updates=980, lr=2.08866e-05, gnorm=0.009, clip=0, loss_scale=256, train_wall=55, gb_free=13.6, wall=6021
-2024-11-03 07:11:39 - progress_bar.py[line:274] - INFO: epoch 003:    208 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.446, ntokens=17918.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.8, wps=3228.8, ups=0.18, wpb=17918.2, bsz=1024, num_updates=990, lr=2.10997e-05, gnorm=0.007, clip=0, loss_scale=256, train_wall=55, gb_free=13.6, wall=6076
-2024-11-03 07:12:35 - progress_bar.py[line:274] - INFO: epoch 003:    218 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.329, ntokens=17536.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.1, wps=3128.5, ups=0.18, wpb=17536.7, bsz=1024, num_updates=1000, lr=2.13129e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=56, gb_free=13.6, wall=6132
-2024-11-03 07:13:33 - progress_bar.py[line:274] - INFO: epoch 003:    228 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.423, ntokens=17771.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.45, wps=3081.1, ups=0.17, wpb=17771.9, bsz=1024, num_updates=1010, lr=2.1526e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=58, gb_free=13.5, wall=6190
-2024-11-03 07:14:30 - progress_bar.py[line:274] - INFO: epoch 003:    238 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.333, ntokens=17462.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.15, wps=3026.9, ups=0.17, wpb=17462.8, bsz=1024, num_updates=1020, lr=2.17391e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=58, gb_free=13.5, wall=6248
-2024-11-03 07:15:27 - progress_bar.py[line:274] - INFO: epoch 003:    248 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.397, ntokens=17702.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.07, wps=3112.4, ups=0.18, wpb=17702.6, bsz=1024, num_updates=1030, lr=2.19523e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=57, gb_free=13.5, wall=6304
-2024-11-03 07:16:23 - progress_bar.py[line:274] - INFO: epoch 003:    258 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.269, ntokens=17357.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=19.28, wps=3101.9, ups=0.18, wpb=17357.6, bsz=1024, num_updates=1040, lr=2.21654e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=56, gb_free=13.5, wall=6360
-2024-11-03 07:17:20 - progress_bar.py[line:274] - INFO: epoch 003:    268 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.447, ntokens=18249.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.81, wps=3189.3, ups=0.17, wpb=18249.6, bsz=1024, num_updates=1050, lr=2.23785e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=57, gb_free=13.5, wall=6418
-2024-11-03 07:18:16 - progress_bar.py[line:274] - INFO: epoch 003:    278 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.333, ntokens=17764.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.16, wps=3200.4, ups=0.18, wpb=17764.1, bsz=1024, num_updates=1060, lr=2.25916e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=55, gb_free=13.4, wall=6473
-2024-11-03 07:19:11 - progress_bar.py[line:274] - INFO: epoch 003:    288 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.296, ntokens=17227.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=19.64, wps=3121.9, ups=0.18, wpb=17227.8, bsz=1024, num_updates=1070, lr=2.28048e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=55, gb_free=13.6, wall=6528
-2024-11-03 07:20:07 - progress_bar.py[line:274] - INFO: epoch 003:    298 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.324, ntokens=17898.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.03, wps=3214.1, ups=0.18, wpb=17898.3, bsz=1024, num_updates=1080, lr=2.30179e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=56, gb_free=13.4, wall=6584
-2024-11-03 07:21:01 - progress_bar.py[line:274] - INFO: epoch 003:    308 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.415, ntokens=18402.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=21.33, wps=3369.1, ups=0.18, wpb=18402.7, bsz=1024, num_updates=1090, lr=2.3231e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=55, gb_free=13.3, wall=6639
-2024-11-03 07:21:58 - progress_bar.py[line:274] - INFO: epoch 003:    318 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.369, ntokens=18312.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.66, wps=3239.2, ups=0.18, wpb=18312.9, bsz=1024, num_updates=1100, lr=2.34442e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=56, gb_free=13.4, wall=6695
-2024-11-03 07:22:54 - progress_bar.py[line:274] - INFO: epoch 003:    328 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.345, ntokens=18118.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.33, wps=3209.5, ups=0.18, wpb=18118.6, bsz=1024, num_updates=1110, lr=2.36573e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=56, gb_free=13.4, wall=6752
-2024-11-03 07:23:51 - progress_bar.py[line:274] - INFO: epoch 003:    338 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.361, ntokens=18372.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.55, wps=3246.8, ups=0.18, wpb=18372.7, bsz=1024, num_updates=1120, lr=2.38704e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=56, gb_free=13.4, wall=6808
-2024-11-03 07:24:52 - progress_bar.py[line:274] - INFO: epoch 003:    348 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.365, ntokens=18510.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.6, wps=3021.6, ups=0.16, wpb=18510.1, bsz=1024, num_updates=1130, lr=2.40835e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=61, gb_free=13.2, wall=6869
-2024-11-03 07:25:58 - progress_bar.py[line:274] - INFO: epoch 003:    358 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.309, ntokens=18306.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=19.82, wps=2782.1, ups=0.15, wpb=18306.6, bsz=1024, num_updates=1140, lr=2.42967e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=66, gb_free=13.2, wall=6935
-2024-11-03 07:27:04 - progress_bar.py[line:274] - INFO: epoch 003:    368 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.333, ntokens=18037.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=20.16, wps=2741.6, ups=0.15, wpb=18037.1, bsz=1024, num_updates=1150, lr=2.45098e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=66, gb_free=13.4, wall=7001
-2024-11-03 07:28:10 - progress_bar.py[line:274] - INFO: epoch 003:    378 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.248, ntokens=17982.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=19.01, wps=2729.9, ups=0.15, wpb=17982.3, bsz=1024, num_updates=1160, lr=2.47229e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=66, gb_free=13.3, wall=7067
-2024-11-03 07:29:16 - progress_bar.py[line:274] - INFO: epoch 003:    388 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.273, ntokens=17965.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=19.34, wps=2728.3, ups=0.15, wpb=17965.8, bsz=1024, num_updates=1170, lr=2.49361e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=66, gb_free=13.4, wall=7133
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
-2024-11-03 07:29:33 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 3 @ 1173 updates
+2024-11-06 15:03:25 - trainer.py[line:703] - INFO: begin training epoch 3
+2024-11-06 15:03:25 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 15:04:09 - progress_bar.py[line:274] - INFO: epoch 003:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.546, ntokens=16694.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=344.3, ups=0.02, wpb=16694.7, bsz=985.6, num_updates=790, lr=1.68372e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=31, gb_free=13.4, wall=4968
+2024-11-06 15:05:01 - progress_bar.py[line:274] - INFO: epoch 003:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.614, ntokens=17901.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3414.2, ups=0.19, wpb=17901.5, bsz=1024, num_updates=800, lr=1.70503e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5020
+2024-11-06 15:05:53 - progress_bar.py[line:274] - INFO: epoch 003:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.615, ntokens=17881.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3463.3, ups=0.19, wpb=17881.7, bsz=1024, num_updates=810, lr=1.72634e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.4, wall=5072
+2024-11-06 15:06:46 - progress_bar.py[line:274] - INFO: epoch 003:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.612, ntokens=17912.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3386.3, ups=0.19, wpb=17912.9, bsz=1024, num_updates=820, lr=1.74766e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.4, wall=5125
+2024-11-06 15:07:38 - progress_bar.py[line:274] - INFO: epoch 003:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.592, ntokens=17721, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3407.6, ups=0.19, wpb=17721, bsz=1024, num_updates=830, lr=1.76897e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5177
+2024-11-06 15:08:30 - progress_bar.py[line:274] - INFO: epoch 003:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.608, ntokens=17823.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3417, ups=0.19, wpb=17823.4, bsz=1024, num_updates=840, lr=1.79028e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5229
+2024-11-06 15:09:23 - progress_bar.py[line:274] - INFO: epoch 003:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.608, ntokens=17643.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3361.1, ups=0.19, wpb=17643.4, bsz=1024, num_updates=850, lr=1.81159e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5281
+2024-11-06 15:10:15 - progress_bar.py[line:274] - INFO: epoch 003:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.601, ntokens=17884.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3418, ups=0.19, wpb=17884.5, bsz=1024, num_updates=860, lr=1.83291e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.4, wall=5334
+2024-11-06 15:11:08 - progress_bar.py[line:274] - INFO: epoch 003:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.534, ntokens=17202.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3252.9, ups=0.19, wpb=17202.1, bsz=1024, num_updates=870, lr=1.85422e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.6, wall=5387
+2024-11-06 15:12:00 - progress_bar.py[line:274] - INFO: epoch 003:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.589, ntokens=17716, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3371.5, ups=0.19, wpb=17716, bsz=1024, num_updates=880, lr=1.87553e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5439
+2024-11-06 15:12:53 - progress_bar.py[line:274] - INFO: epoch 003:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.594, ntokens=17544.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3354.7, ups=0.19, wpb=17544.8, bsz=1024, num_updates=890, lr=1.89685e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5492
+2024-11-06 15:13:45 - progress_bar.py[line:274] - INFO: epoch 003:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.64, ntokens=17929.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.23, wps=3410.5, ups=0.19, wpb=17929.2, bsz=1024, num_updates=900, lr=1.91816e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5544
+2024-11-06 15:14:38 - progress_bar.py[line:274] - INFO: epoch 003:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.555, ntokens=17466, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3307.3, ups=0.19, wpb=17466, bsz=1024, num_updates=910, lr=1.93947e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5597
+2024-11-06 15:15:30 - progress_bar.py[line:274] - INFO: epoch 003:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.545, ntokens=17273.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3306.4, ups=0.19, wpb=17273.5, bsz=1024, num_updates=920, lr=1.96078e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5649
+2024-11-06 15:16:22 - progress_bar.py[line:274] - INFO: epoch 003:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.587, ntokens=17656.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3424.5, ups=0.19, wpb=17656.9, bsz=1024, num_updates=930, lr=1.9821e-05, gnorm=0.003, clip=0, loss_scale=256, train_wall=30, gb_free=13.4, wall=5701
+2024-11-06 15:17:13 - progress_bar.py[line:274] - INFO: epoch 003:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.601, ntokens=17831.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3455, ups=0.19, wpb=17831.6, bsz=1024, num_updates=940, lr=2.00341e-05, gnorm=0.003, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=5752
+2024-11-06 15:18:06 - progress_bar.py[line:274] - INFO: epoch 003:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.612, ntokens=17982.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3452.8, ups=0.19, wpb=17982.9, bsz=1024, num_updates=950, lr=2.02472e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.4, wall=5804
+2024-11-06 15:18:59 - progress_bar.py[line:274] - INFO: epoch 003:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.531, ntokens=17181.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3241, ups=0.19, wpb=17181.7, bsz=1024, num_updates=960, lr=2.04604e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.6, wall=5857
+2024-11-06 15:19:51 - progress_bar.py[line:274] - INFO: epoch 003:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.603, ntokens=17835.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3420.2, ups=0.19, wpb=17835.7, bsz=1024, num_updates=970, lr=2.06735e-05, gnorm=0.005, clip=0, loss_scale=256, train_wall=30, gb_free=13.4, wall=5910
+2024-11-06 15:20:43 - progress_bar.py[line:274] - INFO: epoch 003:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.584, ntokens=17554.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3333.9, ups=0.19, wpb=17554.7, bsz=1024, num_updates=980, lr=2.08866e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=31, gb_free=13.6, wall=5962
+2024-11-06 15:21:36 - progress_bar.py[line:274] - INFO: epoch 003:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.606, ntokens=17918.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=3413.8, ups=0.19, wpb=17918.2, bsz=1024, num_updates=990, lr=2.10997e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.6, wall=6015
+2024-11-06 15:22:28 - progress_bar.py[line:274] - INFO: epoch 003:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.56, ntokens=17536.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3354.8, ups=0.19, wpb=17536.7, bsz=1024, num_updates=1000, lr=2.13129e-05, gnorm=0.004, clip=0, loss_scale=256, train_wall=30, gb_free=13.6, wall=6067
+2024-11-06 15:23:21 - progress_bar.py[line:274] - INFO: epoch 003:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.593, ntokens=17771.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3364.2, ups=0.19, wpb=17771.9, bsz=1024, num_updates=1010, lr=2.1526e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=30, gb_free=13.5, wall=6120
+2024-11-06 15:24:14 - progress_bar.py[line:274] - INFO: epoch 003:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=17462.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3318, ups=0.19, wpb=17462.8, bsz=1024, num_updates=1020, lr=2.17391e-05, gnorm=0.006, clip=0, loss_scale=256, train_wall=31, gb_free=13.5, wall=6172
+2024-11-06 15:25:06 - progress_bar.py[line:274] - INFO: epoch 003:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.618, ntokens=17702.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3399.1, ups=0.19, wpb=17702.6, bsz=1024, num_updates=1030, lr=2.19523e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=6225
+2024-11-06 15:25:58 - progress_bar.py[line:274] - INFO: epoch 003:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.562, ntokens=17357.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3303.3, ups=0.19, wpb=17357.6, bsz=1024, num_updates=1040, lr=2.21654e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=31, gb_free=13.5, wall=6277
+2024-11-06 15:26:50 - progress_bar.py[line:274] - INFO: epoch 003:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.671, ntokens=18249.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.37, wps=3506.3, ups=0.19, wpb=18249.6, bsz=1024, num_updates=1050, lr=2.23785e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=6329
+2024-11-06 15:27:43 - progress_bar.py[line:274] - INFO: epoch 003:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.599, ntokens=17764.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3395.4, ups=0.19, wpb=17764.1, bsz=1024, num_updates=1060, lr=2.25916e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=6381
+2024-11-06 15:28:34 - progress_bar.py[line:274] - INFO: epoch 003:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17227.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3326.3, ups=0.19, wpb=17227.8, bsz=1024, num_updates=1070, lr=2.28048e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=6433
+2024-11-06 15:29:26 - progress_bar.py[line:274] - INFO: epoch 003:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.653, ntokens=17898.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.29, wps=3458.8, ups=0.19, wpb=17898.3, bsz=1024, num_updates=1080, lr=2.30179e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=31, gb_free=13.4, wall=6485
+2024-11-06 15:30:18 - progress_bar.py[line:274] - INFO: epoch 003:    308 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.743, ntokens=18402.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3531.2, ups=0.19, wpb=18402.7, bsz=1024, num_updates=1090, lr=2.3231e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=30, gb_free=13.2, wall=6537
+2024-11-06 15:31:10 - progress_bar.py[line:274] - INFO: epoch 003:    318 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.727, ntokens=18312.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3513.3, ups=0.19, wpb=18312.9, bsz=1024, num_updates=1100, lr=2.34442e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=31, gb_free=13.4, wall=6589
+2024-11-06 15:32:03 - progress_bar.py[line:274] - INFO: epoch 003:    328 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.684, ntokens=18118.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.43, wps=3472.9, ups=0.19, wpb=18118.6, bsz=1024, num_updates=1110, lr=2.36573e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=6641
+2024-11-06 15:32:54 - progress_bar.py[line:274] - INFO: epoch 003:    338 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.724, ntokens=18372.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.61, wps=3546, ups=0.19, wpb=18372.7, bsz=1024, num_updates=1120, lr=2.38704e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=31, gb_free=13.4, wall=6693
+2024-11-06 15:33:46 - progress_bar.py[line:274] - INFO: epoch 003:    348 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.719, ntokens=18510.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3549.8, ups=0.19, wpb=18510.1, bsz=1024, num_updates=1130, lr=2.40835e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=34, gb_free=13.2, wall=6745
+2024-11-06 15:34:39 - progress_bar.py[line:274] - INFO: epoch 003:    358 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.714, ntokens=18306.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.56, wps=3488, ups=0.19, wpb=18306.6, bsz=1024, num_updates=1140, lr=2.42967e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=45, gb_free=13.2, wall=6798
+2024-11-06 15:35:31 - progress_bar.py[line:274] - INFO: epoch 003:    368 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.667, ntokens=18037.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3442, ups=0.19, wpb=18037.1, bsz=1024, num_updates=1150, lr=2.45098e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=52, gb_free=13.4, wall=6850
+2024-11-06 15:36:26 - progress_bar.py[line:274] - INFO: epoch 003:    378 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.65, ntokens=17982.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3319.6, ups=0.18, wpb=17982.3, bsz=1024, num_updates=1160, lr=2.47229e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=54, gb_free=13.4, wall=6904
+2024-11-06 15:37:19 - progress_bar.py[line:274] - INFO: epoch 003:    388 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.678, ntokens=17965.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.4, wps=3349.7, ups=0.19, wpb=17965.8, bsz=1024, num_updates=1170, lr=2.49361e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=54, gb_free=13.4, wall=6958
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
+2024-11-06 15:37:31 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 3 @ 1173 updates
+
+
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
+
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
-2024-11-03 07:29:33 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 15:37:31 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt': File exists
+: File exists
+: File exists
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 07:31:26 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-03 07:33:18 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 3 @ 1173 updates, score 0) (writing took 224.89223005599342 seconds)
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 15:39:20 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+
+
+slice_id 2 seek offset 100000
+slice_id 6 seek offset 300000
+slice_id 7 seek offset 350000
+2024-11-06 15:41:11 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 3 @ 1173 updates, score 0) (writing took 220.05015903500316 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_3.pt
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappingfile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
 slice_id 3 seek offset 150000
-slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
 slice_id 5 seek offset 250000
-2024-11-03 07:34:12 - train.py[line:336] - INFO: end of epoch 3 (average epoch stats below)
-2024-11-03 07:34:12 - progress_bar.py[line:282] - INFO: epoch 003 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 4.419 | ntokens 17784.7 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 21.4 | wps 2606.2 | ups 0.15 | wpb 17784.7 | bsz 1023 | num_updates 1173 | lr 2.5e-05 | gnorm 0.006 | clip 0 | loss_scale 512 | train_wall 2104 | gb_free 13.5 | wall 7429
-2024-11-03 07:34:12 - trainer.py[line:639] - INFO: loading train data for epoch 4
+slice_id 4 seek offset 200000
+2024-11-06 15:42:06 - train.py[line:336] - INFO: end of epoch 3 (average epoch stats below)
+2024-11-06 15:42:06 - progress_bar.py[line:282] - INFO: epoch 003 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.618 | ntokens 17784.7 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.14 | wps 2804.8 | ups 0.16 | wpb 17784.7 | bsz 1023 | num_updates 1173 | lr 2.5e-05 | gnorm 0.005 | clip 0 | loss_scale 512 | train_wall 1278 | gb_free 13.4 | wall 7245
+2024-11-06 15:42:06 - trainer.py[line:639] - INFO: loading train data for epoch 4
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 07:36:57 - trainer.py[line:703] - INFO: begin training epoch 4
-2024-11-03 07:36:57 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 07:37:35 - progress_bar.py[line:274] - INFO: epoch 004:      7 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.256, ntokens=16998.2, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=19.1, wps=340.1, ups=0.02, wpb=16998.2, bsz=985.6, num_updates=1180, lr=2.51492e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=38, gb_free=13.6, wall=7633
-2024-11-03 07:38:27 - progress_bar.py[line:274] - INFO: epoch 004:     17 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.253, ntokens=17917.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=19.07, wps=3449.3, ups=0.19, wpb=17917.9, bsz=1024, num_updates=1190, lr=2.53623e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=31, gb_free=13.6, wall=7684
-2024-11-03 07:39:20 - progress_bar.py[line:274] - INFO: epoch 004:     27 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.225, ntokens=17656.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=18.71, wps=3357.6, ups=0.19, wpb=17656.8, bsz=1024, num_updates=1200, lr=2.55754e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=7737
-2024-11-03 07:40:12 - progress_bar.py[line:274] - INFO: epoch 004:     37 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.272, ntokens=18080.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=19.32, wps=3491.6, ups=0.19, wpb=18080.6, bsz=1024, num_updates=1210, lr=2.57886e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=7789
-2024-11-03 07:41:04 - progress_bar.py[line:274] - INFO: epoch 004:     47 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.197, ntokens=17609.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=18.34, wps=3399.3, ups=0.19, wpb=17609.3, bsz=1024, num_updates=1220, lr=2.60017e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=7841
-2024-11-03 07:41:56 - progress_bar.py[line:274] - INFO: epoch 004:     57 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.209, ntokens=17636.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=18.49, wps=3343, ups=0.19, wpb=17636.2, bsz=1024, num_updates=1230, lr=2.62148e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=7893
-2024-11-03 07:42:49 - progress_bar.py[line:274] - INFO: epoch 004:     67 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.196, ntokens=17847.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=18.33, wps=3405.3, ups=0.19, wpb=17847.3, bsz=1024, num_updates=1240, lr=2.6428e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=31, gb_free=13.4, wall=7946
-2024-11-03 07:43:41 - progress_bar.py[line:274] - INFO: epoch 004:     77 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.153, ntokens=17303.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.79, wps=3319, ups=0.19, wpb=17303.2, bsz=1024, num_updates=1250, lr=2.66411e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=7998
-2024-11-03 07:44:33 - progress_bar.py[line:274] - INFO: epoch 004:     87 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.176, ntokens=17653.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=18.07, wps=3386.2, ups=0.19, wpb=17653.4, bsz=1024, num_updates=1260, lr=2.68542e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=8050
-2024-11-03 07:45:25 - progress_bar.py[line:274] - INFO: epoch 004:     97 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.163, ntokens=17707, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.91, wps=3418.2, ups=0.19, wpb=17707, bsz=1024, num_updates=1270, lr=2.70673e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=31, gb_free=13.4, wall=8102
-2024-11-03 07:46:18 - progress_bar.py[line:274] - INFO: epoch 004:    107 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.14, ntokens=17543.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.64, wps=3322.4, ups=0.19, wpb=17543.2, bsz=1024, num_updates=1280, lr=2.72805e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=8155
-2024-11-03 07:47:09 - progress_bar.py[line:274] - INFO: epoch 004:    117 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.197, ntokens=17831.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=18.34, wps=3438.7, ups=0.19, wpb=17831.1, bsz=1024, num_updates=1290, lr=2.74936e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=31, gb_free=13.5, wall=8207
-2024-11-03 07:48:01 - progress_bar.py[line:274] - INFO: epoch 004:    127 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.12, ntokens=17544.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.39, wps=3374.1, ups=0.19, wpb=17544.4, bsz=1024, num_updates=1300, lr=2.77067e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=8259
-2024-11-03 07:48:53 - progress_bar.py[line:274] - INFO: epoch 004:    137 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.128, ntokens=17554.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.49, wps=3376.3, ups=0.19, wpb=17554.4, bsz=1024, num_updates=1310, lr=2.79199e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.3, wall=8311
-2024-11-03 07:49:46 - progress_bar.py[line:274] - INFO: epoch 004:    147 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.201, ntokens=17976, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=18.39, wps=3408.6, ups=0.19, wpb=17976, bsz=1024, num_updates=1320, lr=2.8133e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=8363
-2024-11-03 07:50:39 - progress_bar.py[line:274] - INFO: epoch 004:    157 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.123, ntokens=17645.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.43, wps=3370, ups=0.19, wpb=17645.7, bsz=1024, num_updates=1330, lr=2.83461e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=8416
-2024-11-03 07:51:30 - progress_bar.py[line:274] - INFO: epoch 004:    167 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.183, ntokens=18192.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=18.17, wps=3503.3, ups=0.19, wpb=18192.6, bsz=1024, num_updates=1340, lr=2.85592e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=8468
-2024-11-03 07:52:23 - progress_bar.py[line:274] - INFO: epoch 004:    177 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.101, ntokens=17242.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.16, wps=3284.6, ups=0.19, wpb=17242.9, bsz=1024, num_updates=1350, lr=2.87724e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=30, gb_free=13.3, wall=8520
-2024-11-03 07:53:16 - progress_bar.py[line:274] - INFO: epoch 004:    187 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.152, ntokens=18062.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.77, wps=3436.6, ups=0.19, wpb=18062.6, bsz=1024, num_updates=1360, lr=2.89855e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=8573
-2024-11-03 07:54:08 - progress_bar.py[line:274] - INFO: epoch 004:    197 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.121, ntokens=17833.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.4, wps=3400.7, ups=0.19, wpb=17833.4, bsz=1024, num_updates=1370, lr=2.91986e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=8625
-2024-11-03 07:55:01 - progress_bar.py[line:274] - INFO: epoch 004:    207 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.157, ntokens=17976.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.84, wps=3414.8, ups=0.19, wpb=17976.4, bsz=1024, num_updates=1380, lr=2.94118e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=8678
-2024-11-03 07:55:53 - progress_bar.py[line:274] - INFO: epoch 004:    217 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.082, ntokens=17835.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.93, wps=3420.2, ups=0.19, wpb=17835.3, bsz=1024, num_updates=1390, lr=2.96249e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=8730
-2024-11-03 07:56:45 - progress_bar.py[line:274] - INFO: epoch 004:    227 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.077, ntokens=17476.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.88, wps=3372.7, ups=0.19, wpb=17476.3, bsz=1024, num_updates=1400, lr=2.9838e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=8782
-2024-11-03 07:57:37 - progress_bar.py[line:274] - INFO: epoch 004:    237 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.068, ntokens=17513.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.77, wps=3360.2, ups=0.19, wpb=17513.1, bsz=1024, num_updates=1410, lr=3.00512e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=8834
-2024-11-03 07:58:29 - progress_bar.py[line:274] - INFO: epoch 004:    247 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.062, ntokens=17357.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.71, wps=3334.8, ups=0.19, wpb=17357.9, bsz=1024, num_updates=1420, lr=3.02643e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=31, gb_free=13.5, wall=8886
-2024-11-03 07:59:21 - progress_bar.py[line:274] - INFO: epoch 004:    257 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.027, ntokens=17406.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.3, wps=3317, ups=0.19, wpb=17406.1, bsz=1024, num_updates=1430, lr=3.04774e-05, gnorm=0.01, clip=0, loss_scale=512, train_wall=31, gb_free=13.4, wall=8938
-2024-11-03 08:00:13 - progress_bar.py[line:274] - INFO: epoch 004:    267 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.041, ntokens=17638.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.46, wps=3386.3, ups=0.19, wpb=17638.7, bsz=1024, num_updates=1440, lr=3.06905e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=8990
-2024-11-03 08:01:05 - progress_bar.py[line:274] - INFO: epoch 004:    277 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.092, ntokens=17655, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.06, wps=3397.2, ups=0.19, wpb=17655, bsz=1024, num_updates=1450, lr=3.09037e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=31, gb_free=13.5, wall=9042
-2024-11-03 08:01:58 - progress_bar.py[line:274] - INFO: epoch 004:    287 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.024, ntokens=17540.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.27, wps=3329.7, ups=0.19, wpb=17540.6, bsz=1024, num_updates=1460, lr=3.11168e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=9095
-2024-11-03 08:02:50 - progress_bar.py[line:274] - INFO: epoch 004:    297 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.079, ntokens=17814.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.9, wps=3422.1, ups=0.19, wpb=17814.3, bsz=1024, num_updates=1470, lr=3.13299e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=9147
-2024-11-03 08:03:41 - progress_bar.py[line:274] - INFO: epoch 004:    307 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.165, ntokens=18544.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.94, wps=3612, ups=0.19, wpb=18544.4, bsz=1024, num_updates=1480, lr=3.15431e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=31, gb_free=13.4, wall=9198
-2024-11-03 08:04:33 - progress_bar.py[line:274] - INFO: epoch 004:    317 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.111, ntokens=18211.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.28, wps=3537.8, ups=0.19, wpb=18211.8, bsz=1024, num_updates=1490, lr=3.17562e-05, gnorm=0.01, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=9250
-2024-11-03 08:05:25 - progress_bar.py[line:274] - INFO: epoch 004:    327 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.08, ntokens=18072.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.91, wps=3495.9, ups=0.19, wpb=18072.1, bsz=1024, num_updates=1500, lr=3.19693e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=31, gb_free=13.4, wall=9302
-2024-11-03 08:06:16 - progress_bar.py[line:274] - INFO: epoch 004:    337 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.099, ntokens=18249.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.14, wps=3515.7, ups=0.19, wpb=18249.8, bsz=1024, num_updates=1510, lr=3.21824e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=33, gb_free=13.5, wall=9354
-2024-11-03 08:07:09 - progress_bar.py[line:274] - INFO: epoch 004:    347 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.118, ntokens=18504.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.37, wps=3553.3, ups=0.19, wpb=18504.8, bsz=1024, num_updates=1520, lr=3.23956e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=33, gb_free=13.4, wall=9406
-2024-11-03 08:08:01 - progress_bar.py[line:274] - INFO: epoch 004:    357 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.116, ntokens=18409.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.34, wps=3524.9, ups=0.19, wpb=18409.7, bsz=1024, num_updates=1530, lr=3.26087e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=36, gb_free=13.4, wall=9458
-2024-11-03 08:08:54 - progress_bar.py[line:274] - INFO: epoch 004:    367 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.054, ntokens=18013.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.61, wps=3414.4, ups=0.19, wpb=18013.3, bsz=1024, num_updates=1540, lr=3.28218e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=49, gb_free=13.3, wall=9511
-2024-11-03 08:09:46 - progress_bar.py[line:274] - INFO: epoch 004:    377 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.104, ntokens=18417.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=17.2, wps=3484.1, ups=0.19, wpb=18417.8, bsz=1024, num_updates=1550, lr=3.3035e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=53, gb_free=13.3, wall=9563
-2024-11-03 08:10:39 - progress_bar.py[line:274] - INFO: epoch 004:    387 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.083, ntokens=17982.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.95, wps=3408.6, ups=0.19, wpb=17982.8, bsz=1024, num_updates=1560, lr=3.32481e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=53, gb_free=13.4, wall=9616
+2024-11-06 15:44:44 - trainer.py[line:703] - INFO: begin training epoch 4
+2024-11-06 15:44:44 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 15:45:23 - progress_bar.py[line:274] - INFO: epoch 004:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=16998.2, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=351.1, ups=0.02, wpb=16998.2, bsz=985.6, num_updates=1180, lr=2.51492e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=33, gb_free=13.6, wall=7442
+2024-11-06 15:46:15 - progress_bar.py[line:274] - INFO: epoch 004:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.617, ntokens=17917.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3430.4, ups=0.19, wpb=17917.9, bsz=1024, num_updates=1190, lr=2.53623e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=7494
+2024-11-06 15:47:08 - progress_bar.py[line:274] - INFO: epoch 004:     27 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.605, ntokens=17656.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3370.5, ups=0.19, wpb=17656.8, bsz=1024, num_updates=1200, lr=2.55754e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=7547
+2024-11-06 15:48:01 - progress_bar.py[line:274] - INFO: epoch 004:     37 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.617, ntokens=18080.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3392.7, ups=0.19, wpb=18080.6, bsz=1024, num_updates=1210, lr=2.57886e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=7600
+2024-11-06 15:48:54 - progress_bar.py[line:274] - INFO: epoch 004:     47 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.593, ntokens=17609.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3359.2, ups=0.19, wpb=17609.3, bsz=1024, num_updates=1220, lr=2.60017e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=7652
+2024-11-06 15:49:46 - progress_bar.py[line:274] - INFO: epoch 004:     57 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.596, ntokens=17636.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3349.4, ups=0.19, wpb=17636.2, bsz=1024, num_updates=1230, lr=2.62148e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=7705
+2024-11-06 15:50:39 - progress_bar.py[line:274] - INFO: epoch 004:     67 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.625, ntokens=17847.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.17, wps=3401.2, ups=0.19, wpb=17847.3, bsz=1024, num_updates=1240, lr=2.6428e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=7758
+2024-11-06 15:51:31 - progress_bar.py[line:274] - INFO: epoch 004:     77 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.558, ntokens=17303.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3303.4, ups=0.19, wpb=17303.2, bsz=1024, num_updates=1250, lr=2.66411e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=7810
+2024-11-06 15:52:23 - progress_bar.py[line:274] - INFO: epoch 004:     87 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.574, ntokens=17653.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3369.9, ups=0.19, wpb=17653.4, bsz=1024, num_updates=1260, lr=2.68542e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=7862
+2024-11-06 15:53:16 - progress_bar.py[line:274] - INFO: epoch 004:     97 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=17707, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3364.7, ups=0.19, wpb=17707, bsz=1024, num_updates=1270, lr=2.70673e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=30, gb_free=13.4, wall=7915
+2024-11-06 15:54:09 - progress_bar.py[line:274] - INFO: epoch 004:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.576, ntokens=17543.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3337.5, ups=0.19, wpb=17543.2, bsz=1024, num_updates=1280, lr=2.72805e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=31, gb_free=13.5, wall=7968
+2024-11-06 15:55:01 - progress_bar.py[line:274] - INFO: epoch 004:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.65, ntokens=17831.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3419.6, ups=0.19, wpb=17831.1, bsz=1024, num_updates=1290, lr=2.74936e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=8020
+2024-11-06 15:55:53 - progress_bar.py[line:274] - INFO: epoch 004:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.56, ntokens=17544.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3353.3, ups=0.19, wpb=17544.4, bsz=1024, num_updates=1300, lr=2.77067e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=8072
+2024-11-06 15:56:46 - progress_bar.py[line:274] - INFO: epoch 004:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.613, ntokens=17554.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3326.4, ups=0.19, wpb=17554.4, bsz=1024, num_updates=1310, lr=2.79199e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=30, gb_free=13.3, wall=8125
+2024-11-06 15:57:41 - progress_bar.py[line:274] - INFO: epoch 004:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.646, ntokens=17976, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3274.5, ups=0.18, wpb=17976, bsz=1024, num_updates=1320, lr=2.8133e-05, gnorm=0.01, clip=0, loss_scale=512, train_wall=44, gb_free=13.5, wall=8180
+2024-11-06 15:58:36 - progress_bar.py[line:274] - INFO: epoch 004:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.599, ntokens=17645.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3175.7, ups=0.18, wpb=17645.7, bsz=1024, num_updates=1330, lr=2.83461e-05, gnorm=0.013, clip=0, loss_scale=512, train_wall=55, gb_free=13.6, wall=8235
+2024-11-06 15:59:32 - progress_bar.py[line:274] - INFO: epoch 004:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.631, ntokens=18192.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3267.3, ups=0.18, wpb=18192.6, bsz=1024, num_updates=1340, lr=2.85592e-05, gnorm=0.012, clip=0, loss_scale=512, train_wall=56, gb_free=13.5, wall=8291
+2024-11-06 16:00:28 - progress_bar.py[line:274] - INFO: epoch 004:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.565, ntokens=17242.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3092.6, ups=0.18, wpb=17242.9, bsz=1024, num_updates=1350, lr=2.87724e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=56, gb_free=13.3, wall=8347
+2024-11-06 16:01:21 - progress_bar.py[line:274] - INFO: epoch 004:    187 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.648, ntokens=18062.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3403, ups=0.19, wpb=18062.6, bsz=1024, num_updates=1360, lr=2.89855e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=53, gb_free=13.5, wall=8400
+2024-11-06 16:02:13 - progress_bar.py[line:274] - INFO: epoch 004:    197 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.618, ntokens=17833.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3414.7, ups=0.19, wpb=17833.4, bsz=1024, num_updates=1370, lr=2.91986e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=52, gb_free=13.4, wall=8452
+2024-11-06 16:03:05 - progress_bar.py[line:274] - INFO: epoch 004:    207 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.616, ntokens=17976.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3443.9, ups=0.19, wpb=17976.4, bsz=1024, num_updates=1380, lr=2.94118e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=52, gb_free=13.5, wall=8504
+2024-11-06 16:03:57 - progress_bar.py[line:274] - INFO: epoch 004:    217 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.599, ntokens=17835.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3460.6, ups=0.19, wpb=17835.3, bsz=1024, num_updates=1390, lr=2.96249e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=51, gb_free=13.5, wall=8556
+2024-11-06 16:04:49 - progress_bar.py[line:274] - INFO: epoch 004:    227 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.562, ntokens=17476.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3377.5, ups=0.19, wpb=17476.3, bsz=1024, num_updates=1400, lr=2.9838e-05, gnorm=0.008, clip=0, loss_scale=512, train_wall=51, gb_free=13.6, wall=8607
+2024-11-06 16:05:40 - progress_bar.py[line:274] - INFO: epoch 004:    237 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.565, ntokens=17513.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3387.5, ups=0.19, wpb=17513.1, bsz=1024, num_updates=1410, lr=3.00512e-05, gnorm=0.009, clip=0, loss_scale=512, train_wall=40, gb_free=13.6, wall=8659
+2024-11-06 16:06:33 - progress_bar.py[line:274] - INFO: epoch 004:    247 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.568, ntokens=17357.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3323.6, ups=0.19, wpb=17357.9, bsz=1024, num_updates=1420, lr=3.02643e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=40, gb_free=13.5, wall=8711
+2024-11-06 16:07:25 - progress_bar.py[line:274] - INFO: epoch 004:    257 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.567, ntokens=17406.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3342.2, ups=0.19, wpb=17406.1, bsz=1024, num_updates=1430, lr=3.04774e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=36, gb_free=13.4, wall=8763
+2024-11-06 16:08:17 - progress_bar.py[line:274] - INFO: epoch 004:    267 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.586, ntokens=17638.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3385.2, ups=0.19, wpb=17638.7, bsz=1024, num_updates=1440, lr=3.06905e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=33, gb_free=13.4, wall=8816
+2024-11-06 16:09:09 - progress_bar.py[line:274] - INFO: epoch 004:    277 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.591, ntokens=17655, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3357.9, ups=0.19, wpb=17655, bsz=1024, num_updates=1450, lr=3.09037e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=30, gb_free=13.5, wall=8868
+2024-11-06 16:10:02 - progress_bar.py[line:274] - INFO: epoch 004:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.571, ntokens=17540.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3353.7, ups=0.19, wpb=17540.6, bsz=1024, num_updates=1460, lr=3.11168e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=30, gb_free=13.6, wall=8920
+2024-11-06 16:10:54 - progress_bar.py[line:274] - INFO: epoch 004:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.637, ntokens=17814.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.22, wps=3369.9, ups=0.19, wpb=17814.3, bsz=1024, num_updates=1470, lr=3.13299e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=33, gb_free=13.4, wall=8973
+2024-11-06 16:11:46 - progress_bar.py[line:274] - INFO: epoch 004:    307 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.747, ntokens=18544.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3571.2, ups=0.19, wpb=18544.4, bsz=1024, num_updates=1480, lr=3.15431e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=45, gb_free=13.4, wall=9025
+2024-11-06 16:12:41 - progress_bar.py[line:274] - INFO: epoch 004:    317 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.72, ntokens=18211.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3353.8, ups=0.18, wpb=18211.8, bsz=1024, num_updates=1490, lr=3.17562e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=50, gb_free=13.4, wall=9080
+2024-11-06 16:13:37 - progress_bar.py[line:274] - INFO: epoch 004:    327 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.691, ntokens=18072.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3199.3, ups=0.18, wpb=18072.1, bsz=1024, num_updates=1500, lr=3.19693e-05, gnorm=0.004, clip=0, loss_scale=512, train_wall=56, gb_free=13.4, wall=9136
+2024-11-06 16:14:34 - progress_bar.py[line:274] - INFO: epoch 004:    337 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.724, ntokens=18249.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.61, wps=3225.3, ups=0.18, wpb=18249.8, bsz=1024, num_updates=1510, lr=3.21824e-05, gnorm=0.007, clip=0, loss_scale=512, train_wall=56, gb_free=13.5, wall=9193
+2024-11-06 16:15:29 - progress_bar.py[line:274] - INFO: epoch 004:    347 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.72, ntokens=18504.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3332.5, ups=0.18, wpb=18504.8, bsz=1024, num_updates=1520, lr=3.23956e-05, gnorm=0.006, clip=0, loss_scale=512, train_wall=55, gb_free=13.4, wall=9248
+2024-11-06 16:16:23 - progress_bar.py[line:274] - INFO: epoch 004:    357 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.736, ntokens=18409.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.66, wps=3428.1, ups=0.19, wpb=18409.7, bsz=1024, num_updates=1530, lr=3.26087e-05, gnorm=0.005, clip=0, loss_scale=512, train_wall=54, gb_free=13.4, wall=9302
+2024-11-06 16:17:15 - progress_bar.py[line:274] - INFO: epoch 004:    367 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.674, ntokens=18013.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.38, wps=3489, ups=0.19, wpb=18013.3, bsz=1024, num_updates=1540, lr=3.28218e-05, gnorm=0.005, clip=0, loss_scale=1024, train_wall=52, gb_free=13.3, wall=9354
+2024-11-06 16:18:07 - progress_bar.py[line:274] - INFO: epoch 004:    377 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.737, ntokens=18417.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.67, wps=3526.6, ups=0.19, wpb=18417.8, bsz=1024, num_updates=1550, lr=3.3035e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=52, gb_free=13.3, wall=9406
+2024-11-06 16:18:59 - progress_bar.py[line:274] - INFO: epoch 004:    387 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17982.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3459.6, ups=0.19, wpb=17982.8, bsz=1024, num_updates=1560, lr=3.32481e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=52, gb_free=13.4, wall=9458
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 08:10:57 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 4 @ 1564 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt
-2024-11-03 08:10:57 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt': File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt2024-11-06 16:19:16 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 4 @ 1564 updates
+
+
+2024-11-06 16:19:16 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt': File exists: File exists
+: File exists
 
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 08:12:48 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-2024-11-03 08:14:40 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 4 @ 1564 updates, score 0) (writing took 222.95293303500512 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 16:21:06 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+
 
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 
-slice_id 7 seek offset 350000
 slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-
+slice_id 6 seek offset 300000
+slice_id 7 seek offset 350000
+2024-11-06 16:22:57 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 4 @ 1564 updates, score 0) (writing took 221.1307931959891 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_4.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
 slice_id 5 seek offset 250000
-slice_id 6 seek offset 300000
-2024-11-03 08:15:34 - train.py[line:336] - INFO: end of epoch 4 (average epoch stats below)
-2024-11-03 08:15:34 - progress_bar.py[line:282] - INFO: epoch 004 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 4.131 | ntokens 17804.6 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 17.53 | wps 2805 | ups 0.16 | wpb 17804.6 | bsz 1023 | num_updates 1564 | lr 3.33333e-05 | gnorm 0.007 | clip 0 | loss_scale 1024 | train_wall 1269 | gb_free 13.4 | wall 9911
-2024-11-03 08:15:34 - trainer.py[line:639] - INFO: loading train data for epoch 5
+slice_id 4 seek offset 200000
+2024-11-06 16:23:52 - train.py[line:336] - INFO: end of epoch 4 (average epoch stats below)
+2024-11-06 16:23:52 - progress_bar.py[line:282] - INFO: epoch 004 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.627 | ntokens 17804.6 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.18 | wps 2778.1 | ups 0.16 | wpb 17804.6 | bsz 1023 | num_updates 1564 | lr 3.33333e-05 | gnorm 0.007 | clip 0 | loss_scale 1024 | train_wall 1618 | gb_free 13.4 | wall 9751
+2024-11-06 16:23:52 - trainer.py[line:639] - INFO: loading train data for epoch 5
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 08:18:19 - trainer.py[line:703] - INFO: begin training epoch 5
-2024-11-03 08:18:19 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 08:18:53 - progress_bar.py[line:274] - INFO: epoch 005:      6 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.993, ntokens=16950.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=15.93, wps=343.5, ups=0.02, wpb=16950.1, bsz=985.6, num_updates=1570, lr=3.34612e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=36, gb_free=13.5, wall=10110
-2024-11-03 08:19:45 - progress_bar.py[line:274] - INFO: epoch 005:     16 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.062, ntokens=18019.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.7, wps=3459.4, ups=0.19, wpb=18019.7, bsz=1024, num_updates=1580, lr=3.36743e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=31, gb_free=13.4, wall=10162
-2024-11-03 08:20:36 - progress_bar.py[line:274] - INFO: epoch 005:     26 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.036, ntokens=17669.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.4, wps=3418.6, ups=0.19, wpb=17669.1, bsz=1024, num_updates=1590, lr=3.38875e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=10213
-2024-11-03 08:21:28 - progress_bar.py[line:274] - INFO: epoch 005:     36 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.053, ntokens=18047.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.6, wps=3489, ups=0.19, wpb=18047.6, bsz=1024, num_updates=1600, lr=3.41006e-05, gnorm=0.005, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=10265
-2024-11-03 08:22:21 - progress_bar.py[line:274] - INFO: epoch 005:     46 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.001, ntokens=17636.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.01, wps=3329.8, ups=0.19, wpb=17636.6, bsz=1024, num_updates=1610, lr=3.43137e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=31, gb_free=13.6, wall=10318
-2024-11-03 08:23:16 - progress_bar.py[line:274] - INFO: epoch 005:     56 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.969, ntokens=17429.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.66, wps=3170.9, ups=0.18, wpb=17429.8, bsz=1024, num_updates=1620, lr=3.45269e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.4, wall=10373
-2024-11-03 08:24:08 - progress_bar.py[line:274] - INFO: epoch 005:     66 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.995, ntokens=17821, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.94, wps=3432.3, ups=0.19, wpb=17821, bsz=1024, num_updates=1630, lr=3.474e-05, gnorm=0.004, clip=0, loss_scale=1024, train_wall=30, gb_free=13.3, wall=10425
-2024-11-03 08:25:00 - progress_bar.py[line:274] - INFO: epoch 005:     76 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.995, ntokens=17513, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.94, wps=3336.5, ups=0.19, wpb=17513, bsz=1024, num_updates=1640, lr=3.49531e-05, gnorm=0.005, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=10477
-2024-11-03 08:25:53 - progress_bar.py[line:274] - INFO: epoch 005:     86 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.955, ntokens=17536.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.51, wps=3355.8, ups=0.19, wpb=17536.2, bsz=1024, num_updates=1650, lr=3.51662e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=10530
-2024-11-03 08:26:45 - progress_bar.py[line:274] - INFO: epoch 005:     96 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.98, ntokens=17748.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.77, wps=3381.9, ups=0.19, wpb=17748.1, bsz=1024, num_updates=1660, lr=3.53794e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=10582
-2024-11-03 08:27:37 - progress_bar.py[line:274] - INFO: epoch 005:    106 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.929, ntokens=17450.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.23, wps=3355.4, ups=0.19, wpb=17450.6, bsz=1024, num_updates=1670, lr=3.55925e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=31, gb_free=13.4, wall=10634
-2024-11-03 08:28:30 - progress_bar.py[line:274] - INFO: epoch 005:    116 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.022, ntokens=17766.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.25, wps=3369.4, ups=0.19, wpb=17766.7, bsz=1024, num_updates=1680, lr=3.58056e-05, gnorm=0.005, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=10687
-2024-11-03 08:29:22 - progress_bar.py[line:274] - INFO: epoch 005:    126 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.927, ntokens=17573.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.21, wps=3342.7, ups=0.19, wpb=17573.9, bsz=1024, num_updates=1690, lr=3.60188e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=10740
-2024-11-03 08:30:15 - progress_bar.py[line:274] - INFO: epoch 005:    136 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.989, ntokens=17717.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.87, wps=3370.5, ups=0.19, wpb=17717.7, bsz=1024, num_updates=1700, lr=3.62319e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=31, gb_free=13.5, wall=10792
-2024-11-03 08:31:07 - progress_bar.py[line:274] - INFO: epoch 005:    146 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.947, ntokens=17535.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.43, wps=3383.3, ups=0.19, wpb=17535.7, bsz=1024, num_updates=1710, lr=3.6445e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=10844
-2024-11-03 08:31:59 - progress_bar.py[line:274] - INFO: epoch 005:    156 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.929, ntokens=17543.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.23, wps=3347.5, ups=0.19, wpb=17543.9, bsz=1024, num_updates=1720, lr=3.66581e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=10896
-2024-11-03 08:32:52 - progress_bar.py[line:274] - INFO: epoch 005:    166 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.985, ntokens=17868.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.83, wps=3384.3, ups=0.19, wpb=17868.5, bsz=1024, num_updates=1730, lr=3.68713e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.4, wall=10949
-2024-11-03 08:33:44 - progress_bar.py[line:274] - INFO: epoch 005:    176 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.954, ntokens=17478.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.5, wps=3332.2, ups=0.19, wpb=17478.3, bsz=1024, num_updates=1740, lr=3.70844e-05, gnorm=0.005, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=11002
-2024-11-03 08:34:37 - progress_bar.py[line:274] - INFO: epoch 005:    186 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.93, ntokens=17764.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.24, wps=3375.2, ups=0.19, wpb=17764.2, bsz=1024, num_updates=1750, lr=3.72975e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=30, gb_free=13.4, wall=11054
-2024-11-03 08:35:30 - progress_bar.py[line:274] - INFO: epoch 005:    196 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.986, ntokens=17853.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.85, wps=3403.6, ups=0.19, wpb=17853.7, bsz=1024, num_updates=1760, lr=3.75107e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=30, gb_free=13.4, wall=11107
-2024-11-03 08:36:22 - progress_bar.py[line:274] - INFO: epoch 005:    206 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.983, ntokens=18052.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.82, wps=3458.4, ups=0.19, wpb=18052.6, bsz=1024, num_updates=1770, lr=3.77238e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=11159
-2024-11-03 08:37:14 - progress_bar.py[line:274] - INFO: epoch 005:    216 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.935, ntokens=17732.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.3, wps=3376, ups=0.19, wpb=17732.1, bsz=1024, num_updates=1780, lr=3.79369e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=11211
-2024-11-03 08:38:06 - progress_bar.py[line:274] - INFO: epoch 005:    226 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.964, ntokens=17857.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.61, wps=3435.8, ups=0.19, wpb=17857.3, bsz=1024, num_updates=1790, lr=3.815e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=11263
-2024-11-03 08:38:59 - progress_bar.py[line:274] - INFO: epoch 005:    236 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.915, ntokens=17412.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.08, wps=3331.6, ups=0.19, wpb=17412.3, bsz=1024, num_updates=1800, lr=3.83632e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=11316
-2024-11-03 08:39:51 - progress_bar.py[line:274] - INFO: epoch 005:    246 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.964, ntokens=17639, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.61, wps=3350.3, ups=0.19, wpb=17639, bsz=1024, num_updates=1810, lr=3.85763e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=11368
-2024-11-03 08:40:44 - progress_bar.py[line:274] - INFO: epoch 005:    256 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.946, ntokens=17767.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.41, wps=3387.8, ups=0.19, wpb=17767.7, bsz=1024, num_updates=1820, lr=3.87894e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.4, wall=11421
-2024-11-03 08:41:36 - progress_bar.py[line:274] - INFO: epoch 005:    266 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.917, ntokens=17693.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.1, wps=3392.4, ups=0.19, wpb=17693.7, bsz=1024, num_updates=1830, lr=3.90026e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=11473
-2024-11-03 08:42:28 - progress_bar.py[line:274] - INFO: epoch 005:    276 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.921, ntokens=17650.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.15, wps=3385.8, ups=0.19, wpb=17650.6, bsz=1024, num_updates=1840, lr=3.92157e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=11525
-2024-11-03 08:43:20 - progress_bar.py[line:274] - INFO: epoch 005:    286 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.904, ntokens=17649.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.97, wps=3392, ups=0.19, wpb=17649.4, bsz=1024, num_updates=1850, lr=3.94288e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=11577
-2024-11-03 08:44:13 - progress_bar.py[line:274] - INFO: epoch 005:    296 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.906, ntokens=17541.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.99, wps=3281.3, ups=0.19, wpb=17541.8, bsz=1024, num_updates=1860, lr=3.96419e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.4, wall=11631
-2024-11-03 08:45:08 - progress_bar.py[line:274] - INFO: epoch 005:    306 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=4.008, ntokens=18415.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=16.08, wps=3350.3, ups=0.18, wpb=18415.1, bsz=1024, num_updates=1870, lr=3.98551e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=31, gb_free=13.4, wall=11685
-2024-11-03 08:46:05 - progress_bar.py[line:274] - INFO: epoch 005:    316 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.976, ntokens=18160, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.74, wps=3204.5, ups=0.18, wpb=18160, bsz=1024, num_updates=1880, lr=4.00682e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=54, gb_free=13.5, wall=11742
-2024-11-03 08:47:02 - progress_bar.py[line:274] - INFO: epoch 005:    326 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.929, ntokens=17951.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.23, wps=3178.8, ups=0.18, wpb=17951.5, bsz=1024, num_updates=1890, lr=4.02813e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=56, gb_free=13.4, wall=11799
-2024-11-03 08:47:58 - progress_bar.py[line:274] - INFO: epoch 005:    336 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.966, ntokens=18088.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.63, wps=3230.8, ups=0.18, wpb=18088.2, bsz=1024, num_updates=1900, lr=4.04945e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=56, gb_free=13.5, wall=11855
-2024-11-03 08:48:53 - progress_bar.py[line:274] - INFO: epoch 005:    346 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.936, ntokens=18104.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.31, wps=3278.6, ups=0.18, wpb=18104.2, bsz=1024, num_updates=1910, lr=4.07076e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=55, gb_free=13.5, wall=11910
-2024-11-03 08:49:45 - progress_bar.py[line:274] - INFO: epoch 005:    356 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.956, ntokens=18218.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.52, wps=3481.6, ups=0.19, wpb=18218.3, bsz=1024, num_updates=1920, lr=4.09207e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=52, gb_free=13.3, wall=11962
-2024-11-03 08:50:37 - progress_bar.py[line:274] - INFO: epoch 005:    366 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.918, ntokens=17962.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.12, wps=3450.9, ups=0.19, wpb=17962.1, bsz=1024, num_updates=1930, lr=4.11338e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=52, gb_free=13.4, wall=12014
-2024-11-03 08:51:29 - progress_bar.py[line:274] - INFO: epoch 005:    376 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.989, ntokens=18236.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.88, wps=3506.2, ups=0.19, wpb=18236.2, bsz=1024, num_updates=1940, lr=4.1347e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=52, gb_free=13.5, wall=12066
-2024-11-03 08:52:21 - progress_bar.py[line:274] - INFO: epoch 005:    386 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.918, ntokens=17753.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.11, wps=3396.1, ups=0.19, wpb=17753.8, bsz=1024, num_updates=1950, lr=4.15601e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=52, gb_free=13.3, wall=12119
+2024-11-06 16:26:31 - trainer.py[line:703] - INFO: begin training epoch 5
+2024-11-06 16:26:31 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 16:27:05 - progress_bar.py[line:274] - INFO: epoch 005:      6 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.608, ntokens=16950.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=348.5, ups=0.02, wpb=16950.1, bsz=985.6, num_updates=1570, lr=3.34612e-05, gnorm=0.005, clip=0, loss_scale=1024, train_wall=35, gb_free=13.5, wall=9944
+2024-11-06 16:27:59 - progress_bar.py[line:274] - INFO: epoch 005:     16 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.628, ntokens=18019.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.18, wps=3344.4, ups=0.19, wpb=18019.7, bsz=1024, num_updates=1580, lr=3.36743e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=35, gb_free=13.4, wall=9998
+2024-11-06 16:28:55 - progress_bar.py[line:274] - INFO: epoch 005:     26 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=17669.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3148.8, ups=0.18, wpb=17669.1, bsz=1024, num_updates=1590, lr=3.38875e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=56, gb_free=13.5, wall=10054
+2024-11-06 16:29:50 - progress_bar.py[line:274] - INFO: epoch 005:     36 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.636, ntokens=18047.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.22, wps=3290.5, ups=0.18, wpb=18047.6, bsz=1024, num_updates=1600, lr=3.41006e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=55, gb_free=13.6, wall=10109
+2024-11-06 16:30:44 - progress_bar.py[line:274] - INFO: epoch 005:     46 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.593, ntokens=17636.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3248.7, ups=0.18, wpb=17636.6, bsz=1024, num_updates=1610, lr=3.43137e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=54, gb_free=13.5, wall=10163
+2024-11-06 16:31:37 - progress_bar.py[line:274] - INFO: epoch 005:     56 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.571, ntokens=17429.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3321.2, ups=0.19, wpb=17429.8, bsz=1024, num_updates=1620, lr=3.45269e-05, gnorm=0.012, clip=0, loss_scale=1024, train_wall=52, gb_free=13.4, wall=10216
+2024-11-06 16:32:32 - progress_bar.py[line:274] - INFO: epoch 005:     66 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.62, ntokens=17821, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.15, wps=3209.1, ups=0.18, wpb=17821, bsz=1024, num_updates=1630, lr=3.474e-05, gnorm=0.012, clip=0, loss_scale=1024, train_wall=55, gb_free=13.3, wall=10271
+2024-11-06 16:33:27 - progress_bar.py[line:274] - INFO: epoch 005:     76 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.607, ntokens=17513, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=3201.7, ups=0.18, wpb=17513, bsz=1024, num_updates=1640, lr=3.49531e-05, gnorm=0.011, clip=0, loss_scale=1024, train_wall=55, gb_free=13.5, wall=10326
+2024-11-06 16:34:19 - progress_bar.py[line:274] - INFO: epoch 005:     86 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.577, ntokens=17536.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3345.3, ups=0.19, wpb=17536.2, bsz=1024, num_updates=1650, lr=3.51662e-05, gnorm=0.011, clip=0, loss_scale=1024, train_wall=52, gb_free=13.5, wall=10378
+2024-11-06 16:35:11 - progress_bar.py[line:274] - INFO: epoch 005:     96 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.617, ntokens=17748.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3417.4, ups=0.19, wpb=17748.1, bsz=1024, num_updates=1660, lr=3.53794e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=52, gb_free=13.6, wall=10430
+2024-11-06 16:36:03 - progress_bar.py[line:274] - INFO: epoch 005:    106 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.583, ntokens=17450.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3399.2, ups=0.19, wpb=17450.6, bsz=1024, num_updates=1670, lr=3.55925e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=51, gb_free=13.4, wall=10482
+2024-11-06 16:36:55 - progress_bar.py[line:274] - INFO: epoch 005:    116 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=17766.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3404.9, ups=0.19, wpb=17766.7, bsz=1024, num_updates=1680, lr=3.58056e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=46, gb_free=13.5, wall=10534
+2024-11-06 16:37:47 - progress_bar.py[line:274] - INFO: epoch 005:    126 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.58, ntokens=17573.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3400.2, ups=0.19, wpb=17573.9, bsz=1024, num_updates=1690, lr=3.60188e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=38, gb_free=13.6, wall=10585
+2024-11-06 16:38:39 - progress_bar.py[line:274] - INFO: epoch 005:    136 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.616, ntokens=17717.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3390.5, ups=0.19, wpb=17717.7, bsz=1024, num_updates=1700, lr=3.62319e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=34, gb_free=13.5, wall=10638
+2024-11-06 16:39:32 - progress_bar.py[line:274] - INFO: epoch 005:    146 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.599, ntokens=17535.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3328.6, ups=0.19, wpb=17535.7, bsz=1024, num_updates=1710, lr=3.6445e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=38, gb_free=13.5, wall=10690
+2024-11-06 16:40:23 - progress_bar.py[line:274] - INFO: epoch 005:    156 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.586, ntokens=17543.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3379.6, ups=0.19, wpb=17543.9, bsz=1024, num_updates=1720, lr=3.66581e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=39, gb_free=13.5, wall=10742
+2024-11-06 16:41:15 - progress_bar.py[line:274] - INFO: epoch 005:    166 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.607, ntokens=17868.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=3447.4, ups=0.19, wpb=17868.5, bsz=1024, num_updates=1730, lr=3.68713e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=34, gb_free=13.4, wall=10794
+2024-11-06 16:42:08 - progress_bar.py[line:274] - INFO: epoch 005:    176 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.577, ntokens=17478.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3295.5, ups=0.19, wpb=17478.3, bsz=1024, num_updates=1740, lr=3.70844e-05, gnorm=0.005, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=10847
+2024-11-06 16:43:00 - progress_bar.py[line:274] - INFO: epoch 005:    186 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.617, ntokens=17764.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3414.2, ups=0.19, wpb=17764.2, bsz=1024, num_updates=1750, lr=3.72975e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.4, wall=10899
+2024-11-06 16:43:53 - progress_bar.py[line:274] - INFO: epoch 005:    196 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.621, ntokens=17853.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.15, wps=3388.3, ups=0.19, wpb=17853.7, bsz=1024, num_updates=1760, lr=3.75107e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=30, gb_free=13.4, wall=10952
+2024-11-06 16:44:45 - progress_bar.py[line:274] - INFO: epoch 005:    206 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.639, ntokens=18052.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.23, wps=3459.5, ups=0.19, wpb=18052.6, bsz=1024, num_updates=1770, lr=3.77238e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=11004
+2024-11-06 16:45:38 - progress_bar.py[line:274] - INFO: epoch 005:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.597, ntokens=17732.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3386.7, ups=0.19, wpb=17732.1, bsz=1024, num_updates=1780, lr=3.79369e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=34, gb_free=13.5, wall=11056
+2024-11-06 16:46:30 - progress_bar.py[line:274] - INFO: epoch 005:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.613, ntokens=17857.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3413.4, ups=0.19, wpb=17857.3, bsz=1024, num_updates=1790, lr=3.815e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=33, gb_free=13.5, wall=11109
+2024-11-06 16:47:23 - progress_bar.py[line:274] - INFO: epoch 005:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.567, ntokens=17412.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3289.9, ups=0.19, wpb=17412.3, bsz=1024, num_updates=1800, lr=3.83632e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=31, gb_free=13.6, wall=11162
+2024-11-06 16:48:16 - progress_bar.py[line:274] - INFO: epoch 005:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=17639, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3309, ups=0.19, wpb=17639, bsz=1024, num_updates=1810, lr=3.85763e-05, gnorm=0.005, clip=0, loss_scale=1024, train_wall=41, gb_free=13.6, wall=11215
+2024-11-06 16:49:09 - progress_bar.py[line:274] - INFO: epoch 005:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.624, ntokens=17767.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.17, wps=3360.2, ups=0.19, wpb=17767.7, bsz=1024, num_updates=1820, lr=3.87894e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=41, gb_free=13.4, wall=11268
+2024-11-06 16:50:02 - progress_bar.py[line:274] - INFO: epoch 005:    266 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.651, ntokens=17693.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3347.5, ups=0.19, wpb=17693.7, bsz=1024, num_updates=1830, lr=3.90026e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=43, gb_free=13.5, wall=11321
+2024-11-06 16:50:54 - progress_bar.py[line:274] - INFO: epoch 005:    276 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.579, ntokens=17650.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3361.1, ups=0.19, wpb=17650.6, bsz=1024, num_updates=1840, lr=3.92157e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=47, gb_free=13.5, wall=11373
+2024-11-06 16:51:47 - progress_bar.py[line:274] - INFO: epoch 005:    286 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.604, ntokens=17649.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3366.7, ups=0.19, wpb=17649.4, bsz=1024, num_updates=1850, lr=3.94288e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=46, gb_free=13.5, wall=11426
+2024-11-06 16:52:39 - progress_bar.py[line:274] - INFO: epoch 005:    296 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.613, ntokens=17541.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3373.2, ups=0.19, wpb=17541.8, bsz=1024, num_updates=1860, lr=3.96419e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=45, gb_free=13.4, wall=11478
+2024-11-06 16:53:31 - progress_bar.py[line:274] - INFO: epoch 005:    306 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.739, ntokens=18415.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.68, wps=3511, ups=0.19, wpb=18415.1, bsz=1024, num_updates=1870, lr=3.98551e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=48, gb_free=13.4, wall=11530
+2024-11-06 16:54:23 - progress_bar.py[line:274] - INFO: epoch 005:    316 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.735, ntokens=18160, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.66, wps=3498.5, ups=0.19, wpb=18160, bsz=1024, num_updates=1880, lr=4.00682e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=52, gb_free=13.5, wall=11582
+2024-11-06 16:55:16 - progress_bar.py[line:274] - INFO: epoch 005:    326 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.685, ntokens=17951.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.43, wps=3406.4, ups=0.19, wpb=17951.5, bsz=1024, num_updates=1890, lr=4.02813e-05, gnorm=0.004, clip=0, loss_scale=1024, train_wall=53, gb_free=13.4, wall=11635
+2024-11-06 16:56:09 - progress_bar.py[line:274] - INFO: epoch 005:    336 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.699, ntokens=18088.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3420.3, ups=0.19, wpb=18088.2, bsz=1024, num_updates=1900, lr=4.04945e-05, gnorm=0.005, clip=0, loss_scale=1024, train_wall=53, gb_free=13.5, wall=11688
+2024-11-06 16:57:01 - progress_bar.py[line:274] - INFO: epoch 005:    346 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.68, ntokens=18104.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3441.7, ups=0.19, wpb=18104.2, bsz=1024, num_updates=1910, lr=4.07076e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=53, gb_free=13.5, wall=11740
+2024-11-06 16:57:53 - progress_bar.py[line:274] - INFO: epoch 005:    356 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.704, ntokens=18218.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3494.5, ups=0.19, wpb=18218.3, bsz=1024, num_updates=1920, lr=4.09207e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=52, gb_free=13.3, wall=11792
+2024-11-06 16:58:46 - progress_bar.py[line:274] - INFO: epoch 005:    366 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.7, ntokens=17962.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3398.4, ups=0.19, wpb=17962.1, bsz=1024, num_updates=1930, lr=4.11338e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=53, gb_free=13.4, wall=11845
+2024-11-06 16:59:38 - progress_bar.py[line:274] - INFO: epoch 005:    376 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.704, ntokens=18236.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3525.5, ups=0.19, wpb=18236.2, bsz=1024, num_updates=1940, lr=4.1347e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=52, gb_free=13.4, wall=11897
+2024-11-06 17:00:30 - progress_bar.py[line:274] - INFO: epoch 005:    386 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.683, ntokens=17753.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3400, ups=0.19, wpb=17753.8, bsz=1024, num_updates=1950, lr=4.15601e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=52, gb_free=13.3, wall=11949
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt2024-11-03 08:52:44 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 5 @ 1955 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt2024-11-06 17:00:53 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 5 @ 1955 updates
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt
-2024-11-03 08:52:44 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt': File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt
+
+2024-11-06 17:00:53 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt': File exists: File exists
+: File exists
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 08:54:35 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-2024-11-03 08:56:27 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 5 @ 1955 updates, score 0) (writing took 222.89019532001112 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 17:02:43 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 
-slice_id 7 seek offset 350000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+slice_id 7 seek offset 350000
+slice_id 6 seek offset 300000
+2024-11-06 17:04:34 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 5 @ 1955 updates, score 0) (writing took 221.5284793380124 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_5.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
 
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-
 slice_id 1 seek offset 50000
 slice_id 5 seek offset 250000
-slice_id 6 seek offset 300000
-slice_id 2 seek offset 100000
-2024-11-03 08:57:22 - train.py[line:336] - INFO: end of epoch 5 (average epoch stats below)
-2024-11-03 08:57:22 - progress_bar.py[line:282] - INFO: epoch 005 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.963 | ntokens 17766 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 15.6 | wps 2770.3 | ups 0.16 | wpb 17766 | bsz 1023 | num_updates 1955 | lr 4.16667e-05 | gnorm 0.007 | clip 0 | loss_scale 1024 | train_wall 1383 | gb_free 13.3 | wall 12419
-2024-11-03 08:57:22 - trainer.py[line:639] - INFO: loading train data for epoch 6
+slice_id 4 seek offset 200000
+2024-11-06 17:05:29 - train.py[line:336] - INFO: end of epoch 5 (average epoch stats below)
+2024-11-06 17:05:29 - progress_bar.py[line:282] - INFO: epoch 005 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.63 | ntokens 17766 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.19 | wps 2781.8 | ups 0.16 | wpb 17766 | bsz 1023 | num_updates 1955 | lr 4.16667e-05 | gnorm 0.008 | clip 0 | loss_scale 1024 | train_wall 1738 | gb_free 13.3 | wall 12248
+2024-11-06 17:05:29 - trainer.py[line:639] - INFO: loading train data for epoch 6
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 09:00:06 - trainer.py[line:703] - INFO: begin training epoch 6
-2024-11-03 09:00:06 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 09:00:34 - progress_bar.py[line:274] - INFO: epoch 006:      5 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.887, ntokens=16956.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=14.79, wps=344, ups=0.02, wpb=16956.5, bsz=985.6, num_updates=1960, lr=4.17732e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=38, gb_free=13.6, wall=12611
-2024-11-03 09:01:28 - progress_bar.py[line:274] - INFO: epoch 006:     15 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.88, ntokens=17600.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.73, wps=3303.1, ups=0.19, wpb=17600.6, bsz=1024, num_updates=1970, lr=4.19864e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=31, gb_free=13.5, wall=12665
-2024-11-03 09:02:19 - progress_bar.py[line:274] - INFO: epoch 006:     25 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.944, ntokens=17796.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.39, wps=3437.5, ups=0.19, wpb=17796.8, bsz=1024, num_updates=1980, lr=4.21995e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=12717
-2024-11-03 09:03:12 - progress_bar.py[line:274] - INFO: epoch 006:     35 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.883, ntokens=17729.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.76, wps=3397.2, ups=0.19, wpb=17729.1, bsz=1024, num_updates=1990, lr=4.24126e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=12769
-2024-11-03 09:04:04 - progress_bar.py[line:274] - INFO: epoch 006:     45 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.847, ntokens=17528.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.39, wps=3354.8, ups=0.19, wpb=17528.1, bsz=1024, num_updates=2000, lr=4.26257e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=12821
-2024-11-03 09:04:56 - progress_bar.py[line:274] - INFO: epoch 006:     55 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.878, ntokens=17694, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.71, wps=3366.5, ups=0.19, wpb=17694, bsz=1024, num_updates=2010, lr=4.28389e-05, gnorm=0.009, clip=0, loss_scale=1024, train_wall=31, gb_free=13.4, wall=12874
-2024-11-03 09:05:48 - progress_bar.py[line:274] - INFO: epoch 006:     65 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.826, ntokens=17532.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.18, wps=3383.8, ups=0.19, wpb=17532.2, bsz=1024, num_updates=2020, lr=4.3052e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=12925
-2024-11-03 09:06:41 - progress_bar.py[line:274] - INFO: epoch 006:     75 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.91, ntokens=18014.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.03, wps=3438.5, ups=0.19, wpb=18014.1, bsz=1024, num_updates=2030, lr=4.32651e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=12978
-2024-11-03 09:07:33 - progress_bar.py[line:274] - INFO: epoch 006:     85 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.843, ntokens=17453, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.35, wps=3348.8, ups=0.19, wpb=17453, bsz=1024, num_updates=2040, lr=4.34783e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=13030
-2024-11-03 09:08:25 - progress_bar.py[line:274] - INFO: epoch 006:     95 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.872, ntokens=17828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.64, wps=3399.1, ups=0.19, wpb=17828, bsz=1024, num_updates=2050, lr=4.36914e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.6, wall=13082
-2024-11-03 09:09:17 - progress_bar.py[line:274] - INFO: epoch 006:    105 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.816, ntokens=17386.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.08, wps=3343.2, ups=0.19, wpb=17386.2, bsz=1024, num_updates=2060, lr=4.39045e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=31, gb_free=13.6, wall=13134
-2024-11-03 09:10:10 - progress_bar.py[line:274] - INFO: epoch 006:    115 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.844, ntokens=17505, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.36, wps=3314, ups=0.19, wpb=17505, bsz=1024, num_updates=2070, lr=4.41176e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=36, gb_free=13.5, wall=13187
-2024-11-03 09:11:02 - progress_bar.py[line:274] - INFO: epoch 006:    125 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.838, ntokens=17815.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.3, wps=3422.4, ups=0.19, wpb=17815.9, bsz=1024, num_updates=2080, lr=4.43308e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=43, gb_free=13.6, wall=13239
-2024-11-03 09:11:54 - progress_bar.py[line:274] - INFO: epoch 006:    135 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.851, ntokens=17692.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.43, wps=3403.3, ups=0.19, wpb=17692.8, bsz=1024, num_updates=2090, lr=4.45439e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=46, gb_free=13.5, wall=13291
-2024-11-03 09:12:45 - progress_bar.py[line:274] - INFO: epoch 006:    145 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.846, ntokens=17534.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.38, wps=3414, ups=0.19, wpb=17534.2, bsz=1024, num_updates=2100, lr=4.4757e-05, gnorm=0.012, clip=0, loss_scale=2048, train_wall=40, gb_free=13.5, wall=13343
-2024-11-03 09:13:38 - progress_bar.py[line:274] - INFO: epoch 006:    155 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.844, ntokens=17791.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.36, wps=3374.7, ups=0.19, wpb=17791.4, bsz=1024, num_updates=2110, lr=4.49702e-05, gnorm=0.013, clip=0, loss_scale=2048, train_wall=31, gb_free=13.4, wall=13395
-2024-11-03 09:14:31 - progress_bar.py[line:274] - INFO: epoch 006:    165 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.879, ntokens=17780.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.71, wps=3376.2, ups=0.19, wpb=17780.2, bsz=1024, num_updates=2120, lr=4.51833e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=31, gb_free=13.6, wall=13448
-2024-11-03 09:15:23 - progress_bar.py[line:274] - INFO: epoch 006:    175 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.826, ntokens=17622.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.18, wps=3363.3, ups=0.19, wpb=17622.4, bsz=1024, num_updates=2130, lr=4.53964e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=13500
-2024-11-03 09:16:15 - progress_bar.py[line:274] - INFO: epoch 006:    185 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.848, ntokens=17660.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.4, wps=3411.2, ups=0.19, wpb=17660.9, bsz=1024, num_updates=2140, lr=4.56095e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=13552
-2024-11-03 09:17:07 - progress_bar.py[line:274] - INFO: epoch 006:    195 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.839, ntokens=17894.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.32, wps=3468.4, ups=0.19, wpb=17894.8, bsz=1024, num_updates=2150, lr=4.58227e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.6, wall=13604
-2024-11-03 09:17:59 - progress_bar.py[line:274] - INFO: epoch 006:    205 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.872, ntokens=17892.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.64, wps=3400.5, ups=0.19, wpb=17892.5, bsz=1024, num_updates=2160, lr=4.60358e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=13656
-2024-11-03 09:18:52 - progress_bar.py[line:274] - INFO: epoch 006:    215 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.766, ntokens=17744.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.6, wps=3360.2, ups=0.19, wpb=17744.5, bsz=1024, num_updates=2170, lr=4.62489e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=31, gb_free=13.6, wall=13709
-2024-11-03 09:19:44 - progress_bar.py[line:274] - INFO: epoch 006:    225 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.854, ntokens=17851.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.46, wps=3442.1, ups=0.19, wpb=17851.1, bsz=1024, num_updates=2180, lr=4.64621e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.6, wall=13761
-2024-11-03 09:20:38 - progress_bar.py[line:274] - INFO: epoch 006:    235 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.82, ntokens=17608.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.12, wps=3263.2, ups=0.19, wpb=17608.7, bsz=1024, num_updates=2190, lr=4.66752e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=38, gb_free=13.4, wall=13815
-2024-11-03 09:21:31 - progress_bar.py[line:274] - INFO: epoch 006:    245 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.823, ntokens=17412.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.15, wps=3293.8, ups=0.19, wpb=17412.8, bsz=1024, num_updates=2200, lr=4.68883e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=50, gb_free=13.6, wall=13868
-2024-11-03 09:22:23 - progress_bar.py[line:274] - INFO: epoch 006:    255 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.801, ntokens=17651.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.94, wps=3384.3, ups=0.19, wpb=17651.2, bsz=1024, num_updates=2210, lr=4.71014e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=50, gb_free=13.4, wall=13920
-2024-11-03 09:23:16 - progress_bar.py[line:274] - INFO: epoch 006:    265 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.85, ntokens=17978.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.42, wps=3408.1, ups=0.19, wpb=17978.2, bsz=1024, num_updates=2220, lr=4.73146e-05, gnorm=0.005, clip=0, loss_scale=2048, train_wall=47, gb_free=13.6, wall=13973
-2024-11-03 09:24:07 - progress_bar.py[line:274] - INFO: epoch 006:    275 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.83, ntokens=17633.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.22, wps=3415.9, ups=0.19, wpb=17633.2, bsz=1024, num_updates=2230, lr=4.75277e-05, gnorm=0.005, clip=0, loss_scale=2048, train_wall=45, gb_free=13.4, wall=14024
-2024-11-03 09:24:59 - progress_bar.py[line:274] - INFO: epoch 006:    285 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.774, ntokens=17538.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.68, wps=3380.1, ups=0.19, wpb=17538.6, bsz=1024, num_updates=2240, lr=4.77408e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=42, gb_free=13.5, wall=14076
-2024-11-03 09:25:51 - progress_bar.py[line:274] - INFO: epoch 006:    295 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.801, ntokens=17736.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.94, wps=3424.8, ups=0.19, wpb=17736.7, bsz=1024, num_updates=2250, lr=4.7954e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=41, gb_free=13.5, wall=14128
-2024-11-03 09:26:43 - progress_bar.py[line:274] - INFO: epoch 006:    305 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.889, ntokens=18116.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.81, wps=3509.8, ups=0.19, wpb=18116.1, bsz=1024, num_updates=2260, lr=4.81671e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=44, gb_free=13.5, wall=14180
-2024-11-03 09:27:34 - progress_bar.py[line:274] - INFO: epoch 006:    315 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.923, ntokens=18532.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=15.16, wps=3613.1, ups=0.19, wpb=18532.7, bsz=1024, num_updates=2270, lr=4.83802e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=43, gb_free=13.4, wall=14231
-2024-11-03 09:28:26 - progress_bar.py[line:274] - INFO: epoch 006:    325 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.873, ntokens=18034.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.66, wps=3451.5, ups=0.19, wpb=18034.4, bsz=1024, num_updates=2280, lr=4.85934e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=40, gb_free=13.4, wall=14283
-2024-11-03 09:29:18 - progress_bar.py[line:274] - INFO: epoch 006:    335 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.879, ntokens=18357.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.71, wps=3514, ups=0.19, wpb=18357.4, bsz=1024, num_updates=2290, lr=4.88065e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=41, gb_free=13.4, wall=14335
-2024-11-03 09:30:12 - progress_bar.py[line:274] - INFO: epoch 006:    345 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.844, ntokens=18123.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.36, wps=3404.4, ups=0.19, wpb=18123.6, bsz=1024, num_updates=2300, lr=4.90196e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=48, gb_free=13.5, wall=14389
-2024-11-03 09:31:04 - progress_bar.py[line:274] - INFO: epoch 006:    355 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.876, ntokens=18383.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.68, wps=3513.1, ups=0.19, wpb=18383.4, bsz=1024, num_updates=2310, lr=4.92327e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=14441
-2024-11-03 09:31:56 - progress_bar.py[line:274] - INFO: epoch 006:    365 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.88, ntokens=18150.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.72, wps=3484.8, ups=0.19, wpb=18150.1, bsz=1024, num_updates=2320, lr=4.94459e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=52, gb_free=13.3, wall=14493
-2024-11-03 09:32:48 - progress_bar.py[line:274] - INFO: epoch 006:    375 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.841, ntokens=18110.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.33, wps=3474.5, ups=0.19, wpb=18110.2, bsz=1024, num_updates=2330, lr=4.9659e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=52, gb_free=13.5, wall=14545
-2024-11-03 09:33:40 - progress_bar.py[line:274] - INFO: epoch 006:    385 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.817, ntokens=17669.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.1, wps=3399.5, ups=0.19, wpb=17669.9, bsz=1024, num_updates=2340, lr=4.98721e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=14597
+2024-11-06 17:08:08 - trainer.py[line:703] - INFO: begin training epoch 6
+2024-11-06 17:08:08 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 17:08:36 - progress_bar.py[line:274] - INFO: epoch 006:      5 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.632, ntokens=16956.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=348.9, ups=0.02, wpb=16956.5, bsz=985.6, num_updates=1960, lr=4.17732e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=38, gb_free=13.6, wall=12435
+2024-11-06 17:09:29 - progress_bar.py[line:274] - INFO: epoch 006:     15 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.587, ntokens=17600.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3366.8, ups=0.19, wpb=17600.6, bsz=1024, num_updates=1970, lr=4.19864e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=12487
+2024-11-06 17:10:20 - progress_bar.py[line:274] - INFO: epoch 006:     25 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.631, ntokens=17796.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3424.6, ups=0.19, wpb=17796.8, bsz=1024, num_updates=1980, lr=4.21995e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=12539
+2024-11-06 17:11:13 - progress_bar.py[line:274] - INFO: epoch 006:     35 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.588, ntokens=17729.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3373, ups=0.19, wpb=17729.1, bsz=1024, num_updates=1990, lr=4.24126e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=12592
+2024-11-06 17:12:05 - progress_bar.py[line:274] - INFO: epoch 006:     45 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.592, ntokens=17528.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3354.9, ups=0.19, wpb=17528.1, bsz=1024, num_updates=2000, lr=4.26257e-05, gnorm=0.007, clip=0, loss_scale=1024, train_wall=30, gb_free=13.6, wall=12644
+2024-11-06 17:12:57 - progress_bar.py[line:274] - INFO: epoch 006:     55 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.623, ntokens=17694, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3400.5, ups=0.19, wpb=17694, bsz=1024, num_updates=2010, lr=4.28389e-05, gnorm=0.01, clip=0, loss_scale=1024, train_wall=30, gb_free=13.3, wall=12696
+2024-11-06 17:13:49 - progress_bar.py[line:274] - INFO: epoch 006:     65 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.581, ntokens=17532.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3392.1, ups=0.19, wpb=17532.2, bsz=1024, num_updates=2020, lr=4.3052e-05, gnorm=0.008, clip=0, loss_scale=1024, train_wall=31, gb_free=13.6, wall=12748
+2024-11-06 17:14:41 - progress_bar.py[line:274] - INFO: epoch 006:     75 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.667, ntokens=18014.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3449.4, ups=0.19, wpb=18014.1, bsz=1024, num_updates=2030, lr=4.32651e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=12800
+2024-11-06 17:15:33 - progress_bar.py[line:274] - INFO: epoch 006:     85 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.579, ntokens=17453, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3355.1, ups=0.19, wpb=17453, bsz=1024, num_updates=2040, lr=4.34783e-05, gnorm=0.006, clip=0, loss_scale=1024, train_wall=30, gb_free=13.5, wall=12852
+2024-11-06 17:16:25 - progress_bar.py[line:274] - INFO: epoch 006:     95 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.633, ntokens=17828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3424.8, ups=0.19, wpb=17828, bsz=1024, num_updates=2050, lr=4.36914e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.6, wall=12904
+2024-11-06 17:17:17 - progress_bar.py[line:274] - INFO: epoch 006:    105 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.59, ntokens=17386.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3333, ups=0.19, wpb=17386.2, bsz=1024, num_updates=2060, lr=4.39045e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=31, gb_free=13.6, wall=12956
+2024-11-06 17:18:09 - progress_bar.py[line:274] - INFO: epoch 006:    115 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.634, ntokens=17505, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3378.6, ups=0.19, wpb=17505, bsz=1024, num_updates=2070, lr=4.41176e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=13008
+2024-11-06 17:19:01 - progress_bar.py[line:274] - INFO: epoch 006:    125 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.629, ntokens=17815.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3415.9, ups=0.19, wpb=17815.9, bsz=1024, num_updates=2080, lr=4.43308e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=13060
+2024-11-06 17:19:54 - progress_bar.py[line:274] - INFO: epoch 006:    135 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.644, ntokens=17692.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3377.3, ups=0.19, wpb=17692.8, bsz=1024, num_updates=2090, lr=4.45439e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=13113
+2024-11-06 17:20:46 - progress_bar.py[line:274] - INFO: epoch 006:    145 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.596, ntokens=17534.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3343.6, ups=0.19, wpb=17534.2, bsz=1024, num_updates=2100, lr=4.4757e-05, gnorm=0.005, clip=0, loss_scale=2048, train_wall=31, gb_free=13.5, wall=13165
+2024-11-06 17:21:39 - progress_bar.py[line:274] - INFO: epoch 006:    155 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.645, ntokens=17791.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3389.5, ups=0.19, wpb=17791.4, bsz=1024, num_updates=2110, lr=4.49702e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=13218
+2024-11-06 17:22:32 - progress_bar.py[line:274] - INFO: epoch 006:    165 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=17780.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3312.9, ups=0.19, wpb=17780.2, bsz=1024, num_updates=2120, lr=4.51833e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=33, gb_free=13.6, wall=13271
+2024-11-06 17:23:27 - progress_bar.py[line:274] - INFO: epoch 006:    175 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.63, ntokens=17622.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3257.1, ups=0.18, wpb=17622.4, bsz=1024, num_updates=2130, lr=4.53964e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=49, gb_free=13.5, wall=13325
+2024-11-06 17:24:22 - progress_bar.py[line:274] - INFO: epoch 006:    185 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.646, ntokens=17660.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3175.1, ups=0.18, wpb=17660.9, bsz=1024, num_updates=2140, lr=4.56095e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=56, gb_free=13.5, wall=13381
+2024-11-06 17:25:15 - progress_bar.py[line:274] - INFO: epoch 006:    195 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.699, ntokens=17894.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3413.9, ups=0.19, wpb=17894.8, bsz=1024, num_updates=2150, lr=4.58227e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=52, gb_free=13.6, wall=13433
+2024-11-06 17:26:07 - progress_bar.py[line:274] - INFO: epoch 006:    205 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.681, ntokens=17892.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3383.9, ups=0.19, wpb=17892.5, bsz=1024, num_updates=2160, lr=4.60358e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=53, gb_free=13.5, wall=13486
+2024-11-06 17:27:02 - progress_bar.py[line:274] - INFO: epoch 006:    215 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.621, ntokens=17744.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.15, wps=3269.5, ups=0.18, wpb=17744.5, bsz=1024, num_updates=2170, lr=4.62489e-05, gnorm=0.012, clip=0, loss_scale=2048, train_wall=54, gb_free=13.6, wall=13541
+2024-11-06 17:27:54 - progress_bar.py[line:274] - INFO: epoch 006:    225 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.662, ntokens=17851.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3446.4, ups=0.19, wpb=17851.1, bsz=1024, num_updates=2180, lr=4.64621e-05, gnorm=0.012, clip=0, loss_scale=2048, train_wall=52, gb_free=13.6, wall=13592
+2024-11-06 17:28:45 - progress_bar.py[line:274] - INFO: epoch 006:    235 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.621, ntokens=17608.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.15, wps=3404.7, ups=0.19, wpb=17608.7, bsz=1024, num_updates=2190, lr=4.66752e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=13644
+2024-11-06 17:29:37 - progress_bar.py[line:274] - INFO: epoch 006:    245 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.618, ntokens=17412.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3343.5, ups=0.19, wpb=17412.8, bsz=1024, num_updates=2200, lr=4.68883e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=52, gb_free=13.6, wall=13696
+2024-11-06 17:30:32 - progress_bar.py[line:274] - INFO: epoch 006:    255 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.622, ntokens=17651.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3223.4, ups=0.18, wpb=17651.2, bsz=1024, num_updates=2210, lr=4.71014e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=55, gb_free=13.4, wall=13751
+2024-11-06 17:31:25 - progress_bar.py[line:274] - INFO: epoch 006:    265 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.704, ntokens=17978.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3418.8, ups=0.19, wpb=17978.2, bsz=1024, num_updates=2220, lr=4.73146e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=53, gb_free=13.5, wall=13804
+2024-11-06 17:32:17 - progress_bar.py[line:274] - INFO: epoch 006:    275 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.619, ntokens=17633.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.15, wps=3368.6, ups=0.19, wpb=17633.2, bsz=1024, num_updates=2230, lr=4.75277e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=13856
+2024-11-06 17:33:09 - progress_bar.py[line:274] - INFO: epoch 006:    285 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.611, ntokens=17538.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3364.2, ups=0.19, wpb=17538.6, bsz=1024, num_updates=2240, lr=4.77408e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=13908
+2024-11-06 17:34:02 - progress_bar.py[line:274] - INFO: epoch 006:    295 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.669, ntokens=17736.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3370.1, ups=0.19, wpb=17736.7, bsz=1024, num_updates=2250, lr=4.7954e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=53, gb_free=13.5, wall=13961
+2024-11-06 17:34:54 - progress_bar.py[line:274] - INFO: epoch 006:    305 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.727, ntokens=18116.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3472, ups=0.19, wpb=18116.1, bsz=1024, num_updates=2260, lr=4.81671e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=52, gb_free=13.5, wall=14013
+2024-11-06 17:35:46 - progress_bar.py[line:274] - INFO: epoch 006:    315 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.793, ntokens=18532.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.93, wps=3548.6, ups=0.19, wpb=18532.7, bsz=1024, num_updates=2270, lr=4.83802e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=14065
+2024-11-06 17:36:38 - progress_bar.py[line:274] - INFO: epoch 006:    325 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.729, ntokens=18034.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3470.6, ups=0.19, wpb=18034.4, bsz=1024, num_updates=2280, lr=4.85934e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=14117
+2024-11-06 17:37:30 - progress_bar.py[line:274] - INFO: epoch 006:    335 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.746, ntokens=18357.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3534.1, ups=0.19, wpb=18357.4, bsz=1024, num_updates=2290, lr=4.88065e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=14169
+2024-11-06 17:38:25 - progress_bar.py[line:274] - INFO: epoch 006:    345 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.717, ntokens=18123.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3307.9, ups=0.18, wpb=18123.6, bsz=1024, num_updates=2300, lr=4.90196e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=55, gb_free=13.4, wall=14224
+2024-11-06 17:39:17 - progress_bar.py[line:274] - INFO: epoch 006:    355 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.766, ntokens=18383.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=3508.4, ups=0.19, wpb=18383.4, bsz=1024, num_updates=2310, lr=4.92327e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=14276
+2024-11-06 17:40:09 - progress_bar.py[line:274] - INFO: epoch 006:    365 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.741, ntokens=18150.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3482.3, ups=0.19, wpb=18150.1, bsz=1024, num_updates=2320, lr=4.94459e-05, gnorm=0.012, clip=0, loss_scale=2048, train_wall=52, gb_free=13.3, wall=14328
+2024-11-06 17:41:02 - progress_bar.py[line:274] - INFO: epoch 006:    375 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=18110.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3477.1, ups=0.19, wpb=18110.2, bsz=1024, num_updates=2330, lr=4.9659e-05, gnorm=0.014, clip=0, loss_scale=2048, train_wall=52, gb_free=13.5, wall=14380
+2024-11-06 17:41:53 - progress_bar.py[line:274] - INFO: epoch 006:    385 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.684, ntokens=17669.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3402.8, ups=0.19, wpb=17669.9, bsz=1024, num_updates=2340, lr=4.98721e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=52, gb_free=13.4, wall=14432
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 09:34:08 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 6 @ 2346 updates
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt
 
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt
-2024-11-03 09:34:08 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt': File exists
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt
+2024-11-06 17:42:21 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 6 @ 2346 updates
+
+2024-11-06 17:42:21 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt': File exists
+: File exists: File exists
 : File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 09:35:59 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 17:44:11 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+slice_id 1 seek offset 50000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 
-slice_id 7 seek offset 350000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+slice_id 6 seek offset 300000
 slice_id 3 seek offset 150000
-2024-11-03 09:37:51 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 6 @ 2346 updates, score 0) (writing took 222.8737999330042 seconds)
+slice_id 7 seek offset 350000
+2024-11-06 17:46:03 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 6 @ 2346 updates, score 0) (writing took 221.55774336399918 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_6.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 5 seek offset 250000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-slice_id 1 seek offset 50000
-2024-11-03 09:38:45 - train.py[line:336] - INFO: end of epoch 6 (average epoch stats below)
-2024-11-03 09:38:45 - progress_bar.py[line:282] - INFO: epoch 006 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.851 | ntokens 17784.1 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 14.43 | wps 2800.1 | ups 0.16 | wpb 17784.1 | bsz 1023 | num_updates 2346 | lr 5e-05 | gnorm 0.008 | clip 0 | loss_scale 2048 | train_wall 1492 | gb_free 13.3 | wall 14902
-2024-11-03 09:38:45 - trainer.py[line:639] - INFO: loading train data for epoch 7
+slice_id 4 seek offset 200000
+2024-11-06 17:46:57 - train.py[line:336] - INFO: end of epoch 6 (average epoch stats below)
+2024-11-06 17:46:57 - progress_bar.py[line:282] - INFO: epoch 006 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.656 | ntokens 17784.1 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.3 | wps 2794.5 | ups 0.16 | wpb 17784.1 | bsz 1023 | num_updates 2346 | lr 5e-05 | gnorm 0.008 | clip 0 | loss_scale 2048 | train_wall 1686 | gb_free 13.3 | wall 14736
+2024-11-06 17:46:57 - trainer.py[line:639] - INFO: loading train data for epoch 7
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 09:41:29 - trainer.py[line:703] - INFO: begin training epoch 7
-2024-11-03 09:41:29 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 09:41:53 - progress_bar.py[line:274] - INFO: epoch 007:      4 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.8, ntokens=17194.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=13.93, wps=349.2, ups=0.02, wpb=17194.7, bsz=985.6, num_updates=2350, lr=4.99946e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=40, gb_free=13.6, wall=15090
-2024-11-03 09:42:45 - progress_bar.py[line:274] - INFO: epoch 007:     14 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.832, ntokens=17776.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.25, wps=3414, ups=0.19, wpb=17776.3, bsz=1024, num_updates=2360, lr=4.9981e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15142
-2024-11-03 09:43:37 - progress_bar.py[line:274] - INFO: epoch 007:     24 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.783, ntokens=17561.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.77, wps=3358.4, ups=0.19, wpb=17561.7, bsz=1024, num_updates=2370, lr=4.99674e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15194
-2024-11-03 09:44:29 - progress_bar.py[line:274] - INFO: epoch 007:     34 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.825, ntokens=18089.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.18, wps=3479.9, ups=0.19, wpb=18089.7, bsz=1024, num_updates=2380, lr=4.99537e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=31, gb_free=13.4, wall=15246
-2024-11-03 09:45:21 - progress_bar.py[line:274] - INFO: epoch 007:     44 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.773, ntokens=17575, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.67, wps=3346.4, ups=0.19, wpb=17575, bsz=1024, num_updates=2390, lr=4.99401e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=30, gb_free=13.6, wall=15298
-2024-11-03 09:46:14 - progress_bar.py[line:274] - INFO: epoch 007:     54 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.787, ntokens=17857.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.81, wps=3422.9, ups=0.19, wpb=17857.9, bsz=1024, num_updates=2400, lr=4.99265e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15351
-2024-11-03 09:47:06 - progress_bar.py[line:274] - INFO: epoch 007:     64 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.821, ntokens=17915.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.13, wps=3395.3, ups=0.19, wpb=17915.1, bsz=1024, num_updates=2410, lr=4.99129e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15403
-2024-11-03 09:47:59 - progress_bar.py[line:274] - INFO: epoch 007:     74 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.759, ntokens=17736.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.54, wps=3373.4, ups=0.19, wpb=17736.8, bsz=1024, num_updates=2420, lr=4.98993e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15456
-2024-11-03 09:48:51 - progress_bar.py[line:274] - INFO: epoch 007:     84 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.77, ntokens=17626.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.64, wps=3385.4, ups=0.19, wpb=17626.6, bsz=1024, num_updates=2430, lr=4.98857e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=31, gb_free=13.5, wall=15508
-2024-11-03 09:49:44 - progress_bar.py[line:274] - INFO: epoch 007:     94 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.767, ntokens=17901.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.62, wps=3386.9, ups=0.19, wpb=17901.8, bsz=1024, num_updates=2440, lr=4.98721e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15561
-2024-11-03 09:50:40 - progress_bar.py[line:274] - INFO: epoch 007:    104 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.746, ntokens=17759.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.42, wps=3147, ups=0.18, wpb=17759.9, bsz=1024, num_updates=2450, lr=4.98585e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=31, gb_free=13.5, wall=15617
-2024-11-03 09:51:32 - progress_bar.py[line:274] - INFO: epoch 007:    114 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.792, ntokens=17760.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.85, wps=3400.9, ups=0.19, wpb=17760.6, bsz=1024, num_updates=2460, lr=4.98449e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15670
-2024-11-03 09:52:25 - progress_bar.py[line:274] - INFO: epoch 007:    124 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.698, ntokens=17529.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.98, wps=3318.8, ups=0.19, wpb=17529.9, bsz=1024, num_updates=2470, lr=4.98313e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15722
-2024-11-03 09:53:18 - progress_bar.py[line:274] - INFO: epoch 007:    134 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.698, ntokens=17282.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.98, wps=3304, ups=0.19, wpb=17282.4, bsz=1024, num_updates=2480, lr=4.98177e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15775
-2024-11-03 09:54:11 - progress_bar.py[line:274] - INFO: epoch 007:    144 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.753, ntokens=17853.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.48, wps=3375.2, ups=0.19, wpb=17853.9, bsz=1024, num_updates=2490, lr=4.98041e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15828
-2024-11-03 09:55:04 - progress_bar.py[line:274] - INFO: epoch 007:    154 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.761, ntokens=17790.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.56, wps=3320.9, ups=0.19, wpb=17790.8, bsz=1024, num_updates=2500, lr=4.97905e-05, gnorm=0.013, clip=0, loss_scale=2048, train_wall=31, gb_free=13.3, wall=15881
-2024-11-03 09:55:57 - progress_bar.py[line:274] - INFO: epoch 007:    164 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.76, ntokens=17768.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.54, wps=3333, ups=0.19, wpb=17768.3, bsz=1024, num_updates=2510, lr=4.97769e-05, gnorm=0.012, clip=0, loss_scale=2048, train_wall=31, gb_free=13.4, wall=15935
-2024-11-03 09:56:49 - progress_bar.py[line:274] - INFO: epoch 007:    174 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.791, ntokens=17920.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.84, wps=3459.2, ups=0.19, wpb=17920.7, bsz=1024, num_updates=2520, lr=4.97633e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15986
-2024-11-03 09:57:41 - progress_bar.py[line:274] - INFO: epoch 007:    184 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.671, ntokens=17346.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.74, wps=3336.9, ups=0.19, wpb=17346.3, bsz=1024, num_updates=2530, lr=4.97497e-05, gnorm=0.012, clip=0, loss_scale=2048, train_wall=31, gb_free=13.4, wall=16038
-2024-11-03 09:58:34 - progress_bar.py[line:274] - INFO: epoch 007:    194 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.755, ntokens=17869.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.51, wps=3404.5, ups=0.19, wpb=17869.5, bsz=1024, num_updates=2540, lr=4.97361e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=16091
-2024-11-03 09:59:27 - progress_bar.py[line:274] - INFO: epoch 007:    204 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.732, ntokens=17821.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.29, wps=3362.6, ups=0.19, wpb=17821.4, bsz=1024, num_updates=2550, lr=4.97225e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=30, gb_free=13.6, wall=16144
-2024-11-03 10:00:19 - progress_bar.py[line:274] - INFO: epoch 007:    214 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.738, ntokens=18120.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.35, wps=3444.6, ups=0.19, wpb=18120.2, bsz=1024, num_updates=2560, lr=4.97089e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=31, gb_free=13.4, wall=16196
-2024-11-03 10:01:12 - progress_bar.py[line:274] - INFO: epoch 007:    224 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.734, ntokens=17745.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.31, wps=3384.1, ups=0.19, wpb=17745.7, bsz=1024, num_updates=2570, lr=4.96953e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=16249
-2024-11-03 10:02:05 - progress_bar.py[line:274] - INFO: epoch 007:    234 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.673, ntokens=17390.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.76, wps=3282.7, ups=0.19, wpb=17390.6, bsz=1024, num_updates=2580, lr=4.96817e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=32, gb_free=13.4, wall=16302
-2024-11-03 10:02:59 - progress_bar.py[line:274] - INFO: epoch 007:    244 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.757, ntokens=17671.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.52, wps=3271.4, ups=0.19, wpb=17671.1, bsz=1024, num_updates=2590, lr=4.96681e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=38, gb_free=13.4, wall=16356
-2024-11-03 10:03:51 - progress_bar.py[line:274] - INFO: epoch 007:    254 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.733, ntokens=17862, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.29, wps=3405.1, ups=0.19, wpb=17862, bsz=1024, num_updates=2600, lr=4.96545e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=52, gb_free=13.3, wall=16408
-2024-11-03 10:04:43 - progress_bar.py[line:274] - INFO: epoch 007:    264 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.694, ntokens=17673.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.94, wps=3403.9, ups=0.19, wpb=17673.5, bsz=1024, num_updates=2610, lr=4.96409e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=47, gb_free=13.5, wall=16460
-2024-11-03 10:05:35 - progress_bar.py[line:274] - INFO: epoch 007:    274 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.752, ntokens=17901.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.48, wps=3447.2, ups=0.19, wpb=17901.4, bsz=1024, num_updates=2620, lr=4.96273e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=43, gb_free=13.5, wall=16512
-2024-11-03 10:06:27 - progress_bar.py[line:274] - INFO: epoch 007:    284 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.634, ntokens=17429.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.42, wps=3375.2, ups=0.19, wpb=17429.2, bsz=1024, num_updates=2630, lr=4.96136e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=39, gb_free=13.6, wall=16564
-2024-11-03 10:07:20 - progress_bar.py[line:274] - INFO: epoch 007:    294 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.693, ntokens=17591.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.93, wps=3323.7, ups=0.19, wpb=17591.7, bsz=1024, num_updates=2640, lr=4.96e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=39, gb_free=13.4, wall=16617
-2024-11-03 10:08:12 - progress_bar.py[line:274] - INFO: epoch 007:    304 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.824, ntokens=18256.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.16, wps=3495.6, ups=0.19, wpb=18256.5, bsz=1024, num_updates=2650, lr=4.95864e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=50, gb_free=13.3, wall=16669
-2024-11-03 10:09:04 - progress_bar.py[line:274] - INFO: epoch 007:    314 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.859, ntokens=18767.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.51, wps=3585.7, ups=0.19, wpb=18767.4, bsz=1024, num_updates=2660, lr=4.95728e-05, gnorm=0.005, clip=0, loss_scale=4096, train_wall=52, gb_free=13.2, wall=16721
-2024-11-03 10:09:56 - progress_bar.py[line:274] - INFO: epoch 007:    324 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.777, ntokens=18130.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.71, wps=3486.4, ups=0.19, wpb=18130.8, bsz=1024, num_updates=2670, lr=4.95592e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=52, gb_free=13.4, wall=16773
-2024-11-03 10:10:49 - progress_bar.py[line:274] - INFO: epoch 007:    334 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.803, ntokens=18209.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.96, wps=3479.2, ups=0.19, wpb=18209.3, bsz=1024, num_updates=2680, lr=4.95456e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=52, gb_free=13.5, wall=16826
-2024-11-03 10:11:40 - progress_bar.py[line:274] - INFO: epoch 007:    344 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.724, ntokens=17916.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.21, wps=3458.6, ups=0.19, wpb=17916.4, bsz=1024, num_updates=2690, lr=4.9532e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=51, gb_free=13.4, wall=16877
-2024-11-03 10:12:33 - progress_bar.py[line:274] - INFO: epoch 007:    354 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.797, ntokens=18730.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.9, wps=3561.2, ups=0.19, wpb=18730.4, bsz=1024, num_updates=2700, lr=4.95184e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=52, gb_free=13.1, wall=16930
-2024-11-03 10:13:25 - progress_bar.py[line:274] - INFO: epoch 007:    364 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.835, ntokens=18436.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=14.27, wps=3555.3, ups=0.19, wpb=18436.4, bsz=1024, num_updates=2710, lr=4.95048e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=52, gb_free=13.5, wall=16982
-2024-11-03 10:14:16 - progress_bar.py[line:274] - INFO: epoch 007:    374 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.724, ntokens=17946, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.22, wps=3477.3, ups=0.19, wpb=17946, bsz=1024, num_updates=2720, lr=4.94912e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=52, gb_free=13.3, wall=17033
-2024-11-03 10:15:08 - progress_bar.py[line:274] - INFO: epoch 007:    384 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.78, ntokens=18220.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.73, wps=3545.1, ups=0.19, wpb=18220.8, bsz=1024, num_updates=2730, lr=4.94776e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=51, gb_free=13.4, wall=17085
+2024-11-06 17:49:36 - trainer.py[line:703] - INFO: begin training epoch 7
+2024-11-06 17:49:36 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 17:50:00 - progress_bar.py[line:274] - INFO: epoch 007:      4 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.682, ntokens=17194.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=353.7, ups=0.02, wpb=17194.7, bsz=985.6, num_updates=2350, lr=4.99946e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=40, gb_free=13.6, wall=14918
+2024-11-06 17:50:52 - progress_bar.py[line:274] - INFO: epoch 007:     14 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.634, ntokens=17776.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3385.9, ups=0.19, wpb=17776.3, bsz=1024, num_updates=2360, lr=4.9981e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=14971
+2024-11-06 17:51:45 - progress_bar.py[line:274] - INFO: epoch 007:     24 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.616, ntokens=17561.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3329.1, ups=0.19, wpb=17561.7, bsz=1024, num_updates=2370, lr=4.99674e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15024
+2024-11-06 17:52:38 - progress_bar.py[line:274] - INFO: epoch 007:     34 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=18089.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3423, ups=0.19, wpb=18089.7, bsz=1024, num_updates=2380, lr=4.99537e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15077
+2024-11-06 17:53:31 - progress_bar.py[line:274] - INFO: epoch 007:     44 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.605, ntokens=17575, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3316.8, ups=0.19, wpb=17575, bsz=1024, num_updates=2390, lr=4.99401e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.6, wall=15130
+2024-11-06 17:54:24 - progress_bar.py[line:274] - INFO: epoch 007:     54 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.636, ntokens=17857.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.22, wps=3350.3, ups=0.19, wpb=17857.9, bsz=1024, num_updates=2400, lr=4.99265e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15183
+2024-11-06 17:55:17 - progress_bar.py[line:274] - INFO: epoch 007:     64 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.668, ntokens=17915.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3402.1, ups=0.19, wpb=17915.1, bsz=1024, num_updates=2410, lr=4.99129e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=31, gb_free=13.5, wall=15235
+2024-11-06 17:56:10 - progress_bar.py[line:274] - INFO: epoch 007:     74 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.645, ntokens=17736.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3320, ups=0.19, wpb=17736.8, bsz=1024, num_updates=2420, lr=4.98993e-05, gnorm=0.014, clip=0, loss_scale=2048, train_wall=31, gb_free=13.4, wall=15289
+2024-11-06 17:57:04 - progress_bar.py[line:274] - INFO: epoch 007:     84 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.604, ntokens=17626.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3254.3, ups=0.18, wpb=17626.6, bsz=1024, num_updates=2430, lr=4.98857e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15343
+2024-11-06 17:57:58 - progress_bar.py[line:274] - INFO: epoch 007:     94 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.659, ntokens=17901.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3317.7, ups=0.19, wpb=17901.8, bsz=1024, num_updates=2440, lr=4.98721e-05, gnorm=0.007, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15397
+2024-11-06 17:58:52 - progress_bar.py[line:274] - INFO: epoch 007:    104 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.636, ntokens=17759.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.22, wps=3329.2, ups=0.19, wpb=17759.9, bsz=1024, num_updates=2450, lr=4.98585e-05, gnorm=0.006, clip=0, loss_scale=2048, train_wall=31, gb_free=13.5, wall=15450
+2024-11-06 17:59:45 - progress_bar.py[line:274] - INFO: epoch 007:    114 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.675, ntokens=17760.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=3350.8, ups=0.19, wpb=17760.6, bsz=1024, num_updates=2460, lr=4.98449e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15503
+2024-11-06 18:00:37 - progress_bar.py[line:274] - INFO: epoch 007:    124 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.583, ntokens=17529.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3355, ups=0.19, wpb=17529.9, bsz=1024, num_updates=2470, lr=4.98313e-05, gnorm=0.01, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15556
+2024-11-06 18:01:29 - progress_bar.py[line:274] - INFO: epoch 007:    134 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.578, ntokens=17282.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3336.2, ups=0.19, wpb=17282.4, bsz=1024, num_updates=2480, lr=4.98177e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15607
+2024-11-06 18:02:21 - progress_bar.py[line:274] - INFO: epoch 007:    144 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=17853.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3419.8, ups=0.19, wpb=17853.9, bsz=1024, num_updates=2490, lr=4.98041e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15660
+2024-11-06 18:03:13 - progress_bar.py[line:274] - INFO: epoch 007:    154 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=17790.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3419.2, ups=0.19, wpb=17790.8, bsz=1024, num_updates=2500, lr=4.97905e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.3, wall=15712
+2024-11-06 18:04:05 - progress_bar.py[line:274] - INFO: epoch 007:    164 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.635, ntokens=17768.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3377.6, ups=0.19, wpb=17768.3, bsz=1024, num_updates=2510, lr=4.97769e-05, gnorm=0.009, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15764
+2024-11-06 18:04:57 - progress_bar.py[line:274] - INFO: epoch 007:    174 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.655, ntokens=17920.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3447, ups=0.19, wpb=17920.7, bsz=1024, num_updates=2520, lr=4.97633e-05, gnorm=0.012, clip=0, loss_scale=2048, train_wall=31, gb_free=13.4, wall=15816
+2024-11-06 18:05:49 - progress_bar.py[line:274] - INFO: epoch 007:    184 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.583, ntokens=17346.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3338.8, ups=0.19, wpb=17346.3, bsz=1024, num_updates=2530, lr=4.97497e-05, gnorm=0.011, clip=0, loss_scale=2048, train_wall=30, gb_free=13.4, wall=15868
+2024-11-06 18:06:42 - progress_bar.py[line:274] - INFO: epoch 007:    194 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.649, ntokens=17869.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3426.7, ups=0.19, wpb=17869.5, bsz=1024, num_updates=2540, lr=4.97361e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.5, wall=15920
+2024-11-06 18:07:34 - progress_bar.py[line:274] - INFO: epoch 007:    204 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.644, ntokens=17821.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3414.7, ups=0.19, wpb=17821.4, bsz=1024, num_updates=2550, lr=4.97225e-05, gnorm=0.008, clip=0, loss_scale=2048, train_wall=30, gb_free=13.6, wall=15973
+2024-11-06 18:08:26 - progress_bar.py[line:274] - INFO: epoch 007:    214 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.666, ntokens=18120.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3434.4, ups=0.19, wpb=18120.2, bsz=1024, num_updates=2560, lr=4.97089e-05, gnorm=0.011, clip=0, loss_scale=4096, train_wall=31, gb_free=13.4, wall=16025
+2024-11-06 18:09:19 - progress_bar.py[line:274] - INFO: epoch 007:    224 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.63, ntokens=17745.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3388.9, ups=0.19, wpb=17745.7, bsz=1024, num_updates=2570, lr=4.96953e-05, gnorm=0.014, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=16078
+2024-11-06 18:10:11 - progress_bar.py[line:274] - INFO: epoch 007:    234 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.594, ntokens=17390.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3319.3, ups=0.19, wpb=17390.6, bsz=1024, num_updates=2580, lr=4.96817e-05, gnorm=0.015, clip=0, loss_scale=4096, train_wall=30, gb_free=13.4, wall=16130
+2024-11-06 18:11:03 - progress_bar.py[line:274] - INFO: epoch 007:    244 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.644, ntokens=17671.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3383.1, ups=0.19, wpb=17671.1, bsz=1024, num_updates=2590, lr=4.96681e-05, gnorm=0.014, clip=0, loss_scale=4096, train_wall=30, gb_free=13.4, wall=16182
+2024-11-06 18:11:56 - progress_bar.py[line:274] - INFO: epoch 007:    254 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.667, ntokens=17862, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3422.6, ups=0.19, wpb=17862, bsz=1024, num_updates=2600, lr=4.96545e-05, gnorm=0.013, clip=0, loss_scale=4096, train_wall=30, gb_free=13.3, wall=16235
+2024-11-06 18:12:47 - progress_bar.py[line:274] - INFO: epoch 007:    264 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.632, ntokens=17673.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3419, ups=0.19, wpb=17673.5, bsz=1024, num_updates=2610, lr=4.96409e-05, gnorm=0.011, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=16286
+2024-11-06 18:13:40 - progress_bar.py[line:274] - INFO: epoch 007:    274 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.646, ntokens=17901.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3420.6, ups=0.19, wpb=17901.4, bsz=1024, num_updates=2620, lr=4.96273e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=16339
+2024-11-06 18:14:32 - progress_bar.py[line:274] - INFO: epoch 007:    284 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.588, ntokens=17429.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3362.3, ups=0.19, wpb=17429.2, bsz=1024, num_updates=2630, lr=4.96136e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=30, gb_free=13.6, wall=16390
+2024-11-06 18:15:24 - progress_bar.py[line:274] - INFO: epoch 007:    294 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.637, ntokens=17591.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.22, wps=3361.5, ups=0.19, wpb=17591.7, bsz=1024, num_updates=2640, lr=4.96e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=30, gb_free=13.4, wall=16443
+2024-11-06 18:16:16 - progress_bar.py[line:274] - INFO: epoch 007:    304 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.733, ntokens=18256.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3525, ups=0.19, wpb=18256.5, bsz=1024, num_updates=2650, lr=4.95864e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=32, gb_free=13.3, wall=16495
+2024-11-06 18:17:09 - progress_bar.py[line:274] - INFO: epoch 007:    314 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.83, ntokens=18767.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3514.4, ups=0.19, wpb=18767.4, bsz=1024, num_updates=2660, lr=4.95728e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=48, gb_free=13.2, wall=16548
+2024-11-06 18:18:02 - progress_bar.py[line:274] - INFO: epoch 007:    324 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.722, ntokens=18130.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.6, wps=3429.3, ups=0.19, wpb=18130.8, bsz=1024, num_updates=2670, lr=4.95592e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=53, gb_free=13.4, wall=16601
+2024-11-06 18:18:55 - progress_bar.py[line:274] - INFO: epoch 007:    334 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.742, ntokens=18209.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3400.8, ups=0.19, wpb=18209.3, bsz=1024, num_updates=2680, lr=4.95456e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=53, gb_free=13.5, wall=16654
+2024-11-06 18:19:48 - progress_bar.py[line:274] - INFO: epoch 007:    344 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.67, ntokens=17916.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3418.8, ups=0.19, wpb=17916.4, bsz=1024, num_updates=2690, lr=4.9532e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=52, gb_free=13.4, wall=16707
+2024-11-06 18:20:40 - progress_bar.py[line:274] - INFO: epoch 007:    354 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.767, ntokens=18730.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.81, wps=3619.4, ups=0.19, wpb=18730.4, bsz=1024, num_updates=2700, lr=4.95184e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=52, gb_free=13.1, wall=16758
+2024-11-06 18:21:34 - progress_bar.py[line:274] - INFO: epoch 007:    364 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.78, ntokens=18436.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.87, wps=3417.3, ups=0.19, wpb=18436.4, bsz=1024, num_updates=2710, lr=4.95048e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=54, gb_free=13.4, wall=16812
+2024-11-06 18:22:26 - progress_bar.py[line:274] - INFO: epoch 007:    374 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.661, ntokens=17946, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3453.8, ups=0.19, wpb=17946, bsz=1024, num_updates=2720, lr=4.94912e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=52, gb_free=13.3, wall=16864
+2024-11-06 18:23:17 - progress_bar.py[line:274] - INFO: epoch 007:    384 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.728, ntokens=18220.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3525.6, ups=0.19, wpb=18220.8, bsz=1024, num_updates=2730, lr=4.94776e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=52, gb_free=13.4, wall=16916
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt
-2024-11-03 10:15:41 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 7 @ 2737 updates
-2024-11-03 10:15:41 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt': File exists: File exists
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt2024-11-06 18:23:50 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 7 @ 2737 updates
 
-: File exists: File exists: File exists
 
+2024-11-06 18:23:50 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt': File exists: File exists: File exists
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 10:17:26 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 18:25:41 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 
-slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+
+slice_id 2 seek offset 100000
+slice_id 7 seek offset 350000
+slice_id 3 seek offset 150000
+2024-11-06 18:27:33 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 7 @ 2737 updates, score 0) (writing took 222.42839722700592 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+slice_id 6 seek offset 300000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+
+
 slice_id 1 seek offset 50000
 slice_id 5 seek offset 250000
-slice_id 6 seek offset 300000
-2024-11-03 10:19:18 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 7 @ 2737 updates, score 0) (writing took 217.5963608849852 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_7.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-slice_id 2 seek offset 100000
-2024-11-03 10:20:13 - train.py[line:336] - INFO: end of epoch 7 (average epoch stats below)
-2024-11-03 10:20:13 - progress_bar.py[line:282] - INFO: epoch 007 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.759 | ntokens 17839.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 13.53 | wps 2804.1 | ups 0.16 | wpb 17839.9 | bsz 1023 | num_updates 2737 | lr 4.94681e-05 | gnorm 0.008 | clip 0 | loss_scale 4096 | train_wall 1469 | gb_free 13.4 | wall 17390
-2024-11-03 10:20:13 - trainer.py[line:639] - INFO: loading train data for epoch 8
+slice_id 4 seek offset 200000
+2024-11-06 18:28:27 - train.py[line:336] - INFO: end of epoch 7 (average epoch stats below)
+2024-11-06 18:28:27 - progress_bar.py[line:282] - INFO: epoch 007 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.658 | ntokens 17839.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.31 | wps 2801.4 | ups 0.16 | wpb 17839.9 | bsz 1023 | num_updates 2737 | lr 4.94681e-05 | gnorm 0.009 | clip 0 | loss_scale 4096 | train_wall 1375 | gb_free 13.4 | wall 17226
+2024-11-06 18:28:27 - trainer.py[line:639] - INFO: loading train data for epoch 8
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 10:22:57 - trainer.py[line:703] - INFO: begin training epoch 8
-2024-11-03 10:22:57 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 10:23:15 - progress_bar.py[line:274] - INFO: epoch 008:      3 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.732, ntokens=17100.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=13.29, wps=350.8, ups=0.02, wpb=17100.5, bsz=985.6, num_updates=2740, lr=4.9464e-05, gnorm=0.011, clip=0, loss_scale=4096, train_wall=41, gb_free=13.5, wall=17572
-2024-11-03 10:24:07 - progress_bar.py[line:274] - INFO: epoch 008:     13 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.677, ntokens=17414.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.79, wps=3343.7, ups=0.19, wpb=17414.2, bsz=1024, num_updates=2750, lr=4.94504e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=31, gb_free=13.4, wall=17624
-2024-11-03 10:25:00 - progress_bar.py[line:274] - INFO: epoch 008:     23 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.725, ntokens=17912, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.23, wps=3399.5, ups=0.19, wpb=17912, bsz=1024, num_updates=2760, lr=4.94368e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=31, gb_free=13.6, wall=17677
-2024-11-03 10:25:52 - progress_bar.py[line:274] - INFO: epoch 008:     33 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.695, ntokens=17865.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.95, wps=3428.2, ups=0.19, wpb=17865.8, bsz=1024, num_updates=2770, lr=4.94232e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=17729
-2024-11-03 10:26:44 - progress_bar.py[line:274] - INFO: epoch 008:     43 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.644, ntokens=17511.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.5, wps=3359.9, ups=0.19, wpb=17511.3, bsz=1024, num_updates=2780, lr=4.94096e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=30, gb_free=13.4, wall=17781
-2024-11-03 10:27:37 - progress_bar.py[line:274] - INFO: epoch 008:     53 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.62, ntokens=17472.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.3, wps=3315.8, ups=0.19, wpb=17472.5, bsz=1024, num_updates=2790, lr=4.9396e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=17834
-2024-11-03 10:28:29 - progress_bar.py[line:274] - INFO: epoch 008:     63 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.739, ntokens=18033.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.35, wps=3464.7, ups=0.19, wpb=18033.8, bsz=1024, num_updates=2800, lr=4.93824e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=30, gb_free=13.4, wall=17886
-2024-11-03 10:29:21 - progress_bar.py[line:274] - INFO: epoch 008:     73 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.679, ntokens=17705.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.81, wps=3407.7, ups=0.19, wpb=17705.5, bsz=1024, num_updates=2810, lr=4.93688e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=31, gb_free=13.5, wall=17938
-2024-11-03 10:30:14 - progress_bar.py[line:274] - INFO: epoch 008:     83 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.607, ntokens=17328.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.19, wps=3239.4, ups=0.19, wpb=17328.2, bsz=1024, num_updates=2820, lr=4.93552e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=37, gb_free=13.5, wall=17992
-2024-11-03 10:31:07 - progress_bar.py[line:274] - INFO: epoch 008:     93 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.652, ntokens=17708, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.57, wps=3390.2, ups=0.19, wpb=17708, bsz=1024, num_updates=2830, lr=4.93416e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=46, gb_free=13.5, wall=18044
-2024-11-03 10:31:58 - progress_bar.py[line:274] - INFO: epoch 008:    103 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.619, ntokens=17518.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.28, wps=3385.2, ups=0.19, wpb=17518.6, bsz=1024, num_updates=2840, lr=4.9328e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=44, gb_free=13.6, wall=18096
-2024-11-03 10:32:51 - progress_bar.py[line:274] - INFO: epoch 008:    113 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.668, ntokens=17769.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.71, wps=3403.7, ups=0.19, wpb=17769.1, bsz=1024, num_updates=2850, lr=4.93144e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=42, gb_free=13.6, wall=18148
-2024-11-03 10:33:42 - progress_bar.py[line:274] - INFO: epoch 008:    123 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.63, ntokens=17828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.38, wps=3447.1, ups=0.19, wpb=17828, bsz=1024, num_updates=2860, lr=4.93008e-05, gnorm=0.005, clip=0, loss_scale=4096, train_wall=38, gb_free=13.6, wall=18199
-2024-11-03 10:34:35 - progress_bar.py[line:274] - INFO: epoch 008:    133 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.667, ntokens=17679.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.7, wps=3372.7, ups=0.19, wpb=17679.5, bsz=1024, num_updates=2870, lr=4.92872e-05, gnorm=0.005, clip=0, loss_scale=4096, train_wall=35, gb_free=13.5, wall=18252
-2024-11-03 10:35:27 - progress_bar.py[line:274] - INFO: epoch 008:    143 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.66, ntokens=17740.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.64, wps=3398.6, ups=0.19, wpb=17740.8, bsz=1024, num_updates=2880, lr=4.92735e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=31, gb_free=13.3, wall=18304
-2024-11-03 10:36:19 - progress_bar.py[line:274] - INFO: epoch 008:    153 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.658, ntokens=17721.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.63, wps=3408, ups=0.19, wpb=17721.6, bsz=1024, num_updates=2890, lr=4.92599e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=30, gb_free=13.6, wall=18356
-2024-11-03 10:37:11 - progress_bar.py[line:274] - INFO: epoch 008:    163 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.632, ntokens=17699.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.4, wps=3375.3, ups=0.19, wpb=17699.2, bsz=1024, num_updates=2900, lr=4.92463e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=18408
-2024-11-03 10:38:03 - progress_bar.py[line:274] - INFO: epoch 008:    173 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.644, ntokens=17743.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.5, wps=3423.2, ups=0.19, wpb=17743.7, bsz=1024, num_updates=2910, lr=4.92327e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=18460
-2024-11-03 10:38:56 - progress_bar.py[line:274] - INFO: epoch 008:    183 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.587, ntokens=17318.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.02, wps=3310.6, ups=0.19, wpb=17318.7, bsz=1024, num_updates=2920, lr=4.92191e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=31, gb_free=13.5, wall=18513
-2024-11-03 10:39:48 - progress_bar.py[line:274] - INFO: epoch 008:    193 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.645, ntokens=17854, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.51, wps=3429.5, ups=0.19, wpb=17854, bsz=1024, num_updates=2930, lr=4.92055e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=30, gb_free=13.3, wall=18565
-2024-11-03 10:40:40 - progress_bar.py[line:274] - INFO: epoch 008:    203 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.673, ntokens=18015.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.75, wps=3438.1, ups=0.19, wpb=18015.2, bsz=1024, num_updates=2940, lr=4.91919e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=31, gb_free=13.4, wall=18617
-2024-11-03 10:41:34 - progress_bar.py[line:274] - INFO: epoch 008:    213 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.603, ntokens=18036.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.15, wps=3341.3, ups=0.19, wpb=18036.2, bsz=1024, num_updates=2950, lr=4.91783e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=31, gb_free=13.5, wall=18671
-2024-11-03 10:42:32 - progress_bar.py[line:274] - INFO: epoch 008:    223 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.64, ntokens=17810.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.47, wps=3091.3, ups=0.17, wpb=17810.8, bsz=1024, num_updates=2960, lr=4.91647e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=31, gb_free=13.6, wall=18729
-2024-11-03 10:43:25 - progress_bar.py[line:274] - INFO: epoch 008:    233 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.653, ntokens=17937.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.58, wps=3330, ups=0.19, wpb=17937.7, bsz=1024, num_updates=2970, lr=4.91511e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=31, gb_free=13.6, wall=18783
-2024-11-03 10:44:18 - progress_bar.py[line:274] - INFO: epoch 008:    243 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.667, ntokens=17626.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.7, wps=3375.8, ups=0.19, wpb=17626.8, bsz=1024, num_updates=2980, lr=4.91375e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=30, gb_free=13.6, wall=18835
-2024-11-03 10:45:11 - progress_bar.py[line:274] - INFO: epoch 008:    253 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.557, ntokens=17410.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.77, wps=3280.2, ups=0.19, wpb=17410.1, bsz=1024, num_updates=2990, lr=4.91239e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=31, gb_free=13.5, wall=18888
-2024-11-03 10:46:03 - progress_bar.py[line:274] - INFO: epoch 008:    263 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.632, ntokens=17756.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.4, wps=3383.7, ups=0.19, wpb=17756.9, bsz=1024, num_updates=3000, lr=4.91103e-05, gnorm=0.005, clip=0, loss_scale=4096, train_wall=31, gb_free=13.6, wall=18940
-2024-11-03 10:46:56 - progress_bar.py[line:274] - INFO: epoch 008:    273 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.655, ntokens=17970.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.6, wps=3429.1, ups=0.19, wpb=17970.2, bsz=1024, num_updates=3010, lr=4.90967e-05, gnorm=0.005, clip=0, loss_scale=4096, train_wall=31, gb_free=13.5, wall=18993
-2024-11-03 10:47:48 - progress_bar.py[line:274] - INFO: epoch 008:    283 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.588, ntokens=17699.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.03, wps=3371.5, ups=0.19, wpb=17699.8, bsz=1024, num_updates=3020, lr=4.90831e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=30, gb_free=13.6, wall=19045
-2024-11-03 10:48:41 - progress_bar.py[line:274] - INFO: epoch 008:    293 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.595, ntokens=17659.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.08, wps=3366.3, ups=0.19, wpb=17659.2, bsz=1024, num_updates=3030, lr=4.90695e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=19098
-2024-11-03 10:49:32 - progress_bar.py[line:274] - INFO: epoch 008:    303 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.698, ntokens=18053, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.98, wps=3511.3, ups=0.19, wpb=18053, bsz=1024, num_updates=3040, lr=4.90559e-05, gnorm=0.005, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=19149
-2024-11-03 10:50:23 - progress_bar.py[line:274] - INFO: epoch 008:    313 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.74, ntokens=18369.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.36, wps=3594.9, ups=0.2, wpb=18369.3, bsz=1024, num_updates=3050, lr=4.90423e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=31, gb_free=13.4, wall=19200
-2024-11-03 10:51:15 - progress_bar.py[line:274] - INFO: epoch 008:    323 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.676, ntokens=17966.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.78, wps=3487.3, ups=0.19, wpb=17966.4, bsz=1024, num_updates=3060, lr=4.90287e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=30, gb_free=13.4, wall=19252
-2024-11-03 10:52:07 - progress_bar.py[line:274] - INFO: epoch 008:    333 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.77, ntokens=18654.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.64, wps=3568.1, ups=0.19, wpb=18654.4, bsz=1024, num_updates=3070, lr=4.90151e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=31, gb_free=13.1, wall=19304
-2024-11-03 10:52:59 - progress_bar.py[line:274] - INFO: epoch 008:    343 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.695, ntokens=18152.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.95, wps=3484.3, ups=0.19, wpb=18152.6, bsz=1024, num_updates=3080, lr=4.90015e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=36, gb_free=13.5, wall=19356
-2024-11-03 10:53:51 - progress_bar.py[line:274] - INFO: epoch 008:    353 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.694, ntokens=18466.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.95, wps=3528.8, ups=0.19, wpb=18466.4, bsz=1024, num_updates=3090, lr=4.89879e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=46, gb_free=13.4, wall=19408
-2024-11-03 10:54:43 - progress_bar.py[line:274] - INFO: epoch 008:    363 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.758, ntokens=18319.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=13.53, wps=3531.6, ups=0.19, wpb=18319.5, bsz=1024, num_updates=3100, lr=4.89743e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=51, gb_free=13.3, wall=19460
-2024-11-03 10:55:35 - progress_bar.py[line:274] - INFO: epoch 008:    373 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.668, ntokens=18187.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.71, wps=3500.5, ups=0.19, wpb=18187.8, bsz=1024, num_updates=3110, lr=4.89607e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=52, gb_free=13.3, wall=19512
-2024-11-03 10:56:27 - progress_bar.py[line:274] - INFO: epoch 008:    383 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.655, ntokens=17865.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.6, wps=3440.5, ups=0.19, wpb=17865.5, bsz=1024, num_updates=3120, lr=4.89471e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=52, gb_free=13.5, wall=19564
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-03 10:57:06 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 8 @ 3128 updates
+2024-11-06 18:31:06 - trainer.py[line:703] - INFO: begin training epoch 8
+2024-11-06 18:31:06 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 18:31:24 - progress_bar.py[line:274] - INFO: epoch 008:      3 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.648, ntokens=17100.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=351.1, ups=0.02, wpb=17100.5, bsz=985.6, num_updates=2740, lr=4.9464e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=42, gb_free=13.5, wall=17403
+2024-11-06 18:32:16 - progress_bar.py[line:274] - INFO: epoch 008:     13 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.576, ntokens=17414.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3354, ups=0.19, wpb=17414.2, bsz=1024, num_updates=2750, lr=4.94504e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=30, gb_free=13.4, wall=17455
+2024-11-06 18:33:10 - progress_bar.py[line:274] - INFO: epoch 008:     23 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=17912, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3321.6, ups=0.19, wpb=17912, bsz=1024, num_updates=2760, lr=4.94368e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=37, gb_free=13.6, wall=17509
+2024-11-06 18:34:02 - progress_bar.py[line:274] - INFO: epoch 008:     33 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.603, ntokens=17865.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3433.4, ups=0.19, wpb=17865.8, bsz=1024, num_updates=2770, lr=4.94232e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=52, gb_free=13.5, wall=17561
+2024-11-06 18:34:54 - progress_bar.py[line:274] - INFO: epoch 008:     43 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.592, ntokens=17511.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3386.9, ups=0.19, wpb=17511.3, bsz=1024, num_updates=2780, lr=4.94096e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=50, gb_free=13.4, wall=17613
+2024-11-06 18:35:46 - progress_bar.py[line:274] - INFO: epoch 008:     53 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.556, ntokens=17472.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3334.6, ups=0.19, wpb=17472.5, bsz=1024, num_updates=2790, lr=4.9396e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=50, gb_free=13.5, wall=17665
+2024-11-06 18:36:38 - progress_bar.py[line:274] - INFO: epoch 008:     63 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=18033.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3471.1, ups=0.19, wpb=18033.8, bsz=1024, num_updates=2800, lr=4.93824e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=50, gb_free=13.4, wall=17717
+2024-11-06 18:37:31 - progress_bar.py[line:274] - INFO: epoch 008:     73 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.648, ntokens=17705.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3374.4, ups=0.19, wpb=17705.5, bsz=1024, num_updates=2810, lr=4.93688e-05, gnorm=0.011, clip=0, loss_scale=4096, train_wall=52, gb_free=13.5, wall=17770
+2024-11-06 18:38:23 - progress_bar.py[line:274] - INFO: epoch 008:     83 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.547, ntokens=17328.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3316.8, ups=0.19, wpb=17328.2, bsz=1024, num_updates=2820, lr=4.93552e-05, gnorm=0.013, clip=0, loss_scale=4096, train_wall=52, gb_free=13.5, wall=17822
+2024-11-06 18:39:15 - progress_bar.py[line:274] - INFO: epoch 008:     93 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.627, ntokens=17708, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.18, wps=3415.6, ups=0.19, wpb=17708, bsz=1024, num_updates=2830, lr=4.93416e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=49, gb_free=13.4, wall=17874
+2024-11-06 18:40:07 - progress_bar.py[line:274] - INFO: epoch 008:    103 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.581, ntokens=17518.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3366.4, ups=0.19, wpb=17518.6, bsz=1024, num_updates=2840, lr=4.9328e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=46, gb_free=13.6, wall=17926
+2024-11-06 18:41:02 - progress_bar.py[line:274] - INFO: epoch 008:    113 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.654, ntokens=17769.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3240.9, ups=0.18, wpb=17769.1, bsz=1024, num_updates=2850, lr=4.93144e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=53, gb_free=13.5, wall=17981
+2024-11-06 18:41:53 - progress_bar.py[line:274] - INFO: epoch 008:    123 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.629, ntokens=17828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3459.6, ups=0.19, wpb=17828, bsz=1024, num_updates=2860, lr=4.93008e-05, gnorm=0.011, clip=0, loss_scale=4096, train_wall=51, gb_free=13.6, wall=18032
+2024-11-06 18:42:45 - progress_bar.py[line:274] - INFO: epoch 008:    133 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.622, ntokens=17679.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3401.1, ups=0.19, wpb=17679.5, bsz=1024, num_updates=2870, lr=4.92872e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=40, gb_free=13.5, wall=18084
+2024-11-06 18:43:40 - progress_bar.py[line:274] - INFO: epoch 008:    143 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.631, ntokens=17740.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3244.9, ups=0.18, wpb=17740.8, bsz=1024, num_updates=2880, lr=4.92735e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=36, gb_free=13.3, wall=18139
+2024-11-06 18:44:33 - progress_bar.py[line:274] - INFO: epoch 008:    153 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.618, ntokens=17721.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3348.6, ups=0.19, wpb=17721.6, bsz=1024, num_updates=2890, lr=4.92599e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=42, gb_free=13.6, wall=18192
+2024-11-06 18:45:26 - progress_bar.py[line:274] - INFO: epoch 008:    163 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.568, ntokens=17699.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3330.8, ups=0.19, wpb=17699.2, bsz=1024, num_updates=2900, lr=4.92463e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=32, gb_free=13.5, wall=18245
+2024-11-06 18:46:19 - progress_bar.py[line:274] - INFO: epoch 008:    173 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.624, ntokens=17743.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3343.1, ups=0.19, wpb=17743.7, bsz=1024, num_updates=2910, lr=4.92327e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=18298
+2024-11-06 18:47:12 - progress_bar.py[line:274] - INFO: epoch 008:    183 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.58, ntokens=17318.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3278.5, ups=0.19, wpb=17318.7, bsz=1024, num_updates=2920, lr=4.92191e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=18351
+2024-11-06 18:48:04 - progress_bar.py[line:274] - INFO: epoch 008:    193 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.646, ntokens=17854, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3402.4, ups=0.19, wpb=17854, bsz=1024, num_updates=2930, lr=4.92055e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=30, gb_free=13.3, wall=18403
+2024-11-06 18:48:57 - progress_bar.py[line:274] - INFO: epoch 008:    203 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=18015.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3440.2, ups=0.19, wpb=18015.2, bsz=1024, num_updates=2940, lr=4.91919e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=30, gb_free=13.4, wall=18456
+2024-11-06 18:49:49 - progress_bar.py[line:274] - INFO: epoch 008:    213 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.623, ntokens=18036.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3443.7, ups=0.19, wpb=18036.2, bsz=1024, num_updates=2950, lr=4.91783e-05, gnorm=0.011, clip=0, loss_scale=4096, train_wall=31, gb_free=13.5, wall=18508
+2024-11-06 18:50:42 - progress_bar.py[line:274] - INFO: epoch 008:    223 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.606, ntokens=17810.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=3384.8, ups=0.19, wpb=17810.8, bsz=1024, num_updates=2960, lr=4.91647e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=30, gb_free=13.6, wall=18561
+2024-11-06 18:51:34 - progress_bar.py[line:274] - INFO: epoch 008:    233 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.635, ntokens=17937.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3418.2, ups=0.19, wpb=17937.7, bsz=1024, num_updates=2970, lr=4.91511e-05, gnorm=0.011, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=18613
+2024-11-06 18:52:26 - progress_bar.py[line:274] - INFO: epoch 008:    243 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.618, ntokens=17626.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3382.7, ups=0.19, wpb=17626.8, bsz=1024, num_updates=2980, lr=4.91375e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=30, gb_free=13.6, wall=18665
+2024-11-06 18:53:18 - progress_bar.py[line:274] - INFO: epoch 008:    253 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=17410.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3336.7, ups=0.19, wpb=17410.1, bsz=1024, num_updates=2990, lr=4.91239e-05, gnorm=0.008, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=18717
+2024-11-06 18:54:11 - progress_bar.py[line:274] - INFO: epoch 008:    263 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.659, ntokens=17756.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3396.7, ups=0.19, wpb=17756.9, bsz=1024, num_updates=3000, lr=4.91103e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=30, gb_free=13.5, wall=18770
+2024-11-06 18:55:03 - progress_bar.py[line:274] - INFO: epoch 008:    273 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.618, ntokens=17970.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3452.6, ups=0.19, wpb=17970.2, bsz=1024, num_updates=3010, lr=4.90967e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=31, gb_free=13.5, wall=18822
+2024-11-06 18:55:55 - progress_bar.py[line:274] - INFO: epoch 008:    283 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.601, ntokens=17699.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3376.5, ups=0.19, wpb=17699.8, bsz=1024, num_updates=3020, lr=4.90831e-05, gnorm=0.007, clip=0, loss_scale=4096, train_wall=31, gb_free=13.6, wall=18874
+2024-11-06 18:56:49 - progress_bar.py[line:274] - INFO: epoch 008:    293 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.622, ntokens=17659.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.15, wps=3251.4, ups=0.18, wpb=17659.2, bsz=1024, num_updates=3030, lr=4.90695e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=47, gb_free=13.5, wall=18928
+2024-11-06 18:57:42 - progress_bar.py[line:274] - INFO: epoch 008:    303 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.673, ntokens=18053, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.38, wps=3446, ups=0.19, wpb=18053, bsz=1024, num_updates=3040, lr=4.90559e-05, gnorm=0.006, clip=0, loss_scale=4096, train_wall=52, gb_free=13.5, wall=18981
+2024-11-06 18:58:35 - progress_bar.py[line:274] - INFO: epoch 008:    313 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.761, ntokens=18369.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.78, wps=3474.8, ups=0.19, wpb=18369.3, bsz=1024, num_updates=3050, lr=4.90423e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=53, gb_free=13.4, wall=19034
+2024-11-06 18:59:26 - progress_bar.py[line:274] - INFO: epoch 008:    323 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.692, ntokens=17966.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3481.2, ups=0.19, wpb=17966.4, bsz=1024, num_updates=3060, lr=4.90287e-05, gnorm=0.01, clip=0, loss_scale=4096, train_wall=52, gb_free=13.4, wall=19085
+2024-11-06 19:00:19 - progress_bar.py[line:274] - INFO: epoch 008:    333 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.763, ntokens=18654.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3575.1, ups=0.19, wpb=18654.4, bsz=1024, num_updates=3070, lr=4.90151e-05, gnorm=0.009, clip=0, loss_scale=4096, train_wall=52, gb_free=13.1, wall=19137
+2024-11-06 19:01:11 - progress_bar.py[line:274] - INFO: epoch 008:    343 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.684, ntokens=18152.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3489.7, ups=0.19, wpb=18152.6, bsz=1024, num_updates=3080, lr=4.90015e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=52, gb_free=13.5, wall=19189
+2024-11-06 19:02:02 - progress_bar.py[line:274] - INFO: epoch 008:    353 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.734, ntokens=18466.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3580.5, ups=0.19, wpb=18466.4, bsz=1024, num_updates=3090, lr=4.89879e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=51, gb_free=13.4, wall=19241
+2024-11-06 19:02:55 - progress_bar.py[line:274] - INFO: epoch 008:    363 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.729, ntokens=18319.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3488.3, ups=0.19, wpb=18319.5, bsz=1024, num_updates=3100, lr=4.89743e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=52, gb_free=13.3, wall=19294
+2024-11-06 19:03:47 - progress_bar.py[line:274] - INFO: epoch 008:    373 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.686, ntokens=18187.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3483, ups=0.19, wpb=18187.8, bsz=1024, num_updates=3110, lr=4.89607e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=52, gb_free=13.3, wall=19346
+2024-11-06 19:04:40 - progress_bar.py[line:274] - INFO: epoch 008:    383 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.671, ntokens=17865.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.37, wps=3366.4, ups=0.19, wpb=17865.5, bsz=1024, num_updates=3120, lr=4.89471e-05, gnorm=0.01, clip=0, loss_scale=8192, train_wall=53, gb_free=13.5, wall=19399
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt
+
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt2024-11-06 19:05:18 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 8 @ 3128 updates
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt
-
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt
-2024-11-03 10:57:06 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+2024-11-06 19:05:18 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
 cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt': File exists: File exists
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 10:58:57 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 19:07:09 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
 slice_id 3 seek offset 150000
-slice_id 4 seek offset 200000
-2024-11-03 11:00:48 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 8 @ 3128 updates, score 0) (writing took 222.74049048501183 seconds)
+2024-11-06 19:09:01 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 8 @ 3128 updates, score 0) (writing took 222.50548460400023 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_8.pt
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+slice_id 6 seek offset 300000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
 slice_id 5 seek offset 250000
-2024-11-03 11:01:43 - train.py[line:336] - INFO: end of epoch 8 (average epoch stats below)
-2024-11-03 11:01:43 - progress_bar.py[line:282] - INFO: epoch 008 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.661 | ntokens 17819.1 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 12.65 | wps 2797.4 | ups 0.16 | wpb 17819.1 | bsz 1023 | num_updates 3128 | lr 4.89362e-05 | gnorm 0.007 | clip 0 | loss_scale 8192 | train_wall 1351 | gb_free 13.3 | wall 19880
-2024-11-03 11:01:43 - trainer.py[line:639] - INFO: loading train data for epoch 9
+slice_id 4 seek offset 200000
+2024-11-06 19:09:55 - train.py[line:336] - INFO: end of epoch 8 (average epoch stats below)
+2024-11-06 19:09:55 - progress_bar.py[line:282] - INFO: epoch 008 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.638 | ntokens 17819.1 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.22 | wps 2800.2 | ups 0.16 | wpb 17819.1 | bsz 1023 | num_updates 3128 | lr 4.89362e-05 | gnorm 0.009 | clip 0 | loss_scale 8192 | train_wall 1652 | gb_free 13.3 | wall 19714
+2024-11-06 19:09:55 - trainer.py[line:639] - INFO: loading train data for epoch 9
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 11:04:28 - trainer.py[line:703] - INFO: begin training epoch 9
-2024-11-03 11:04:28 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 11:04:41 - progress_bar.py[line:274] - INFO: epoch 009:      2 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.655, ntokens=17141.2, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=12.6, wps=347.4, ups=0.02, wpb=17141.2, bsz=985.6, num_updates=3130, lr=4.89334e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=45, gb_free=13.6, wall=20058
-2024-11-03 11:05:33 - progress_bar.py[line:274] - INFO: epoch 009:     12 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.585, ntokens=17444.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12, wps=3338.7, ups=0.19, wpb=17444.1, bsz=1024, num_updates=3140, lr=4.89198e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=30, gb_free=13.3, wall=20110
-2024-11-03 11:06:25 - progress_bar.py[line:274] - INFO: epoch 009:     22 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.629, ntokens=17755.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.37, wps=3423.9, ups=0.19, wpb=17755.6, bsz=1024, num_updates=3150, lr=4.89062e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20162
-2024-11-03 11:07:17 - progress_bar.py[line:274] - INFO: epoch 009:     32 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.612, ntokens=17875.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.23, wps=3424.3, ups=0.19, wpb=17875.9, bsz=1024, num_updates=3160, lr=4.88926e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20214
-2024-11-03 11:08:10 - progress_bar.py[line:274] - INFO: epoch 009:     42 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.611, ntokens=17766.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.22, wps=3371.3, ups=0.19, wpb=17766.8, bsz=1024, num_updates=3170, lr=4.8879e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=30, gb_free=13.6, wall=20267
-2024-11-03 11:09:02 - progress_bar.py[line:274] - INFO: epoch 009:     52 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.512, ntokens=17315.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.4, wps=3316.2, ups=0.19, wpb=17315.9, bsz=1024, num_updates=3180, lr=4.88654e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=31, gb_free=13.4, wall=20319
-2024-11-03 11:09:54 - progress_bar.py[line:274] - INFO: epoch 009:     62 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.634, ntokens=18011.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.42, wps=3440.7, ups=0.19, wpb=18011.3, bsz=1024, num_updates=3190, lr=4.88518e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20371
-2024-11-03 11:10:47 - progress_bar.py[line:274] - INFO: epoch 009:     72 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.562, ntokens=17506.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.81, wps=3336.7, ups=0.19, wpb=17506.1, bsz=1024, num_updates=3200, lr=4.88382e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=30, gb_free=13.4, wall=20424
-2024-11-03 11:11:39 - progress_bar.py[line:274] - INFO: epoch 009:     82 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.59, ntokens=17833.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.04, wps=3375, ups=0.19, wpb=17833.5, bsz=1024, num_updates=3210, lr=4.88246e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=30, gb_free=13.4, wall=20477
-2024-11-03 11:12:32 - progress_bar.py[line:274] - INFO: epoch 009:     92 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.58, ntokens=17695.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.96, wps=3390.8, ups=0.19, wpb=17695.7, bsz=1024, num_updates=3220, lr=4.8811e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20529
-2024-11-03 11:13:24 - progress_bar.py[line:274] - INFO: epoch 009:    102 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.574, ntokens=17789.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.91, wps=3418.4, ups=0.19, wpb=17789.9, bsz=1024, num_updates=3230, lr=4.87974e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20581
-2024-11-03 11:14:16 - progress_bar.py[line:274] - INFO: epoch 009:    112 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.535, ntokens=17357.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.59, wps=3317.8, ups=0.19, wpb=17357.9, bsz=1024, num_updates=3240, lr=4.87838e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=31, gb_free=13.6, wall=20633
-2024-11-03 11:15:09 - progress_bar.py[line:274] - INFO: epoch 009:    122 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.515, ntokens=17711.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.43, wps=3352, ups=0.19, wpb=17711.1, bsz=1024, num_updates=3250, lr=4.87702e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20686
-2024-11-03 11:16:08 - progress_bar.py[line:274] - INFO: epoch 009:    132 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.586, ntokens=17570, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.01, wps=2944.8, ups=0.17, wpb=17570, bsz=1024, num_updates=3260, lr=4.87566e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=35, gb_free=13.5, wall=20746
-2024-11-03 11:17:02 - progress_bar.py[line:274] - INFO: epoch 009:    142 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.548, ntokens=17685.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.7, wps=3308.8, ups=0.19, wpb=17685.9, bsz=1024, num_updates=3270, lr=4.8743e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=42, gb_free=13.4, wall=20799
-2024-11-03 11:17:57 - progress_bar.py[line:274] - INFO: epoch 009:    152 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.575, ntokens=17679.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.92, wps=3234.1, ups=0.18, wpb=17679.9, bsz=1024, num_updates=3280, lr=4.87294e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=42, gb_free=13.5, wall=20854
-2024-11-03 11:18:50 - progress_bar.py[line:274] - INFO: epoch 009:    162 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.526, ntokens=17683.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.52, wps=3296.3, ups=0.19, wpb=17683.1, bsz=1024, num_updates=3290, lr=4.87158e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=42, gb_free=13.5, wall=20907
-2024-11-03 11:19:44 - progress_bar.py[line:274] - INFO: epoch 009:    172 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.597, ntokens=17846, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.1, wps=3322.1, ups=0.19, wpb=17846, bsz=1024, num_updates=3300, lr=4.87022e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=43, gb_free=13.6, wall=20961
-2024-11-03 11:20:41 - progress_bar.py[line:274] - INFO: epoch 009:    182 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.545, ntokens=17672.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.67, wps=3117, ups=0.18, wpb=17672.3, bsz=1024, num_updates=3310, lr=4.86886e-05, gnorm=0.01, clip=0, loss_scale=8192, train_wall=55, gb_free=13.6, wall=21018
-2024-11-03 11:21:38 - progress_bar.py[line:274] - INFO: epoch 009:    192 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.552, ntokens=17843.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.73, wps=3121.2, ups=0.17, wpb=17843.8, bsz=1024, num_updates=3320, lr=4.8675e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=57, gb_free=13.6, wall=21075
-2024-11-03 11:22:34 - progress_bar.py[line:274] - INFO: epoch 009:    202 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.6, ntokens=17942.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.13, wps=3193.7, ups=0.18, wpb=17942.9, bsz=1024, num_updates=3330, lr=4.86614e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=56, gb_free=13.5, wall=21131
-2024-11-03 11:23:30 - progress_bar.py[line:274] - INFO: epoch 009:    212 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.524, ntokens=17887.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.5, wps=3184, ups=0.18, wpb=17887.5, bsz=1024, num_updates=3340, lr=4.86478e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=56, gb_free=13.4, wall=21187
-2024-11-03 11:24:26 - progress_bar.py[line:274] - INFO: epoch 009:    222 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.545, ntokens=17811.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.67, wps=3174.3, ups=0.18, wpb=17811.8, bsz=1024, num_updates=3350, lr=4.86342e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=56, gb_free=13.5, wall=21243
-2024-11-03 11:25:23 - progress_bar.py[line:274] - INFO: epoch 009:    232 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.559, ntokens=17888.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.78, wps=3182.1, ups=0.18, wpb=17888.1, bsz=1024, num_updates=3360, lr=4.86206e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=56, gb_free=13.5, wall=21300
-2024-11-03 11:26:18 - progress_bar.py[line:274] - INFO: epoch 009:    242 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.517, ntokens=17238.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.45, wps=3085, ups=0.18, wpb=17238.4, bsz=1024, num_updates=3370, lr=4.8607e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=56, gb_free=13.5, wall=21356
-2024-11-03 11:27:15 - progress_bar.py[line:274] - INFO: epoch 009:    252 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.55, ntokens=17685.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.72, wps=3140, ups=0.18, wpb=17685.6, bsz=1024, num_updates=3380, lr=4.85934e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=56, gb_free=13.5, wall=21412
-2024-11-03 11:28:11 - progress_bar.py[line:274] - INFO: epoch 009:    262 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.567, ntokens=17868.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.85, wps=3158.8, ups=0.18, wpb=17868.6, bsz=1024, num_updates=3390, lr=4.85797e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=56, gb_free=13.5, wall=21468
-2024-11-03 11:29:06 - progress_bar.py[line:274] - INFO: epoch 009:    272 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.577, ntokens=17859.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.94, wps=3277.6, ups=0.18, wpb=17859.5, bsz=1024, num_updates=3400, lr=4.85661e-05, gnorm=0.012, clip=0, loss_scale=8192, train_wall=54, gb_free=13.5, wall=21523
-2024-11-03 11:30:02 - progress_bar.py[line:274] - INFO: epoch 009:    282 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.49, ntokens=17557, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.24, wps=3106.9, ups=0.18, wpb=17557, bsz=1024, num_updates=3410, lr=4.85525e-05, gnorm=0.012, clip=0, loss_scale=8192, train_wall=56, gb_free=13.6, wall=21579
-2024-11-03 11:31:00 - progress_bar.py[line:274] - INFO: epoch 009:    292 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.54, ntokens=17677.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.63, wps=3085.5, ups=0.17, wpb=17677.9, bsz=1024, num_updates=3420, lr=4.85389e-05, gnorm=0.01, clip=0, loss_scale=8192, train_wall=57, gb_free=13.4, wall=21637
-2024-11-03 11:31:56 - progress_bar.py[line:274] - INFO: epoch 009:    302 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.597, ntokens=17791.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.1, wps=3176.1, ups=0.18, wpb=17791.6, bsz=1024, num_updates=3430, lr=4.85253e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=56, gb_free=13.4, wall=21693
-2024-11-03 11:32:53 - progress_bar.py[line:274] - INFO: epoch 009:    312 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.672, ntokens=18392.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.74, wps=3198.3, ups=0.17, wpb=18392.2, bsz=1024, num_updates=3440, lr=4.85117e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=57, gb_free=13.4, wall=21750
-2024-11-03 11:33:49 - progress_bar.py[line:274] - INFO: epoch 009:    322 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.637, ntokens=18254.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.44, wps=3254.1, ups=0.18, wpb=18254.3, bsz=1024, num_updates=3450, lr=4.84981e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=56, gb_free=13.3, wall=21806
-2024-11-03 11:34:46 - progress_bar.py[line:274] - INFO: epoch 009:    332 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.636, ntokens=18282.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.43, wps=3225.8, ups=0.18, wpb=18282.3, bsz=1024, num_updates=3460, lr=4.84845e-05, gnorm=0.005, clip=0, loss_scale=8192, train_wall=57, gb_free=13.5, wall=21863
-2024-11-03 11:35:42 - progress_bar.py[line:274] - INFO: epoch 009:    342 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.606, ntokens=17979.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.18, wps=3189.9, ups=0.18, wpb=17979.4, bsz=1024, num_updates=3470, lr=4.84709e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=56, gb_free=13.4, wall=21919
-2024-11-03 11:36:38 - progress_bar.py[line:274] - INFO: epoch 009:    352 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.629, ntokens=18383.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.37, wps=3276.2, ups=0.18, wpb=18383.8, bsz=1024, num_updates=3480, lr=4.84573e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=56, gb_free=13.4, wall=21976
-2024-11-03 11:37:33 - progress_bar.py[line:274] - INFO: epoch 009:    362 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.657, ntokens=18115.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.62, wps=3344.7, ups=0.18, wpb=18115.8, bsz=1024, num_updates=3490, lr=4.84437e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=54, gb_free=13.3, wall=22030
-2024-11-03 11:38:30 - progress_bar.py[line:274] - INFO: epoch 009:    372 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.69, ntokens=18733.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.91, wps=3271.3, ups=0.17, wpb=18733.8, bsz=1024, num_updates=3500, lr=4.84301e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=57, gb_free=13.3, wall=22087
-2024-11-03 11:39:26 - progress_bar.py[line:274] - INFO: epoch 009:    382 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.533, ntokens=17772.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.57, wps=3153.5, ups=0.18, wpb=17772.4, bsz=1024, num_updates=3510, lr=4.84165e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=56, gb_free=13.5, wall=22143
+2024-11-06 19:12:34 - trainer.py[line:703] - INFO: begin training epoch 9
+2024-11-06 19:12:34 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 19:12:47 - progress_bar.py[line:274] - INFO: epoch 009:      2 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.643, ntokens=17141.2, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=351.7, ups=0.02, wpb=17141.2, bsz=985.6, num_updates=3130, lr=4.89334e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=44, gb_free=13.6, wall=19886
+2024-11-06 19:13:39 - progress_bar.py[line:274] - INFO: epoch 009:     12 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.578, ntokens=17444.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3346.4, ups=0.19, wpb=17444.1, bsz=1024, num_updates=3140, lr=4.89198e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=30, gb_free=13.3, wall=19938
+2024-11-06 19:14:32 - progress_bar.py[line:274] - INFO: epoch 009:     22 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.606, ntokens=17755.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=3399.3, ups=0.19, wpb=17755.6, bsz=1024, num_updates=3150, lr=4.89062e-05, gnorm=0.01, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=19990
+2024-11-06 19:15:24 - progress_bar.py[line:274] - INFO: epoch 009:     32 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=17875.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3419.8, ups=0.19, wpb=17875.9, bsz=1024, num_updates=3160, lr=4.88926e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20043
+2024-11-06 19:16:16 - progress_bar.py[line:274] - INFO: epoch 009:     42 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.617, ntokens=17766.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3378.7, ups=0.19, wpb=17766.8, bsz=1024, num_updates=3170, lr=4.8879e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=30, gb_free=13.6, wall=20095
+2024-11-06 19:17:08 - progress_bar.py[line:274] - INFO: epoch 009:     52 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.525, ntokens=17315.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3344.9, ups=0.19, wpb=17315.9, bsz=1024, num_updates=3180, lr=4.88654e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=30, gb_free=13.3, wall=20147
+2024-11-06 19:18:01 - progress_bar.py[line:274] - INFO: epoch 009:     62 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.652, ntokens=18011.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3438.9, ups=0.19, wpb=18011.3, bsz=1024, num_updates=3190, lr=4.88518e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20200
+2024-11-06 19:18:53 - progress_bar.py[line:274] - INFO: epoch 009:     72 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.586, ntokens=17506.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3345.2, ups=0.19, wpb=17506.1, bsz=1024, num_updates=3200, lr=4.88382e-05, gnorm=0.013, clip=0, loss_scale=8192, train_wall=30, gb_free=13.4, wall=20252
+2024-11-06 19:19:45 - progress_bar.py[line:274] - INFO: epoch 009:     82 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.62, ntokens=17833.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.15, wps=3401.3, ups=0.19, wpb=17833.5, bsz=1024, num_updates=3210, lr=4.88246e-05, gnorm=0.013, clip=0, loss_scale=8192, train_wall=30, gb_free=13.4, wall=20304
+2024-11-06 19:20:38 - progress_bar.py[line:274] - INFO: epoch 009:     92 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.604, ntokens=17695.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3388.3, ups=0.19, wpb=17695.7, bsz=1024, num_updates=3220, lr=4.8811e-05, gnorm=0.013, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20356
+2024-11-06 19:21:30 - progress_bar.py[line:274] - INFO: epoch 009:    102 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.613, ntokens=17789.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3394.1, ups=0.19, wpb=17789.9, bsz=1024, num_updates=3230, lr=4.87974e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=31, gb_free=13.5, wall=20409
+2024-11-06 19:22:22 - progress_bar.py[line:274] - INFO: epoch 009:    112 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.57, ntokens=17357.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3321.7, ups=0.19, wpb=17357.9, bsz=1024, num_updates=3240, lr=4.87838e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=30, gb_free=13.6, wall=20461
+2024-11-06 19:23:15 - progress_bar.py[line:274] - INFO: epoch 009:    122 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.595, ntokens=17711.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3386.7, ups=0.19, wpb=17711.1, bsz=1024, num_updates=3250, lr=4.87702e-05, gnorm=0.01, clip=0, loss_scale=8192, train_wall=31, gb_free=13.5, wall=20513
+2024-11-06 19:24:07 - progress_bar.py[line:274] - INFO: epoch 009:    132 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.585, ntokens=17570, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3364.8, ups=0.19, wpb=17570, bsz=1024, num_updates=3260, lr=4.87566e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=30, gb_free=13.4, wall=20566
+2024-11-06 19:24:59 - progress_bar.py[line:274] - INFO: epoch 009:    142 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=17685.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3383.5, ups=0.19, wpb=17685.9, bsz=1024, num_updates=3270, lr=4.8743e-05, gnorm=0.01, clip=0, loss_scale=8192, train_wall=30, gb_free=13.4, wall=20618
+2024-11-06 19:25:51 - progress_bar.py[line:274] - INFO: epoch 009:    152 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.615, ntokens=17679.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3372.2, ups=0.19, wpb=17679.9, bsz=1024, num_updates=3280, lr=4.87294e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=20670
+2024-11-06 19:26:44 - progress_bar.py[line:274] - INFO: epoch 009:    162 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.556, ntokens=17683.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3359, ups=0.19, wpb=17683.1, bsz=1024, num_updates=3290, lr=4.87158e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=34, gb_free=13.5, wall=20723
+2024-11-06 19:27:36 - progress_bar.py[line:274] - INFO: epoch 009:    172 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.643, ntokens=17846, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3422.7, ups=0.19, wpb=17846, bsz=1024, num_updates=3300, lr=4.87022e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=43, gb_free=13.6, wall=20775
+2024-11-06 19:28:29 - progress_bar.py[line:274] - INFO: epoch 009:    182 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.6, ntokens=17672.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3336.5, ups=0.19, wpb=17672.3, bsz=1024, num_updates=3310, lr=4.86886e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=45, gb_free=13.6, wall=20828
+2024-11-06 19:29:22 - progress_bar.py[line:274] - INFO: epoch 009:    192 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.62, ntokens=17843.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.15, wps=3394.7, ups=0.19, wpb=17843.8, bsz=1024, num_updates=3320, lr=4.8675e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=49, gb_free=13.6, wall=20881
+2024-11-06 19:30:14 - progress_bar.py[line:274] - INFO: epoch 009:    202 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.63, ntokens=17942.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3450, ups=0.19, wpb=17942.9, bsz=1024, num_updates=3330, lr=4.86614e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=49, gb_free=13.5, wall=20933
+2024-11-06 19:31:07 - progress_bar.py[line:274] - INFO: epoch 009:    212 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.594, ntokens=17887.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3383, ups=0.19, wpb=17887.5, bsz=1024, num_updates=3340, lr=4.86478e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=53, gb_free=13.4, wall=20986
+2024-11-06 19:31:59 - progress_bar.py[line:274] - INFO: epoch 009:    222 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.63, ntokens=17811.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3425.6, ups=0.19, wpb=17811.8, bsz=1024, num_updates=3350, lr=4.86342e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=52, gb_free=13.5, wall=21038
+2024-11-06 19:32:52 - progress_bar.py[line:274] - INFO: epoch 009:    232 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=17888.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3378.2, ups=0.19, wpb=17888.1, bsz=1024, num_updates=3360, lr=4.86206e-05, gnorm=0.005, clip=0, loss_scale=8192, train_wall=53, gb_free=13.5, wall=21091
+2024-11-06 19:33:44 - progress_bar.py[line:274] - INFO: epoch 009:    242 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.546, ntokens=17238.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3271.9, ups=0.19, wpb=17238.4, bsz=1024, num_updates=3370, lr=4.8607e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=53, gb_free=13.5, wall=21143
+2024-11-06 19:34:37 - progress_bar.py[line:274] - INFO: epoch 009:    252 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.593, ntokens=17685.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3328.7, ups=0.19, wpb=17685.6, bsz=1024, num_updates=3380, lr=4.85934e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=53, gb_free=13.5, wall=21196
+2024-11-06 19:35:31 - progress_bar.py[line:274] - INFO: epoch 009:    262 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.628, ntokens=17868.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.18, wps=3362, ups=0.19, wpb=17868.6, bsz=1024, num_updates=3390, lr=4.85797e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=53, gb_free=13.5, wall=21249
+2024-11-06 19:36:23 - progress_bar.py[line:274] - INFO: epoch 009:    272 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=17859.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3435.9, ups=0.19, wpb=17859.5, bsz=1024, num_updates=3400, lr=4.85661e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=52, gb_free=13.5, wall=21301
+2024-11-06 19:37:14 - progress_bar.py[line:274] - INFO: epoch 009:    282 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.576, ntokens=17557, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3408, ups=0.19, wpb=17557, bsz=1024, num_updates=3410, lr=4.85525e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=51, gb_free=13.6, wall=21353
+2024-11-06 19:38:06 - progress_bar.py[line:274] - INFO: epoch 009:    292 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=17677.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3377.4, ups=0.19, wpb=17677.9, bsz=1024, num_updates=3420, lr=4.85389e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=52, gb_free=13.4, wall=21405
+2024-11-06 19:38:59 - progress_bar.py[line:274] - INFO: epoch 009:    302 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.646, ntokens=17791.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3380.1, ups=0.19, wpb=17791.6, bsz=1024, num_updates=3430, lr=4.85253e-05, gnorm=0.01, clip=0, loss_scale=8192, train_wall=53, gb_free=13.4, wall=21458
+2024-11-06 19:39:51 - progress_bar.py[line:274] - INFO: epoch 009:    312 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.745, ntokens=18392.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3537.3, ups=0.19, wpb=18392.2, bsz=1024, num_updates=3440, lr=4.85117e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=52, gb_free=13.4, wall=21510
+2024-11-06 19:40:43 - progress_bar.py[line:274] - INFO: epoch 009:    322 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.687, ntokens=18254.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3535.6, ups=0.19, wpb=18254.3, bsz=1024, num_updates=3450, lr=4.84981e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=52, gb_free=13.3, wall=21562
+2024-11-06 19:41:36 - progress_bar.py[line:274] - INFO: epoch 009:    332 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.718, ntokens=18282.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3424.9, ups=0.19, wpb=18282.3, bsz=1024, num_updates=3460, lr=4.84845e-05, gnorm=0.013, clip=0, loss_scale=8192, train_wall=53, gb_free=13.5, wall=21615
+2024-11-06 19:42:28 - progress_bar.py[line:274] - INFO: epoch 009:    342 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.649, ntokens=17979.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3445.6, ups=0.19, wpb=17979.4, bsz=1024, num_updates=3470, lr=4.84709e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=52, gb_free=13.4, wall=21667
+2024-11-06 19:43:21 - progress_bar.py[line:274] - INFO: epoch 009:    352 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.718, ntokens=18383.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3516.1, ups=0.19, wpb=18383.8, bsz=1024, num_updates=3480, lr=4.84573e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=52, gb_free=13.4, wall=21719
+2024-11-06 19:44:13 - progress_bar.py[line:274] - INFO: epoch 009:    362 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.686, ntokens=18115.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3475.5, ups=0.19, wpb=18115.8, bsz=1024, num_updates=3490, lr=4.84437e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=52, gb_free=13.3, wall=21772
+2024-11-06 19:45:05 - progress_bar.py[line:274] - INFO: epoch 009:    372 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.769, ntokens=18733.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=3575.5, ups=0.19, wpb=18733.8, bsz=1024, num_updates=3500, lr=4.84301e-05, gnorm=0.01, clip=0, loss_scale=8192, train_wall=52, gb_free=13.3, wall=21824
+2024-11-06 19:45:57 - progress_bar.py[line:274] - INFO: epoch 009:    382 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.639, ntokens=17772.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.23, wps=3409.1, ups=0.19, wpb=17772.4, bsz=1024, num_updates=3510, lr=4.84165e-05, gnorm=0.01, clip=0, loss_scale=8192, train_wall=52, gb_free=13.5, wall=21876
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
-2024-11-03 11:40:14 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 9 @ 3519 updates
-2024-11-03 11:40:14 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
+2024-11-06 19:46:41 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 9 @ 3519 updates
+2024-11-06 19:46:41 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt': File exists: File exists: File exists
+
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 11:42:05 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-03 11:43:57 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 9 @ 3519 updates, score 0) (writing took 223.0600704709941 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+2024-11-06 19:48:32 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+
+
+slice_id 2 seek offset 100000
 slice_id 7 seek offset 350000
 slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+2024-11-06 19:50:23 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 9 @ 3519 updates, score 0) (writing took 222.20814892199996 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_9.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+slice_id 6 seek offset 300000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+
+
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
 slice_id 5 seek offset 250000
-slice_id 6 seek offset 300000
-2024-11-03 11:44:52 - train.py[line:336] - INFO: end of epoch 9 (average epoch stats below)
-2024-11-03 11:44:52 - progress_bar.py[line:282] - INFO: epoch 009 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.581 | ntokens 17807.6 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 11.97 | wps 2690.1 | ups 0.15 | wpb 17807.6 | bsz 1023 | num_updates 3519 | lr 4.84043e-05 | gnorm 0.008 | clip 0 | loss_scale 8192 | train_wall 1801 | gb_free 13.3 | wall 22469
-2024-11-03 11:44:52 - trainer.py[line:639] - INFO: loading train data for epoch 10
+slice_id 4 seek offset 200000
+2024-11-06 19:51:18 - train.py[line:336] - INFO: end of epoch 9 (average epoch stats below)
+2024-11-06 19:51:18 - progress_bar.py[line:282] - INFO: epoch 009 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.625 | ntokens 17807.6 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.17 | wps 2804.8 | ups 0.16 | wpb 17807.6 | bsz 1023 | num_updates 3519 | lr 4.84043e-05 | gnorm 0.009 | clip 0 | loss_scale 8192 | train_wall 1668 | gb_free 13.3 | wall 22197
+2024-11-06 19:51:18 - trainer.py[line:639] - INFO: loading train data for epoch 10
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 11:47:37 - trainer.py[line:703] - INFO: begin training epoch 10
-2024-11-03 11:47:37 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 11:47:46 - progress_bar.py[line:274] - INFO: epoch 010:      1 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.669, ntokens=17453.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=12.72, wps=349.3, ups=0.02, wpb=17453.4, bsz=985.6, num_updates=3520, lr=4.84029e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=52, gb_free=13.4, wall=22643
-2024-11-03 11:48:39 - progress_bar.py[line:274] - INFO: epoch 010:     11 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.528, ntokens=17511.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.54, wps=3302.6, ups=0.19, wpb=17511.6, bsz=1024, num_updates=3530, lr=4.83893e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=44, gb_free=13.5, wall=22696
-2024-11-03 11:49:36 - progress_bar.py[line:274] - INFO: epoch 010:     21 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.541, ntokens=17794.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.64, wps=3137.3, ups=0.18, wpb=17794.2, bsz=1024, num_updates=3540, lr=4.83757e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=55, gb_free=13.4, wall=22753
-2024-11-03 11:50:31 - progress_bar.py[line:274] - INFO: epoch 010:     31 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.542, ntokens=17737.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.65, wps=3185.3, ups=0.18, wpb=17737.9, bsz=1024, num_updates=3550, lr=4.83621e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=56, gb_free=13.3, wall=22809
-2024-11-03 11:51:34 - progress_bar.py[line:274] - INFO: epoch 010:     41 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.477, ntokens=17723.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.13, wps=2846.8, ups=0.16, wpb=17723.7, bsz=1024, num_updates=3560, lr=4.83485e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=62, gb_free=13.4, wall=22871
-2024-11-03 11:52:39 - progress_bar.py[line:274] - INFO: epoch 010:     51 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.468, ntokens=17516.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.07, wps=2669.8, ups=0.15, wpb=17516.5, bsz=1024, num_updates=3570, lr=4.83349e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=66, gb_free=13.5, wall=22936
-2024-11-03 11:53:45 - progress_bar.py[line:274] - INFO: epoch 010:     61 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.532, ntokens=17826.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.56, wps=2710.1, ups=0.15, wpb=17826.6, bsz=1024, num_updates=3580, lr=4.83213e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=66, gb_free=13.5, wall=23002
-2024-11-03 11:54:50 - progress_bar.py[line:274] - INFO: epoch 010:     71 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.484, ntokens=17606.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.19, wps=2691.7, ups=0.15, wpb=17606.6, bsz=1024, num_updates=3590, lr=4.83077e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=65, gb_free=13.6, wall=23068
-2024-11-03 11:55:56 - progress_bar.py[line:274] - INFO: epoch 010:     81 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.506, ntokens=17723.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.36, wps=2690.2, ups=0.15, wpb=17723.1, bsz=1024, num_updates=3600, lr=4.82941e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=66, gb_free=13.6, wall=23133
-2024-11-03 11:57:03 - progress_bar.py[line:274] - INFO: epoch 010:     91 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.509, ntokens=17804.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.38, wps=2691, ups=0.15, wpb=17804.9, bsz=1024, num_updates=3610, lr=4.82805e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=66, gb_free=13.5, wall=23200
-2024-11-03 11:58:09 - progress_bar.py[line:274] - INFO: epoch 010:    101 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.458, ntokens=17542.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.99, wps=2655, ups=0.15, wpb=17542.5, bsz=1024, num_updates=3620, lr=4.82669e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=66, gb_free=13.5, wall=23266
-2024-11-03 11:59:15 - progress_bar.py[line:274] - INFO: epoch 010:    111 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.501, ntokens=17597.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.32, wps=2658.5, ups=0.15, wpb=17597.9, bsz=1024, num_updates=3630, lr=4.82533e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=66, gb_free=13.4, wall=23332
-2024-11-03 12:00:21 - progress_bar.py[line:274] - INFO: epoch 010:    121 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.449, ntokens=17656, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.92, wps=2653.2, ups=0.15, wpb=17656, bsz=1024, num_updates=3640, lr=4.82396e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=66, gb_free=13.5, wall=23398
-2024-11-03 12:01:27 - progress_bar.py[line:274] - INFO: epoch 010:    131 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.475, ntokens=17395.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.12, wps=2637.1, ups=0.15, wpb=17395.8, bsz=1024, num_updates=3650, lr=4.8226e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=66, gb_free=13.6, wall=23464
-2024-11-03 12:02:34 - progress_bar.py[line:274] - INFO: epoch 010:    141 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.497, ntokens=17768.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.29, wps=2672.4, ups=0.15, wpb=17768.9, bsz=1024, num_updates=3660, lr=4.82124e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=66, gb_free=13.4, wall=23531
-2024-11-03 12:03:27 - progress_bar.py[line:274] - INFO: epoch 010:    151 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.529, ntokens=17862, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.54, wps=3368.1, ups=0.19, wpb=17862, bsz=1024, num_updates=3670, lr=4.81988e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=47, gb_free=13.5, wall=23584
-2024-11-03 12:04:21 - progress_bar.py[line:274] - INFO: epoch 010:    161 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.497, ntokens=17723, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.29, wps=3303.4, ups=0.19, wpb=17723, bsz=1024, num_updates=3680, lr=4.81852e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=54, gb_free=13.6, wall=23638
-2024-11-03 12:05:13 - progress_bar.py[line:274] - INFO: epoch 010:    171 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.534, ntokens=18031, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.59, wps=3453.8, ups=0.19, wpb=18031, bsz=1024, num_updates=3690, lr=4.81716e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=23690
-2024-11-03 12:06:05 - progress_bar.py[line:274] - INFO: epoch 010:    181 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.467, ntokens=17524.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.05, wps=3378, ups=0.19, wpb=17524.8, bsz=1024, num_updates=3700, lr=4.8158e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.6, wall=23742
-2024-11-03 12:06:57 - progress_bar.py[line:274] - INFO: epoch 010:    191 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.51, ntokens=17937.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.39, wps=3432.5, ups=0.19, wpb=17937.8, bsz=1024, num_updates=3710, lr=4.81444e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=23794
-2024-11-03 12:07:49 - progress_bar.py[line:274] - INFO: epoch 010:    201 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.483, ntokens=17760.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.18, wps=3400.4, ups=0.19, wpb=17760.5, bsz=1024, num_updates=3720, lr=4.81308e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=23846
-2024-11-03 12:08:41 - progress_bar.py[line:274] - INFO: epoch 010:    211 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.464, ntokens=18022.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.03, wps=3461.2, ups=0.19, wpb=18022.8, bsz=1024, num_updates=3730, lr=4.81172e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=49, gb_free=13.4, wall=23898
-2024-11-03 12:09:33 - progress_bar.py[line:274] - INFO: epoch 010:    221 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.465, ntokens=17742.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.04, wps=3410.6, ups=0.19, wpb=17742.3, bsz=1024, num_updates=3740, lr=4.81036e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=35, gb_free=13.5, wall=23950
-2024-11-03 12:10:26 - progress_bar.py[line:274] - INFO: epoch 010:    231 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.554, ntokens=18208.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.75, wps=3441.9, ups=0.19, wpb=18208.9, bsz=1024, num_updates=3750, lr=4.809e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=42, gb_free=13.5, wall=24003
-2024-11-03 12:11:18 - progress_bar.py[line:274] - INFO: epoch 010:    241 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.487, ntokens=17420.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.21, wps=3349.1, ups=0.19, wpb=17420.4, bsz=1024, num_updates=3760, lr=4.80764e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=45, gb_free=13.6, wall=24055
-2024-11-03 12:12:10 - progress_bar.py[line:274] - INFO: epoch 010:    251 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.426, ntokens=17454.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.75, wps=3337.2, ups=0.19, wpb=17454.9, bsz=1024, num_updates=3770, lr=4.80628e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=45, gb_free=13.5, wall=24108
-2024-11-03 12:13:03 - progress_bar.py[line:274] - INFO: epoch 010:    261 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.445, ntokens=17489.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.89, wps=3352.9, ups=0.19, wpb=17489.7, bsz=1024, num_updates=3780, lr=4.80492e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=43, gb_free=13.5, wall=24160
-2024-11-03 12:13:55 - progress_bar.py[line:274] - INFO: epoch 010:    271 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.483, ntokens=17692.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.18, wps=3360.7, ups=0.19, wpb=17692.3, bsz=1024, num_updates=3790, lr=4.80356e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=24212
-2024-11-03 12:14:48 - progress_bar.py[line:274] - INFO: epoch 010:    281 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.422, ntokens=17650.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.72, wps=3366.6, ups=0.19, wpb=17650.2, bsz=1024, num_updates=3800, lr=4.8022e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=24265
-2024-11-03 12:15:40 - progress_bar.py[line:274] - INFO: epoch 010:    291 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.459, ntokens=17732.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11, wps=3399, ups=0.19, wpb=17732.6, bsz=1024, num_updates=3810, lr=4.80084e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=24317
-2024-11-03 12:16:32 - progress_bar.py[line:274] - INFO: epoch 010:    301 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.587, ntokens=18175.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.02, wps=3514.2, ups=0.19, wpb=18175.7, bsz=1024, num_updates=3820, lr=4.79948e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=30, gb_free=13.3, wall=24369
-2024-11-03 12:17:24 - progress_bar.py[line:274] - INFO: epoch 010:    311 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.652, ntokens=18509.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.57, wps=3514.7, ups=0.19, wpb=18509.4, bsz=1024, num_updates=3830, lr=4.79812e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=24421
-2024-11-03 12:18:16 - progress_bar.py[line:274] - INFO: epoch 010:    321 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.589, ntokens=18219.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.04, wps=3511.4, ups=0.19, wpb=18219.5, bsz=1024, num_updates=3840, lr=4.79676e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=24473
-2024-11-03 12:19:08 - progress_bar.py[line:274] - INFO: epoch 010:    331 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.598, ntokens=18186.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.11, wps=3524.4, ups=0.19, wpb=18186.7, bsz=1024, num_updates=3850, lr=4.7954e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.3, wall=24525
-2024-11-03 12:20:00 - progress_bar.py[line:274] - INFO: epoch 010:    341 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.552, ntokens=18129.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.73, wps=3494.1, ups=0.19, wpb=18129.2, bsz=1024, num_updates=3860, lr=4.79404e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.2, wall=24577
-2024-11-03 12:20:52 - progress_bar.py[line:274] - INFO: epoch 010:    351 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.625, ntokens=18677.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.34, wps=3554.6, ups=0.19, wpb=18677.2, bsz=1024, num_updates=3870, lr=4.79268e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=38, gb_free=13.4, wall=24629
-2024-11-03 12:21:44 - progress_bar.py[line:274] - INFO: epoch 010:    361 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.593, ntokens=18294.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.07, wps=3544.6, ups=0.19, wpb=18294.1, bsz=1024, num_updates=3880, lr=4.79132e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=43, gb_free=13.4, wall=24681
-2024-11-03 12:22:36 - progress_bar.py[line:274] - INFO: epoch 010:    371 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.574, ntokens=18055.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.91, wps=3467.9, ups=0.19, wpb=18055.6, bsz=1024, num_updates=3890, lr=4.78995e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=42, gb_free=13.5, wall=24733
-2024-11-03 12:23:28 - progress_bar.py[line:274] - INFO: epoch 010:    381 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.578, ntokens=18334.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.94, wps=3532, ups=0.19, wpb=18334.2, bsz=1024, num_updates=3900, lr=4.78859e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=43, gb_free=13.5, wall=24785
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-03 12:24:17 - progress_bar.py[line:274] - INFO: epoch 010:    391 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.604, ntokens=17446.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=12.16, wps=3573.8, ups=0.2, wpb=17446.6, bsz=985.6, num_updates=3910, lr=4.78723e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=42, gb_free=13.3, wall=24834
-2024-11-03 12:24:17 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 10 @ 3910 updates
+2024-11-06 19:53:57 - trainer.py[line:703] - INFO: begin training epoch 10
+2024-11-06 19:53:57 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 19:54:05 - progress_bar.py[line:274] - INFO: epoch 010:      1 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17453.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=358.1, ups=0.02, wpb=17453.4, bsz=985.6, num_updates=3520, lr=4.84029e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=47, gb_free=13.4, wall=22363
+2024-11-06 19:54:57 - progress_bar.py[line:274] - INFO: epoch 010:     11 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.586, ntokens=17511.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3366.1, ups=0.19, wpb=17511.6, bsz=1024, num_updates=3530, lr=4.83893e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=22416
+2024-11-06 19:55:49 - progress_bar.py[line:274] - INFO: epoch 010:     21 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.595, ntokens=17794.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3383.3, ups=0.19, wpb=17794.2, bsz=1024, num_updates=3540, lr=4.83757e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=31, gb_free=13.4, wall=22468
+2024-11-06 19:56:42 - progress_bar.py[line:274] - INFO: epoch 010:     31 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.578, ntokens=17737.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3361.6, ups=0.19, wpb=17737.9, bsz=1024, num_updates=3550, lr=4.83621e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=30, gb_free=13.3, wall=22521
+2024-11-06 19:57:34 - progress_bar.py[line:274] - INFO: epoch 010:     41 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.556, ntokens=17723.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3414.6, ups=0.19, wpb=17723.7, bsz=1024, num_updates=3560, lr=4.83485e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=30, gb_free=13.4, wall=22573
+2024-11-06 19:58:26 - progress_bar.py[line:274] - INFO: epoch 010:     51 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.575, ntokens=17516.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3369.9, ups=0.19, wpb=17516.5, bsz=1024, num_updates=3570, lr=4.83349e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=22625
+2024-11-06 19:59:18 - progress_bar.py[line:274] - INFO: epoch 010:     61 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.595, ntokens=17826.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3432.6, ups=0.19, wpb=17826.6, bsz=1024, num_updates=3580, lr=4.83213e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=30, gb_free=13.5, wall=22677
+2024-11-06 20:00:10 - progress_bar.py[line:274] - INFO: epoch 010:     71 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.6, ntokens=17606.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3350.2, ups=0.19, wpb=17606.6, bsz=1024, num_updates=3590, lr=4.83077e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=22729
+2024-11-06 20:01:03 - progress_bar.py[line:274] - INFO: epoch 010:     81 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.588, ntokens=17723.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3387.3, ups=0.19, wpb=17723.1, bsz=1024, num_updates=3600, lr=4.82941e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=22782
+2024-11-06 20:01:55 - progress_bar.py[line:274] - INFO: epoch 010:     91 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.604, ntokens=17804.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3406.1, ups=0.19, wpb=17804.9, bsz=1024, num_updates=3610, lr=4.82805e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=22834
+2024-11-06 20:02:48 - progress_bar.py[line:274] - INFO: epoch 010:    101 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.576, ntokens=17542.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3293.3, ups=0.19, wpb=17542.5, bsz=1024, num_updates=3620, lr=4.82669e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=39, gb_free=13.5, wall=22887
+2024-11-06 20:03:40 - progress_bar.py[line:274] - INFO: epoch 010:    111 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.601, ntokens=17597.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3382.8, ups=0.19, wpb=17597.9, bsz=1024, num_updates=3630, lr=4.82533e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=40, gb_free=13.4, wall=22939
+2024-11-06 20:04:32 - progress_bar.py[line:274] - INFO: epoch 010:    121 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.596, ntokens=17656, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3385.8, ups=0.19, wpb=17656, bsz=1024, num_updates=3640, lr=4.82396e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=33, gb_free=13.5, wall=22991
+2024-11-06 20:05:25 - progress_bar.py[line:274] - INFO: epoch 010:    131 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.561, ntokens=17395.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3310.3, ups=0.19, wpb=17395.8, bsz=1024, num_updates=3650, lr=4.8226e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=23044
+2024-11-06 20:06:17 - progress_bar.py[line:274] - INFO: epoch 010:    141 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=17768.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3398.7, ups=0.19, wpb=17768.9, bsz=1024, num_updates=3660, lr=4.82124e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=23096
+2024-11-06 20:07:10 - progress_bar.py[line:274] - INFO: epoch 010:    151 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.613, ntokens=17862, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3395.2, ups=0.19, wpb=17862, bsz=1024, num_updates=3670, lr=4.81988e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=23149
+2024-11-06 20:08:03 - progress_bar.py[line:274] - INFO: epoch 010:    161 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.588, ntokens=17723, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3337.1, ups=0.19, wpb=17723, bsz=1024, num_updates=3680, lr=4.81852e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=23202
+2024-11-06 20:08:58 - progress_bar.py[line:274] - INFO: epoch 010:    171 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.627, ntokens=18031, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.18, wps=3258.9, ups=0.18, wpb=18031, bsz=1024, num_updates=3690, lr=4.81716e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=44, gb_free=13.5, wall=23257
+2024-11-06 20:09:55 - progress_bar.py[line:274] - INFO: epoch 010:    181 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.582, ntokens=17524.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3087.2, ups=0.18, wpb=17524.8, bsz=1024, num_updates=3700, lr=4.8158e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=48, gb_free=13.6, wall=23314
+2024-11-06 20:10:52 - progress_bar.py[line:274] - INFO: epoch 010:    191 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.643, ntokens=17937.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3169.3, ups=0.18, wpb=17937.8, bsz=1024, num_updates=3710, lr=4.81444e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=57, gb_free=13.4, wall=23371
+2024-11-06 20:11:49 - progress_bar.py[line:274] - INFO: epoch 010:    201 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=17760.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3098.3, ups=0.17, wpb=17760.5, bsz=1024, num_updates=3720, lr=4.81308e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=57, gb_free=13.4, wall=23428
+2024-11-06 20:12:45 - progress_bar.py[line:274] - INFO: epoch 010:    211 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.605, ntokens=18022.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3227.1, ups=0.18, wpb=18022.8, bsz=1024, num_updates=3730, lr=4.81172e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=56, gb_free=13.4, wall=23484
+2024-11-06 20:13:37 - progress_bar.py[line:274] - INFO: epoch 010:    221 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.592, ntokens=17742.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3388.4, ups=0.19, wpb=17742.3, bsz=1024, num_updates=3740, lr=4.81036e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=23536
+2024-11-06 20:14:30 - progress_bar.py[line:274] - INFO: epoch 010:    231 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.645, ntokens=18208.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3464.8, ups=0.19, wpb=18208.9, bsz=1024, num_updates=3750, lr=4.809e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=23589
+2024-11-06 20:15:22 - progress_bar.py[line:274] - INFO: epoch 010:    241 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.563, ntokens=17420.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3341.2, ups=0.19, wpb=17420.4, bsz=1024, num_updates=3760, lr=4.80764e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.6, wall=23641
+2024-11-06 20:16:14 - progress_bar.py[line:274] - INFO: epoch 010:    251 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.555, ntokens=17454.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3356, ups=0.19, wpb=17454.9, bsz=1024, num_updates=3770, lr=4.80628e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=23693
+2024-11-06 20:17:05 - progress_bar.py[line:274] - INFO: epoch 010:    261 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.573, ntokens=17489.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3403.9, ups=0.19, wpb=17489.7, bsz=1024, num_updates=3780, lr=4.80492e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=50, gb_free=13.5, wall=23744
+2024-11-06 20:17:57 - progress_bar.py[line:274] - INFO: epoch 010:    271 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.58, ntokens=17692.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3422.1, ups=0.19, wpb=17692.3, bsz=1024, num_updates=3790, lr=4.80356e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=39, gb_free=13.5, wall=23796
+2024-11-06 20:18:50 - progress_bar.py[line:274] - INFO: epoch 010:    281 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.577, ntokens=17650.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3338.8, ups=0.19, wpb=17650.2, bsz=1024, num_updates=3800, lr=4.8022e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=23849
+2024-11-06 20:19:43 - progress_bar.py[line:274] - INFO: epoch 010:    291 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.602, ntokens=17732.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3354.9, ups=0.19, wpb=17732.6, bsz=1024, num_updates=3810, lr=4.80084e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=23902
+2024-11-06 20:20:35 - progress_bar.py[line:274] - INFO: epoch 010:    301 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.69, ntokens=18175.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3472.5, ups=0.19, wpb=18175.7, bsz=1024, num_updates=3820, lr=4.79948e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=32, gb_free=13.3, wall=23954
+2024-11-06 20:21:29 - progress_bar.py[line:274] - INFO: epoch 010:    311 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.734, ntokens=18509.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3459.5, ups=0.19, wpb=18509.4, bsz=1024, num_updates=3830, lr=4.79812e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=50, gb_free=13.4, wall=24007
+2024-11-06 20:22:20 - progress_bar.py[line:274] - INFO: epoch 010:    321 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.698, ntokens=18219.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3523.2, ups=0.19, wpb=18219.5, bsz=1024, num_updates=3840, lr=4.79676e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=24059
+2024-11-06 20:23:13 - progress_bar.py[line:274] - INFO: epoch 010:    331 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.716, ntokens=18186.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3461.8, ups=0.19, wpb=18186.7, bsz=1024, num_updates=3850, lr=4.7954e-05, gnorm=0.013, clip=0, loss_scale=16384, train_wall=52, gb_free=13.3, wall=24112
+2024-11-06 20:24:04 - progress_bar.py[line:274] - INFO: epoch 010:    341 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.715, ntokens=18129.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3514.5, ups=0.19, wpb=18129.2, bsz=1024, num_updates=3860, lr=4.79404e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=52, gb_free=13.2, wall=24163
+2024-11-06 20:24:56 - progress_bar.py[line:274] - INFO: epoch 010:    351 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.788, ntokens=18677.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.91, wps=3597.7, ups=0.19, wpb=18677.2, bsz=1024, num_updates=3870, lr=4.79268e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=24215
+2024-11-06 20:25:48 - progress_bar.py[line:274] - INFO: epoch 010:    361 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.723, ntokens=18294.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.6, wps=3558.3, ups=0.19, wpb=18294.1, bsz=1024, num_updates=3880, lr=4.79132e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=51, gb_free=13.4, wall=24267
+2024-11-06 20:26:40 - progress_bar.py[line:274] - INFO: epoch 010:    371 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.702, ntokens=18055.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3422.7, ups=0.19, wpb=18055.6, bsz=1024, num_updates=3890, lr=4.78995e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=24319
+2024-11-06 20:27:33 - progress_bar.py[line:274] - INFO: epoch 010:    381 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.75, ntokens=18334.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3502.8, ups=0.19, wpb=18334.2, bsz=1024, num_updates=3900, lr=4.78859e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=24372
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+2024-11-06 20:28:22 - progress_bar.py[line:274] - INFO: epoch 010:    391 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.718, ntokens=17446.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3537.4, ups=0.2, wpb=17446.6, bsz=985.6, num_updates=3910, lr=4.78723e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=49, gb_free=13.3, wall=24421
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt
-2024-11-03 12:24:17 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt': File exists: File exists
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt
+
+2024-11-06 20:28:22 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 10 @ 3910 updates
+2024-11-06 20:28:22 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt': File exists: File exists: File exists
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 12:26:07 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-2024-11-03 12:27:59 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 10 @ 3910 updates, score 0) (writing took 222.57917487202212 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 20:30:11 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
+slice_id 6 seek offset 300000
+slice_id 3 seek offset 150000
+2024-11-06 20:32:03 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 10 @ 3910 updates, score 0) (writing took 220.60931834900111 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_10.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+slice_id 7 seek offset 350000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 5 seek offset 250000
-slice_id 2 seek offset 100000
+slice_id 4 seek offset 200000
 slice_id 1 seek offset 50000
-slice_id 6 seek offset 300000
-2024-11-03 12:28:54 - train.py[line:336] - INFO: end of epoch 10 (average epoch stats below)
-2024-11-03 12:28:54 - progress_bar.py[line:282] - INFO: epoch 010 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.517 | ntokens 17831.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 11.45 | wps 2638.8 | ups 0.15 | wpb 17831.9 | bsz 1023 | num_updates 3910 | lr 4.78723e-05 | gnorm 0.007 | clip 0 | loss_scale 16384 | train_wall 1901 | gb_free 13.3 | wall 25111
-2024-11-03 12:28:54 - trainer.py[line:639] - INFO: loading train data for epoch 11
+2024-11-06 20:32:57 - train.py[line:336] - INFO: end of epoch 10 (average epoch stats below)
+2024-11-06 20:32:57 - progress_bar.py[line:282] - INFO: epoch 010 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.627 | ntokens 17831.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.18 | wps 2789.5 | ups 0.16 | wpb 17831.9 | bsz 1023 | num_updates 3910 | lr 4.78723e-05 | gnorm 0.008 | clip 0 | loss_scale 16384 | train_wall 1626 | gb_free 13.3 | wall 24696
+2024-11-06 20:32:57 - trainer.py[line:639] - INFO: loading train data for epoch 11
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 12:31:38 - trainer.py[line:703] - INFO: begin training epoch 11
-2024-11-03 12:31:38 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 12:32:32 - progress_bar.py[line:274] - INFO: epoch 011:     10 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.443, ntokens=17420.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.87, wps=351.3, ups=0.02, wpb=17420.1, bsz=1024, num_updates=3920, lr=4.78587e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25330
-2024-11-03 12:33:25 - progress_bar.py[line:274] - INFO: epoch 011:     20 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.452, ntokens=17675.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.95, wps=3339.4, ups=0.19, wpb=17675.8, bsz=1024, num_updates=3930, lr=4.78451e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25383
-2024-11-03 12:34:18 - progress_bar.py[line:274] - INFO: epoch 011:     30 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.451, ntokens=17745.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.94, wps=3376.7, ups=0.19, wpb=17745.3, bsz=1024, num_updates=3940, lr=4.78315e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=25435
-2024-11-03 12:35:10 - progress_bar.py[line:274] - INFO: epoch 011:     40 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.477, ntokens=17891, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.13, wps=3446.8, ups=0.19, wpb=17891, bsz=1024, num_updates=3950, lr=4.78179e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25487
-2024-11-03 12:36:03 - progress_bar.py[line:274] - INFO: epoch 011:     50 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.403, ntokens=17564, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.58, wps=3333.6, ups=0.19, wpb=17564, bsz=1024, num_updates=3960, lr=4.78043e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25540
-2024-11-03 12:36:55 - progress_bar.py[line:274] - INFO: epoch 011:     60 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.45, ntokens=17881.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.92, wps=3439.6, ups=0.19, wpb=17881.1, bsz=1024, num_updates=3970, lr=4.77907e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25592
-2024-11-03 12:37:47 - progress_bar.py[line:274] - INFO: epoch 011:     70 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.443, ntokens=17633.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.88, wps=3368.6, ups=0.19, wpb=17633.7, bsz=1024, num_updates=3980, lr=4.77771e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=25644
-2024-11-03 12:38:40 - progress_bar.py[line:274] - INFO: epoch 011:     80 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.472, ntokens=17886.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.1, wps=3379.1, ups=0.19, wpb=17886.6, bsz=1024, num_updates=3990, lr=4.77635e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=25697
-2024-11-03 12:39:32 - progress_bar.py[line:274] - INFO: epoch 011:     90 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.387, ntokens=17440.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.46, wps=3357.6, ups=0.19, wpb=17440.4, bsz=1024, num_updates=4000, lr=4.77499e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=25749
-2024-11-03 12:40:24 - progress_bar.py[line:274] - INFO: epoch 011:    100 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.458, ntokens=17900.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.99, wps=3419.2, ups=0.19, wpb=17900.2, bsz=1024, num_updates=4010, lr=4.77363e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=25801
-2024-11-03 12:41:17 - progress_bar.py[line:274] - INFO: epoch 011:    110 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.424, ntokens=17790.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.73, wps=3382.2, ups=0.19, wpb=17790.2, bsz=1024, num_updates=4020, lr=4.77227e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25854
-2024-11-03 12:42:10 - progress_bar.py[line:274] - INFO: epoch 011:    120 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.436, ntokens=17724.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.82, wps=3327.4, ups=0.19, wpb=17724.2, bsz=1024, num_updates=4030, lr=4.77091e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25907
-2024-11-03 12:43:03 - progress_bar.py[line:274] - INFO: epoch 011:    130 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.374, ntokens=17222, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.37, wps=3253.5, ups=0.19, wpb=17222, bsz=1024, num_updates=4040, lr=4.76955e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=25960
-2024-11-03 12:43:55 - progress_bar.py[line:274] - INFO: epoch 011:    140 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.374, ntokens=17347.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.36, wps=3340.7, ups=0.19, wpb=17347.5, bsz=1024, num_updates=4050, lr=4.76819e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=26012
-2024-11-03 12:44:47 - progress_bar.py[line:274] - INFO: epoch 011:    150 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.423, ntokens=17626.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.73, wps=3378.2, ups=0.19, wpb=17626.7, bsz=1024, num_updates=4060, lr=4.76683e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=26064
-2024-11-03 12:45:40 - progress_bar.py[line:274] - INFO: epoch 011:    160 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.436, ntokens=17634.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.83, wps=3353.1, ups=0.19, wpb=17634.8, bsz=1024, num_updates=4070, lr=4.76547e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=26117
-2024-11-03 12:46:33 - progress_bar.py[line:274] - INFO: epoch 011:    170 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.465, ntokens=18081.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.04, wps=3416.2, ups=0.19, wpb=18081.6, bsz=1024, num_updates=4080, lr=4.76411e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=26170
-2024-11-03 12:47:25 - progress_bar.py[line:274] - INFO: epoch 011:    180 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.441, ntokens=17767.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.86, wps=3385.2, ups=0.19, wpb=17767.2, bsz=1024, num_updates=4090, lr=4.76275e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=26222
-2024-11-03 12:48:17 - progress_bar.py[line:274] - INFO: epoch 011:    190 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.428, ntokens=17950.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.76, wps=3434.9, ups=0.19, wpb=17950.3, bsz=1024, num_updates=4100, lr=4.76139e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=26274
-2024-11-03 12:49:10 - progress_bar.py[line:274] - INFO: epoch 011:    200 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.476, ntokens=17939.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.13, wps=3387.6, ups=0.19, wpb=17939.3, bsz=1024, num_updates=4110, lr=4.76003e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=26327
-2024-11-03 12:50:03 - progress_bar.py[line:274] - INFO: epoch 011:    210 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.376, ntokens=17644.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.38, wps=3368.2, ups=0.19, wpb=17644.2, bsz=1024, num_updates=4120, lr=4.75867e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=26380
-2024-11-03 12:50:55 - progress_bar.py[line:274] - INFO: epoch 011:    220 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.429, ntokens=17997.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.77, wps=3406.5, ups=0.19, wpb=17997.2, bsz=1024, num_updates=4130, lr=4.75731e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=26433
-2024-11-03 12:51:48 - progress_bar.py[line:274] - INFO: epoch 011:    230 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.396, ntokens=17635.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.53, wps=3376.6, ups=0.19, wpb=17635.3, bsz=1024, num_updates=4140, lr=4.75594e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=26485
-2024-11-03 12:52:40 - progress_bar.py[line:274] - INFO: epoch 011:    240 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.432, ntokens=17489.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.8, wps=3323.3, ups=0.19, wpb=17489.8, bsz=1024, num_updates=4150, lr=4.75458e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=26537
-2024-11-03 12:53:33 - progress_bar.py[line:274] - INFO: epoch 011:    250 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.397, ntokens=17612.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.54, wps=3342, ups=0.19, wpb=17612.8, bsz=1024, num_updates=4160, lr=4.75322e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=26590
-2024-11-03 12:54:27 - progress_bar.py[line:274] - INFO: epoch 011:    260 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.428, ntokens=17726.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.76, wps=3297, ups=0.19, wpb=17726.1, bsz=1024, num_updates=4170, lr=4.75186e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=33, gb_free=13.6, wall=26644
-2024-11-03 12:55:19 - progress_bar.py[line:274] - INFO: epoch 011:    270 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.464, ntokens=18057.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.03, wps=3487.6, ups=0.19, wpb=18057.4, bsz=1024, num_updates=4180, lr=4.7505e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=44, gb_free=13.6, wall=26696
-2024-11-03 12:56:10 - progress_bar.py[line:274] - INFO: epoch 011:    280 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.368, ntokens=17572.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.33, wps=3429.6, ups=0.2, wpb=17572.2, bsz=1024, num_updates=4190, lr=4.74914e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=36, gb_free=13.5, wall=26747
-2024-11-03 12:57:02 - progress_bar.py[line:274] - INFO: epoch 011:    290 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.368, ntokens=17455, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.33, wps=3333.3, ups=0.19, wpb=17455, bsz=1024, num_updates=4200, lr=4.74778e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=26799
-2024-11-03 12:57:54 - progress_bar.py[line:274] - INFO: epoch 011:    300 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.49, ntokens=17839.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.23, wps=3432, ups=0.19, wpb=17839.5, bsz=1024, num_updates=4210, lr=4.74642e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=34, gb_free=13.2, wall=26851
-2024-11-03 12:58:46 - progress_bar.py[line:274] - INFO: epoch 011:    310 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.602, ntokens=18562.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.14, wps=3573.7, ups=0.19, wpb=18562.4, bsz=1024, num_updates=4220, lr=4.74506e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=40, gb_free=13.4, wall=26903
-2024-11-03 12:59:37 - progress_bar.py[line:274] - INFO: epoch 011:    320 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.522, ntokens=18050.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.49, wps=3511.6, ups=0.19, wpb=18050.7, bsz=1024, num_updates=4230, lr=4.7437e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=39, gb_free=13.4, wall=26955
-2024-11-03 13:00:30 - progress_bar.py[line:274] - INFO: epoch 011:    330 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.552, ntokens=18245.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.73, wps=3500.7, ups=0.19, wpb=18245.5, bsz=1024, num_updates=4240, lr=4.74234e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=36, gb_free=13.4, wall=27007
-2024-11-03 13:01:22 - progress_bar.py[line:274] - INFO: epoch 011:    340 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.525, ntokens=18322.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.51, wps=3486.2, ups=0.19, wpb=18322.9, bsz=1024, num_updates=4250, lr=4.74098e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=45, gb_free=13.3, wall=27059
-2024-11-03 13:02:14 - progress_bar.py[line:274] - INFO: epoch 011:    350 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.507, ntokens=18309.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.37, wps=3551.1, ups=0.19, wpb=18309.9, bsz=1024, num_updates=4260, lr=4.73962e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=51, gb_free=13.3, wall=27111
-2024-11-03 13:03:06 - progress_bar.py[line:274] - INFO: epoch 011:    360 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.588, ntokens=18400, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=12.03, wps=3530.5, ups=0.19, wpb=18400, bsz=1024, num_updates=4270, lr=4.73826e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.2, wall=27163
-2024-11-03 13:03:58 - progress_bar.py[line:274] - INFO: epoch 011:    370 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.542, ntokens=18262.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.64, wps=3498.9, ups=0.19, wpb=18262.4, bsz=1024, num_updates=4280, lr=4.7369e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=27215
-2024-11-03 13:04:50 - progress_bar.py[line:274] - INFO: epoch 011:    380 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.543, ntokens=18451.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.66, wps=3563.9, ups=0.19, wpb=18451.1, bsz=1024, num_updates=4290, lr=4.73554e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=27267
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-03 13:05:42 - progress_bar.py[line:274] - INFO: epoch 011:    390 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.515, ntokens=17753.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.43, wps=3395.4, ups=0.19, wpb=17753.6, bsz=1024, num_updates=4300, lr=4.73418e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=27319
-2024-11-03 13:05:44 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 11 @ 4301 updates
+2024-11-06 20:35:36 - trainer.py[line:703] - INFO: begin training epoch 11
+2024-11-06 20:35:36 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 20:36:30 - progress_bar.py[line:274] - INFO: epoch 011:     10 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.599, ntokens=17420.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=356.7, ups=0.02, wpb=17420.1, bsz=1024, num_updates=3920, lr=4.78587e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=24909
+2024-11-06 20:37:23 - progress_bar.py[line:274] - INFO: epoch 011:     20 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.59, ntokens=17675.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3383.7, ups=0.19, wpb=17675.8, bsz=1024, num_updates=3930, lr=4.78451e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=24962
+2024-11-06 20:38:15 - progress_bar.py[line:274] - INFO: epoch 011:     30 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.607, ntokens=17745.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=3398, ups=0.19, wpb=17745.3, bsz=1024, num_updates=3940, lr=4.78315e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=25014
+2024-11-06 20:39:07 - progress_bar.py[line:274] - INFO: epoch 011:     40 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=17891, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3442, ups=0.19, wpb=17891, bsz=1024, num_updates=3950, lr=4.78179e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=25066
+2024-11-06 20:39:59 - progress_bar.py[line:274] - INFO: epoch 011:     50 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.587, ntokens=17564, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3372.9, ups=0.19, wpb=17564, bsz=1024, num_updates=3960, lr=4.78043e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25118
+2024-11-06 20:40:51 - progress_bar.py[line:274] - INFO: epoch 011:     60 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.644, ntokens=17881.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3428.9, ups=0.19, wpb=17881.1, bsz=1024, num_updates=3970, lr=4.77907e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25170
+2024-11-06 20:41:43 - progress_bar.py[line:274] - INFO: epoch 011:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.606, ntokens=17633.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=3372.7, ups=0.19, wpb=17633.7, bsz=1024, num_updates=3980, lr=4.77771e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=25222
+2024-11-06 20:42:36 - progress_bar.py[line:274] - INFO: epoch 011:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.622, ntokens=17886.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3415.3, ups=0.19, wpb=17886.6, bsz=1024, num_updates=3990, lr=4.77635e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=25275
+2024-11-06 20:43:30 - progress_bar.py[line:274] - INFO: epoch 011:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.554, ntokens=17440.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3236.6, ups=0.19, wpb=17440.4, bsz=1024, num_updates=4000, lr=4.77499e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=25329
+2024-11-06 20:44:22 - progress_bar.py[line:274] - INFO: epoch 011:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.628, ntokens=17900.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.18, wps=3427.5, ups=0.19, wpb=17900.2, bsz=1024, num_updates=4010, lr=4.77363e-05, gnorm=0.013, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=25381
+2024-11-06 20:45:14 - progress_bar.py[line:274] - INFO: epoch 011:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.618, ntokens=17790.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3423.1, ups=0.19, wpb=17790.2, bsz=1024, num_updates=4020, lr=4.77227e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25433
+2024-11-06 20:46:06 - progress_bar.py[line:274] - INFO: epoch 011:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.625, ntokens=17724.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.17, wps=3374.6, ups=0.19, wpb=17724.2, bsz=1024, num_updates=4030, lr=4.77091e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25485
+2024-11-06 20:46:59 - progress_bar.py[line:274] - INFO: epoch 011:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.552, ntokens=17222, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3268.7, ups=0.19, wpb=17222, bsz=1024, num_updates=4040, lr=4.76955e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=25538
+2024-11-06 20:47:51 - progress_bar.py[line:274] - INFO: epoch 011:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.539, ntokens=17347.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3320, ups=0.19, wpb=17347.5, bsz=1024, num_updates=4050, lr=4.76819e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25590
+2024-11-06 20:48:44 - progress_bar.py[line:274] - INFO: epoch 011:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.602, ntokens=17626.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3361.3, ups=0.19, wpb=17626.7, bsz=1024, num_updates=4060, lr=4.76683e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25643
+2024-11-06 20:49:36 - progress_bar.py[line:274] - INFO: epoch 011:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.591, ntokens=17634.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3381.1, ups=0.19, wpb=17634.8, bsz=1024, num_updates=4070, lr=4.76547e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25695
+2024-11-06 20:50:28 - progress_bar.py[line:274] - INFO: epoch 011:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.629, ntokens=18081.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3481.4, ups=0.19, wpb=18081.6, bsz=1024, num_updates=4080, lr=4.76411e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25747
+2024-11-06 20:51:20 - progress_bar.py[line:274] - INFO: epoch 011:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.622, ntokens=17767.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3395.7, ups=0.19, wpb=17767.2, bsz=1024, num_updates=4090, lr=4.76275e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=25799
+2024-11-06 20:52:12 - progress_bar.py[line:274] - INFO: epoch 011:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.605, ntokens=17950.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3439.2, ups=0.19, wpb=17950.3, bsz=1024, num_updates=4100, lr=4.76139e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=25851
+2024-11-06 20:53:05 - progress_bar.py[line:274] - INFO: epoch 011:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.632, ntokens=17939.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3442.9, ups=0.19, wpb=17939.3, bsz=1024, num_updates=4110, lr=4.76003e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=25903
+2024-11-06 20:53:57 - progress_bar.py[line:274] - INFO: epoch 011:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.557, ntokens=17644.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3358.8, ups=0.19, wpb=17644.2, bsz=1024, num_updates=4120, lr=4.75867e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=25956
+2024-11-06 20:54:50 - progress_bar.py[line:274] - INFO: epoch 011:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.622, ntokens=17997.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3411.8, ups=0.19, wpb=17997.2, bsz=1024, num_updates=4130, lr=4.75731e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=26009
+2024-11-06 20:55:42 - progress_bar.py[line:274] - INFO: epoch 011:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.566, ntokens=17635.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3349.6, ups=0.19, wpb=17635.3, bsz=1024, num_updates=4140, lr=4.75594e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=26061
+2024-11-06 20:56:34 - progress_bar.py[line:274] - INFO: epoch 011:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.551, ntokens=17489.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.86, wps=3370.9, ups=0.19, wpb=17489.8, bsz=1024, num_updates=4150, lr=4.75458e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=26113
+2024-11-06 20:57:27 - progress_bar.py[line:274] - INFO: epoch 011:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.581, ntokens=17612.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3358.4, ups=0.19, wpb=17612.8, bsz=1024, num_updates=4160, lr=4.75322e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=26166
+2024-11-06 20:58:19 - progress_bar.py[line:274] - INFO: epoch 011:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.6, ntokens=17726.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3409.2, ups=0.19, wpb=17726.1, bsz=1024, num_updates=4170, lr=4.75186e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=26218
+2024-11-06 20:59:11 - progress_bar.py[line:274] - INFO: epoch 011:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.632, ntokens=18057.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3472.4, ups=0.19, wpb=18057.4, bsz=1024, num_updates=4180, lr=4.7505e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=26270
+2024-11-06 21:00:03 - progress_bar.py[line:274] - INFO: epoch 011:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.546, ntokens=17572.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3342, ups=0.19, wpb=17572.2, bsz=1024, num_updates=4190, lr=4.74914e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=26322
+2024-11-06 21:00:56 - progress_bar.py[line:274] - INFO: epoch 011:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.559, ntokens=17455, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3334.9, ups=0.19, wpb=17455, bsz=1024, num_updates=4200, lr=4.74778e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=26375
+2024-11-06 21:01:47 - progress_bar.py[line:274] - INFO: epoch 011:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.622, ntokens=17839.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3446.9, ups=0.19, wpb=17839.5, bsz=1024, num_updates=4210, lr=4.74642e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.2, wall=26426
+2024-11-06 21:02:39 - progress_bar.py[line:274] - INFO: epoch 011:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.749, ntokens=18562.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.72, wps=3620.5, ups=0.2, wpb=18562.4, bsz=1024, num_updates=4220, lr=4.74506e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=26478
+2024-11-06 21:03:31 - progress_bar.py[line:274] - INFO: epoch 011:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.685, ntokens=18050.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.43, wps=3451, ups=0.19, wpb=18050.7, bsz=1024, num_updates=4230, lr=4.7437e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=26530
+2024-11-06 21:04:23 - progress_bar.py[line:274] - INFO: epoch 011:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.696, ntokens=18245.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.48, wps=3506.5, ups=0.19, wpb=18245.5, bsz=1024, num_updates=4240, lr=4.74234e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=26582
+2024-11-06 21:05:15 - progress_bar.py[line:274] - INFO: epoch 011:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.676, ntokens=18322.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=3502.3, ups=0.19, wpb=18322.9, bsz=1024, num_updates=4250, lr=4.74098e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=26634
+2024-11-06 21:06:07 - progress_bar.py[line:274] - INFO: epoch 011:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.715, ntokens=18309.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3538.3, ups=0.19, wpb=18309.9, bsz=1024, num_updates=4260, lr=4.73962e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=26686
+2024-11-06 21:06:59 - progress_bar.py[line:274] - INFO: epoch 011:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=18400, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3540, ups=0.19, wpb=18400, bsz=1024, num_updates=4270, lr=4.73826e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=26738
+2024-11-06 21:07:51 - progress_bar.py[line:274] - INFO: epoch 011:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.695, ntokens=18262.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.48, wps=3521.1, ups=0.19, wpb=18262.4, bsz=1024, num_updates=4280, lr=4.7369e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=26790
+2024-11-06 21:08:43 - progress_bar.py[line:274] - INFO: epoch 011:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.72, ntokens=18451.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3517.8, ups=0.19, wpb=18451.1, bsz=1024, num_updates=4290, lr=4.73554e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=36, gb_free=13.4, wall=26842
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+2024-11-06 21:09:35 - progress_bar.py[line:274] - INFO: epoch 011:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.616, ntokens=17753.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3416.7, ups=0.19, wpb=17753.6, bsz=1024, num_updates=4300, lr=4.73418e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=40, gb_free=13.3, wall=26894
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt2024-11-06 21:09:37 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 11 @ 4301 updates
 
-2024-11-03 13:05:44 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt': File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 13:07:31 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-2024-11-03 13:09:22 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 11 @ 4301 updates, score 0) (writing took 218.2200506569934 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+2024-11-06 21:09:37 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt': File exists
+: File exists: File exists
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 
-slice_id 3 seek offset 150000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 21:11:26 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 
 
-slice_id 1 seek offset 50000
-slice_id 5 seek offset 250000
 slice_id 2 seek offset 100000
 slice_id 6 seek offset 300000
-2024-11-03 13:10:17 - train.py[line:336] - INFO: end of epoch 11 (average epoch stats below)
-2024-11-03 13:10:17 - progress_bar.py[line:282] - INFO: epoch 011 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.456 | ntokens 17815.1 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 10.97 | wps 2805.4 | ups 0.16 | wpb 17815.1 | bsz 1023 | num_updates 4301 | lr 4.73404e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 1360 | gb_free 13.4 | wall 27594
-2024-11-03 13:10:17 - trainer.py[line:639] - INFO: loading train data for epoch 12
+slice_id 7 seek offset 350000
+2024-11-06 21:13:18 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 11 @ 4301 updates, score 0) (writing took 220.4491288470017 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_11.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+slice_id 3 seek offset 150000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+slice_id 5 seek offset 250000
+slice_id 4 seek offset 200000
+slice_id 1 seek offset 50000
+2024-11-06 21:14:12 - train.py[line:336] - INFO: end of epoch 11 (average epoch stats below)
+2024-11-06 21:14:12 - progress_bar.py[line:282] - INFO: epoch 011 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.62 | ntokens 17815.1 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.15 | wps 2814.5 | ups 0.16 | wpb 17815.1 | bsz 1023 | num_updates 4301 | lr 4.73404e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1205 | gb_free 13.4 | wall 27171
+2024-11-06 21:14:12 - trainer.py[line:639] - INFO: loading train data for epoch 12
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 13:13:01 - trainer.py[line:703] - INFO: begin training epoch 12
-2024-11-03 13:13:01 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 13:13:50 - progress_bar.py[line:274] - INFO: epoch 012:      9 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.383, ntokens=16806.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=10.43, wps=344.3, ups=0.02, wpb=16806.3, bsz=985.6, num_updates=4310, lr=4.73282e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=29, gb_free=13.6, wall=27807
-2024-11-03 13:14:43 - progress_bar.py[line:274] - INFO: epoch 012:     19 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.412, ntokens=17724.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.65, wps=3377.9, ups=0.19, wpb=17724.7, bsz=1024, num_updates=4320, lr=4.73146e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=27860
-2024-11-03 13:15:36 - progress_bar.py[line:274] - INFO: epoch 012:     29 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.386, ntokens=17678, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.46, wps=3341.1, ups=0.19, wpb=17678, bsz=1024, num_updates=4330, lr=4.7301e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=27913
-2024-11-03 13:16:28 - progress_bar.py[line:274] - INFO: epoch 012:     39 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.402, ntokens=17727.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.57, wps=3369.8, ups=0.19, wpb=17727.6, bsz=1024, num_updates=4340, lr=4.72874e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=27965
-2024-11-03 13:17:21 - progress_bar.py[line:274] - INFO: epoch 012:     49 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.373, ntokens=17686, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.36, wps=3366.7, ups=0.19, wpb=17686, bsz=1024, num_updates=4350, lr=4.72738e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=28018
-2024-11-03 13:18:15 - progress_bar.py[line:274] - INFO: epoch 012:     59 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.384, ntokens=17735.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.44, wps=3264.2, ups=0.18, wpb=17735.7, bsz=1024, num_updates=4360, lr=4.72602e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=36, gb_free=13.5, wall=28072
-2024-11-03 13:19:10 - progress_bar.py[line:274] - INFO: epoch 012:     69 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.375, ntokens=17553.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.37, wps=3206.3, ups=0.18, wpb=17553.7, bsz=1024, num_updates=4370, lr=4.72466e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=28127
-2024-11-03 13:20:02 - progress_bar.py[line:274] - INFO: epoch 012:     79 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.413, ntokens=17770.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.65, wps=3441.7, ups=0.19, wpb=17770.2, bsz=1024, num_updates=4380, lr=4.7233e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=28179
-2024-11-03 13:20:58 - progress_bar.py[line:274] - INFO: epoch 012:     89 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.382, ntokens=17780.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.42, wps=3152.5, ups=0.18, wpb=17780.9, bsz=1024, num_updates=4390, lr=4.72194e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=28235
-2024-11-03 13:21:50 - progress_bar.py[line:274] - INFO: epoch 012:     99 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.311, ntokens=17368.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.92, wps=3365.9, ups=0.19, wpb=17368.8, bsz=1024, num_updates=4400, lr=4.72057e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=28287
-2024-11-03 13:22:42 - progress_bar.py[line:274] - INFO: epoch 012:    109 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.371, ntokens=17590.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.34, wps=3342.9, ups=0.19, wpb=17590.7, bsz=1024, num_updates=4410, lr=4.71921e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.3, wall=28339
-2024-11-03 13:23:35 - progress_bar.py[line:274] - INFO: epoch 012:    119 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.366, ntokens=17550.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.31, wps=3339.2, ups=0.19, wpb=17550.6, bsz=1024, num_updates=4420, lr=4.71785e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=28392
-2024-11-03 13:24:27 - progress_bar.py[line:274] - INFO: epoch 012:    129 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.335, ntokens=17397.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.09, wps=3310.2, ups=0.19, wpb=17397.8, bsz=1024, num_updates=4430, lr=4.71649e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=28444
-2024-11-03 13:25:19 - progress_bar.py[line:274] - INFO: epoch 012:    139 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.348, ntokens=17614.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.18, wps=3415.6, ups=0.19, wpb=17614.1, bsz=1024, num_updates=4440, lr=4.71513e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=50, gb_free=13.5, wall=28496
-2024-11-03 13:26:11 - progress_bar.py[line:274] - INFO: epoch 012:    149 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.361, ntokens=17529.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.28, wps=3331.3, ups=0.19, wpb=17529.5, bsz=1024, num_updates=4450, lr=4.71377e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=50, gb_free=13.5, wall=28549
-2024-11-03 13:27:03 - progress_bar.py[line:274] - INFO: epoch 012:    159 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.425, ntokens=17924.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.74, wps=3483.5, ups=0.19, wpb=17924.9, bsz=1024, num_updates=4460, lr=4.71241e-05, gnorm=0.013, clip=0, loss_scale=32768, train_wall=43, gb_free=13.5, wall=28600
-2024-11-03 13:27:55 - progress_bar.py[line:274] - INFO: epoch 012:    169 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.363, ntokens=17841, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.29, wps=3409.8, ups=0.19, wpb=17841, bsz=1024, num_updates=4470, lr=4.71105e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=28652
-2024-11-03 13:28:48 - progress_bar.py[line:274] - INFO: epoch 012:    179 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.359, ntokens=17410.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.26, wps=3310.6, ups=0.19, wpb=17410.8, bsz=1024, num_updates=4480, lr=4.70969e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=28705
-2024-11-03 13:29:41 - progress_bar.py[line:274] - INFO: epoch 012:    189 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.325, ntokens=17548, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.02, wps=3300.5, ups=0.19, wpb=17548, bsz=1024, num_updates=4490, lr=4.70833e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=28758
-2024-11-03 13:30:34 - progress_bar.py[line:274] - INFO: epoch 012:    199 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.354, ntokens=17483.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.22, wps=3326, ups=0.19, wpb=17483.8, bsz=1024, num_updates=4500, lr=4.70697e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=28811
-2024-11-03 13:31:27 - progress_bar.py[line:274] - INFO: epoch 012:    209 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.352, ntokens=17799.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.21, wps=3346.1, ups=0.19, wpb=17799.2, bsz=1024, num_updates=4510, lr=4.70561e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=28864
-2024-11-03 13:32:19 - progress_bar.py[line:274] - INFO: epoch 012:    219 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.354, ntokens=17863.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.23, wps=3390.2, ups=0.19, wpb=17863.8, bsz=1024, num_updates=4520, lr=4.70425e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=28917
-2024-11-03 13:33:13 - progress_bar.py[line:274] - INFO: epoch 012:    229 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.374, ntokens=17836.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.37, wps=3335.1, ups=0.19, wpb=17836.4, bsz=1024, num_updates=4530, lr=4.70289e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=28970
-2024-11-03 13:34:06 - progress_bar.py[line:274] - INFO: epoch 012:    239 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.419, ntokens=17776.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.7, wps=3349.8, ups=0.19, wpb=17776.2, bsz=1024, num_updates=4540, lr=4.70153e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=29023
-2024-11-03 13:34:58 - progress_bar.py[line:274] - INFO: epoch 012:    249 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.349, ntokens=17473.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.19, wps=3365.3, ups=0.19, wpb=17473.2, bsz=1024, num_updates=4550, lr=4.70017e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=29075
-2024-11-03 13:35:50 - progress_bar.py[line:274] - INFO: epoch 012:    259 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.364, ntokens=17734.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.3, wps=3377.4, ups=0.19, wpb=17734.7, bsz=1024, num_updates=4560, lr=4.69881e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=29128
-2024-11-03 13:36:43 - progress_bar.py[line:274] - INFO: epoch 012:    269 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.378, ntokens=17717.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.4, wps=3377.2, ups=0.19, wpb=17717.5, bsz=1024, num_updates=4570, lr=4.69745e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.2, wall=29180
-2024-11-03 13:37:35 - progress_bar.py[line:274] - INFO: epoch 012:    279 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.284, ntokens=17403.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.74, wps=3357.3, ups=0.19, wpb=17403.3, bsz=1024, num_updates=4580, lr=4.69609e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=29232
-2024-11-03 13:38:27 - progress_bar.py[line:274] - INFO: epoch 012:    289 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.383, ntokens=17767.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.43, wps=3411.3, ups=0.19, wpb=17767.1, bsz=1024, num_updates=4590, lr=4.69473e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=29284
-2024-11-03 13:39:19 - progress_bar.py[line:274] - INFO: epoch 012:    299 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.445, ntokens=17805.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.89, wps=3437.7, ups=0.19, wpb=17805.7, bsz=1024, num_updates=4600, lr=4.69337e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=29336
-2024-11-03 13:40:10 - progress_bar.py[line:274] - INFO: epoch 012:    309 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.565, ntokens=18678.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.84, wps=3618.1, ups=0.19, wpb=18678.1, bsz=1024, num_updates=4610, lr=4.69201e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=35, gb_free=13.3, wall=29387
-2024-11-03 13:41:03 - progress_bar.py[line:274] - INFO: epoch 012:    319 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.489, ntokens=18152.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.23, wps=3472.8, ups=0.19, wpb=18152.7, bsz=1024, num_updates=4620, lr=4.69065e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=40, gb_free=13.5, wall=29440
-2024-11-03 13:41:55 - progress_bar.py[line:274] - INFO: epoch 012:    329 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.505, ntokens=18142.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.36, wps=3440.5, ups=0.19, wpb=18142.8, bsz=1024, num_updates=4630, lr=4.68929e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=47, gb_free=13.3, wall=29492
-2024-11-03 13:42:48 - progress_bar.py[line:274] - INFO: epoch 012:    339 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.451, ntokens=18105.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.93, wps=3418.3, ups=0.19, wpb=18105.5, bsz=1024, num_updates=4640, lr=4.68793e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=29545
-2024-11-03 13:43:42 - progress_bar.py[line:274] - INFO: epoch 012:    349 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.562, ntokens=18874.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.81, wps=3524.7, ups=0.19, wpb=18874.7, bsz=1024, num_updates=4650, lr=4.68656e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=29599
-2024-11-03 13:44:35 - progress_bar.py[line:274] - INFO: epoch 012:    359 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.538, ntokens=18340.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.62, wps=3476.9, ups=0.19, wpb=18340.4, bsz=1024, num_updates=4660, lr=4.6852e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=53, gb_free=13.3, wall=29652
-2024-11-03 13:45:26 - progress_bar.py[line:274] - INFO: epoch 012:    369 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.493, ntokens=18140.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.26, wps=3492.9, ups=0.19, wpb=18140.3, bsz=1024, num_updates=4670, lr=4.68384e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=29704
-2024-11-03 13:46:19 - progress_bar.py[line:274] - INFO: epoch 012:    379 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.44, ntokens=18041.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.85, wps=3438.9, ups=0.19, wpb=18041.8, bsz=1024, num_updates=4680, lr=4.68248e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=29756
-2024-11-03 13:47:11 - progress_bar.py[line:274] - INFO: epoch 012:    389 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.527, ntokens=18180.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.53, wps=3482.5, ups=0.19, wpb=18180.7, bsz=1024, num_updates=4690, lr=4.68112e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=29808
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-03 13:47:18 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 12 @ 4692 updates
+2024-11-06 21:16:52 - trainer.py[line:703] - INFO: begin training epoch 12
+2024-11-06 21:16:52 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 21:17:42 - progress_bar.py[line:274] - INFO: epoch 012:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.577, ntokens=16806.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=345, ups=0.02, wpb=16806.3, bsz=985.6, num_updates=4310, lr=4.73282e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=29, gb_free=13.6, wall=27381
+2024-11-06 21:18:35 - progress_bar.py[line:274] - INFO: epoch 012:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.575, ntokens=17724.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3400.6, ups=0.19, wpb=17724.7, bsz=1024, num_updates=4320, lr=4.73146e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=27433
+2024-11-06 21:19:27 - progress_bar.py[line:274] - INFO: epoch 012:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.585, ntokens=17678, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3396.2, ups=0.19, wpb=17678, bsz=1024, num_updates=4330, lr=4.7301e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=27486
+2024-11-06 21:20:19 - progress_bar.py[line:274] - INFO: epoch 012:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.571, ntokens=17727.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3387.6, ups=0.19, wpb=17727.6, bsz=1024, num_updates=4340, lr=4.72874e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=27538
+2024-11-06 21:21:11 - progress_bar.py[line:274] - INFO: epoch 012:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.591, ntokens=17686, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3402.6, ups=0.19, wpb=17686, bsz=1024, num_updates=4350, lr=4.72738e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=27590
+2024-11-06 21:22:04 - progress_bar.py[line:274] - INFO: epoch 012:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.589, ntokens=17735.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3372.5, ups=0.19, wpb=17735.7, bsz=1024, num_updates=4360, lr=4.72602e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=27642
+2024-11-06 21:22:35 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
+2024-11-06 21:23:02 - progress_bar.py[line:274] - INFO: epoch 012:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.557, ntokens=17545.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3012.8, ups=0.17, wpb=17545.6, bsz=1024, num_updates=4370, lr=4.72466e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=33, gb_free=13.5, wall=27701
+2024-11-06 21:23:56 - progress_bar.py[line:274] - INFO: epoch 012:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.613, ntokens=17855.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3316, ups=0.19, wpb=17855.9, bsz=1024, num_updates=4380, lr=4.7233e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=33, gb_free=13.5, wall=27755
+2024-11-06 21:24:48 - progress_bar.py[line:274] - INFO: epoch 012:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.562, ntokens=17694.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3353.6, ups=0.19, wpb=17694.3, bsz=1024, num_updates=4390, lr=4.72194e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=43, gb_free=13.5, wall=27807
+2024-11-06 21:25:41 - progress_bar.py[line:274] - INFO: epoch 012:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.582, ntokens=17525.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3346, ups=0.19, wpb=17525.8, bsz=1024, num_updates=4400, lr=4.72057e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=41, gb_free=13.4, wall=27860
+2024-11-06 21:26:33 - progress_bar.py[line:274] - INFO: epoch 012:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.553, ntokens=17467.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3336.3, ups=0.19, wpb=17467.5, bsz=1024, num_updates=4410, lr=4.71921e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=44, gb_free=13.5, wall=27912
+2024-11-06 21:27:27 - progress_bar.py[line:274] - INFO: epoch 012:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.563, ntokens=17515, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3260.1, ups=0.19, wpb=17515, bsz=1024, num_updates=4420, lr=4.71785e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=48, gb_free=13.5, wall=27966
+2024-11-06 21:28:22 - progress_bar.py[line:274] - INFO: epoch 012:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.549, ntokens=17453.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=3182.2, ups=0.18, wpb=17453.1, bsz=1024, num_updates=4430, lr=4.71649e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=55, gb_free=13.6, wall=28021
+2024-11-06 21:29:17 - progress_bar.py[line:274] - INFO: epoch 012:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.593, ntokens=17630, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3214, ups=0.18, wpb=17630, bsz=1024, num_updates=4440, lr=4.71513e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=55, gb_free=13.5, wall=28075
+2024-11-06 21:30:12 - progress_bar.py[line:274] - INFO: epoch 012:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.542, ntokens=17515.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.82, wps=3173.6, ups=0.18, wpb=17515.3, bsz=1024, num_updates=4450, lr=4.71377e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=55, gb_free=13.4, wall=28131
+2024-11-06 21:31:06 - progress_bar.py[line:274] - INFO: epoch 012:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.6, ntokens=17873.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3265.3, ups=0.18, wpb=17873.8, bsz=1024, num_updates=4460, lr=4.71241e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=55, gb_free=13.5, wall=28185
+2024-11-06 21:32:02 - progress_bar.py[line:274] - INFO: epoch 012:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.605, ntokens=17938.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3249.7, ups=0.18, wpb=17938.3, bsz=1024, num_updates=4470, lr=4.71105e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=55, gb_free=13.5, wall=28241
+2024-11-06 21:32:56 - progress_bar.py[line:274] - INFO: epoch 012:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.528, ntokens=17298.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3197.4, ups=0.18, wpb=17298.8, bsz=1024, num_updates=4480, lr=4.70969e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=54, gb_free=13.5, wall=28295
+2024-11-06 21:33:50 - progress_bar.py[line:274] - INFO: epoch 012:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.572, ntokens=17632.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3278.7, ups=0.19, wpb=17632.9, bsz=1024, num_updates=4490, lr=4.70833e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=54, gb_free=13.3, wall=28348
+2024-11-06 21:34:44 - progress_bar.py[line:274] - INFO: epoch 012:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.545, ntokens=17490.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3243.1, ups=0.19, wpb=17490.1, bsz=1024, num_updates=4500, lr=4.70697e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=54, gb_free=13.3, wall=28402
+2024-11-06 21:35:37 - progress_bar.py[line:274] - INFO: epoch 012:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.578, ntokens=17881.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3331.9, ups=0.19, wpb=17881.5, bsz=1024, num_updates=4510, lr=4.70561e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=54, gb_free=13.3, wall=28456
+2024-11-06 21:36:29 - progress_bar.py[line:274] - INFO: epoch 012:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.571, ntokens=17657.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3379.1, ups=0.19, wpb=17657.2, bsz=1024, num_updates=4520, lr=4.70425e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.6, wall=28508
+2024-11-06 21:37:21 - progress_bar.py[line:274] - INFO: epoch 012:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.592, ntokens=17865.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3456.1, ups=0.19, wpb=17865.9, bsz=1024, num_updates=4530, lr=4.70289e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=28560
+2024-11-06 21:38:13 - progress_bar.py[line:274] - INFO: epoch 012:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=17866.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3419.6, ups=0.19, wpb=17866.9, bsz=1024, num_updates=4540, lr=4.70153e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=28612
+2024-11-06 21:39:05 - progress_bar.py[line:274] - INFO: epoch 012:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.545, ntokens=17373.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3347.1, ups=0.19, wpb=17373.8, bsz=1024, num_updates=4550, lr=4.70017e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=28664
+2024-11-06 21:39:58 - progress_bar.py[line:274] - INFO: epoch 012:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.598, ntokens=17749.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3382.2, ups=0.19, wpb=17749.4, bsz=1024, num_updates=4560, lr=4.69881e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=28717
+2024-11-06 21:40:51 - progress_bar.py[line:274] - INFO: epoch 012:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.569, ntokens=17713.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3300.8, ups=0.19, wpb=17713.2, bsz=1024, num_updates=4570, lr=4.69745e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=48, gb_free=13.6, wall=28770
+2024-11-06 21:41:43 - progress_bar.py[line:274] - INFO: epoch 012:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.556, ntokens=17484.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3360, ups=0.19, wpb=17484.7, bsz=1024, num_updates=4580, lr=4.69609e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=28822
+2024-11-06 21:42:35 - progress_bar.py[line:274] - INFO: epoch 012:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=17708.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3419.8, ups=0.19, wpb=17708.5, bsz=1024, num_updates=4590, lr=4.69473e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=28874
+2024-11-06 21:43:27 - progress_bar.py[line:274] - INFO: epoch 012:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.659, ntokens=18008.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3470.3, ups=0.19, wpb=18008.3, bsz=1024, num_updates=4600, lr=4.69337e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=32, gb_free=13.2, wall=28926
+2024-11-06 21:44:19 - progress_bar.py[line:274] - INFO: epoch 012:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.756, ntokens=18623.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.76, wps=3582.6, ups=0.19, wpb=18623.4, bsz=1024, num_updates=4610, lr=4.69201e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=39, gb_free=13.4, wall=28978
+2024-11-06 21:45:12 - progress_bar.py[line:274] - INFO: epoch 012:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.664, ntokens=18040, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3409.9, ups=0.19, wpb=18040, bsz=1024, num_updates=4620, lr=4.69065e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=48, gb_free=13.4, wall=29031
+2024-11-06 21:46:05 - progress_bar.py[line:274] - INFO: epoch 012:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.681, ntokens=18312.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3458.7, ups=0.19, wpb=18312.9, bsz=1024, num_updates=4630, lr=4.68929e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=53, gb_free=13.4, wall=29084
+2024-11-06 21:46:58 - progress_bar.py[line:274] - INFO: epoch 012:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.648, ntokens=17978.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3421.2, ups=0.19, wpb=17978.5, bsz=1024, num_updates=4640, lr=4.68793e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=29136
+2024-11-06 21:47:50 - progress_bar.py[line:274] - INFO: epoch 012:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.755, ntokens=18880.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=3617.1, ups=0.19, wpb=18880.2, bsz=1024, num_updates=4650, lr=4.68656e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.3, wall=29189
+2024-11-06 21:48:42 - progress_bar.py[line:274] - INFO: epoch 012:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.679, ntokens=18306.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3483.8, ups=0.19, wpb=18306.2, bsz=1024, num_updates=4660, lr=4.6852e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.2, wall=29241
+2024-11-06 21:49:35 - progress_bar.py[line:274] - INFO: epoch 012:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.654, ntokens=18129.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3458.2, ups=0.19, wpb=18129.6, bsz=1024, num_updates=4670, lr=4.68384e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=52, gb_free=13.2, wall=29294
+2024-11-06 21:50:26 - progress_bar.py[line:274] - INFO: epoch 012:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.635, ntokens=18046.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3491.5, ups=0.19, wpb=18046.4, bsz=1024, num_updates=4680, lr=4.68248e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=29345
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+2024-11-06 21:51:22 - progress_bar.py[line:274] - INFO: epoch 012:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.662, ntokens=18230.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3291.8, ups=0.18, wpb=18230.1, bsz=1024, num_updates=4690, lr=4.68112e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=55, gb_free=13.5, wall=29401
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt2024-11-06 21:51:24 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 12 @ 4691 updates
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt
-2024-11-03 13:47:18 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt': File exists
-: File exists: File exists
-: File exists: File exists
+2024-11-06 21:51:24 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt': File exists: File exists
+
+: File exists
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 
 
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 13:49:02 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+2024-11-06 21:53:13 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
 
-slice_id 7 seek offset 350000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
 slice_id 2 seek offset 100000
-slice_id 1 seek offset 50000
-2024-11-03 13:50:53 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 12 @ 4692 updates, score 0) (writing took 215.34788353100885 seconds)
+slice_id 3 seek offset 150000
+slice_id 6 seek offset 300000
+2024-11-06 21:55:04 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 12 @ 4691 updates, score 0) (writing took 220.51554317999398 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_12.pt
-2024-11-03 13:51:48 - train.py[line:336] - INFO: end of epoch 12 (average epoch stats below)
-2024-11-03 13:51:48 - progress_bar.py[line:282] - INFO: epoch 012 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.404 | ntokens 17788.1 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 10.59 | wps 2791.7 | ups 0.16 | wpb 17788.1 | bsz 1023 | num_updates 4692 | lr 4.68085e-05 | gnorm 0.007 | clip 0 | loss_scale 65536 | train_wall 1569 | gb_free 13.4 | wall 30085
-2024-11-03 13:51:48 - trainer.py[line:639] - INFO: loading train data for epoch 13
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+slice_id 7 seek offset 350000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
+
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 4 seek offset 200000
+slice_id 5 seek offset 250000
+slice_id 1 seek offset 50000
+2024-11-06 21:55:59 - train.py[line:336] - INFO: end of epoch 12 (average epoch stats below)
+2024-11-06 21:55:59 - progress_bar.py[line:282] - INFO: epoch 012 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.602 | ntokens 17789.3 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.07 | wps 2767.9 | ups 0.16 | wpb 17789.3 | bsz 1023 | num_updates 4691 | lr 4.68099e-05 | gnorm 0.008 | clip 0 | loss_scale 16384 | train_wall 1770 | gb_free 13.4 | wall 29677
+2024-11-06 21:55:59 - trainer.py[line:639] - INFO: loading train data for epoch 13
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 13:54:30 - trainer.py[line:703] - INFO: begin training epoch 13
-2024-11-03 13:54:30 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 13:55:15 - progress_bar.py[line:274] - INFO: epoch 013:      8 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.356, ntokens=16846.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=10.24, wps=348.5, ups=0.02, wpb=16846.6, bsz=985.6, num_updates=4700, lr=4.67976e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=30292
-2024-11-03 13:56:07 - progress_bar.py[line:274] - INFO: epoch 013:     18 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.34, ntokens=17592.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.12, wps=3357, ups=0.19, wpb=17592.1, bsz=1024, num_updates=4710, lr=4.6784e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=30344
-2024-11-03 13:57:00 - progress_bar.py[line:274] - INFO: epoch 013:     28 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.324, ntokens=17650.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.01, wps=3354.4, ups=0.19, wpb=17650.2, bsz=1024, num_updates=4720, lr=4.67704e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=30397
-2024-11-03 13:57:53 - progress_bar.py[line:274] - INFO: epoch 013:     38 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.393, ntokens=17921.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.5, wps=3378.9, ups=0.19, wpb=17921.7, bsz=1024, num_updates=4730, lr=4.67568e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=30450
-2024-11-03 13:58:45 - progress_bar.py[line:274] - INFO: epoch 013:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.345, ntokens=17801.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.16, wps=3394.8, ups=0.19, wpb=17801.4, bsz=1024, num_updates=4740, lr=4.67432e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=30502
-2024-11-03 13:59:38 - progress_bar.py[line:274] - INFO: epoch 013:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.359, ntokens=17935.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.26, wps=3400.8, ups=0.19, wpb=17935.6, bsz=1024, num_updates=4750, lr=4.67296e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=30555
-2024-11-03 14:00:30 - progress_bar.py[line:274] - INFO: epoch 013:     68 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.328, ntokens=17601.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.04, wps=3358.2, ups=0.19, wpb=17601.4, bsz=1024, num_updates=4760, lr=4.6716e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=30607
-2024-11-03 14:01:23 - progress_bar.py[line:274] - INFO: epoch 013:     78 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.336, ntokens=17483.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.1, wps=3296.3, ups=0.19, wpb=17483.1, bsz=1024, num_updates=4770, lr=4.67024e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=30660
-2024-11-03 14:02:16 - progress_bar.py[line:274] - INFO: epoch 013:     88 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.283, ntokens=17441.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.73, wps=3326.1, ups=0.19, wpb=17441.6, bsz=1024, num_updates=4780, lr=4.66888e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=30713
-2024-11-03 14:03:08 - progress_bar.py[line:274] - INFO: epoch 013:     98 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.324, ntokens=17718.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.02, wps=3385.7, ups=0.19, wpb=17718.7, bsz=1024, num_updates=4790, lr=4.66752e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=30765
-2024-11-03 14:04:01 - progress_bar.py[line:274] - INFO: epoch 013:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.297, ntokens=17419.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.83, wps=3278.8, ups=0.19, wpb=17419.6, bsz=1024, num_updates=4800, lr=4.66616e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=30818
-2024-11-03 14:04:54 - progress_bar.py[line:274] - INFO: epoch 013:    118 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.346, ntokens=17550.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.17, wps=3317.7, ups=0.19, wpb=17550.1, bsz=1024, num_updates=4810, lr=4.6648e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=30871
-2024-11-03 14:05:47 - progress_bar.py[line:274] - INFO: epoch 013:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.28, ntokens=17469.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.71, wps=3312.1, ups=0.19, wpb=17469.9, bsz=1024, num_updates=4820, lr=4.66344e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=30924
-2024-11-03 14:06:40 - progress_bar.py[line:274] - INFO: epoch 013:    138 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.367, ntokens=17853.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.32, wps=3379.2, ups=0.19, wpb=17853.5, bsz=1024, num_updates=4830, lr=4.66208e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=30977
-2024-11-03 14:07:33 - progress_bar.py[line:274] - INFO: epoch 013:    148 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.381, ntokens=17891.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.42, wps=3337.8, ups=0.19, wpb=17891.6, bsz=1024, num_updates=4840, lr=4.66072e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=31030
-2024-11-03 14:08:27 - progress_bar.py[line:274] - INFO: epoch 013:    158 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.36, ntokens=17839.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.27, wps=3318.1, ups=0.19, wpb=17839.2, bsz=1024, num_updates=4850, lr=4.65936e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=31084
-2024-11-03 14:09:20 - progress_bar.py[line:274] - INFO: epoch 013:    168 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.307, ntokens=17701.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.9, wps=3359.2, ups=0.19, wpb=17701.3, bsz=1024, num_updates=4860, lr=4.658e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=31137
-2024-11-03 14:10:13 - progress_bar.py[line:274] - INFO: epoch 013:    178 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.291, ntokens=17340.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.79, wps=3278.3, ups=0.19, wpb=17340.1, bsz=1024, num_updates=4870, lr=4.65664e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=31190
-2024-11-03 14:11:05 - progress_bar.py[line:274] - INFO: epoch 013:    188 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.313, ntokens=17678.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.94, wps=3383.7, ups=0.19, wpb=17678.7, bsz=1024, num_updates=4880, lr=4.65528e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=31242
-2024-11-03 14:11:57 - progress_bar.py[line:274] - INFO: epoch 013:    198 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.302, ntokens=17493.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.86, wps=3323.9, ups=0.19, wpb=17493.3, bsz=1024, num_updates=4890, lr=4.65392e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=31295
-2024-11-03 14:12:50 - progress_bar.py[line:274] - INFO: epoch 013:    208 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.354, ntokens=18056.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.22, wps=3428.4, ups=0.19, wpb=18056.6, bsz=1024, num_updates=4900, lr=4.65255e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=31347
-2024-11-03 14:13:42 - progress_bar.py[line:274] - INFO: epoch 013:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.327, ntokens=17978.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.03, wps=3440.9, ups=0.19, wpb=17978.6, bsz=1024, num_updates=4910, lr=4.65119e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=31399
-2024-11-03 14:13:48 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-03 14:14:41 - progress_bar.py[line:274] - INFO: epoch 013:    229 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.321, ntokens=17638.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.99, wps=3017.8, ups=0.17, wpb=17638.6, bsz=1024, num_updates=4920, lr=4.64983e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=31458
-2024-11-03 14:15:34 - progress_bar.py[line:274] - INFO: epoch 013:    239 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.337, ntokens=17440.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.11, wps=3278.9, ups=0.19, wpb=17440.2, bsz=1024, num_updates=4930, lr=4.64847e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=31511
-2024-11-03 14:16:27 - progress_bar.py[line:274] - INFO: epoch 013:    249 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.295, ntokens=17378.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.81, wps=3300.1, ups=0.19, wpb=17378.7, bsz=1024, num_updates=4940, lr=4.64711e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=31564
-2024-11-03 14:17:20 - progress_bar.py[line:274] - INFO: epoch 013:    259 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.322, ntokens=17687.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10, wps=3333, ups=0.19, wpb=17687.1, bsz=1024, num_updates=4950, lr=4.64575e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=31617
-2024-11-03 14:18:13 - progress_bar.py[line:274] - INFO: epoch 013:    269 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.375, ntokens=17911.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.38, wps=3391.4, ups=0.19, wpb=17911.8, bsz=1024, num_updates=4960, lr=4.64439e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=31670
-2024-11-03 14:19:05 - progress_bar.py[line:274] - INFO: epoch 013:    279 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.338, ntokens=17922, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.11, wps=3407.5, ups=0.19, wpb=17922, bsz=1024, num_updates=4970, lr=4.64303e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=31722
-2024-11-03 14:19:58 - progress_bar.py[line:274] - INFO: epoch 013:    289 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.312, ntokens=17584.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.93, wps=3349.7, ups=0.19, wpb=17584.8, bsz=1024, num_updates=4980, lr=4.64167e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=31775
-2024-11-03 14:20:49 - progress_bar.py[line:274] - INFO: epoch 013:    299 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.366, ntokens=17508, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.31, wps=3382.7, ups=0.19, wpb=17508, bsz=1024, num_updates=4990, lr=4.64031e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=31826
-2024-11-03 14:21:41 - progress_bar.py[line:274] - INFO: epoch 013:    309 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.507, ntokens=18523, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.37, wps=3559.3, ups=0.19, wpb=18523, bsz=1024, num_updates=5000, lr=4.63895e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=31879
-loss_reg: 0.024014921928474313 loss_cls: 0.004517079796642065
-loss_reg: 0.01595003665017753 loss_cls: 0.0037904137279838324
-loss_reg: 0.028996004720944803 loss_cls: 0.0032494834158569574
-loss_reg: 0.023470642760042747 loss_cls: 0.0038906075060367584
-loss_reg: 0.020287728485290365 loss_cls: 0.003400663612410426
-loss_reg: 0.01478539197903547 loss_cls: 0.003950794693082571
-loss_reg: 0.01704928168673187 loss_cls: 0.004171249456703663
-loss_reg: 0.02248719821965025 loss_cls: 0.0033316148910671473
-loss_reg: 0.020500359044608573 loss_cls: 0.0036610609386116266
-loss_reg: 0.03284385382468263 loss_cls: 0.004742177668958902
-loss_reg: 0.023542953698726665 loss_cls: 0.002936617936939001
-loss_reg: 0.035754302128018785 loss_cls: 0.003564282553270459
-loss_reg: 0.020077591548833994 loss_cls: 0.004809455946087837
-loss_reg: 0.020109244115683305 loss_cls: 0.004402155056595802
-loss_reg: 0.0275759369203289 loss_cls: 0.004530644975602627
-loss_reg: 0.020330003064882547 loss_cls: 0.004014622885733843
-loss_reg: 0.02190486672777415 loss_cls: 0.004193136002868414
-loss_reg: 0.02822272796224007 loss_cls: 0.004502866882830858
-loss_reg: 0.01714882678104538 loss_cls: 0.004169507417827845
-loss_reg: 0.02473096783950987 loss_cls: 0.00406315503641963
-loss_reg: 0.020250592975834078 loss_cls: 0.004693592432886362
-loss_reg: 0.026291496826232857 loss_cls: 0.004150781314820051
-loss_reg: 0.02004457240428834 loss_cls: 0.003070040373131633
-loss_reg: 0.021651174020022694 loss_cls: 0.003640944603830576
-loss_reg: 0.02140805669724949 loss_cls: 0.0043005864135921
-loss_reg: 0.027124626357420082 loss_cls: 0.005204975605010986
-loss_reg: 0.02673046130729513 loss_cls: 0.004760494455695152
-loss_reg: 0.017468779547045187 loss_cls: 0.004255740903317928
-loss_reg: 0.022727101168981995 loss_cls: 0.00567263038828969
-loss_reg: 0.02172201293009531 loss_cls: 0.0036928460467606783
-loss_reg: 0.015243577791408214 loss_cls: 0.003760796971619129
-loss_reg: 0.024635387056007957 loss_cls: 0.004511329345405102
-loss_reg: 0.020341744018481923 loss_cls: 0.0037356135435402393
-loss_reg: 0.02462488009096172 loss_cls: 0.0031773915980011225
-loss_reg: 0.014784150249544837 loss_cls: 0.002850350458174944
-loss_reg: 0.019419981794043837 loss_cls: 0.0034033695701509714
-loss_reg: 0.026107558620977155 loss_cls: 0.0035959542728960514
-loss_reg: 0.030995370543219828 loss_cls: 0.004855619743466377
-loss_reg: 0.03280706250378581 loss_cls: 0.0026113649364560843
-loss_reg: 0.03076143015830546 loss_cls: 0.004114043898880482
-loss_reg: 0.030006768532357467 loss_cls: 0.0036530361976474524
-loss_reg: 0.022839261203946623 loss_cls: 0.003300816984847188
-loss_reg: 0.01564495606035539 loss_cls: 0.003428309690207243
-loss_reg: 0.027006355457079988 loss_cls: 0.003104186849668622
-loss_reg: 0.02769961321876013 loss_cls: 0.004907713271677494
-loss_reg: 0.021618442304686264 loss_cls: 0.003964536357671022
-loss_reg: 0.023969664737003977 loss_cls: 0.003255118615925312
-loss_reg: 0.01963272319888025 loss_cls: 0.003912233281880617
-loss_reg: 0.026402901448900354 loss_cls: 0.0033941813744604588
-loss_reg: 0.028140449946258354 loss_cls: 0.003931805491447449
-loss_reg: 0.016446055368795905 loss_cls: 0.003528635250404477
-loss_reg: 0.024818325182157715 loss_cls: 0.0037058359012007713
-loss_reg: 0.013223613929537385 loss_cls: 0.0033441982232034206
-loss_reg: 0.02051540919452994 loss_cls: 0.0030727824196219444
-loss_reg: 0.02414892210591332 loss_cls: 0.004082343075424433
-loss_reg: 0.018477569744324997 loss_cls: 0.004207496996968985
-loss_reg: 0.02796636288597047 loss_cls: 0.004485605284571648
-loss_reg: 0.01848296862870398 loss_cls: 0.0038765270728617907
-loss_reg: 0.02433242711171724 loss_cls: 0.0035567807499319315
-loss_reg: 0.021780295666227144 loss_cls: 0.004234215244650841
-loss_reg: 0.026343841824142112 loss_cls: 0.0040841009467840195
-loss_reg: 0.02973240017856267 loss_cls: 0.0030358312651515007
-loss_reg: 0.021076496474065627 loss_cls: 0.003419196233153343
-loss_reg: 0.02665021854755985 loss_cls: 0.004082303028553724
-2024-11-03 14:22:37 - progress_bar.py[line:274] - INFO: epoch 013:    319 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.46, ntokens=18160.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.01, wps=3246, ups=0.18, wpb=18160.5, bsz=1024, num_updates=5010, lr=4.63759e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=43, gb_free=13.5, wall=31934
-2024-11-03 14:23:33 - progress_bar.py[line:274] - INFO: epoch 013:    329 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.45, ntokens=18232, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.93, wps=3293.8, ups=0.18, wpb=18232, bsz=1024, num_updates=5020, lr=4.63623e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=31990
-2024-11-03 14:24:28 - progress_bar.py[line:274] - INFO: epoch 013:    339 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.434, ntokens=18118.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.81, wps=3292.4, ups=0.18, wpb=18118.4, bsz=1024, num_updates=5030, lr=4.63487e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=32045
-2024-11-03 14:25:22 - progress_bar.py[line:274] - INFO: epoch 013:    349 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.483, ntokens=18506.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.18, wps=3416.2, ups=0.18, wpb=18506.3, bsz=1024, num_updates=5040, lr=4.63351e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=32099
-2024-11-03 14:26:17 - progress_bar.py[line:274] - INFO: epoch 013:    359 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.447, ntokens=18133.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.91, wps=3289.2, ups=0.18, wpb=18133.1, bsz=1024, num_updates=5050, lr=4.63215e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=32154
-2024-11-03 14:27:09 - progress_bar.py[line:274] - INFO: epoch 013:    369 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.42, ntokens=17843.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.7, wps=3407.9, ups=0.19, wpb=17843.6, bsz=1024, num_updates=5060, lr=4.63079e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=32207
-2024-11-03 14:28:04 - progress_bar.py[line:274] - INFO: epoch 013:    379 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.473, ntokens=18273.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.1, wps=3329.6, ups=0.18, wpb=18273.6, bsz=1024, num_updates=5070, lr=4.62943e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=32261
-2024-11-03 14:28:59 - progress_bar.py[line:274] - INFO: epoch 013:    389 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.462, ntokens=18208.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.02, wps=3312, ups=0.18, wpb=18208.6, bsz=1024, num_updates=5080, lr=4.62807e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=32316
+2024-11-06 21:58:38 - trainer.py[line:703] - INFO: begin training epoch 13
+2024-11-06 21:58:38 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 21:59:27 - progress_bar.py[line:274] - INFO: epoch 013:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.567, ntokens=16762.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=345.4, ups=0.02, wpb=16762.1, bsz=985.6, num_updates=4700, lr=4.67976e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=29, gb_free=13.6, wall=29886
+2024-11-06 22:00:19 - progress_bar.py[line:274] - INFO: epoch 013:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.537, ntokens=17627.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3390.9, ups=0.19, wpb=17627.8, bsz=1024, num_updates=4710, lr=4.6784e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=29938
+2024-11-06 22:01:12 - progress_bar.py[line:274] - INFO: epoch 013:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.548, ntokens=17631.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=3359.5, ups=0.19, wpb=17631.1, bsz=1024, num_updates=4720, lr=4.67704e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=29990
+2024-11-06 22:02:04 - progress_bar.py[line:274] - INFO: epoch 013:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.573, ntokens=17890.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3410.2, ups=0.19, wpb=17890.4, bsz=1024, num_updates=4730, lr=4.67568e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=30043
+2024-11-06 22:02:56 - progress_bar.py[line:274] - INFO: epoch 013:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.602, ntokens=17869.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3442.5, ups=0.19, wpb=17869.3, bsz=1024, num_updates=4740, lr=4.67432e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=30095
+2024-11-06 22:03:48 - progress_bar.py[line:274] - INFO: epoch 013:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.574, ntokens=17834.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3434.7, ups=0.19, wpb=17834.3, bsz=1024, num_updates=4750, lr=4.67296e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=30147
+2024-11-06 22:04:40 - progress_bar.py[line:274] - INFO: epoch 013:     69 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.555, ntokens=17611.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3406.7, ups=0.19, wpb=17611.5, bsz=1024, num_updates=4760, lr=4.6716e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=30198
+2024-11-06 22:05:32 - progress_bar.py[line:274] - INFO: epoch 013:     79 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.534, ntokens=17503.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3323.5, ups=0.19, wpb=17503.1, bsz=1024, num_updates=4770, lr=4.67024e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=30251
+2024-11-06 22:06:25 - progress_bar.py[line:274] - INFO: epoch 013:     89 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.534, ntokens=17538.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3337.7, ups=0.19, wpb=17538.4, bsz=1024, num_updates=4780, lr=4.66888e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=30304
+2024-11-06 22:07:17 - progress_bar.py[line:274] - INFO: epoch 013:     99 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.554, ntokens=17648.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3388.1, ups=0.19, wpb=17648.9, bsz=1024, num_updates=4790, lr=4.66752e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=30356
+2024-11-06 22:08:09 - progress_bar.py[line:274] - INFO: epoch 013:    109 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17403.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3337.8, ups=0.19, wpb=17403.4, bsz=1024, num_updates=4800, lr=4.66616e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=30408
+2024-11-06 22:09:01 - progress_bar.py[line:274] - INFO: epoch 013:    119 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.561, ntokens=17582.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3357.1, ups=0.19, wpb=17582.1, bsz=1024, num_updates=4810, lr=4.6648e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=30460
+2024-11-06 22:09:55 - progress_bar.py[line:274] - INFO: epoch 013:    129 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.545, ntokens=17477.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3241.6, ups=0.19, wpb=17477.6, bsz=1024, num_updates=4820, lr=4.66344e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=40, gb_free=13.5, wall=30514
+2024-11-06 22:10:49 - progress_bar.py[line:274] - INFO: epoch 013:    139 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.626, ntokens=18064.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.17, wps=3357.1, ups=0.19, wpb=18064.7, bsz=1024, num_updates=4830, lr=4.66208e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=30568
+2024-11-06 22:11:44 - progress_bar.py[line:274] - INFO: epoch 013:    149 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.577, ntokens=17781.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3248.8, ups=0.18, wpb=17781.5, bsz=1024, num_updates=4840, lr=4.66072e-05, gnorm=0.013, clip=0, loss_scale=16384, train_wall=55, gb_free=13.5, wall=30623
+2024-11-06 22:12:37 - progress_bar.py[line:274] - INFO: epoch 013:    159 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.585, ntokens=17786.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3322.7, ups=0.19, wpb=17786.3, bsz=1024, num_updates=4850, lr=4.65936e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=30676
+2024-11-06 22:13:31 - progress_bar.py[line:274] - INFO: epoch 013:    169 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.565, ntokens=17698.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3304.3, ups=0.19, wpb=17698.8, bsz=1024, num_updates=4860, lr=4.658e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=30730
+2024-11-06 22:14:24 - progress_bar.py[line:274] - INFO: epoch 013:    179 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.539, ntokens=17337.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3276.4, ups=0.19, wpb=17337.2, bsz=1024, num_updates=4870, lr=4.65664e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.6, wall=30783
+2024-11-06 22:15:16 - progress_bar.py[line:274] - INFO: epoch 013:    189 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.535, ntokens=17583.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3358.6, ups=0.19, wpb=17583.3, bsz=1024, num_updates=4880, lr=4.65528e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=30835
+2024-11-06 22:16:09 - progress_bar.py[line:274] - INFO: epoch 013:    199 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.585, ntokens=17646.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3318.3, ups=0.19, wpb=17646.6, bsz=1024, num_updates=4890, lr=4.65392e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=50, gb_free=13.5, wall=30888
+2024-11-06 22:17:01 - progress_bar.py[line:274] - INFO: epoch 013:    209 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.581, ntokens=17933.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3450.3, ups=0.19, wpb=17933.3, bsz=1024, num_updates=4900, lr=4.65255e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=46, gb_free=13.4, wall=30940
+2024-11-06 22:17:53 - progress_bar.py[line:274] - INFO: epoch 013:    219 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.592, ntokens=18020.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3462.7, ups=0.19, wpb=18020.9, bsz=1024, num_updates=4910, lr=4.65119e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=45, gb_free=13.5, wall=30992
+2024-11-06 22:18:46 - progress_bar.py[line:274] - INFO: epoch 013:    229 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.55, ntokens=17638.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.86, wps=3385.4, ups=0.19, wpb=17638.6, bsz=1024, num_updates=4920, lr=4.64983e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=43, gb_free=13.5, wall=31044
+2024-11-06 22:19:38 - progress_bar.py[line:274] - INFO: epoch 013:    239 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.513, ntokens=17440.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3354.5, ups=0.19, wpb=17440.2, bsz=1024, num_updates=4930, lr=4.64847e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=40, gb_free=13.4, wall=31096
+2024-11-06 22:20:30 - progress_bar.py[line:274] - INFO: epoch 013:    249 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.542, ntokens=17378.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.82, wps=3332.1, ups=0.19, wpb=17378.7, bsz=1024, num_updates=4940, lr=4.64711e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=42, gb_free=13.5, wall=31149
+2024-11-06 22:21:21 - progress_bar.py[line:274] - INFO: epoch 013:    259 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=17687.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3427.2, ups=0.19, wpb=17687.1, bsz=1024, num_updates=4950, lr=4.64575e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=40, gb_free=13.6, wall=31200
+2024-11-06 22:22:13 - progress_bar.py[line:274] - INFO: epoch 013:    269 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.604, ntokens=17911.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.08, wps=3480.8, ups=0.19, wpb=17911.8, bsz=1024, num_updates=4960, lr=4.64439e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=39, gb_free=13.3, wall=31252
+2024-11-06 22:23:05 - progress_bar.py[line:274] - INFO: epoch 013:    279 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.575, ntokens=17922, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3447.3, ups=0.19, wpb=17922, bsz=1024, num_updates=4970, lr=4.64303e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=36, gb_free=13.5, wall=31304
+2024-11-06 22:23:58 - progress_bar.py[line:274] - INFO: epoch 013:    289 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.543, ntokens=17584.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3312.5, ups=0.19, wpb=17584.8, bsz=1024, num_updates=4980, lr=4.64167e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=40, gb_free=13.4, wall=31357
+2024-11-06 22:24:51 - progress_bar.py[line:274] - INFO: epoch 013:    299 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.59, ntokens=17508, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3308, ups=0.19, wpb=17508, bsz=1024, num_updates=4990, lr=4.64031e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=49, gb_free=13.5, wall=31410
+2024-11-06 22:25:44 - progress_bar.py[line:274] - INFO: epoch 013:    309 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.708, ntokens=18523, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3498.2, ups=0.19, wpb=18523, bsz=1024, num_updates=5000, lr=4.63895e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=31463
+loss_reg: 0.01371661686701624 loss_cls: 0.0031999179627746344
+loss_reg: 0.01546810929731631 loss_cls: 0.004155462142080069
+loss_reg: 0.020677162917150323 loss_cls: 0.003050205297768116
+loss_reg: 0.009692380419354309 loss_cls: 0.0036115599796175957
+loss_reg: 0.011814246689947393 loss_cls: 0.0035622422583401203
+loss_reg: 0.012816680010544427 loss_cls: 0.0035774887073785067
+loss_reg: 0.012699140482210398 loss_cls: 0.003415712621062994
+loss_reg: 0.012842890728452957 loss_cls: 0.003977042157202959
+loss_reg: 0.0182060165197422 loss_cls: 0.0025799269787967205
+loss_reg: 0.0149602597222693 loss_cls: 0.0031678760424256325
+loss_reg: 0.02265295157481819 loss_cls: 0.004339650273323059
+loss_reg: 0.016193055329361168 loss_cls: 0.003202456282451749
+loss_reg: 0.02425943885639123 loss_cls: 0.004520528484135866
+loss_reg: 0.014691715793301764 loss_cls: 0.002744094468653202
+loss_reg: 0.014839134290115403 loss_cls: 0.004534281324595213
+loss_reg: 0.014397479441294823 loss_cls: 0.0039008944295346737
+loss_reg: 0.024003220302664524 loss_cls: 0.004171252250671387
+loss_reg: 0.016202961998968472 loss_cls: 0.003729133866727352
+loss_reg: 0.016514621988894782 loss_cls: 0.0038731195963919163
+loss_reg: 0.012425913383087201 loss_cls: 0.003739406121894717
+loss_reg: 0.01913431730954543 loss_cls: 0.00446197297424078
+loss_reg: 0.012278234403096045 loss_cls: 0.003851885674521327
+loss_reg: 0.017069840575418327 loss_cls: 0.0038591863121837378
+loss_reg: 0.014867098275800513 loss_cls: 0.00444171205163002
+loss_reg: 0.018319569260746495 loss_cls: 0.0037218001671135426
+loss_reg: 0.01390489326559432 loss_cls: 0.00301563274115324
+loss_reg: 0.014288889372070637 loss_cls: 0.0037061315961182117
+loss_reg: 0.00941902919552759 loss_cls: 0.0029771558474749327
+loss_reg: 0.01560203965808278 loss_cls: 0.003866473911330104
+loss_reg: 0.01811548980177785 loss_cls: 0.004769590683281422
+loss_reg: 0.01814551325208829 loss_cls: 0.00421040877699852
+loss_reg: 0.01337034987423303 loss_cls: 0.004224575124680996
+loss_reg: 0.015303118928802463 loss_cls: 0.005169191397726536
+loss_reg: 0.015984099075731548 loss_cls: 0.003952597733587027
+loss_reg: 0.016464827773476846 loss_cls: 0.003819927340373397
+loss_reg: 0.01766032845952449 loss_cls: 0.004267076961696148
+loss_reg: 0.01794988277137548 loss_cls: 0.0029481658712029457
+loss_reg: 0.014030041303698854 loss_cls: 0.003557977033779025
+loss_reg: 0.00883847879334395 loss_cls: 0.002649585483595729
+loss_reg: 0.020115630722730454 loss_cls: 0.0027355721686035395
+loss_reg: 0.011637083879534866 loss_cls: 0.0030943697784096003
+loss_reg: 0.019710911406632173 loss_cls: 0.004125168547034264
+loss_reg: 0.018772240230705284 loss_cls: 0.0032720433082431555
+loss_reg: 0.016644670502957964 loss_cls: 0.0029676370322704315
+loss_reg: 0.023679372410256652 loss_cls: 0.003705955110490322
+loss_reg: 0.025670070675991506 loss_cls: 0.00211633974686265
+loss_reg: 0.012875137965582806 loss_cls: 0.00318109430372715
+loss_reg: 0.01486882100166004 loss_cls: 0.0033472126815468073
+loss_reg: 0.008406330430459014 loss_cls: 0.003070804290473461
+loss_reg: 0.012394431657941619 loss_cls: 0.003620947478339076
+loss_reg: 0.01598960518499753 loss_cls: 0.0028950769919902086
+loss_reg: 0.01821989213492124 loss_cls: 0.0037469964008778334
+loss_reg: 0.015139598003539412 loss_cls: 0.0037321513518691063
+loss_reg: 0.013331770637969111 loss_cls: 0.0030536146368831396
+loss_reg: 0.017013799969210843 loss_cls: 0.0037234555929899216
+loss_reg: 0.020058742467869765 loss_cls: 0.0038905469700694084
+loss_reg: 0.022646155970886428 loss_cls: 0.0035183350555598736
+loss_reg: 0.02724376864553613 loss_cls: 0.003199436003342271
+loss_reg: 0.02045431692524385 loss_cls: 0.004243470262736082
+loss_reg: 0.012087560845532155 loss_cls: 0.003464183770120144
+loss_reg: 0.015068138371516182 loss_cls: 0.0034942361526191235
+loss_reg: 0.017364603166321615 loss_cls: 0.0033361439127475023
+loss_reg: 0.02212407362240475 loss_cls: 0.002972640795633197
+loss_reg: 0.0157434702954352 loss_cls: 0.0034931327681988478
+2024-11-06 22:26:37 - progress_bar.py[line:274] - INFO: epoch 013:    319 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.652, ntokens=18160.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.29, wps=3390.7, ups=0.19, wpb=18160.5, bsz=1024, num_updates=5010, lr=4.63759e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=31516
+2024-11-06 22:27:29 - progress_bar.py[line:274] - INFO: epoch 013:    329 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.655, ntokens=18232, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3493, ups=0.19, wpb=18232, bsz=1024, num_updates=5020, lr=4.63623e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=31568
+2024-11-06 22:28:22 - progress_bar.py[line:274] - INFO: epoch 013:    339 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.651, ntokens=18118.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3439.3, ups=0.19, wpb=18118.4, bsz=1024, num_updates=5030, lr=4.63487e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=31621
+2024-11-06 22:29:14 - progress_bar.py[line:274] - INFO: epoch 013:    349 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.686, ntokens=18506.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3539, ups=0.19, wpb=18506.3, bsz=1024, num_updates=5040, lr=4.63351e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=31673
+2024-11-06 22:30:06 - progress_bar.py[line:274] - INFO: epoch 013:    359 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.651, ntokens=18133.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3525.4, ups=0.19, wpb=18133.1, bsz=1024, num_updates=5050, lr=4.63215e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=51, gb_free=13.3, wall=31725
+2024-11-06 22:30:58 - progress_bar.py[line:274] - INFO: epoch 013:    369 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.617, ntokens=17843.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3426.8, ups=0.19, wpb=17843.6, bsz=1024, num_updates=5060, lr=4.63079e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=31777
+2024-11-06 22:31:49 - progress_bar.py[line:274] - INFO: epoch 013:    379 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.658, ntokens=18273.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.31, wps=3554.5, ups=0.19, wpb=18273.6, bsz=1024, num_updates=5070, lr=4.62943e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=51, gb_free=13.4, wall=31828
+2024-11-06 22:32:41 - progress_bar.py[line:274] - INFO: epoch 013:    389 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.661, ntokens=18208.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3513.6, ups=0.19, wpb=18208.6, bsz=1024, num_updates=5080, lr=4.62807e-05, gnorm=0.014, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=31880
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 14:29:04 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
-2024-11-03 14:29:06 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 13 @ 5081 updates
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt
 
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt
-2024-11-03 14:29:06 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+
+2024-11-06 22:32:48 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 13 @ 5082 updates
+2024-11-06 22:32:48 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt': File exists: File exists
+: File exists
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 14:30:57 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-03 14:32:49 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 13 @ 5081 updates, score 0) (writing took 222.6235513529973 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 22:34:38 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
 
 
-slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
+slice_id 2 seek offset 100000
+slice_id 6 seek offset 300000
 slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+2024-11-06 22:36:29 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 13 @ 5082 updates, score 0) (writing took 220.6883977020043 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_13.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+slice_id 7 seek offset 350000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
 slice_id 1 seek offset 50000
 slice_id 5 seek offset 250000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-03 14:33:43 - train.py[line:336] - INFO: end of epoch 13 (average epoch stats below)
-2024-11-03 14:33:44 - progress_bar.py[line:282] - INFO: epoch 013 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.362 | ntokens 17775.3 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 10.28 | wps 2748.9 | ups 0.15 | wpb 17775.3 | bsz 1023 | num_updates 5081 | lr 4.62793e-05 | gnorm 0.008 | clip 0 | loss_scale 16384 | train_wall 1369 | gb_free 13.3 | wall 32601
-2024-11-03 14:33:44 - trainer.py[line:639] - INFO: loading train data for epoch 14
+slice_id 4 seek offset 200000
+2024-11-06 22:37:23 - train.py[line:336] - INFO: end of epoch 13 (average epoch stats below)
+2024-11-06 22:37:23 - progress_bar.py[line:282] - INFO: epoch 013 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.586 | ntokens 17777.3 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6 | wps 2797.3 | ups 0.16 | wpb 17777.3 | bsz 1023 | num_updates 5082 | lr 4.6278e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1666 | gb_free 13.3 | wall 32162
+2024-11-06 22:37:23 - trainer.py[line:639] - INFO: loading train data for epoch 14
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 14:36:28 - trainer.py[line:703] - INFO: begin training epoch 14
-2024-11-03 14:36:28 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 14:37:18 - progress_bar.py[line:274] - INFO: epoch 014:      9 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.293, ntokens=16737.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.8, wps=335.5, ups=0.02, wpb=16737.5, bsz=985.6, num_updates=5090, lr=4.62671e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=34, gb_free=13.5, wall=32815
-2024-11-03 14:38:11 - progress_bar.py[line:274] - INFO: epoch 014:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.34, ntokens=17894.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.13, wps=3388.8, ups=0.19, wpb=17894.5, bsz=1024, num_updates=5100, lr=4.62535e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=32868
-2024-11-03 14:39:04 - progress_bar.py[line:274] - INFO: epoch 014:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.308, ntokens=17705.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.9, wps=3345.3, ups=0.19, wpb=17705.3, bsz=1024, num_updates=5110, lr=4.62399e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=32921
-2024-11-03 14:39:57 - progress_bar.py[line:274] - INFO: epoch 014:     39 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.335, ntokens=17853.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.09, wps=3379.9, ups=0.19, wpb=17853.4, bsz=1024, num_updates=5120, lr=4.62263e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.3, wall=32974
-2024-11-03 14:40:50 - progress_bar.py[line:274] - INFO: epoch 014:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.273, ntokens=17488.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.67, wps=3300.2, ups=0.19, wpb=17488.5, bsz=1024, num_updates=5130, lr=4.62127e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=33027
-2024-11-03 14:41:43 - progress_bar.py[line:274] - INFO: epoch 014:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.331, ntokens=17968.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.06, wps=3400, ups=0.19, wpb=17968.1, bsz=1024, num_updates=5140, lr=4.61991e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33080
-2024-11-03 14:42:35 - progress_bar.py[line:274] - INFO: epoch 014:     69 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.327, ntokens=17784.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.03, wps=3386.7, ups=0.19, wpb=17784.4, bsz=1024, num_updates=5150, lr=4.61854e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33132
-2024-11-03 14:43:28 - progress_bar.py[line:274] - INFO: epoch 014:     79 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.32, ntokens=17730.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.99, wps=3357.5, ups=0.19, wpb=17730.8, bsz=1024, num_updates=5160, lr=4.61718e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=33185
-2024-11-03 14:44:20 - progress_bar.py[line:274] - INFO: epoch 014:     89 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.268, ntokens=17556.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.64, wps=3337.7, ups=0.19, wpb=17556.3, bsz=1024, num_updates=5170, lr=4.61582e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33238
-2024-11-03 14:45:13 - progress_bar.py[line:274] - INFO: epoch 014:     99 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.308, ntokens=17702.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.91, wps=3364.1, ups=0.19, wpb=17702.4, bsz=1024, num_updates=5180, lr=4.61446e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33290
-2024-11-03 14:46:05 - progress_bar.py[line:274] - INFO: epoch 014:    109 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.262, ntokens=17579.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.59, wps=3365.2, ups=0.19, wpb=17579.8, bsz=1024, num_updates=5190, lr=4.6131e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=33342
-2024-11-03 14:46:58 - progress_bar.py[line:274] - INFO: epoch 014:    119 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.332, ntokens=17728.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.07, wps=3368.3, ups=0.19, wpb=17728.8, bsz=1024, num_updates=5200, lr=4.61174e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.3, wall=33395
-2024-11-03 14:47:51 - progress_bar.py[line:274] - INFO: epoch 014:    129 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.251, ntokens=17332.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.52, wps=3297.8, ups=0.19, wpb=17332.4, bsz=1024, num_updates=5210, lr=4.61038e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33448
-2024-11-03 14:48:43 - progress_bar.py[line:274] - INFO: epoch 014:    139 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.33, ntokens=17959.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.05, wps=3409.3, ups=0.19, wpb=17959.9, bsz=1024, num_updates=5220, lr=4.60902e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33500
-2024-11-03 14:49:36 - progress_bar.py[line:274] - INFO: epoch 014:    149 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.287, ntokens=17585.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.76, wps=3360.4, ups=0.19, wpb=17585.3, bsz=1024, num_updates=5230, lr=4.60766e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33553
-2024-11-03 14:50:28 - progress_bar.py[line:274] - INFO: epoch 014:    159 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.367, ntokens=17866.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.32, wps=3414.9, ups=0.19, wpb=17866.6, bsz=1024, num_updates=5240, lr=4.6063e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=33605
-2024-11-03 14:51:21 - progress_bar.py[line:274] - INFO: epoch 014:    169 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.282, ntokens=17743.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.73, wps=3356.8, ups=0.19, wpb=17743.7, bsz=1024, num_updates=5250, lr=4.60494e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=33658
-2024-11-03 14:52:14 - progress_bar.py[line:274] - INFO: epoch 014:    179 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.257, ntokens=17469.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.56, wps=3305.6, ups=0.19, wpb=17469.6, bsz=1024, num_updates=5260, lr=4.60358e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=33711
-2024-11-03 14:53:06 - progress_bar.py[line:274] - INFO: epoch 014:    189 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.284, ntokens=17741.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.74, wps=3389.5, ups=0.19, wpb=17741.8, bsz=1024, num_updates=5270, lr=4.60222e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33763
-2024-11-03 14:53:59 - progress_bar.py[line:274] - INFO: epoch 014:    199 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.311, ntokens=17692.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.93, wps=3347, ups=0.19, wpb=17692.3, bsz=1024, num_updates=5280, lr=4.60086e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=33816
-2024-11-03 14:54:51 - progress_bar.py[line:274] - INFO: epoch 014:    209 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.329, ntokens=18134.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.05, wps=3451.6, ups=0.19, wpb=18134.5, bsz=1024, num_updates=5290, lr=4.5995e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33868
-2024-11-03 14:55:44 - progress_bar.py[line:274] - INFO: epoch 014:    219 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.27, ntokens=17906.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.65, wps=3402.9, ups=0.19, wpb=17906.9, bsz=1024, num_updates=5300, lr=4.59814e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=33921
-2024-11-03 14:56:37 - progress_bar.py[line:274] - INFO: epoch 014:    229 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.295, ntokens=17631.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.81, wps=3347.4, ups=0.19, wpb=17631.3, bsz=1024, num_updates=5310, lr=4.59678e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=33974
-2024-11-03 14:57:29 - progress_bar.py[line:274] - INFO: epoch 014:    239 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.309, ntokens=17507.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.91, wps=3325, ups=0.19, wpb=17507.4, bsz=1024, num_updates=5320, lr=4.59542e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=34026
-2024-11-03 14:58:22 - progress_bar.py[line:274] - INFO: epoch 014:    249 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.332, ntokens=17680.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.07, wps=3355, ups=0.19, wpb=17680.6, bsz=1024, num_updates=5330, lr=4.59406e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=34079
-2024-11-03 14:59:15 - progress_bar.py[line:274] - INFO: epoch 014:    259 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.261, ntokens=17605.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.58, wps=3338.9, ups=0.19, wpb=17605.6, bsz=1024, num_updates=5340, lr=4.5927e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=34132
-2024-11-03 15:00:07 - progress_bar.py[line:274] - INFO: epoch 014:    269 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.353, ntokens=18079.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.22, wps=3426.1, ups=0.19, wpb=18079.7, bsz=1024, num_updates=5350, lr=4.59134e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=33, gb_free=13.3, wall=34185
-2024-11-03 15:01:01 - progress_bar.py[line:274] - INFO: epoch 014:    279 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.273, ntokens=17696.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.67, wps=3337.3, ups=0.19, wpb=17696.4, bsz=1024, num_updates=5360, lr=4.58998e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=38, gb_free=13.4, wall=34238
-2024-11-03 15:01:54 - progress_bar.py[line:274] - INFO: epoch 014:    289 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.29, ntokens=17741.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.78, wps=3302, ups=0.19, wpb=17741.1, bsz=1024, num_updates=5370, lr=4.58862e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=43, gb_free=13.4, wall=34291
-2024-11-03 15:02:48 - progress_bar.py[line:274] - INFO: epoch 014:    299 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.368, ntokens=17807.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.33, wps=3322.2, ups=0.19, wpb=17807.3, bsz=1024, num_updates=5380, lr=4.58726e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=34345
-2024-11-03 15:03:41 - progress_bar.py[line:274] - INFO: epoch 014:    309 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.462, ntokens=18286, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.02, wps=3441.9, ups=0.19, wpb=18286, bsz=1024, num_updates=5390, lr=4.5859e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=34398
-2024-11-03 15:04:36 - progress_bar.py[line:274] - INFO: epoch 014:    319 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.408, ntokens=18027.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.62, wps=3299.7, ups=0.18, wpb=18027.4, bsz=1024, num_updates=5400, lr=4.58454e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=36, gb_free=13.5, wall=34453
-2024-11-03 15:05:32 - progress_bar.py[line:274] - INFO: epoch 014:    329 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.392, ntokens=17945.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.5, wps=3180.3, ups=0.18, wpb=17945.9, bsz=1024, num_updates=5410, lr=4.58317e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=34509
-2024-11-03 15:06:26 - progress_bar.py[line:274] - INFO: epoch 014:    339 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.365, ntokens=18111.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.3, wps=3384.4, ups=0.19, wpb=18111.7, bsz=1024, num_updates=5420, lr=4.58181e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=32, gb_free=13.4, wall=34563
-2024-11-03 15:07:18 - progress_bar.py[line:274] - INFO: epoch 014:    349 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.45, ntokens=18439.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.93, wps=3529, ups=0.19, wpb=18439.7, bsz=1024, num_updates=5430, lr=4.58045e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=36, gb_free=13.4, wall=34615
-2024-11-03 15:08:11 - progress_bar.py[line:274] - INFO: epoch 014:    359 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.445, ntokens=18353.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.89, wps=3476.6, ups=0.19, wpb=18353.3, bsz=1024, num_updates=5440, lr=4.57909e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=49, gb_free=13.5, wall=34668
-2024-11-03 15:09:03 - progress_bar.py[line:274] - INFO: epoch 014:    369 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.431, ntokens=18123.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.78, wps=3436, ups=0.19, wpb=18123.2, bsz=1024, num_updates=5450, lr=4.57773e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=34720
-2024-11-03 15:09:56 - progress_bar.py[line:274] - INFO: epoch 014:    379 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.395, ntokens=18086.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.52, wps=3436.5, ups=0.19, wpb=18086.3, bsz=1024, num_updates=5460, lr=4.57637e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=34773
-2024-11-03 15:10:49 - progress_bar.py[line:274] - INFO: epoch 014:    389 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.368, ntokens=17540.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.33, wps=3314.1, ups=0.19, wpb=17540.6, bsz=1024, num_updates=5470, lr=4.57501e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.4, wall=34826
+2024-11-06 22:40:02 - trainer.py[line:703] - INFO: begin training epoch 14
+2024-11-06 22:40:02 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 22:40:46 - progress_bar.py[line:274] - INFO: epoch 014:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.549, ntokens=16881.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=347.9, ups=0.02, wpb=16881.7, bsz=985.6, num_updates=5090, lr=4.62671e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=32365
+2024-11-06 22:41:38 - progress_bar.py[line:274] - INFO: epoch 014:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.583, ntokens=17891.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3464.1, ups=0.19, wpb=17891.7, bsz=1024, num_updates=5100, lr=4.62535e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=32417
+2024-11-06 22:42:30 - progress_bar.py[line:274] - INFO: epoch 014:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.552, ntokens=17637.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.86, wps=3400.9, ups=0.19, wpb=17637.7, bsz=1024, num_updates=5110, lr=4.62399e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=32469
+2024-11-06 22:43:22 - progress_bar.py[line:274] - INFO: epoch 014:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.536, ntokens=17771.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3396.7, ups=0.19, wpb=17771.1, bsz=1024, num_updates=5120, lr=4.62263e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=32521
+2024-11-06 22:44:14 - progress_bar.py[line:274] - INFO: epoch 014:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.553, ntokens=17644.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3397.2, ups=0.19, wpb=17644.5, bsz=1024, num_updates=5130, lr=4.62127e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=32573
+2024-11-06 22:45:06 - progress_bar.py[line:274] - INFO: epoch 014:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.58, ntokens=17894.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3447.7, ups=0.19, wpb=17894.5, bsz=1024, num_updates=5140, lr=4.61991e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=32625
+2024-11-06 22:45:58 - progress_bar.py[line:274] - INFO: epoch 014:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.569, ntokens=17797.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3393.4, ups=0.19, wpb=17797.9, bsz=1024, num_updates=5150, lr=4.61854e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=32677
+2024-11-06 22:46:51 - progress_bar.py[line:274] - INFO: epoch 014:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.561, ntokens=17727.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3382.9, ups=0.19, wpb=17727.2, bsz=1024, num_updates=5160, lr=4.61718e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=32730
+2024-11-06 22:47:44 - progress_bar.py[line:274] - INFO: epoch 014:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.522, ntokens=17433.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3273.6, ups=0.19, wpb=17433.2, bsz=1024, num_updates=5170, lr=4.61582e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=32783
+2024-11-06 22:48:39 - progress_bar.py[line:274] - INFO: epoch 014:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.565, ntokens=17823.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3219.3, ups=0.18, wpb=17823.7, bsz=1024, num_updates=5180, lr=4.61446e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=40, gb_free=13.5, wall=32838
+2024-11-06 22:49:36 - progress_bar.py[line:274] - INFO: epoch 014:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.538, ntokens=17535.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3128.4, ups=0.18, wpb=17535.8, bsz=1024, num_updates=5190, lr=4.6131e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=32894
+2024-11-06 22:50:31 - progress_bar.py[line:274] - INFO: epoch 014:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.596, ntokens=17782.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3223.9, ups=0.18, wpb=17782.3, bsz=1024, num_updates=5200, lr=4.61174e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=32950
+2024-11-06 22:51:26 - progress_bar.py[line:274] - INFO: epoch 014:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.515, ntokens=17358, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3123.9, ups=0.18, wpb=17358, bsz=1024, num_updates=5210, lr=4.61038e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=33005
+2024-11-06 22:52:21 - progress_bar.py[line:274] - INFO: epoch 014:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.583, ntokens=17799.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3224.3, ups=0.18, wpb=17799.4, bsz=1024, num_updates=5220, lr=4.60902e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=33060
+2024-11-06 22:53:18 - progress_bar.py[line:274] - INFO: epoch 014:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.54, ntokens=17639.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.82, wps=3134.1, ups=0.18, wpb=17639.9, bsz=1024, num_updates=5230, lr=4.60766e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=33117
+2024-11-06 22:54:14 - progress_bar.py[line:274] - INFO: epoch 014:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.58, ntokens=17806.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3182.8, ups=0.18, wpb=17806.9, bsz=1024, num_updates=5240, lr=4.6063e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=33173
+2024-11-06 22:55:10 - progress_bar.py[line:274] - INFO: epoch 014:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=17874.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3202.3, ups=0.18, wpb=17874.2, bsz=1024, num_updates=5250, lr=4.60494e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=33228
+2024-11-06 22:56:06 - progress_bar.py[line:274] - INFO: epoch 014:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.527, ntokens=17524.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3125.1, ups=0.18, wpb=17524.6, bsz=1024, num_updates=5260, lr=4.60358e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=33284
+2024-11-06 22:56:28 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
+2024-11-06 22:57:07 - progress_bar.py[line:274] - INFO: epoch 014:    189 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.539, ntokens=17660.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=2880, ups=0.16, wpb=17660.5, bsz=1024, num_updates=5270, lr=4.60222e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=61, gb_free=13.5, wall=33346
+2024-11-06 22:58:00 - progress_bar.py[line:274] - INFO: epoch 014:    199 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.563, ntokens=17692.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3341.4, ups=0.19, wpb=17692.3, bsz=1024, num_updates=5280, lr=4.60086e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.6, wall=33399
+2024-11-06 22:58:52 - progress_bar.py[line:274] - INFO: epoch 014:    209 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.598, ntokens=18134.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3447.4, ups=0.19, wpb=18134.5, bsz=1024, num_updates=5290, lr=4.5995e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=33451
+2024-11-06 22:59:45 - progress_bar.py[line:274] - INFO: epoch 014:    219 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.575, ntokens=17906.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3421.8, ups=0.19, wpb=17906.9, bsz=1024, num_updates=5300, lr=4.59814e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=33504
+2024-11-06 23:00:38 - progress_bar.py[line:274] - INFO: epoch 014:    229 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.537, ntokens=17631.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3335.5, ups=0.19, wpb=17631.3, bsz=1024, num_updates=5310, lr=4.59678e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=33557
+2024-11-06 23:01:30 - progress_bar.py[line:274] - INFO: epoch 014:    239 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.516, ntokens=17507.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.72, wps=3313.6, ups=0.19, wpb=17507.4, bsz=1024, num_updates=5320, lr=4.59542e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=33609
+2024-11-06 23:02:23 - progress_bar.py[line:274] - INFO: epoch 014:    249 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.582, ntokens=17680.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3369.5, ups=0.19, wpb=17680.6, bsz=1024, num_updates=5330, lr=4.59406e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=33662
+2024-11-06 23:03:15 - progress_bar.py[line:274] - INFO: epoch 014:    259 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.53, ntokens=17605.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3401.4, ups=0.19, wpb=17605.6, bsz=1024, num_updates=5340, lr=4.5927e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=33714
+2024-11-06 23:04:07 - progress_bar.py[line:274] - INFO: epoch 014:    269 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.618, ntokens=18079.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3444.3, ups=0.19, wpb=18079.7, bsz=1024, num_updates=5350, lr=4.59134e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.3, wall=33766
+2024-11-06 23:05:01 - progress_bar.py[line:274] - INFO: epoch 014:    279 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.543, ntokens=17696.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3298.2, ups=0.19, wpb=17696.4, bsz=1024, num_updates=5360, lr=4.58998e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=54, gb_free=13.4, wall=33820
+2024-11-06 23:05:57 - progress_bar.py[line:274] - INFO: epoch 014:    289 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.554, ntokens=17741.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3182.9, ups=0.18, wpb=17741.1, bsz=1024, num_updates=5370, lr=4.58862e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=56, gb_free=13.4, wall=33875
+2024-11-06 23:06:50 - progress_bar.py[line:274] - INFO: epoch 014:    299 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=17807.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3345.3, ups=0.19, wpb=17807.3, bsz=1024, num_updates=5380, lr=4.58726e-05, gnorm=0.004, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=33929
+2024-11-06 23:07:43 - progress_bar.py[line:274] - INFO: epoch 014:    309 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.656, ntokens=18286, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3449.3, ups=0.19, wpb=18286, bsz=1024, num_updates=5390, lr=4.5859e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=33982
+2024-11-06 23:08:36 - progress_bar.py[line:274] - INFO: epoch 014:    319 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.628, ntokens=18027.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.18, wps=3397.9, ups=0.19, wpb=18027.4, bsz=1024, num_updates=5400, lr=4.58454e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=34035
+2024-11-06 23:09:30 - progress_bar.py[line:274] - INFO: epoch 014:    329 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.64, ntokens=17945.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.23, wps=3320.7, ups=0.19, wpb=17945.9, bsz=1024, num_updates=5410, lr=4.58317e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=54, gb_free=13.4, wall=34089
+2024-11-06 23:10:24 - progress_bar.py[line:274] - INFO: epoch 014:    339 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.623, ntokens=18111.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3373, ups=0.19, wpb=18111.7, bsz=1024, num_updates=5420, lr=4.58181e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=54, gb_free=13.4, wall=34143
+2024-11-06 23:11:17 - progress_bar.py[line:274] - INFO: epoch 014:    349 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.683, ntokens=18439.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3425.8, ups=0.19, wpb=18439.7, bsz=1024, num_updates=5430, lr=4.58045e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=54, gb_free=13.4, wall=34196
+2024-11-06 23:12:11 - progress_bar.py[line:274] - INFO: epoch 014:    359 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=18353.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3412.9, ups=0.19, wpb=18353.3, bsz=1024, num_updates=5440, lr=4.57909e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=54, gb_free=13.5, wall=34250
+2024-11-06 23:13:05 - progress_bar.py[line:274] - INFO: epoch 014:    369 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.649, ntokens=18123.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3374.1, ups=0.19, wpb=18123.2, bsz=1024, num_updates=5450, lr=4.57773e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=54, gb_free=13.5, wall=34304
+2024-11-06 23:13:58 - progress_bar.py[line:274] - INFO: epoch 014:    379 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.627, ntokens=18086.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.18, wps=3425, ups=0.19, wpb=18086.3, bsz=1024, num_updates=5460, lr=4.57637e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=34357
+2024-11-06 23:14:50 - progress_bar.py[line:274] - INFO: epoch 014:    389 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.581, ntokens=17540.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3381.8, ups=0.19, wpb=17540.6, bsz=1024, num_updates=5470, lr=4.57501e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=34409
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt2024-11-03 15:10:56 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 14 @ 5472 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt
-2024-11-03 15:10:56 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt': File exists: File exists
+2024-11-06 23:14:57 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 14 @ 5472 updates
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt
+
+2024-11-06 23:14:57 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt': File exists: File exists: File exists
+: File exists
+: File exists
+: File exists
+
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 15:12:36 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+2024-11-06 23:16:06 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
 
-slice_id 2 seek offset 100000
-slice_id 5 seek offset 250000
-2024-11-03 15:14:33 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 14 @ 5472 updates, score 0) (writing took 217.0062127049896 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt
-2024-11-03 15:15:28 - train.py[line:336] - INFO: end of epoch 14 (average epoch stats below)
-2024-11-03 15:15:28 - progress_bar.py[line:282] - INFO: epoch 014 | loss 0.002 | loss_v1 0 | loss_v2 0 | nll_loss 3.33 | ntokens 17790.3 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 10.06 | wps 2777.9 | ups 0.16 | wpb 17790.3 | bsz 1023 | num_updates 5472 | lr 4.57474e-05 | gnorm 0.007 | clip 0 | loss_scale 16384 | train_wall 1356 | gb_free 13.3 | wall 35105
-2024-11-03 15:15:28 - trainer.py[line:639] - INFO: loading train data for epoch 15
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+slice_id 5 seek offset 250000
+slice_id 1 seek offset 50000
 slice_id 4 seek offset 200000
-slice_id 3 seek offset 150000
+slice_id 2 seek offset 100000
 slice_id 7 seek offset 350000
+slice_id 3 seek offset 150000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+slice_id 6 seek offset 300000
+2024-11-06 23:18:28 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 14 @ 5472 updates, score 0) (writing took 211.57376466400456 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_14.pt
+2024-11-06 23:19:23 - train.py[line:336] - INFO: end of epoch 14 (average epoch stats below)
+2024-11-06 23:19:23 - progress_bar.py[line:282] - INFO: epoch 014 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.578 | ntokens 17790.2 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.97 | wps 2754.1 | ups 0.15 | wpb 17790.2 | bsz 1023 | num_updates 5472 | lr 4.57474e-05 | gnorm 0.008 | clip 0 | loss_scale 16384 | train_wall 1882 | gb_free 13.3 | wall 34682
+2024-11-06 23:19:23 - trainer.py[line:639] - INFO: loading train data for epoch 15
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 15:18:09 - trainer.py[line:703] - INFO: begin training epoch 15
-2024-11-03 15:18:09 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 15:18:53 - progress_bar.py[line:274] - INFO: epoch 015:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.305, ntokens=16888.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.89, wps=348.7, ups=0.02, wpb=16888.4, bsz=985.6, num_updates=5480, lr=4.57365e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=35310
-2024-11-03 15:19:46 - progress_bar.py[line:274] - INFO: epoch 015:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.25, ntokens=17788, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.52, wps=3377.5, ups=0.19, wpb=17788, bsz=1024, num_updates=5490, lr=4.57229e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=35363
-2024-11-03 15:20:38 - progress_bar.py[line:274] - INFO: epoch 015:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.252, ntokens=17398.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.53, wps=3318.6, ups=0.19, wpb=17398.6, bsz=1024, num_updates=5500, lr=4.57093e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=35415
-2024-11-03 15:21:30 - progress_bar.py[line:274] - INFO: epoch 015:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.311, ntokens=17952.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.92, wps=3438.1, ups=0.19, wpb=17952.9, bsz=1024, num_updates=5510, lr=4.56957e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=35468
-2024-11-03 15:22:22 - progress_bar.py[line:274] - INFO: epoch 015:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.255, ntokens=17658.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.55, wps=3398.8, ups=0.19, wpb=17658.1, bsz=1024, num_updates=5520, lr=4.56821e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=35520
-2024-11-03 15:23:15 - progress_bar.py[line:274] - INFO: epoch 015:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.256, ntokens=17716.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.55, wps=3370.6, ups=0.19, wpb=17716.8, bsz=1024, num_updates=5530, lr=4.56685e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=35572
-2024-11-03 15:24:08 - progress_bar.py[line:274] - INFO: epoch 015:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.297, ntokens=17849.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.83, wps=3397, ups=0.19, wpb=17849.3, bsz=1024, num_updates=5540, lr=4.56549e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=35625
-2024-11-03 15:25:00 - progress_bar.py[line:274] - INFO: epoch 015:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.297, ntokens=17716.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.83, wps=3381.2, ups=0.19, wpb=17716.6, bsz=1024, num_updates=5550, lr=4.56413e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=35677
-2024-11-03 15:25:53 - progress_bar.py[line:274] - INFO: epoch 015:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.249, ntokens=17641.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.5, wps=3342.8, ups=0.19, wpb=17641.6, bsz=1024, num_updates=5560, lr=4.56277e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=35730
-2024-11-03 15:26:45 - progress_bar.py[line:274] - INFO: epoch 015:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.273, ntokens=17892.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.66, wps=3395.6, ups=0.19, wpb=17892.3, bsz=1024, num_updates=5570, lr=4.56141e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=35782
-2024-11-03 15:27:38 - progress_bar.py[line:274] - INFO: epoch 015:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.211, ntokens=17209.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.26, wps=3249.1, ups=0.19, wpb=17209.8, bsz=1024, num_updates=5580, lr=4.56005e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=35835
-2024-11-03 15:28:31 - progress_bar.py[line:274] - INFO: epoch 015:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.333, ntokens=17748.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.08, wps=3345.6, ups=0.19, wpb=17748.6, bsz=1024, num_updates=5590, lr=4.55869e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=35889
-2024-11-03 15:29:19 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
-2024-11-03 15:29:29 - progress_bar.py[line:274] - INFO: epoch 015:    129 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.191, ntokens=17339.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.13, wps=3000.3, ups=0.17, wpb=17339.8, bsz=1024, num_updates=5600, lr=4.55733e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=33, gb_free=13.5, wall=35946
-2024-11-03 15:30:22 - progress_bar.py[line:274] - INFO: epoch 015:    139 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.308, ntokens=17893.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.9, wps=3384.2, ups=0.19, wpb=17893.4, bsz=1024, num_updates=5610, lr=4.55597e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=35999
-2024-11-03 15:31:15 - progress_bar.py[line:274] - INFO: epoch 015:    149 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.276, ntokens=17665.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.69, wps=3350.1, ups=0.19, wpb=17665.4, bsz=1024, num_updates=5620, lr=4.55461e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=36052
-2024-11-03 15:32:08 - progress_bar.py[line:274] - INFO: epoch 015:    159 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.263, ntokens=17606.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.6, wps=3327.4, ups=0.19, wpb=17606.9, bsz=1024, num_updates=5630, lr=4.55325e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=36105
-2024-11-03 15:33:01 - progress_bar.py[line:274] - INFO: epoch 015:    169 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.347, ntokens=18231.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.18, wps=3419.6, ups=0.19, wpb=18231.1, bsz=1024, num_updates=5640, lr=4.55189e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=40, gb_free=13.5, wall=36158
-2024-11-03 15:33:56 - progress_bar.py[line:274] - INFO: epoch 015:    179 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.206, ntokens=17425, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.22, wps=3180.6, ups=0.18, wpb=17425, bsz=1024, num_updates=5650, lr=4.55053e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.6, wall=36213
-2024-11-03 15:34:50 - progress_bar.py[line:274] - INFO: epoch 015:    189 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.253, ntokens=17628.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.53, wps=3276.9, ups=0.19, wpb=17628.5, bsz=1024, num_updates=5660, lr=4.54916e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=54, gb_free=13.6, wall=36267
-2024-11-03 15:35:42 - progress_bar.py[line:274] - INFO: epoch 015:    199 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.309, ntokens=17924.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.91, wps=3432.1, ups=0.19, wpb=17924.1, bsz=1024, num_updates=5670, lr=4.5478e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.6, wall=36319
-2024-11-03 15:36:34 - progress_bar.py[line:274] - INFO: epoch 015:    209 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.265, ntokens=18000, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.61, wps=3474.5, ups=0.19, wpb=18000, bsz=1024, num_updates=5680, lr=4.54644e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=36371
-2024-11-03 15:37:26 - progress_bar.py[line:274] - INFO: epoch 015:    219 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.282, ntokens=17942.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.72, wps=3399.5, ups=0.19, wpb=17942.1, bsz=1024, num_updates=5690, lr=4.54508e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=50, gb_free=13.6, wall=36424
-2024-11-03 15:38:19 - progress_bar.py[line:274] - INFO: epoch 015:    229 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.251, ntokens=17772.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.52, wps=3392.1, ups=0.19, wpb=17772.8, bsz=1024, num_updates=5700, lr=4.54372e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=36476
-2024-11-03 15:39:11 - progress_bar.py[line:274] - INFO: epoch 015:    239 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.271, ntokens=17466.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.65, wps=3329.5, ups=0.19, wpb=17466.9, bsz=1024, num_updates=5710, lr=4.54236e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=36528
-2024-11-03 15:40:04 - progress_bar.py[line:274] - INFO: epoch 015:    249 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.279, ntokens=17582.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.71, wps=3365.4, ups=0.19, wpb=17582.7, bsz=1024, num_updates=5720, lr=4.541e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=36581
-2024-11-03 15:40:55 - progress_bar.py[line:274] - INFO: epoch 015:    259 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.24, ntokens=17559.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.44, wps=3420.8, ups=0.19, wpb=17559.9, bsz=1024, num_updates=5730, lr=4.53964e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=47, gb_free=13.6, wall=36632
-2024-11-03 15:41:46 - progress_bar.py[line:274] - INFO: epoch 015:    269 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.312, ntokens=17940.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.93, wps=3492, ups=0.19, wpb=17940.2, bsz=1024, num_updates=5740, lr=4.53828e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=36, gb_free=13.2, wall=36683
-2024-11-03 15:42:39 - progress_bar.py[line:274] - INFO: epoch 015:    279 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.221, ntokens=17690.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.33, wps=3370, ups=0.19, wpb=17690.6, bsz=1024, num_updates=5750, lr=4.53692e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=36736
-2024-11-03 15:43:31 - progress_bar.py[line:274] - INFO: epoch 015:    289 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.306, ntokens=17788.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.89, wps=3425.7, ups=0.19, wpb=17788.7, bsz=1024, num_updates=5760, lr=4.53556e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=36788
-2024-11-03 15:44:22 - progress_bar.py[line:274] - INFO: epoch 015:    299 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.302, ntokens=17659.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.86, wps=3425.4, ups=0.19, wpb=17659.3, bsz=1024, num_updates=5770, lr=4.5342e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=36839
-2024-11-03 15:45:14 - progress_bar.py[line:274] - INFO: epoch 015:    309 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.468, ntokens=18562.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=11.06, wps=3603.3, ups=0.19, wpb=18562.6, bsz=1024, num_updates=5780, lr=4.53284e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=36891
-2024-11-03 15:46:06 - progress_bar.py[line:274] - INFO: epoch 015:    319 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.429, ntokens=18258.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.77, wps=3513.8, ups=0.19, wpb=18258.5, bsz=1024, num_updates=5790, lr=4.53148e-05, gnorm=0.004, clip=0, loss_scale=16384, train_wall=35, gb_free=13.5, wall=36943
-2024-11-03 15:46:58 - progress_bar.py[line:274] - INFO: epoch 015:    329 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.393, ntokens=18220.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.51, wps=3484.7, ups=0.19, wpb=18220.3, bsz=1024, num_updates=5800, lr=4.53012e-05, gnorm=0.004, clip=0, loss_scale=16384, train_wall=34, gb_free=13.4, wall=36995
-2024-11-03 15:47:53 - progress_bar.py[line:274] - INFO: epoch 015:    339 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.409, ntokens=18404.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.62, wps=3340.7, ups=0.18, wpb=18404.7, bsz=1024, num_updates=5810, lr=4.52876e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=45, gb_free=13.5, wall=37050
-2024-11-03 15:48:47 - progress_bar.py[line:274] - INFO: epoch 015:    349 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.453, ntokens=18668.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.95, wps=3480.9, ups=0.19, wpb=18668.7, bsz=1024, num_updates=5820, lr=4.5274e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=54, gb_free=13.4, wall=37104
-2024-11-03 15:49:38 - progress_bar.py[line:274] - INFO: epoch 015:    359 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.453, ntokens=18375.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.95, wps=3556.3, ups=0.19, wpb=18375.7, bsz=1024, num_updates=5830, lr=4.52604e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=37155
-2024-11-03 15:50:30 - progress_bar.py[line:274] - INFO: epoch 015:    369 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.375, ntokens=17984.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.38, wps=3504.8, ups=0.19, wpb=17984.1, bsz=1024, num_updates=5840, lr=4.52468e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=51, gb_free=13.5, wall=37207
-2024-11-03 15:51:21 - progress_bar.py[line:274] - INFO: epoch 015:    379 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.369, ntokens=18247.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.33, wps=3528.7, ups=0.19, wpb=18247.4, bsz=1024, num_updates=5850, lr=4.52332e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=37259
-2024-11-03 15:52:13 - progress_bar.py[line:274] - INFO: epoch 015:    389 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.401, ntokens=17940.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.56, wps=3473.9, ups=0.19, wpb=17940.9, bsz=1024, num_updates=5860, lr=4.52196e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=52, gb_free=13.6, wall=37310
+2024-11-06 23:22:06 - trainer.py[line:703] - INFO: begin training epoch 15
+2024-11-06 23:22:06 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-06 23:22:50 - progress_bar.py[line:274] - INFO: epoch 015:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.559, ntokens=16888.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=351.4, ups=0.02, wpb=16888.4, bsz=985.6, num_updates=5480, lr=4.57365e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=34889
+2024-11-06 23:23:43 - progress_bar.py[line:274] - INFO: epoch 015:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.558, ntokens=17788, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3357.1, ups=0.19, wpb=17788, bsz=1024, num_updates=5490, lr=4.57229e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=34942
+2024-11-06 23:24:36 - progress_bar.py[line:274] - INFO: epoch 015:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.533, ntokens=17398.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3297.7, ups=0.19, wpb=17398.6, bsz=1024, num_updates=5500, lr=4.57093e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=34995
+2024-11-06 23:25:28 - progress_bar.py[line:274] - INFO: epoch 015:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.563, ntokens=17952.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3449.7, ups=0.19, wpb=17952.9, bsz=1024, num_updates=5510, lr=4.56957e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=35047
+2024-11-06 23:26:20 - progress_bar.py[line:274] - INFO: epoch 015:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.558, ntokens=17658.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3396.4, ups=0.19, wpb=17658.1, bsz=1024, num_updates=5520, lr=4.56821e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=35099
+2024-11-06 23:27:12 - progress_bar.py[line:274] - INFO: epoch 015:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.533, ntokens=17716.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3409, ups=0.19, wpb=17716.8, bsz=1024, num_updates=5530, lr=4.56685e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=35151
+2024-11-06 23:28:05 - progress_bar.py[line:274] - INFO: epoch 015:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=17849.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3391.8, ups=0.19, wpb=17849.3, bsz=1024, num_updates=5540, lr=4.56549e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=35203
+2024-11-06 23:28:57 - progress_bar.py[line:274] - INFO: epoch 015:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.544, ntokens=17716.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3371.1, ups=0.19, wpb=17716.6, bsz=1024, num_updates=5550, lr=4.56413e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=35256
+2024-11-06 23:29:49 - progress_bar.py[line:274] - INFO: epoch 015:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.536, ntokens=17641.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3393.7, ups=0.19, wpb=17641.6, bsz=1024, num_updates=5560, lr=4.56277e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=35308
+2024-11-06 23:30:41 - progress_bar.py[line:274] - INFO: epoch 015:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.578, ntokens=17892.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3452.6, ups=0.19, wpb=17892.3, bsz=1024, num_updates=5570, lr=4.56141e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=35360
+2024-11-06 23:31:33 - progress_bar.py[line:274] - INFO: epoch 015:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.499, ntokens=17209.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3296.3, ups=0.19, wpb=17209.8, bsz=1024, num_updates=5580, lr=4.56005e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=35412
+2024-11-06 23:32:26 - progress_bar.py[line:274] - INFO: epoch 015:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.576, ntokens=17748.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3377.3, ups=0.19, wpb=17748.6, bsz=1024, num_updates=5590, lr=4.55869e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=35465
+2024-11-06 23:33:18 - progress_bar.py[line:274] - INFO: epoch 015:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.532, ntokens=17337.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3332.9, ups=0.19, wpb=17337.9, bsz=1024, num_updates=5600, lr=4.55733e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=35517
+2024-11-06 23:34:10 - progress_bar.py[line:274] - INFO: epoch 015:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.55, ntokens=17750.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=3400, ups=0.19, wpb=17750.4, bsz=1024, num_updates=5610, lr=4.55597e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=35569
+2024-11-06 23:35:03 - progress_bar.py[line:274] - INFO: epoch 015:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.549, ntokens=17636.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=3311, ups=0.19, wpb=17636.3, bsz=1024, num_updates=5620, lr=4.55461e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=34, gb_free=13.3, wall=35622
+2024-11-06 23:35:57 - progress_bar.py[line:274] - INFO: epoch 015:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.542, ntokens=17748.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.82, wps=3298.8, ups=0.19, wpb=17748.3, bsz=1024, num_updates=5630, lr=4.55325e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=37, gb_free=13.5, wall=35676
+2024-11-06 23:36:51 - progress_bar.py[line:274] - INFO: epoch 015:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.584, ntokens=18143.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3380.4, ups=0.19, wpb=18143.1, bsz=1024, num_updates=5640, lr=4.55189e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=44, gb_free=13.4, wall=35730
+2024-11-06 23:37:44 - progress_bar.py[line:274] - INFO: epoch 015:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.533, ntokens=17474.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3270.4, ups=0.19, wpb=17474.8, bsz=1024, num_updates=5650, lr=4.55053e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=53, gb_free=13.6, wall=35783
+2024-11-06 23:38:38 - progress_bar.py[line:274] - INFO: epoch 015:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.538, ntokens=17724.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3285, ups=0.19, wpb=17724.8, bsz=1024, num_updates=5660, lr=4.54916e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=54, gb_free=13.5, wall=35837
+2024-11-06 23:39:31 - progress_bar.py[line:274] - INFO: epoch 015:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.561, ntokens=17671.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3319.9, ups=0.19, wpb=17671.2, bsz=1024, num_updates=5670, lr=4.5478e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=35890
+2024-11-06 23:40:24 - progress_bar.py[line:274] - INFO: epoch 015:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.58, ntokens=18129.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3469.8, ups=0.19, wpb=18129.3, bsz=1024, num_updates=5680, lr=4.54644e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=35942
+2024-11-06 23:41:16 - progress_bar.py[line:274] - INFO: epoch 015:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.568, ntokens=17835.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3409.1, ups=0.19, wpb=17835.3, bsz=1024, num_updates=5690, lr=4.54508e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=52, gb_free=13.6, wall=35995
+2024-11-06 23:42:08 - progress_bar.py[line:274] - INFO: epoch 015:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.576, ntokens=17940.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3436.3, ups=0.19, wpb=17940.6, bsz=1024, num_updates=5700, lr=4.54372e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=36047
+2024-11-06 23:42:59 - progress_bar.py[line:274] - INFO: epoch 015:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.497, ntokens=17362.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3409.9, ups=0.2, wpb=17362.1, bsz=1024, num_updates=5710, lr=4.54236e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=51, gb_free=13.5, wall=36098
+2024-11-06 23:43:51 - progress_bar.py[line:274] - INFO: epoch 015:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.566, ntokens=17673.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3398.4, ups=0.19, wpb=17673.1, bsz=1024, num_updates=5720, lr=4.541e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=46, gb_free=13.5, wall=36150
+2024-11-06 23:44:43 - progress_bar.py[line:274] - INFO: epoch 015:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.502, ntokens=17425.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.67, wps=3326.5, ups=0.19, wpb=17425.4, bsz=1024, num_updates=5730, lr=4.53964e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=47, gb_free=13.5, wall=36202
+2024-11-06 23:45:35 - progress_bar.py[line:274] - INFO: epoch 015:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.598, ntokens=18134.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3487.3, ups=0.19, wpb=18134.9, bsz=1024, num_updates=5740, lr=4.53828e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=48, gb_free=13.5, wall=36254
+2024-11-06 23:46:28 - progress_bar.py[line:274] - INFO: epoch 015:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.536, ntokens=17620.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3373.5, ups=0.19, wpb=17620.5, bsz=1024, num_updates=5750, lr=4.53692e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=42, gb_free=13.4, wall=36306
+2024-11-06 23:47:20 - progress_bar.py[line:274] - INFO: epoch 015:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.563, ntokens=17771.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3413.4, ups=0.19, wpb=17771.4, bsz=1024, num_updates=5760, lr=4.53556e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=39, gb_free=13.6, wall=36359
+2024-11-06 23:48:12 - progress_bar.py[line:274] - INFO: epoch 015:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.566, ntokens=17657.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3377.9, ups=0.19, wpb=17657.5, bsz=1024, num_updates=5770, lr=4.5342e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=38, gb_free=13.3, wall=36411
+2024-11-06 23:49:04 - progress_bar.py[line:274] - INFO: epoch 015:    308 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.7, ntokens=18611.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3607.9, ups=0.19, wpb=18611.1, bsz=1024, num_updates=5780, lr=4.53284e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=44, gb_free=13.2, wall=36462
+2024-11-06 23:49:55 - progress_bar.py[line:274] - INFO: epoch 015:    318 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.653, ntokens=18197.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.29, wps=3513.9, ups=0.19, wpb=18197.7, bsz=1024, num_updates=5790, lr=4.53148e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=44, gb_free=13.3, wall=36514
+2024-11-06 23:50:49 - progress_bar.py[line:274] - INFO: epoch 015:    328 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.634, ntokens=18193.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3372.4, ups=0.19, wpb=18193.1, bsz=1024, num_updates=5800, lr=4.53012e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=49, gb_free=13.4, wall=36568
+2024-11-06 23:51:43 - progress_bar.py[line:274] - INFO: epoch 015:    338 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.678, ntokens=18373.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.4, wps=3395.1, ups=0.18, wpb=18373.1, bsz=1024, num_updates=5810, lr=4.52876e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=36622
+2024-11-06 23:52:37 - progress_bar.py[line:274] - INFO: epoch 015:    348 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.689, ntokens=18694.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3466.6, ups=0.19, wpb=18694.8, bsz=1024, num_updates=5820, lr=4.5274e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.3, wall=36676
+2024-11-06 23:53:31 - progress_bar.py[line:274] - INFO: epoch 015:    358 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.659, ntokens=18297.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3380.5, ups=0.18, wpb=18297.4, bsz=1024, num_updates=5830, lr=4.52604e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.2, wall=36730
+2024-11-06 23:54:25 - progress_bar.py[line:274] - INFO: epoch 015:    368 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.618, ntokens=18049.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3357, ups=0.19, wpb=18049.9, bsz=1024, num_updates=5840, lr=4.52468e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=36784
+2024-11-06 23:55:19 - progress_bar.py[line:274] - INFO: epoch 015:    378 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.655, ntokens=18343.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3398, ups=0.19, wpb=18343.5, bsz=1024, num_updates=5850, lr=4.52332e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=36838
+2024-11-06 23:56:11 - progress_bar.py[line:274] - INFO: epoch 015:    388 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.607, ntokens=17862.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=3463.2, ups=0.19, wpb=17862.8, bsz=1024, num_updates=5860, lr=4.52196e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=36890
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 15:52:20 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 15 @ 5862 updates
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
-2024-11-03 15:52:20 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt': File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 
+2024-11-06 23:56:23 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 15 @ 5863 updates
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
+2024-11-06 23:56:23 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt': File exists: File exists
 
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 15:54:12 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-2024-11-03 15:56:03 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 15 @ 5862 updates, score 0) (writing took 222.9289869730128 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-06 23:58:14 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
 slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+slice_id 3 seek offset 150000
+2024-11-07 00:00:06 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 15 @ 5863 updates, score 0) (writing took 222.57859106099932 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_15.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
+slice_id 6 seek offset 300000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 
 
-2024-11-03 15:56:58 - train.py[line:336] - INFO: end of epoch 15 (average epoch stats below)
-2024-11-03 15:56:58 - progress_bar.py[line:282] - INFO: epoch 015 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.307 | ntokens 17826.7 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 9.9 | wps 2791.7 | ups 0.16 | wpb 17826.7 | bsz 1023 | num_updates 5862 | lr 4.52168e-05 | gnorm 0.007 | clip 0 | loss_scale 16384 | train_wall 1524 | gb_free 13.4 | wall 37595
-2024-11-03 15:56:58 - trainer.py[line:639] - INFO: loading train data for epoch 16
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 5 seek offset 250000
-slice_id 6 seek offset 300000
+slice_id 5 seek offset 250000slice_id 4 seek offset 200000
+
+2024-11-07 00:01:00 - train.py[line:336] - INFO: end of epoch 15 (average epoch stats below)
+2024-11-07 00:01:00 - progress_bar.py[line:282] - INFO: epoch 015 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.576 | ntokens 17824.1 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.96 | wps 2790.5 | ups 0.16 | wpb 17824.1 | bsz 1023 | num_updates 5863 | lr 4.52155e-05 | gnorm 0.009 | clip 0 | loss_scale 32768 | train_wall 1632 | gb_free 13.3 | wall 37179
+2024-11-07 00:01:00 - trainer.py[line:639] - INFO: loading train data for epoch 16
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 15:59:42 - trainer.py[line:703] - INFO: begin training epoch 16
-2024-11-03 15:59:42 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 16:00:26 - progress_bar.py[line:274] - INFO: epoch 016:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.314, ntokens=17029.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.94, wps=345.4, ups=0.02, wpb=17029.4, bsz=985.6, num_updates=5870, lr=4.5206e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=37803
-2024-11-03 16:01:18 - progress_bar.py[line:274] - INFO: epoch 016:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.207, ntokens=17549, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.23, wps=3368.8, ups=0.19, wpb=17549, bsz=1024, num_updates=5880, lr=4.51924e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=37855
-2024-11-03 16:02:10 - progress_bar.py[line:274] - INFO: epoch 016:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.242, ntokens=17551.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.46, wps=3379.1, ups=0.19, wpb=17551.5, bsz=1024, num_updates=5890, lr=4.51788e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=37907
-2024-11-03 16:03:02 - progress_bar.py[line:274] - INFO: epoch 016:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.313, ntokens=18113.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.94, wps=3476.9, ups=0.19, wpb=18113.6, bsz=1024, num_updates=5900, lr=4.51652e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=37959
-2024-11-03 16:03:55 - progress_bar.py[line:274] - INFO: epoch 016:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.194, ntokens=17532.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.15, wps=3350.5, ups=0.19, wpb=17532.8, bsz=1024, num_updates=5910, lr=4.51515e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=38012
-2024-11-03 16:04:47 - progress_bar.py[line:274] - INFO: epoch 016:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.261, ntokens=17730.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.59, wps=3400.3, ups=0.19, wpb=17730.5, bsz=1024, num_updates=5920, lr=4.51379e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=38064
-2024-11-03 16:05:39 - progress_bar.py[line:274] - INFO: epoch 016:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.225, ntokens=17631.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.35, wps=3385.3, ups=0.19, wpb=17631.9, bsz=1024, num_updates=5930, lr=4.51243e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=38116
-2024-11-03 16:06:31 - progress_bar.py[line:274] - INFO: epoch 016:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.31, ntokens=17808, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.92, wps=3386.6, ups=0.19, wpb=17808, bsz=1024, num_updates=5940, lr=4.51107e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=38169
-2024-11-03 16:07:24 - progress_bar.py[line:274] - INFO: epoch 016:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.196, ntokens=17328.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.16, wps=3293.6, ups=0.19, wpb=17328.2, bsz=1024, num_updates=5950, lr=4.50971e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=38221
-2024-11-03 16:08:16 - progress_bar.py[line:274] - INFO: epoch 016:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.221, ntokens=17600.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.32, wps=3368, ups=0.19, wpb=17600.2, bsz=1024, num_updates=5960, lr=4.50835e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=38273
-2024-11-03 16:09:08 - progress_bar.py[line:274] - INFO: epoch 016:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.188, ntokens=17369.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.11, wps=3342.9, ups=0.19, wpb=17369.9, bsz=1024, num_updates=5970, lr=4.50699e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=38325
-2024-11-03 16:10:01 - progress_bar.py[line:274] - INFO: epoch 016:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.334, ntokens=17952.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.08, wps=3434.2, ups=0.19, wpb=17952.4, bsz=1024, num_updates=5980, lr=4.50563e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=38378
-2024-11-03 16:10:53 - progress_bar.py[line:274] - INFO: epoch 016:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.191, ntokens=17409.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.13, wps=3340.9, ups=0.19, wpb=17409.8, bsz=1024, num_updates=5990, lr=4.50427e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=38430
-2024-11-03 16:11:45 - progress_bar.py[line:274] - INFO: epoch 016:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.233, ntokens=17519.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.4, wps=3325.1, ups=0.19, wpb=17519.6, bsz=1024, num_updates=6000, lr=4.50291e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=38482
-2024-11-03 16:12:37 - progress_bar.py[line:274] - INFO: epoch 016:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.283, ntokens=17871.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.73, wps=3429, ups=0.19, wpb=17871.4, bsz=1024, num_updates=6010, lr=4.50155e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.3, wall=38535
-2024-11-03 16:13:30 - progress_bar.py[line:274] - INFO: epoch 016:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.257, ntokens=17828.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.56, wps=3416.5, ups=0.19, wpb=17828.9, bsz=1024, num_updates=6020, lr=4.50019e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=38587
-2024-11-03 16:14:22 - progress_bar.py[line:274] - INFO: epoch 016:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.306, ntokens=18114.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.89, wps=3459.2, ups=0.19, wpb=18114.8, bsz=1024, num_updates=6030, lr=4.49883e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=38639
-2024-11-03 16:15:14 - progress_bar.py[line:274] - INFO: epoch 016:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.201, ntokens=17423.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.2, wps=3347.9, ups=0.19, wpb=17423.3, bsz=1024, num_updates=6040, lr=4.49747e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=38691
-2024-11-03 16:16:06 - progress_bar.py[line:274] - INFO: epoch 016:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.257, ntokens=17871.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.56, wps=3427.5, ups=0.19, wpb=17871.2, bsz=1024, num_updates=6050, lr=4.49611e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=38743
-2024-11-03 16:16:59 - progress_bar.py[line:274] - INFO: epoch 016:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.269, ntokens=17832.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.64, wps=3368.7, ups=0.19, wpb=17832.4, bsz=1024, num_updates=6060, lr=4.49475e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=38796
-2024-11-03 16:17:52 - progress_bar.py[line:274] - INFO: epoch 016:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.223, ntokens=17796.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.34, wps=3358.6, ups=0.19, wpb=17796.1, bsz=1024, num_updates=6070, lr=4.49339e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=38849
-2024-11-03 16:18:45 - progress_bar.py[line:274] - INFO: epoch 016:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.243, ntokens=18024.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.47, wps=3425.8, ups=0.19, wpb=18024.7, bsz=1024, num_updates=6080, lr=4.49203e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=38902
-2024-11-03 16:19:37 - progress_bar.py[line:274] - INFO: epoch 016:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.203, ntokens=17472.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.21, wps=3334.1, ups=0.19, wpb=17472.5, bsz=1024, num_updates=6090, lr=4.49067e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=38954
-2024-11-03 16:20:30 - progress_bar.py[line:274] - INFO: epoch 016:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.207, ntokens=17322.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.23, wps=3298.2, ups=0.19, wpb=17322.7, bsz=1024, num_updates=6100, lr=4.48931e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=39007
-2024-11-03 16:21:22 - progress_bar.py[line:274] - INFO: epoch 016:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.298, ntokens=17602.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.84, wps=3366.9, ups=0.19, wpb=17602.4, bsz=1024, num_updates=6110, lr=4.48795e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=39059
-2024-11-03 16:22:14 - progress_bar.py[line:274] - INFO: epoch 016:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.18, ntokens=17568, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.06, wps=3348.3, ups=0.19, wpb=17568, bsz=1024, num_updates=6120, lr=4.48659e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=39112
-2024-11-03 16:23:06 - progress_bar.py[line:274] - INFO: epoch 016:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.335, ntokens=18115.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.09, wps=3492.4, ups=0.19, wpb=18115.2, bsz=1024, num_updates=6130, lr=4.48523e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=39163
-2024-11-03 16:23:58 - progress_bar.py[line:274] - INFO: epoch 016:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.184, ntokens=17616.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.09, wps=3388.2, ups=0.19, wpb=17616.6, bsz=1024, num_updates=6140, lr=4.48387e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=39215
-2024-11-03 16:24:51 - progress_bar.py[line:274] - INFO: epoch 016:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.227, ntokens=17457.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.37, wps=3321.1, ups=0.19, wpb=17457.8, bsz=1024, num_updates=6150, lr=4.48251e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=39268
-2024-11-03 16:25:43 - progress_bar.py[line:274] - INFO: epoch 016:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.316, ntokens=17952.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.96, wps=3461.2, ups=0.19, wpb=17952.3, bsz=1024, num_updates=6160, lr=4.48114e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=39320
-2024-11-03 16:26:35 - progress_bar.py[line:274] - INFO: epoch 016:    308 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.44, ntokens=18375.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.85, wps=3503.4, ups=0.19, wpb=18375.3, bsz=1024, num_updates=6170, lr=4.47978e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=39372
-2024-11-03 16:27:27 - progress_bar.py[line:274] - INFO: epoch 016:    318 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.388, ntokens=18182.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.47, wps=3483.2, ups=0.19, wpb=18182.9, bsz=1024, num_updates=6180, lr=4.47842e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=39424
-2024-11-03 16:28:20 - progress_bar.py[line:274] - INFO: epoch 016:    328 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.339, ntokens=17991, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.12, wps=3429.4, ups=0.19, wpb=17991, bsz=1024, num_updates=6190, lr=4.47706e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=39477
-2024-11-03 16:29:12 - progress_bar.py[line:274] - INFO: epoch 016:    338 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.305, ntokens=17931.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.88, wps=3422.3, ups=0.19, wpb=17931.4, bsz=1024, num_updates=6200, lr=4.4757e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=39529
-2024-11-03 16:30:05 - progress_bar.py[line:274] - INFO: epoch 016:    348 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.443, ntokens=18732.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.87, wps=3557.8, ups=0.19, wpb=18732.7, bsz=1024, num_updates=6210, lr=4.47434e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=39582
-2024-11-03 16:30:57 - progress_bar.py[line:274] - INFO: epoch 016:    358 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.399, ntokens=18228.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.55, wps=3484.4, ups=0.19, wpb=18228.1, bsz=1024, num_updates=6220, lr=4.47298e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=39634
-2024-11-03 16:31:51 - progress_bar.py[line:274] - INFO: epoch 016:    368 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.409, ntokens=18371.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.62, wps=3393.2, ups=0.18, wpb=18371.1, bsz=1024, num_updates=6230, lr=4.47162e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=40, gb_free=13.4, wall=39688
-2024-11-03 16:32:45 - progress_bar.py[line:274] - INFO: epoch 016:    378 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.334, ntokens=18038.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.08, wps=3332, ups=0.18, wpb=18038.2, bsz=1024, num_updates=6240, lr=4.47026e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=39743
-2024-11-03 16:33:40 - progress_bar.py[line:274] - INFO: epoch 016:    388 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.361, ntokens=17902.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.28, wps=3311.3, ups=0.18, wpb=17902.7, bsz=1024, num_updates=6250, lr=4.4689e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=39797
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-03 16:33:52 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 16 @ 6253 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
+2024-11-07 00:03:40 - trainer.py[line:703] - INFO: begin training epoch 16
+2024-11-07 00:03:40 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 00:04:18 - progress_bar.py[line:274] - INFO: epoch 016:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.534, ntokens=16874.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=346, ups=0.02, wpb=16874.7, bsz=985.6, num_updates=5870, lr=4.5206e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=37377
+2024-11-07 00:05:11 - progress_bar.py[line:274] - INFO: epoch 016:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.547, ntokens=17724.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3389.5, ups=0.19, wpb=17724.1, bsz=1024, num_updates=5880, lr=4.51924e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=37430
+2024-11-07 00:06:03 - progress_bar.py[line:274] - INFO: epoch 016:     27 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.545, ntokens=17606.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3336.8, ups=0.19, wpb=17606.8, bsz=1024, num_updates=5890, lr=4.51788e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=37482
+2024-11-07 00:06:56 - progress_bar.py[line:274] - INFO: epoch 016:     37 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.578, ntokens=18099.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3456.6, ups=0.19, wpb=18099.8, bsz=1024, num_updates=5900, lr=4.51652e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=37535
+2024-11-07 00:07:48 - progress_bar.py[line:274] - INFO: epoch 016:     47 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.544, ntokens=17565.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3401.5, ups=0.19, wpb=17565.9, bsz=1024, num_updates=5910, lr=4.51515e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=37586
+2024-11-07 00:08:40 - progress_bar.py[line:274] - INFO: epoch 016:     57 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.499, ntokens=17510.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3330.1, ups=0.19, wpb=17510.9, bsz=1024, num_updates=5920, lr=4.51379e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=37639
+2024-11-07 00:09:33 - progress_bar.py[line:274] - INFO: epoch 016:     67 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.538, ntokens=17756.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3341.9, ups=0.19, wpb=17756.7, bsz=1024, num_updates=5930, lr=4.51243e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=37692
+2024-11-07 00:10:25 - progress_bar.py[line:274] - INFO: epoch 016:     77 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.539, ntokens=17649.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3392.1, ups=0.19, wpb=17649.9, bsz=1024, num_updates=5940, lr=4.51107e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=37744
+2024-11-07 00:11:17 - progress_bar.py[line:274] - INFO: epoch 016:     87 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.497, ntokens=17583.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3400.2, ups=0.19, wpb=17583.4, bsz=1024, num_updates=5950, lr=4.50971e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=37796
+2024-11-07 00:12:10 - progress_bar.py[line:274] - INFO: epoch 016:     97 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.536, ntokens=17481.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3317, ups=0.19, wpb=17481.7, bsz=1024, num_updates=5960, lr=4.50835e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=37849
+2024-11-07 00:13:02 - progress_bar.py[line:274] - INFO: epoch 016:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.529, ntokens=17497.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3347.4, ups=0.19, wpb=17497.8, bsz=1024, num_updates=5970, lr=4.50699e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=37901
+2024-11-07 00:13:55 - progress_bar.py[line:274] - INFO: epoch 016:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.585, ntokens=17805.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3384.1, ups=0.19, wpb=17805.2, bsz=1024, num_updates=5980, lr=4.50563e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=37953
+2024-11-07 00:14:47 - progress_bar.py[line:274] - INFO: epoch 016:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.511, ntokens=17482.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.7, wps=3350.8, ups=0.19, wpb=17482.3, bsz=1024, num_updates=5990, lr=4.50427e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=38006
+2024-11-07 00:15:39 - progress_bar.py[line:274] - INFO: epoch 016:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.555, ntokens=17593.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3340, ups=0.19, wpb=17593.5, bsz=1024, num_updates=6000, lr=4.50291e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=38058
+2024-11-07 00:16:32 - progress_bar.py[line:274] - INFO: epoch 016:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.558, ntokens=17763.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3375, ups=0.19, wpb=17763.6, bsz=1024, num_updates=6010, lr=4.50155e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=38111
+2024-11-07 00:17:24 - progress_bar.py[line:274] - INFO: epoch 016:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.551, ntokens=17809.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.86, wps=3418.1, ups=0.19, wpb=17809.5, bsz=1024, num_updates=6020, lr=4.50019e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=38163
+2024-11-07 00:18:18 - progress_bar.py[line:274] - INFO: epoch 016:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.598, ntokens=18219.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3361.7, ups=0.18, wpb=18219.5, bsz=1024, num_updates=6030, lr=4.49883e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=38217
+2024-11-07 00:19:10 - progress_bar.py[line:274] - INFO: epoch 016:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.495, ntokens=17357.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3338.4, ups=0.19, wpb=17357.7, bsz=1024, num_updates=6040, lr=4.49747e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=38269
+2024-11-07 00:20:03 - progress_bar.py[line:274] - INFO: epoch 016:    187 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.565, ntokens=17900.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3398.2, ups=0.19, wpb=17900.6, bsz=1024, num_updates=6050, lr=4.49611e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=38322
+2024-11-07 00:20:55 - progress_bar.py[line:274] - INFO: epoch 016:    197 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.539, ntokens=17719.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3399.9, ups=0.19, wpb=17719.7, bsz=1024, num_updates=6060, lr=4.49475e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=38374
+2024-11-07 00:21:48 - progress_bar.py[line:274] - INFO: epoch 016:    207 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.53, ntokens=17867.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3384.6, ups=0.19, wpb=17867.5, bsz=1024, num_updates=6070, lr=4.49339e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=38427
+2024-11-07 00:22:40 - progress_bar.py[line:274] - INFO: epoch 016:    217 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.575, ntokens=18008.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3454.4, ups=0.19, wpb=18008.3, bsz=1024, num_updates=6080, lr=4.49203e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=38479
+2024-11-07 00:23:33 - progress_bar.py[line:274] - INFO: epoch 016:    227 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.514, ntokens=17647.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3349.3, ups=0.19, wpb=17647.5, bsz=1024, num_updates=6090, lr=4.49067e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=38532
+2024-11-07 00:24:26 - progress_bar.py[line:274] - INFO: epoch 016:    237 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.462, ntokens=17179.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.51, wps=3240.9, ups=0.19, wpb=17179.2, bsz=1024, num_updates=6100, lr=4.48931e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=38585
+2024-11-07 00:25:18 - progress_bar.py[line:274] - INFO: epoch 016:    247 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.54, ntokens=17611.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3393.5, ups=0.19, wpb=17611.1, bsz=1024, num_updates=6110, lr=4.48795e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=38637
+2024-11-07 00:26:10 - progress_bar.py[line:274] - INFO: epoch 016:    257 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.518, ntokens=17585.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3352, ups=0.19, wpb=17585.6, bsz=1024, num_updates=6120, lr=4.48659e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=38689
+2024-11-07 00:27:02 - progress_bar.py[line:274] - INFO: epoch 016:    267 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.594, ntokens=18103.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3468.6, ups=0.19, wpb=18103.3, bsz=1024, num_updates=6130, lr=4.48523e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=38741
+2024-11-07 00:27:54 - progress_bar.py[line:274] - INFO: epoch 016:    277 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.531, ntokens=17710.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3395.4, ups=0.19, wpb=17710.9, bsz=1024, num_updates=6140, lr=4.48387e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=38793
+2024-11-07 00:28:48 - progress_bar.py[line:274] - INFO: epoch 016:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.496, ntokens=17454.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3290.1, ups=0.19, wpb=17454.1, bsz=1024, num_updates=6150, lr=4.48251e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=38846
+2024-11-07 00:29:40 - progress_bar.py[line:274] - INFO: epoch 016:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.571, ntokens=17716.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3403.4, ups=0.19, wpb=17716.7, bsz=1024, num_updates=6160, lr=4.48114e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=38898
+2024-11-07 00:30:32 - progress_bar.py[line:274] - INFO: epoch 016:    307 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.693, ntokens=18525.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3557, ups=0.19, wpb=18525.8, bsz=1024, num_updates=6170, lr=4.47978e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=38951
+2024-11-07 00:31:23 - progress_bar.py[line:274] - INFO: epoch 016:    317 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.623, ntokens=18116.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3495.6, ups=0.19, wpb=18116.6, bsz=1024, num_updates=6180, lr=4.47842e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=39002
+2024-11-07 00:32:15 - progress_bar.py[line:274] - INFO: epoch 016:    327 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.598, ntokens=17904.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3445.7, ups=0.19, wpb=17904.2, bsz=1024, num_updates=6190, lr=4.47706e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=39054
+2024-11-07 00:33:08 - progress_bar.py[line:274] - INFO: epoch 016:    337 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.652, ntokens=18129.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3473.1, ups=0.19, wpb=18129.8, bsz=1024, num_updates=6200, lr=4.4757e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=39107
+2024-11-07 00:34:02 - progress_bar.py[line:274] - INFO: epoch 016:    347 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.682, ntokens=18613.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3405.1, ups=0.18, wpb=18613.2, bsz=1024, num_updates=6210, lr=4.47434e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=43, gb_free=13.4, wall=39161
+2024-11-07 00:34:57 - progress_bar.py[line:274] - INFO: epoch 016:    357 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.65, ntokens=18229.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3307.9, ups=0.18, wpb=18229.4, bsz=1024, num_updates=6220, lr=4.47298e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=39216
+2024-11-07 00:35:53 - progress_bar.py[line:274] - INFO: epoch 016:    367 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=18483.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3349.8, ups=0.18, wpb=18483.7, bsz=1024, num_updates=6230, lr=4.47162e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.2, wall=39271
+2024-11-07 00:36:48 - progress_bar.py[line:274] - INFO: epoch 016:    377 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.571, ntokens=17855.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3223.9, ups=0.18, wpb=17855.2, bsz=1024, num_updates=6240, lr=4.47026e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=39327
+2024-11-07 00:37:44 - progress_bar.py[line:274] - INFO: epoch 016:    387 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.633, ntokens=18040.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3236.8, ups=0.18, wpb=18040.3, bsz=1024, num_updates=6250, lr=4.4689e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=39383
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
+2024-11-07 00:38:01 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 16 @ 6254 updates
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
-2024-11-03 16:33:52 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
+2024-11-07 00:38:01 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt': File exists: File exists
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 16:35:51 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-03 16:37:51 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 16 @ 6253 updates, score 0) (writing took 238.81939381200937 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 00:39:52 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+
+slice_id 6 seek offset 300000
+slice_id 2 seek offset 100000
+2024-11-07 00:41:44 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 16 @ 6254 updates, score 0) (writing took 222.36170212901197 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_16.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-slice_id 4 seek offset 200000
 slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+slice_id 1 seek offset 50000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+slice_id 3 seek offset 150000
 slice_id 5 seek offset 250000
-slice_id 1 seek offset 50000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-2024-11-03 16:38:50 - train.py[line:336] - INFO: end of epoch 16 (average epoch stats below)
-2024-11-03 16:38:50 - progress_bar.py[line:282] - INFO: epoch 016 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.279 | ntokens 17786.6 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 9.71 | wps 2768.2 | ups 0.16 | wpb 17786.6 | bsz 1023 | num_updates 6253 | lr 4.46849e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1249 | gb_free 13.4 | wall 40107
-2024-11-03 16:38:50 - trainer.py[line:639] - INFO: loading train data for epoch 17
+slice_id 4 seek offset 200000
+2024-11-07 00:42:38 - train.py[line:336] - INFO: end of epoch 16 (average epoch stats below)
+2024-11-07 00:42:38 - progress_bar.py[line:282] - INFO: epoch 016 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.563 | ntokens 17786.6 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.91 | wps 2783.8 | ups 0.16 | wpb 17786.6 | bsz 1023 | num_updates 6254 | lr 4.46836e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1309 | gb_free 13.4 | wall 39677
+2024-11-07 00:42:38 - trainer.py[line:639] - INFO: loading train data for epoch 17
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 16:41:35 - trainer.py[line:703] - INFO: begin training epoch 17
-2024-11-03 16:41:35 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 16:42:14 - progress_bar.py[line:274] - INFO: epoch 017:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.221, ntokens=16740.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.33, wps=325.5, ups=0.02, wpb=16740.1, bsz=985.6, num_updates=6260, lr=4.46754e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=34, gb_free=13.6, wall=40311
-2024-11-03 16:43:07 - progress_bar.py[line:274] - INFO: epoch 017:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.22, ntokens=17637.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.32, wps=3288.9, ups=0.19, wpb=17637.6, bsz=1024, num_updates=6270, lr=4.46618e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=33, gb_free=13.6, wall=40365
-2024-11-03 16:43:46 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
-2024-11-03 16:44:08 - progress_bar.py[line:274] - INFO: epoch 017:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.215, ntokens=17727.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.29, wps=2912.8, ups=0.16, wpb=17727.7, bsz=1024, num_updates=6280, lr=4.46482e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=58, gb_free=13.4, wall=40425
-2024-11-03 16:45:04 - progress_bar.py[line:274] - INFO: epoch 017:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.237, ntokens=17876.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.43, wps=3212.2, ups=0.18, wpb=17876.3, bsz=1024, num_updates=6290, lr=4.46346e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=56, gb_free=13.5, wall=40481
-2024-11-03 16:45:56 - progress_bar.py[line:274] - INFO: epoch 017:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.179, ntokens=17466.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.06, wps=3361.9, ups=0.19, wpb=17466.4, bsz=1024, num_updates=6300, lr=4.4621e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=40533
-2024-11-03 16:46:48 - progress_bar.py[line:274] - INFO: epoch 017:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.228, ntokens=17756.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.37, wps=3428.3, ups=0.19, wpb=17756.4, bsz=1024, num_updates=6310, lr=4.46074e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=40585
-2024-11-03 16:47:40 - progress_bar.py[line:274] - INFO: epoch 017:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.257, ntokens=17839.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.56, wps=3421.9, ups=0.19, wpb=17839.2, bsz=1024, num_updates=6320, lr=4.45938e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=40637
-2024-11-03 16:48:31 - progress_bar.py[line:274] - INFO: epoch 017:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.223, ntokens=17517.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.34, wps=3399.5, ups=0.19, wpb=17517.5, bsz=1024, num_updates=6330, lr=4.45802e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=47, gb_free=13.4, wall=40688
-2024-11-03 16:49:23 - progress_bar.py[line:274] - INFO: epoch 017:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.231, ntokens=17747.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.39, wps=3414.8, ups=0.19, wpb=17747.2, bsz=1024, num_updates=6340, lr=4.45666e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=36, gb_free=13.6, wall=40740
-2024-11-03 16:50:15 - progress_bar.py[line:274] - INFO: epoch 017:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.193, ntokens=17577, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.15, wps=3383.1, ups=0.19, wpb=17577, bsz=1024, num_updates=6350, lr=4.4553e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=40792
-2024-11-03 16:51:08 - progress_bar.py[line:274] - INFO: epoch 017:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.157, ntokens=17493.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.92, wps=3312.4, ups=0.19, wpb=17493.5, bsz=1024, num_updates=6360, lr=4.45394e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=40845
-2024-11-03 16:52:00 - progress_bar.py[line:274] - INFO: epoch 017:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.308, ntokens=17937.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.9, wps=3427.8, ups=0.19, wpb=17937.5, bsz=1024, num_updates=6370, lr=4.45258e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=40898
-2024-11-03 16:52:54 - progress_bar.py[line:274] - INFO: epoch 017:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.186, ntokens=17598.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.1, wps=3302.5, ups=0.19, wpb=17598.1, bsz=1024, num_updates=6380, lr=4.45122e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=40951
-2024-11-03 16:53:46 - progress_bar.py[line:274] - INFO: epoch 017:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.233, ntokens=17614.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.4, wps=3367.2, ups=0.19, wpb=17614.3, bsz=1024, num_updates=6390, lr=4.44986e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=41003
-2024-11-03 16:54:39 - progress_bar.py[line:274] - INFO: epoch 017:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.223, ntokens=17723.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.33, wps=3342.2, ups=0.19, wpb=17723.5, bsz=1024, num_updates=6400, lr=4.4485e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=36, gb_free=13.4, wall=41056
-2024-11-03 16:55:32 - progress_bar.py[line:274] - INFO: epoch 017:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.233, ntokens=17647.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.4, wps=3336.2, ups=0.19, wpb=17647.2, bsz=1024, num_updates=6410, lr=4.44714e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=39, gb_free=13.5, wall=41109
-2024-11-03 16:56:25 - progress_bar.py[line:274] - INFO: epoch 017:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.202, ntokens=17612.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.2, wps=3324, ups=0.19, wpb=17612.1, bsz=1024, num_updates=6420, lr=4.44577e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=38, gb_free=13.5, wall=41162
-2024-11-03 16:57:18 - progress_bar.py[line:274] - INFO: epoch 017:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.165, ntokens=17394.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.97, wps=3272.6, ups=0.19, wpb=17394.6, bsz=1024, num_updates=6430, lr=4.44441e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=46, gb_free=13.6, wall=41215
-2024-11-03 16:58:11 - progress_bar.py[line:274] - INFO: epoch 017:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.24, ntokens=17845.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.45, wps=3359, ups=0.19, wpb=17845.2, bsz=1024, num_updates=6440, lr=4.44305e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=41268
-2024-11-03 16:59:04 - progress_bar.py[line:274] - INFO: epoch 017:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.311, ntokens=18121.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.92, wps=3413.2, ups=0.19, wpb=18121.3, bsz=1024, num_updates=6450, lr=4.44169e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=46, gb_free=13.6, wall=41321
-2024-11-03 16:59:57 - progress_bar.py[line:274] - INFO: epoch 017:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.208, ntokens=17928.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.24, wps=3414.9, ups=0.19, wpb=17928.2, bsz=1024, num_updates=6460, lr=4.44033e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=41374
-2024-11-03 17:00:49 - progress_bar.py[line:274] - INFO: epoch 017:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.197, ntokens=17835.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.17, wps=3396.7, ups=0.19, wpb=17835.9, bsz=1024, num_updates=6470, lr=4.43897e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=41426
-2024-11-03 17:01:44 - progress_bar.py[line:274] - INFO: epoch 017:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.248, ntokens=17981.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.5, wps=3304.2, ups=0.18, wpb=17981.1, bsz=1024, num_updates=6480, lr=4.43761e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=34, gb_free=13.6, wall=41481
-2024-11-03 17:02:36 - progress_bar.py[line:274] - INFO: epoch 017:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.224, ntokens=17520.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.34, wps=3358.1, ups=0.19, wpb=17520.2, bsz=1024, num_updates=6490, lr=4.43625e-05, gnorm=0.013, clip=0, loss_scale=16384, train_wall=43, gb_free=13.4, wall=41533
-2024-11-03 17:03:28 - progress_bar.py[line:274] - INFO: epoch 017:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.24, ntokens=17506.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.45, wps=3373, ups=0.19, wpb=17506.5, bsz=1024, num_updates=6500, lr=4.43489e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=37, gb_free=13.5, wall=41585
-2024-11-03 17:04:20 - progress_bar.py[line:274] - INFO: epoch 017:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.126, ntokens=17371.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.73, wps=3299.4, ups=0.19, wpb=17371.3, bsz=1024, num_updates=6510, lr=4.43353e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=32, gb_free=13.6, wall=41638
-2024-11-03 17:05:13 - progress_bar.py[line:274] - INFO: epoch 017:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.321, ntokens=18095, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.99, wps=3448, ups=0.19, wpb=18095, bsz=1024, num_updates=6520, lr=4.43217e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=41690
-2024-11-03 17:06:05 - progress_bar.py[line:274] - INFO: epoch 017:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.173, ntokens=17684.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.02, wps=3374.2, ups=0.19, wpb=17684.2, bsz=1024, num_updates=6530, lr=4.43081e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=41742
-2024-11-03 17:06:58 - progress_bar.py[line:274] - INFO: epoch 017:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.205, ntokens=17601.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.22, wps=3316.9, ups=0.19, wpb=17601.6, bsz=1024, num_updates=6540, lr=4.42945e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=41796
-2024-11-03 17:07:51 - progress_bar.py[line:274] - INFO: epoch 017:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.294, ntokens=17715.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.81, wps=3387, ups=0.19, wpb=17715.4, bsz=1024, num_updates=6550, lr=4.42809e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=41848
-2024-11-03 17:08:42 - progress_bar.py[line:274] - INFO: epoch 017:    308 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.389, ntokens=18521, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.48, wps=3608.9, ups=0.19, wpb=18521, bsz=1024, num_updates=6560, lr=4.42673e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.1, wall=41899
-2024-11-03 17:09:36 - progress_bar.py[line:274] - INFO: epoch 017:    318 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.418, ntokens=18222.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.69, wps=3398.6, ups=0.19, wpb=18222.2, bsz=1024, num_updates=6570, lr=4.42537e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=40, gb_free=13.3, wall=41953
-2024-11-03 17:10:29 - progress_bar.py[line:274] - INFO: epoch 017:    328 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.366, ntokens=18454.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.31, wps=3429.9, ups=0.19, wpb=18454.5, bsz=1024, num_updates=6580, lr=4.42401e-05, gnorm=0.004, clip=0, loss_scale=16384, train_wall=53, gb_free=13.4, wall=42007
-2024-11-03 17:11:24 - progress_bar.py[line:274] - INFO: epoch 017:    338 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.343, ntokens=18164.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.15, wps=3331.1, ups=0.18, wpb=18164.4, bsz=1024, num_updates=6590, lr=4.42265e-05, gnorm=0.004, clip=0, loss_scale=16384, train_wall=54, gb_free=13.5, wall=42061
-2024-11-03 17:12:20 - progress_bar.py[line:274] - INFO: epoch 017:    348 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.398, ntokens=18700.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.54, wps=3345.1, ups=0.18, wpb=18700.5, bsz=1024, num_updates=6600, lr=4.42129e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=56, gb_free=13.4, wall=42117
-2024-11-03 17:13:13 - progress_bar.py[line:274] - INFO: epoch 017:    358 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.364, ntokens=18176.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.29, wps=3455.1, ups=0.19, wpb=18176.6, bsz=1024, num_updates=6610, lr=4.41993e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=53, gb_free=13.1, wall=42170
-2024-11-03 17:14:04 - progress_bar.py[line:274] - INFO: epoch 017:    368 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.378, ntokens=18231.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.39, wps=3523.4, ups=0.19, wpb=18231.1, bsz=1024, num_updates=6620, lr=4.41857e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=42221
-2024-11-03 17:14:56 - progress_bar.py[line:274] - INFO: epoch 017:    378 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.256, ntokens=17776.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.55, wps=3439.1, ups=0.19, wpb=17776.1, bsz=1024, num_updates=6630, lr=4.41721e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=42273
-2024-11-03 17:15:47 - progress_bar.py[line:274] - INFO: epoch 017:    388 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.373, ntokens=17966.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.36, wps=3494, ups=0.19, wpb=17966.3, bsz=1024, num_updates=6640, lr=4.41585e-05, gnorm=0.015, clip=0, loss_scale=16384, train_wall=51, gb_free=13.4, wall=42324
+2024-11-07 00:45:18 - trainer.py[line:703] - INFO: begin training epoch 17
+2024-11-07 00:45:18 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 00:45:51 - progress_bar.py[line:274] - INFO: epoch 017:      6 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.511, ntokens=16807, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.7, wps=344.8, ups=0.02, wpb=16807, bsz=985.6, num_updates=6260, lr=4.46754e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=36, gb_free=13.5, wall=39870
+2024-11-07 00:46:44 - progress_bar.py[line:274] - INFO: epoch 017:     16 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.528, ntokens=17664.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3342.7, ups=0.19, wpb=17664.7, bsz=1024, num_updates=6270, lr=4.46618e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=39923
+2024-11-07 00:47:37 - progress_bar.py[line:274] - INFO: epoch 017:     26 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.502, ntokens=17441.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.66, wps=3312.9, ups=0.19, wpb=17441.1, bsz=1024, num_updates=6280, lr=4.46482e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=39975
+2024-11-07 00:48:29 - progress_bar.py[line:274] - INFO: epoch 017:     36 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.54, ntokens=17889.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.82, wps=3409.5, ups=0.19, wpb=17889.3, bsz=1024, num_updates=6290, lr=4.46346e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=40028
+2024-11-07 00:49:21 - progress_bar.py[line:274] - INFO: epoch 017:     46 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.506, ntokens=17516, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.68, wps=3369.5, ups=0.19, wpb=17516, bsz=1024, num_updates=6300, lr=4.4621e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=40080
+2024-11-07 00:50:14 - progress_bar.py[line:274] - INFO: epoch 017:     56 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.53, ntokens=17646.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3362.6, ups=0.19, wpb=17646.8, bsz=1024, num_updates=6310, lr=4.46074e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=40132
+2024-11-07 00:51:06 - progress_bar.py[line:274] - INFO: epoch 017:     66 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.565, ntokens=17989.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3456.5, ups=0.19, wpb=17989.1, bsz=1024, num_updates=6320, lr=4.45938e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=40184
+2024-11-07 00:51:57 - progress_bar.py[line:274] - INFO: epoch 017:     76 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.506, ntokens=17409.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.68, wps=3365.1, ups=0.19, wpb=17409.8, bsz=1024, num_updates=6330, lr=4.45802e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=40236
+2024-11-07 00:52:49 - progress_bar.py[line:274] - INFO: epoch 017:     86 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.515, ntokens=17724.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.72, wps=3402.3, ups=0.19, wpb=17724.2, bsz=1024, num_updates=6340, lr=4.45666e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=40288
+2024-11-07 00:53:42 - progress_bar.py[line:274] - INFO: epoch 017:     96 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.534, ntokens=17665.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3355.9, ups=0.19, wpb=17665.6, bsz=1024, num_updates=6350, lr=4.4553e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=40341
+2024-11-07 00:54:34 - progress_bar.py[line:274] - INFO: epoch 017:    106 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17642.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3387.4, ups=0.19, wpb=17642.3, bsz=1024, num_updates=6360, lr=4.45394e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=40393
+2024-11-07 00:55:26 - progress_bar.py[line:274] - INFO: epoch 017:    116 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.588, ntokens=17817.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3423.6, ups=0.19, wpb=17817.6, bsz=1024, num_updates=6370, lr=4.45258e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=40445
+2024-11-07 00:56:20 - progress_bar.py[line:274] - INFO: epoch 017:    126 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.502, ntokens=17590.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.67, wps=3253.1, ups=0.18, wpb=17590.9, bsz=1024, num_updates=6380, lr=4.45122e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=37, gb_free=13.6, wall=40499
+2024-11-07 00:57:14 - progress_bar.py[line:274] - INFO: epoch 017:    136 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.544, ntokens=17663.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3300.8, ups=0.19, wpb=17663.8, bsz=1024, num_updates=6390, lr=4.44986e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=44, gb_free=13.5, wall=40553
+2024-11-07 00:58:07 - progress_bar.py[line:274] - INFO: epoch 017:    146 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.545, ntokens=17698.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3296.5, ups=0.19, wpb=17698.3, bsz=1024, num_updates=6400, lr=4.4485e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=40606
+2024-11-07 00:59:01 - progress_bar.py[line:274] - INFO: epoch 017:    156 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.519, ntokens=17502.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3284.2, ups=0.19, wpb=17502.9, bsz=1024, num_updates=6410, lr=4.44714e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=40660
+2024-11-07 00:59:53 - progress_bar.py[line:274] - INFO: epoch 017:    166 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17786.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3399.4, ups=0.19, wpb=17786.7, bsz=1024, num_updates=6420, lr=4.44577e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=52, gb_free=13.3, wall=40712
+2024-11-07 01:00:45 - progress_bar.py[line:274] - INFO: epoch 017:    176 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.492, ntokens=17432.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.62, wps=3364.4, ups=0.19, wpb=17432.7, bsz=1024, num_updates=6430, lr=4.44441e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=40764
+2024-11-07 01:01:37 - progress_bar.py[line:274] - INFO: epoch 017:    186 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.554, ntokens=17821.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3414, ups=0.19, wpb=17821.6, bsz=1024, num_updates=6440, lr=4.44305e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=40816
+2024-11-07 01:02:29 - progress_bar.py[line:274] - INFO: epoch 017:    196 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.595, ntokens=18043, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3459.1, ups=0.19, wpb=18043, bsz=1024, num_updates=6450, lr=4.44169e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=40868
+2024-11-07 01:03:21 - progress_bar.py[line:274] - INFO: epoch 017:    206 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.524, ntokens=17859.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3430.4, ups=0.19, wpb=17859.1, bsz=1024, num_updates=6460, lr=4.44033e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=40920
+2024-11-07 01:04:14 - progress_bar.py[line:274] - INFO: epoch 017:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17918.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3430.8, ups=0.19, wpb=17918.5, bsz=1024, num_updates=6470, lr=4.43897e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=52, gb_free=13.6, wall=40972
+2024-11-07 01:05:05 - progress_bar.py[line:274] - INFO: epoch 017:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=17941.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3461.5, ups=0.19, wpb=17941.3, bsz=1024, num_updates=6480, lr=4.43761e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=41024
+2024-11-07 01:05:57 - progress_bar.py[line:274] - INFO: epoch 017:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.514, ntokens=17611.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3394.8, ups=0.19, wpb=17611.9, bsz=1024, num_updates=6490, lr=4.43625e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=41076
+2024-11-07 01:06:49 - progress_bar.py[line:274] - INFO: epoch 017:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.484, ntokens=17354.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3326.9, ups=0.19, wpb=17354.7, bsz=1024, num_updates=6500, lr=4.43489e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=52, gb_free=13.6, wall=41128
+2024-11-07 01:07:42 - progress_bar.py[line:274] - INFO: epoch 017:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.516, ntokens=17608, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.72, wps=3350, ups=0.19, wpb=17608, bsz=1024, num_updates=6510, lr=4.43353e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=41181
+2024-11-07 01:08:24 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
+2024-11-07 01:08:40 - progress_bar.py[line:274] - INFO: epoch 017:    267 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.555, ntokens=17858.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3097.6, ups=0.17, wpb=17858.3, bsz=1024, num_updates=6520, lr=4.43217e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=41239
+2024-11-07 01:09:32 - progress_bar.py[line:274] - INFO: epoch 017:    277 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.514, ntokens=17611.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3335.9, ups=0.19, wpb=17611.1, bsz=1024, num_updates=6530, lr=4.43081e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=41291
+2024-11-07 01:10:25 - progress_bar.py[line:274] - INFO: epoch 017:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.531, ntokens=17626, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3352.4, ups=0.19, wpb=17626, bsz=1024, num_updates=6540, lr=4.42945e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.6, wall=41344
+2024-11-07 01:11:18 - progress_bar.py[line:274] - INFO: epoch 017:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.546, ntokens=17679.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3307.9, ups=0.19, wpb=17679.5, bsz=1024, num_updates=6550, lr=4.42809e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=41397
+2024-11-07 01:12:12 - progress_bar.py[line:274] - INFO: epoch 017:    307 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.671, ntokens=18479.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.37, wps=3422.2, ups=0.19, wpb=18479.6, bsz=1024, num_updates=6560, lr=4.42673e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.2, wall=41451
+2024-11-07 01:13:06 - progress_bar.py[line:274] - INFO: epoch 017:    317 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.632, ntokens=18188.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3398.9, ups=0.19, wpb=18188.5, bsz=1024, num_updates=6570, lr=4.42537e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=41505
+2024-11-07 01:13:59 - progress_bar.py[line:274] - INFO: epoch 017:    327 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.643, ntokens=18555.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3474.5, ups=0.19, wpb=18555.9, bsz=1024, num_updates=6580, lr=4.42401e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=41558
+2024-11-07 01:14:53 - progress_bar.py[line:274] - INFO: epoch 017:    337 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.644, ntokens=18213.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3414.3, ups=0.19, wpb=18213.6, bsz=1024, num_updates=6590, lr=4.42265e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=41612
+2024-11-07 01:15:46 - progress_bar.py[line:274] - INFO: epoch 017:    347 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.655, ntokens=18627.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3467.8, ups=0.19, wpb=18627.4, bsz=1024, num_updates=6600, lr=4.42129e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=41665
+2024-11-07 01:16:39 - progress_bar.py[line:274] - INFO: epoch 017:    357 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=18216.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3443.6, ups=0.19, wpb=18216.5, bsz=1024, num_updates=6610, lr=4.41993e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=41718
+2024-11-07 01:17:33 - progress_bar.py[line:274] - INFO: epoch 017:    367 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.619, ntokens=18217.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3413.3, ups=0.19, wpb=18217.5, bsz=1024, num_updates=6620, lr=4.41857e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=53, gb_free=13.2, wall=41772
+2024-11-07 01:18:25 - progress_bar.py[line:274] - INFO: epoch 017:    377 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.55, ntokens=17754.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.86, wps=3384.1, ups=0.19, wpb=17754.9, bsz=1024, num_updates=6630, lr=4.41721e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=41824
+2024-11-07 01:19:19 - progress_bar.py[line:274] - INFO: epoch 017:    387 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.615, ntokens=18047.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3364.7, ups=0.19, wpb=18047.6, bsz=1024, num_updates=6640, lr=4.41585e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=41878
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 17:16:00 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 17 @ 6643 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt
+
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt2024-11-07 01:19:36 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 17 @ 6644 updates
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt
-2024-11-03 17:16:00 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt': File exists: File exists
+2024-11-07 01:19:36 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt': File exists: File exists: File exists
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 17:18:00 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 01:21:27 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 
-slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+
+slice_id 2 seek offset 100000
+slice_id 6 seek offset 300000
 slice_id 3 seek offset 150000
-2024-11-03 17:20:00 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 17 @ 6643 updates, score 0) (writing took 240.35216320399195 seconds)
+2024-11-07 01:23:19 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 17 @ 6644 updates, score 0) (writing took 222.2265964919934 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_17.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+slice_id 7 seek offset 350000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+
 slice_id 1 seek offset 50000
 slice_id 5 seek offset 250000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-03 17:21:00 - train.py[line:336] - INFO: end of epoch 17 (average epoch stats below)
-2024-11-03 17:21:00 - progress_bar.py[line:282] - INFO: epoch 017 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.257 | ntokens 17806.6 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 9.56 | wps 2745.6 | ups 0.15 | wpb 17806.6 | bsz 1023 | num_updates 6643 | lr 4.41544e-05 | gnorm 0.008 | clip 0 | loss_scale 16384 | train_wall 1598 | gb_free 13.4 | wall 42637
-2024-11-03 17:21:00 - trainer.py[line:639] - INFO: loading train data for epoch 18
+slice_id 4 seek offset 200000
+2024-11-07 01:24:13 - train.py[line:336] - INFO: end of epoch 17 (average epoch stats below)
+2024-11-07 01:24:13 - progress_bar.py[line:282] - INFO: epoch 017 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.554 | ntokens 17800.4 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.87 | wps 2782.7 | ups 0.16 | wpb 17800.4 | bsz 1023 | num_updates 6644 | lr 4.4153e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1774 | gb_free 13.4 | wall 42172
+2024-11-07 01:24:13 - trainer.py[line:639] - INFO: loading train data for epoch 18
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 17:23:44 - trainer.py[line:703] - INFO: begin training epoch 18
-2024-11-03 17:23:44 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 17:24:23 - progress_bar.py[line:274] - INFO: epoch 018:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.228, ntokens=16891.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.37, wps=327.7, ups=0.02, wpb=16891.1, bsz=985.6, num_updates=6650, lr=4.41449e-05, gnorm=0.013, clip=0, loss_scale=16384, train_wall=34, gb_free=13.6, wall=42840
-2024-11-03 17:25:16 - progress_bar.py[line:274] - INFO: epoch 018:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.2, ntokens=17656.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.19, wps=3345.2, ups=0.19, wpb=17656.9, bsz=1024, num_updates=6660, lr=4.41313e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=42893
-2024-11-03 17:26:08 - progress_bar.py[line:274] - INFO: epoch 018:     27 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.209, ntokens=17634.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.25, wps=3348.8, ups=0.19, wpb=17634.6, bsz=1024, num_updates=6670, lr=4.41176e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=42945
-2024-11-03 17:27:01 - progress_bar.py[line:274] - INFO: epoch 018:     37 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.185, ntokens=17692.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.1, wps=3349.1, ups=0.19, wpb=17692.8, bsz=1024, num_updates=6680, lr=4.4104e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=42998
-2024-11-03 17:27:54 - progress_bar.py[line:274] - INFO: epoch 018:     47 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.186, ntokens=17638.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.1, wps=3350, ups=0.19, wpb=17638.8, bsz=1024, num_updates=6690, lr=4.40904e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=43051
-2024-11-03 17:28:47 - progress_bar.py[line:274] - INFO: epoch 018:     57 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.199, ntokens=17675.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.18, wps=3309, ups=0.19, wpb=17675.5, bsz=1024, num_updates=6700, lr=4.40768e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=43104
-2024-11-03 17:29:39 - progress_bar.py[line:274] - INFO: epoch 018:     67 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.232, ntokens=17866.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.4, wps=3418.9, ups=0.19, wpb=17866.1, bsz=1024, num_updates=6710, lr=4.40632e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=43156
-2024-11-03 17:30:32 - progress_bar.py[line:274] - INFO: epoch 018:     77 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.206, ntokens=17631.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.23, wps=3343.9, ups=0.19, wpb=17631.4, bsz=1024, num_updates=6720, lr=4.40496e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=43209
-2024-11-03 17:31:25 - progress_bar.py[line:274] - INFO: epoch 018:     87 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.215, ntokens=17752.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.29, wps=3384.3, ups=0.19, wpb=17752.9, bsz=1024, num_updates=6730, lr=4.4036e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=43262
-2024-11-03 17:32:19 - progress_bar.py[line:274] - INFO: epoch 018:     97 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.195, ntokens=17920, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.16, wps=3274.2, ups=0.18, wpb=17920, bsz=1024, num_updates=6740, lr=4.40224e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=43, gb_free=13.4, wall=43316
-2024-11-03 17:33:11 - progress_bar.py[line:274] - INFO: epoch 018:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.185, ntokens=17505, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.09, wps=3381.4, ups=0.19, wpb=17505, bsz=1024, num_updates=6750, lr=4.40088e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=47, gb_free=13.6, wall=43368
-2024-11-03 17:34:04 - progress_bar.py[line:274] - INFO: epoch 018:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.238, ntokens=17742.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.44, wps=3342.6, ups=0.19, wpb=17742.5, bsz=1024, num_updates=6760, lr=4.39952e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=38, gb_free=13.5, wall=43421
-2024-11-03 17:35:01 - progress_bar.py[line:274] - INFO: epoch 018:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.14, ntokens=17418.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.81, wps=3076.1, ups=0.18, wpb=17418.9, bsz=1024, num_updates=6770, lr=4.39816e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=55, gb_free=13.4, wall=43478
-2024-11-03 17:35:54 - progress_bar.py[line:274] - INFO: epoch 018:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.178, ntokens=17476.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.05, wps=3260.8, ups=0.19, wpb=17476.2, bsz=1024, num_updates=6780, lr=4.3968e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=54, gb_free=13.4, wall=43531
-2024-11-03 17:36:47 - progress_bar.py[line:274] - INFO: epoch 018:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.268, ntokens=18082.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.63, wps=3447.2, ups=0.19, wpb=18082.6, bsz=1024, num_updates=6790, lr=4.39544e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=43584
-2024-11-03 17:37:39 - progress_bar.py[line:274] - INFO: epoch 018:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.216, ntokens=17690.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.29, wps=3392.6, ups=0.19, wpb=17690.2, bsz=1024, num_updates=6800, lr=4.39408e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=43636
-2024-11-03 17:38:31 - progress_bar.py[line:274] - INFO: epoch 018:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.249, ntokens=18100.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.51, wps=3449.1, ups=0.19, wpb=18100.6, bsz=1024, num_updates=6810, lr=4.39272e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=43689
-2024-11-03 17:39:24 - progress_bar.py[line:274] - INFO: epoch 018:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.16, ntokens=17505.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.94, wps=3333.8, ups=0.19, wpb=17505.4, bsz=1024, num_updates=6820, lr=4.39136e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=43741
-2024-11-03 17:40:16 - progress_bar.py[line:274] - INFO: epoch 018:    187 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.168, ntokens=17689.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.99, wps=3411.7, ups=0.19, wpb=17689.6, bsz=1024, num_updates=6830, lr=4.39e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=49, gb_free=13.5, wall=43793
-2024-11-03 17:41:08 - progress_bar.py[line:274] - INFO: epoch 018:    197 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.192, ntokens=17660.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.14, wps=3399, ups=0.19, wpb=17660.8, bsz=1024, num_updates=6840, lr=4.38864e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=42, gb_free=13.4, wall=43845
-2024-11-03 17:42:00 - progress_bar.py[line:274] - INFO: epoch 018:    207 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.252, ntokens=18140.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.52, wps=3474.2, ups=0.19, wpb=18140.9, bsz=1024, num_updates=6850, lr=4.38728e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=38, gb_free=13.6, wall=43897
-2024-11-03 17:42:52 - progress_bar.py[line:274] - INFO: epoch 018:    217 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.162, ntokens=17699.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.95, wps=3380, ups=0.19, wpb=17699.3, bsz=1024, num_updates=6860, lr=4.38592e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=39, gb_free=13.5, wall=43949
-2024-11-03 17:43:45 - progress_bar.py[line:274] - INFO: epoch 018:    227 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.205, ntokens=17877.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.22, wps=3395.7, ups=0.19, wpb=17877.3, bsz=1024, num_updates=6870, lr=4.38456e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=40, gb_free=13.5, wall=44002
-2024-11-03 17:44:38 - progress_bar.py[line:274] - INFO: epoch 018:    237 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.169, ntokens=17406.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.99, wps=3294.3, ups=0.19, wpb=17406.8, bsz=1024, num_updates=6880, lr=4.3832e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=44055
-2024-11-03 17:45:30 - progress_bar.py[line:274] - INFO: epoch 018:    247 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.205, ntokens=17422.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.22, wps=3310.3, ups=0.19, wpb=17422.6, bsz=1024, num_updates=6890, lr=4.38184e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=44108
-2024-11-03 17:46:23 - progress_bar.py[line:274] - INFO: epoch 018:    257 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.143, ntokens=17506.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.83, wps=3304.3, ups=0.19, wpb=17506.7, bsz=1024, num_updates=6900, lr=4.38048e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=44161
-2024-11-03 17:47:16 - progress_bar.py[line:274] - INFO: epoch 018:    267 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.288, ntokens=18126.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.76, wps=3445.5, ups=0.19, wpb=18126.7, bsz=1024, num_updates=6910, lr=4.37912e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=44213
-2024-11-03 17:48:09 - progress_bar.py[line:274] - INFO: epoch 018:    277 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.176, ntokens=17674, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.04, wps=3353, ups=0.19, wpb=17674, bsz=1024, num_updates=6920, lr=4.37775e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=44266
-2024-11-03 17:49:02 - progress_bar.py[line:274] - INFO: epoch 018:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.157, ntokens=17534.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.92, wps=3325.9, ups=0.19, wpb=17534.6, bsz=1024, num_updates=6930, lr=4.37639e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=44319
-2024-11-03 17:49:53 - progress_bar.py[line:274] - INFO: epoch 018:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.299, ntokens=17862.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.84, wps=3455.2, ups=0.19, wpb=17862.9, bsz=1024, num_updates=6940, lr=4.37503e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=44370
-2024-11-03 17:50:45 - progress_bar.py[line:274] - INFO: epoch 018:    307 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.382, ntokens=18390.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.42, wps=3555.3, ups=0.19, wpb=18390.4, bsz=1024, num_updates=6950, lr=4.37367e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.2, wall=44422
-2024-11-03 17:51:38 - progress_bar.py[line:274] - INFO: epoch 018:    317 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.325, ntokens=18094.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.02, wps=3433.2, ups=0.19, wpb=18094.1, bsz=1024, num_updates=6960, lr=4.37231e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=44475
-2024-11-03 17:52:31 - progress_bar.py[line:274] - INFO: epoch 018:    327 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.297, ntokens=17872.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.83, wps=3354, ups=0.19, wpb=17872.6, bsz=1024, num_updates=6970, lr=4.37095e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=48, gb_free=13.4, wall=44528
-2024-11-03 17:53:04 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
-2024-11-03 17:53:30 - progress_bar.py[line:274] - INFO: epoch 018:    338 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.307, ntokens=18002.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.9, wps=3043.7, ups=0.17, wpb=18002.6, bsz=1024, num_updates=6980, lr=4.36959e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=59, gb_free=13.4, wall=44587
-2024-11-03 17:54:23 - progress_bar.py[line:274] - INFO: epoch 018:    348 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.354, ntokens=18671.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.22, wps=3526.8, ups=0.19, wpb=18671.4, bsz=1024, num_updates=6990, lr=4.36823e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.2, wall=44640
-2024-11-03 17:55:15 - progress_bar.py[line:274] - INFO: epoch 018:    358 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.381, ntokens=18453, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.42, wps=3517.6, ups=0.19, wpb=18453, bsz=1024, num_updates=7000, lr=4.36687e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.2, wall=44693
-2024-11-03 17:56:08 - progress_bar.py[line:274] - INFO: epoch 018:    368 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.397, ntokens=18600.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.54, wps=3553, ups=0.19, wpb=18600.9, bsz=1024, num_updates=7010, lr=4.36551e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=52, gb_free=13.3, wall=44745
-2024-11-03 17:57:00 - progress_bar.py[line:274] - INFO: epoch 018:    378 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.36, ntokens=18344.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.27, wps=3525.9, ups=0.19, wpb=18344.7, bsz=1024, num_updates=7020, lr=4.36415e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=44797
-2024-11-03 17:57:52 - progress_bar.py[line:274] - INFO: epoch 018:    388 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.345, ntokens=18037.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.16, wps=3474.1, ups=0.19, wpb=18037.3, bsz=1024, num_updates=7030, lr=4.36279e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=44849
+2024-11-07 01:26:52 - trainer.py[line:703] - INFO: begin training epoch 18
+2024-11-07 01:26:52 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 01:27:25 - progress_bar.py[line:274] - INFO: epoch 018:      6 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.537, ntokens=16905.2, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=347.5, ups=0.02, wpb=16905.2, bsz=985.6, num_updates=6650, lr=4.41449e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=36, gb_free=13.6, wall=42364
+2024-11-07 01:28:18 - progress_bar.py[line:274] - INFO: epoch 018:     16 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.533, ntokens=17690.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3365.8, ups=0.19, wpb=17690.4, bsz=1024, num_updates=6660, lr=4.41313e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=42417
+2024-11-07 01:29:10 - progress_bar.py[line:274] - INFO: epoch 018:     26 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.475, ntokens=17411.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3306.9, ups=0.19, wpb=17411.3, bsz=1024, num_updates=6670, lr=4.41176e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=42469
+2024-11-07 01:30:02 - progress_bar.py[line:274] - INFO: epoch 018:     36 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.538, ntokens=17778.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3424.7, ups=0.19, wpb=17778.9, bsz=1024, num_updates=6680, lr=4.4104e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=42521
+2024-11-07 01:30:54 - progress_bar.py[line:274] - INFO: epoch 018:     46 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.497, ntokens=17597.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3393.4, ups=0.19, wpb=17597.8, bsz=1024, num_updates=6690, lr=4.40904e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=42573
+2024-11-07 01:31:46 - progress_bar.py[line:274] - INFO: epoch 018:     56 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.521, ntokens=17756.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=3423.5, ups=0.19, wpb=17756.2, bsz=1024, num_updates=6700, lr=4.40768e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=42625
+2024-11-07 01:32:39 - progress_bar.py[line:274] - INFO: epoch 018:     66 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.544, ntokens=17912, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3407.9, ups=0.19, wpb=17912, bsz=1024, num_updates=6710, lr=4.40632e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=42678
+2024-11-07 01:33:31 - progress_bar.py[line:274] - INFO: epoch 018:     76 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.508, ntokens=17547.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.69, wps=3364.5, ups=0.19, wpb=17547.1, bsz=1024, num_updates=6720, lr=4.40496e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=42730
+2024-11-07 01:34:24 - progress_bar.py[line:274] - INFO: epoch 018:     86 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.52, ntokens=17763.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=3347.6, ups=0.19, wpb=17763.4, bsz=1024, num_updates=6730, lr=4.4036e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=42783
+2024-11-07 01:35:16 - progress_bar.py[line:274] - INFO: epoch 018:     96 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.551, ntokens=17948.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.86, wps=3441.1, ups=0.19, wpb=17948.6, bsz=1024, num_updates=6740, lr=4.40224e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=42835
+2024-11-07 01:36:08 - progress_bar.py[line:274] - INFO: epoch 018:    106 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.509, ntokens=17604.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.69, wps=3391.2, ups=0.19, wpb=17604.8, bsz=1024, num_updates=6750, lr=4.40088e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=42887
+2024-11-07 01:37:00 - progress_bar.py[line:274] - INFO: epoch 018:    116 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.563, ntokens=17684.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3375.3, ups=0.19, wpb=17684.2, bsz=1024, num_updates=6760, lr=4.39952e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=42939
+2024-11-07 01:37:54 - progress_bar.py[line:274] - INFO: epoch 018:    126 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.454, ntokens=17398.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.48, wps=3226.5, ups=0.19, wpb=17398.1, bsz=1024, num_updates=6770, lr=4.39816e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=42993
+2024-11-07 01:38:50 - progress_bar.py[line:274] - INFO: epoch 018:    136 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.508, ntokens=17447.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.69, wps=3120.1, ups=0.18, wpb=17447.3, bsz=1024, num_updates=6780, lr=4.3968e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=49, gb_free=13.5, wall=43049
+2024-11-07 01:39:43 - progress_bar.py[line:274] - INFO: epoch 018:    146 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.585, ntokens=18061.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3435.2, ups=0.19, wpb=18061.2, bsz=1024, num_updates=6790, lr=4.39544e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=43102
+2024-11-07 01:40:35 - progress_bar.py[line:274] - INFO: epoch 018:    156 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.546, ntokens=17689.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3370.2, ups=0.19, wpb=17689.6, bsz=1024, num_updates=6800, lr=4.39408e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=49, gb_free=13.5, wall=43154
+2024-11-07 01:41:28 - progress_bar.py[line:274] - INFO: epoch 018:    166 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=18074.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3429, ups=0.19, wpb=18074.8, bsz=1024, num_updates=6810, lr=4.39272e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=36, gb_free=13.4, wall=43207
+2024-11-07 01:42:24 - progress_bar.py[line:274] - INFO: epoch 018:    176 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.53, ntokens=17661, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3135.8, ups=0.18, wpb=17661, bsz=1024, num_updates=6820, lr=4.39136e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=43263
+2024-11-07 01:43:20 - progress_bar.py[line:274] - INFO: epoch 018:    186 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.531, ntokens=17578.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3138.4, ups=0.18, wpb=17578.5, bsz=1024, num_updates=6830, lr=4.39e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=43319
+2024-11-07 01:44:13 - progress_bar.py[line:274] - INFO: epoch 018:    196 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.538, ntokens=17651.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3321.9, ups=0.19, wpb=17651.4, bsz=1024, num_updates=6840, lr=4.38864e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=43372
+2024-11-07 01:45:06 - progress_bar.py[line:274] - INFO: epoch 018:    206 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.583, ntokens=18082.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3433.7, ups=0.19, wpb=18082.2, bsz=1024, num_updates=6850, lr=4.38728e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=43425
+2024-11-07 01:45:59 - progress_bar.py[line:274] - INFO: epoch 018:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.538, ntokens=17840.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3393.1, ups=0.19, wpb=17840.8, bsz=1024, num_updates=6860, lr=4.38592e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=43478
+2024-11-07 01:46:51 - progress_bar.py[line:274] - INFO: epoch 018:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.556, ntokens=17811.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3408.4, ups=0.19, wpb=17811.4, bsz=1024, num_updates=6870, lr=4.38456e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=43530
+2024-11-07 01:47:43 - progress_bar.py[line:274] - INFO: epoch 018:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.502, ntokens=17427.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.67, wps=3330.4, ups=0.19, wpb=17427.3, bsz=1024, num_updates=6880, lr=4.3832e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=43582
+2024-11-07 01:48:36 - progress_bar.py[line:274] - INFO: epoch 018:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.497, ntokens=17351.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3307.8, ups=0.19, wpb=17351.4, bsz=1024, num_updates=6890, lr=4.38184e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=43635
+2024-11-07 01:49:28 - progress_bar.py[line:274] - INFO: epoch 018:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.546, ntokens=17591.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3338.2, ups=0.19, wpb=17591.6, bsz=1024, num_updates=6900, lr=4.38048e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=43687
+2024-11-07 01:50:21 - progress_bar.py[line:274] - INFO: epoch 018:    266 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.571, ntokens=18000.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3440.8, ups=0.19, wpb=18000.2, bsz=1024, num_updates=6910, lr=4.37912e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=43740
+2024-11-07 01:51:13 - progress_bar.py[line:274] - INFO: epoch 018:    276 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.53, ntokens=17755.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3375.6, ups=0.19, wpb=17755.5, bsz=1024, num_updates=6920, lr=4.37775e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=43792
+2024-11-07 01:52:06 - progress_bar.py[line:274] - INFO: epoch 018:    286 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.511, ntokens=17508.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.7, wps=3349.6, ups=0.19, wpb=17508.1, bsz=1024, num_updates=6930, lr=4.37639e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=43845
+2024-11-07 01:52:58 - progress_bar.py[line:274] - INFO: epoch 018:    296 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.57, ntokens=17919.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3446.6, ups=0.19, wpb=17919.1, bsz=1024, num_updates=6940, lr=4.37503e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=43896
+2024-11-07 01:53:50 - progress_bar.py[line:274] - INFO: epoch 018:    306 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.662, ntokens=18323.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3465.1, ups=0.19, wpb=18323.3, bsz=1024, num_updates=6950, lr=4.37367e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=34, gb_free=13.4, wall=43949
+2024-11-07 01:54:42 - progress_bar.py[line:274] - INFO: epoch 018:    316 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.623, ntokens=18077.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3502.1, ups=0.19, wpb=18077.7, bsz=1024, num_updates=6960, lr=4.37231e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=46, gb_free=13.4, wall=44001
+2024-11-07 01:55:35 - progress_bar.py[line:274] - INFO: epoch 018:    326 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.597, ntokens=17846, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3401.2, ups=0.19, wpb=17846, bsz=1024, num_updates=6970, lr=4.37095e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=51, gb_free=13.3, wall=44053
+2024-11-07 01:56:26 - progress_bar.py[line:274] - INFO: epoch 018:    336 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.636, ntokens=18218.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3546.1, ups=0.19, wpb=18218.2, bsz=1024, num_updates=6980, lr=4.36959e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=43, gb_free=13.5, wall=44105
+2024-11-07 01:57:18 - progress_bar.py[line:274] - INFO: epoch 018:    346 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.656, ntokens=18517.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3545, ups=0.19, wpb=18517.8, bsz=1024, num_updates=6990, lr=4.36823e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=42, gb_free=13.3, wall=44157
+2024-11-07 01:58:10 - progress_bar.py[line:274] - INFO: epoch 018:    356 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.633, ntokens=18404.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3544.9, ups=0.19, wpb=18404.7, bsz=1024, num_updates=7000, lr=4.36687e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=37, gb_free=13.4, wall=44209
+2024-11-07 01:59:02 - progress_bar.py[line:274] - INFO: epoch 018:    366 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.679, ntokens=18572, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3548.9, ups=0.19, wpb=18572, bsz=1024, num_updates=7010, lr=4.36551e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=40, gb_free=13.4, wall=44261
+2024-11-07 01:59:55 - progress_bar.py[line:274] - INFO: epoch 018:    376 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.632, ntokens=18409.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3505.9, ups=0.19, wpb=18409.7, bsz=1024, num_updates=7020, lr=4.36415e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=45, gb_free=13.4, wall=44314
+2024-11-07 02:00:47 - progress_bar.py[line:274] - INFO: epoch 018:    386 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.619, ntokens=18058.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3468, ups=0.19, wpb=18058.9, bsz=1024, num_updates=7030, lr=4.36279e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=48, gb_free=13.4, wall=44366
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 17:58:04 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 18 @ 7033 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt
-2024-11-03 17:58:04 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt': File exists
-: File exists
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt
+
+
+2024-11-07 02:01:10 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 18 @ 7035 updates
+2024-11-07 02:01:10 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 18:00:03 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 02:03:01 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+2024-11-07 02:04:52 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 18 @ 7035 updates, score 0) (writing took 222.36915990599664 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 
+slice_id 1 seek offset 50000
+slice_id 5 seek offset 250000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 
-2024-11-03 18:02:03 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 18 @ 7033 updates, score 0) (writing took 238.90756183999474 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_18.pt
-slice_id 5 seek offset 250000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+slice_id 7 seek offset 350000
+slice_id 3 seek offset 150000
 slice_id 6 seek offset 300000
-2024-11-03 18:03:02 - train.py[line:336] - INFO: end of epoch 18 (average epoch stats below)
-2024-11-03 18:03:02 - progress_bar.py[line:282] - INFO: epoch 018 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.238 | ntokens 17816.9 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 9.44 | wps 2754.9 | ups 0.15 | wpb 17816.9 | bsz 1023 | num_updates 7033 | lr 4.36238e-05 | gnorm 0.008 | clip 0 | loss_scale 16384 | train_wall 1574 | gb_free 13.4 | wall 45159
-2024-11-03 18:03:02 - trainer.py[line:639] - INFO: loading train data for epoch 19
+slice_id 2 seek offset 100000
+slice_id 4 seek offset 200000
+2024-11-07 02:05:47 - train.py[line:336] - INFO: end of epoch 18 (average epoch stats below)
+2024-11-07 02:05:47 - progress_bar.py[line:282] - INFO: epoch 018 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.557 | ntokens 17820 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.89 | wps 2794 | ups 0.16 | wpb 17820 | bsz 1023 | num_updates 7035 | lr 4.36211e-05 | gnorm 0.007 | clip 0 | loss_scale 65536 | train_wall 1372 | gb_free 13.4 | wall 44666
+2024-11-07 02:05:47 - trainer.py[line:639] - INFO: loading train data for epoch 19
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 18:05:46 - trainer.py[line:703] - INFO: begin training epoch 19
-2024-11-03 18:05:46 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 18:06:25 - progress_bar.py[line:274] - INFO: epoch 019:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.153, ntokens=16583.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.89, wps=322.9, ups=0.02, wpb=16583.3, bsz=985.6, num_updates=7040, lr=4.36143e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=33, gb_free=13.6, wall=45362
-2024-11-03 18:07:19 - progress_bar.py[line:274] - INFO: epoch 019:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.223, ntokens=17954.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.34, wps=3376.3, ups=0.19, wpb=17954.1, bsz=1024, num_updates=7050, lr=4.36007e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=45416
-2024-11-03 18:08:11 - progress_bar.py[line:274] - INFO: epoch 019:     27 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.151, ntokens=17592.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.88, wps=3325.4, ups=0.19, wpb=17592.7, bsz=1024, num_updates=7060, lr=4.35871e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=45469
-2024-11-03 18:09:04 - progress_bar.py[line:274] - INFO: epoch 019:     37 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.172, ntokens=17680.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.02, wps=3350, ups=0.19, wpb=17680.6, bsz=1024, num_updates=7070, lr=4.35735e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=45521
-2024-11-03 18:09:57 - progress_bar.py[line:274] - INFO: epoch 019:     47 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.162, ntokens=17682.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.95, wps=3355.7, ups=0.19, wpb=17682.6, bsz=1024, num_updates=7080, lr=4.35599e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=45574
-2024-11-03 18:10:49 - progress_bar.py[line:274] - INFO: epoch 019:     57 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.193, ntokens=17698.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.15, wps=3375.2, ups=0.19, wpb=17698.4, bsz=1024, num_updates=7090, lr=4.35463e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=45626
-2024-11-03 18:11:42 - progress_bar.py[line:274] - INFO: epoch 019:     67 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.19, ntokens=17861.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.12, wps=3363.3, ups=0.19, wpb=17861.9, bsz=1024, num_updates=7100, lr=4.35327e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=45680
-2024-11-03 18:12:35 - progress_bar.py[line:274] - INFO: epoch 019:     77 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.201, ntokens=17721.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.19, wps=3367.4, ups=0.19, wpb=17721.8, bsz=1024, num_updates=7110, lr=4.35191e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=45732
-2024-11-03 18:13:27 - progress_bar.py[line:274] - INFO: epoch 019:     87 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.214, ntokens=17808.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.28, wps=3404, ups=0.19, wpb=17808.6, bsz=1024, num_updates=7120, lr=4.35055e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=45785
-2024-11-03 18:14:20 - progress_bar.py[line:274] - INFO: epoch 019:     97 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.139, ntokens=17550.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.81, wps=3348.5, ups=0.19, wpb=17550.6, bsz=1024, num_updates=7130, lr=4.34919e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=45837
-2024-11-03 18:15:12 - progress_bar.py[line:274] - INFO: epoch 019:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.226, ntokens=17904.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.36, wps=3408.2, ups=0.19, wpb=17904.3, bsz=1024, num_updates=7140, lr=4.34783e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=45889
-2024-11-03 18:16:06 - progress_bar.py[line:274] - INFO: epoch 019:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.2, ntokens=17537, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.19, wps=3289.8, ups=0.19, wpb=17537, bsz=1024, num_updates=7150, lr=4.34647e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=45943
-2024-11-03 18:16:59 - progress_bar.py[line:274] - INFO: epoch 019:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.121, ntokens=17628.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.7, wps=3333.2, ups=0.19, wpb=17628.2, bsz=1024, num_updates=7160, lr=4.34511e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=45996
-2024-11-03 18:17:52 - progress_bar.py[line:274] - INFO: epoch 019:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.213, ntokens=17638.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.27, wps=3307.4, ups=0.19, wpb=17638.9, bsz=1024, num_updates=7170, lr=4.34374e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.3, wall=46049
-2024-11-03 18:18:44 - progress_bar.py[line:274] - INFO: epoch 019:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.199, ntokens=17749.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.18, wps=3388.9, ups=0.19, wpb=17749.6, bsz=1024, num_updates=7180, lr=4.34238e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=46101
-2024-11-03 18:19:37 - progress_bar.py[line:274] - INFO: epoch 019:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.2, ntokens=17677.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.19, wps=3344.9, ups=0.19, wpb=17677.2, bsz=1024, num_updates=7190, lr=4.34102e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=46154
-2024-11-03 18:20:30 - progress_bar.py[line:274] - INFO: epoch 019:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.245, ntokens=18092.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.48, wps=3452.2, ups=0.19, wpb=18092.2, bsz=1024, num_updates=7200, lr=4.33966e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=46207
-2024-11-03 18:21:22 - progress_bar.py[line:274] - INFO: epoch 019:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.108, ntokens=17341.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.62, wps=3297.8, ups=0.19, wpb=17341.3, bsz=1024, num_updates=7210, lr=4.3383e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=46259
-2024-11-03 18:22:15 - progress_bar.py[line:274] - INFO: epoch 019:    187 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.212, ntokens=17908.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.27, wps=3398.2, ups=0.19, wpb=17908.5, bsz=1024, num_updates=7220, lr=4.33694e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=46312
-2024-11-03 18:23:08 - progress_bar.py[line:274] - INFO: epoch 019:    197 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.214, ntokens=18019.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.28, wps=3410.7, ups=0.19, wpb=18019.8, bsz=1024, num_updates=7230, lr=4.33558e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=46365
-2024-11-03 18:24:00 - progress_bar.py[line:274] - INFO: epoch 019:    207 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.247, ntokens=18183, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.49, wps=3454.7, ups=0.19, wpb=18183, bsz=1024, num_updates=7240, lr=4.33422e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=46417
-2024-11-03 18:24:54 - progress_bar.py[line:274] - INFO: epoch 019:    217 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.176, ntokens=17832.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.04, wps=3292.5, ups=0.18, wpb=17832.1, bsz=1024, num_updates=7250, lr=4.33286e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=46472
-2024-11-03 18:25:49 - progress_bar.py[line:274] - INFO: epoch 019:    227 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.162, ntokens=17819.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.95, wps=3279.8, ups=0.18, wpb=17819.5, bsz=1024, num_updates=7260, lr=4.3315e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=46526
-2024-11-03 18:26:42 - progress_bar.py[line:274] - INFO: epoch 019:    237 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.206, ntokens=17593.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.23, wps=3329.8, ups=0.19, wpb=17593.1, bsz=1024, num_updates=7270, lr=4.33014e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=46579
-2024-11-03 18:27:34 - progress_bar.py[line:274] - INFO: epoch 019:    247 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.212, ntokens=17724.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.26, wps=3379.9, ups=0.19, wpb=17724.7, bsz=1024, num_updates=7280, lr=4.32878e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=46631
-2024-11-03 18:28:27 - progress_bar.py[line:274] - INFO: epoch 019:    257 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.122, ntokens=17363.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.7, wps=3291.7, ups=0.19, wpb=17363.6, bsz=1024, num_updates=7290, lr=4.32742e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=46684
-2024-11-03 18:29:21 - progress_bar.py[line:274] - INFO: epoch 019:    267 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.235, ntokens=17969.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.41, wps=3335.6, ups=0.19, wpb=17969.2, bsz=1024, num_updates=7300, lr=4.32606e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=46738
-2024-11-03 18:30:15 - progress_bar.py[line:274] - INFO: epoch 019:    277 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.167, ntokens=17860, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.98, wps=3303.5, ups=0.18, wpb=17860, bsz=1024, num_updates=7310, lr=4.3247e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=36, gb_free=13.5, wall=46792
-2024-11-03 18:31:07 - progress_bar.py[line:274] - INFO: epoch 019:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.162, ntokens=17571.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.95, wps=3359.4, ups=0.19, wpb=17571.4, bsz=1024, num_updates=7320, lr=4.32334e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=41, gb_free=13.6, wall=46844
-2024-11-03 18:31:59 - progress_bar.py[line:274] - INFO: epoch 019:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.246, ntokens=17822.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.49, wps=3439.7, ups=0.19, wpb=17822.3, bsz=1024, num_updates=7330, lr=4.32198e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=38, gb_free=13.4, wall=46896
-2024-11-03 18:32:51 - progress_bar.py[line:274] - INFO: epoch 019:    307 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.392, ntokens=18568, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.5, wps=3584.7, ups=0.19, wpb=18568, bsz=1024, num_updates=7340, lr=4.32062e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=38, gb_free=13.2, wall=46948
-2024-11-03 18:33:43 - progress_bar.py[line:274] - INFO: epoch 019:    317 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.383, ntokens=18468.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.43, wps=3539.1, ups=0.19, wpb=18468.9, bsz=1024, num_updates=7350, lr=4.31926e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=42, gb_free=13.4, wall=47000
-2024-11-03 18:34:34 - progress_bar.py[line:274] - INFO: epoch 019:    327 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.293, ntokens=18059, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.8, wps=3496.7, ups=0.19, wpb=18059, bsz=1024, num_updates=7360, lr=4.3179e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=44, gb_free=13.4, wall=47052
-2024-11-03 18:35:26 - progress_bar.py[line:274] - INFO: epoch 019:    337 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.316, ntokens=18270.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.96, wps=3526.2, ups=0.19, wpb=18270.6, bsz=1024, num_updates=7370, lr=4.31654e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=42, gb_free=13.5, wall=47103
-2024-11-03 18:36:20 - progress_bar.py[line:274] - INFO: epoch 019:    347 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.315, ntokens=18281.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.95, wps=3415.3, ups=0.19, wpb=18281.6, bsz=1024, num_updates=7380, lr=4.31518e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=49, gb_free=13.4, wall=47157
-2024-11-03 18:37:12 - progress_bar.py[line:274] - INFO: epoch 019:    357 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.354, ntokens=18452.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.22, wps=3541.6, ups=0.19, wpb=18452.8, bsz=1024, num_updates=7390, lr=4.31382e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=47209
-2024-11-03 18:38:04 - progress_bar.py[line:274] - INFO: epoch 019:    367 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.311, ntokens=17926.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.92, wps=3442.3, ups=0.19, wpb=17926.2, bsz=1024, num_updates=7400, lr=4.31246e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=51, gb_free=13.3, wall=47261
-2024-11-03 18:38:55 - progress_bar.py[line:274] - INFO: epoch 019:    377 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.309, ntokens=18335.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.91, wps=3563.4, ups=0.19, wpb=18335.1, bsz=1024, num_updates=7410, lr=4.3111e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=50, gb_free=13.3, wall=47313
-2024-11-03 18:39:49 - progress_bar.py[line:274] - INFO: epoch 019:    387 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.319, ntokens=17816.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.98, wps=3297.7, ups=0.19, wpb=17816.4, bsz=1024, num_updates=7420, lr=4.30973e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=51, gb_free=13.4, wall=47367
+2024-11-07 02:08:27 - trainer.py[line:703] - INFO: begin training epoch 19
+2024-11-07 02:08:27 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 02:08:55 - progress_bar.py[line:274] - INFO: epoch 019:      5 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.531, ntokens=16966.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=347.7, ups=0.02, wpb=16966.3, bsz=985.6, num_updates=7040, lr=4.36143e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=37, gb_free=13.5, wall=44854
+2024-11-07 02:09:48 - progress_bar.py[line:274] - INFO: epoch 019:     15 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.525, ntokens=17702.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3355.6, ups=0.19, wpb=17702.5, bsz=1024, num_updates=7050, lr=4.36007e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=44907
+2024-11-07 02:10:40 - progress_bar.py[line:274] - INFO: epoch 019:     25 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.482, ntokens=17532, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3341, ups=0.19, wpb=17532, bsz=1024, num_updates=7060, lr=4.35871e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=44959
+2024-11-07 02:11:32 - progress_bar.py[line:274] - INFO: epoch 019:     35 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.496, ntokens=17750, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3412.9, ups=0.19, wpb=17750, bsz=1024, num_updates=7070, lr=4.35735e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=45011
+2024-11-07 02:12:27 - progress_bar.py[line:274] - INFO: epoch 019:     45 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.497, ntokens=17622.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3210.9, ups=0.18, wpb=17622.6, bsz=1024, num_updates=7080, lr=4.35599e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=45066
+2024-11-07 02:13:23 - progress_bar.py[line:274] - INFO: epoch 019:     55 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.528, ntokens=17785.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3197.6, ups=0.18, wpb=17785.6, bsz=1024, num_updates=7090, lr=4.35463e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=45122
+2024-11-07 02:14:16 - progress_bar.py[line:274] - INFO: epoch 019:     65 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.512, ntokens=17919.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.7, wps=3364, ups=0.19, wpb=17919.3, bsz=1024, num_updates=7100, lr=4.35327e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=45175
+2024-11-07 02:15:11 - progress_bar.py[line:274] - INFO: epoch 019:     75 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.546, ntokens=17685.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3213.3, ups=0.18, wpb=17685.5, bsz=1024, num_updates=7110, lr=4.35191e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=49, gb_free=13.5, wall=45230
+2024-11-07 02:16:07 - progress_bar.py[line:274] - INFO: epoch 019:     85 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.529, ntokens=17855.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3208.8, ups=0.18, wpb=17855.3, bsz=1024, num_updates=7120, lr=4.35055e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=45286
+2024-11-07 02:17:03 - progress_bar.py[line:274] - INFO: epoch 019:     95 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.533, ntokens=17514.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3135.2, ups=0.18, wpb=17514.6, bsz=1024, num_updates=7130, lr=4.34919e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=45341
+2024-11-07 02:17:57 - progress_bar.py[line:274] - INFO: epoch 019:    105 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.552, ntokens=17907.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3262.8, ups=0.18, wpb=17907.8, bsz=1024, num_updates=7140, lr=4.34783e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=45396
+2024-11-07 02:18:49 - progress_bar.py[line:274] - INFO: epoch 019:    115 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.513, ntokens=17342.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3337.6, ups=0.19, wpb=17342.1, bsz=1024, num_updates=7150, lr=4.34647e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=45448
+2024-11-07 02:19:41 - progress_bar.py[line:274] - INFO: epoch 019:    125 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.523, ntokens=17863, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3469, ups=0.19, wpb=17863, bsz=1024, num_updates=7160, lr=4.34511e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=51, gb_free=13.5, wall=45500
+2024-11-07 02:20:33 - progress_bar.py[line:274] - INFO: epoch 019:    135 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.518, ntokens=17514.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3354.3, ups=0.19, wpb=17514.4, bsz=1024, num_updates=7170, lr=4.34374e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=45552
+2024-11-07 02:21:25 - progress_bar.py[line:274] - INFO: epoch 019:    145 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.532, ntokens=17813.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3452.9, ups=0.19, wpb=17813.2, bsz=1024, num_updates=7180, lr=4.34238e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=44, gb_free=13.6, wall=45604
+2024-11-07 02:22:17 - progress_bar.py[line:274] - INFO: epoch 019:    155 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.53, ntokens=17706.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3400.7, ups=0.19, wpb=17706.6, bsz=1024, num_updates=7190, lr=4.34102e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=33, gb_free=13.5, wall=45656
+2024-11-07 02:23:10 - progress_bar.py[line:274] - INFO: epoch 019:    165 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.532, ntokens=18027.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3418.3, ups=0.19, wpb=18027.7, bsz=1024, num_updates=7200, lr=4.33966e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=45708
+2024-11-07 02:24:02 - progress_bar.py[line:274] - INFO: epoch 019:    175 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.498, ntokens=17610.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3339.2, ups=0.19, wpb=17610.1, bsz=1024, num_updates=7210, lr=4.3383e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=45761
+2024-11-07 02:24:55 - progress_bar.py[line:274] - INFO: epoch 019:    185 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.516, ntokens=17586.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.72, wps=3355.5, ups=0.19, wpb=17586.1, bsz=1024, num_updates=7220, lr=4.33694e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=45814
+2024-11-07 02:25:48 - progress_bar.py[line:274] - INFO: epoch 019:    195 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.572, ntokens=18088.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3382.1, ups=0.19, wpb=18088.7, bsz=1024, num_updates=7230, lr=4.33558e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=37, gb_free=13.6, wall=45867
+2024-11-07 02:26:41 - progress_bar.py[line:274] - INFO: epoch 019:    205 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.565, ntokens=18086.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3391.3, ups=0.19, wpb=18086.4, bsz=1024, num_updates=7240, lr=4.33422e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=47, gb_free=13.5, wall=45920
+2024-11-07 02:27:35 - progress_bar.py[line:274] - INFO: epoch 019:    215 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.506, ntokens=17929.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.68, wps=3371.4, ups=0.19, wpb=17929.7, bsz=1024, num_updates=7250, lr=4.33286e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=37, gb_free=13.6, wall=45974
+2024-11-07 02:28:28 - progress_bar.py[line:274] - INFO: epoch 019:    225 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.527, ntokens=17805.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3341.5, ups=0.19, wpb=17805.8, bsz=1024, num_updates=7260, lr=4.3315e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=41, gb_free=13.6, wall=46027
+2024-11-07 02:29:20 - progress_bar.py[line:274] - INFO: epoch 019:    235 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.528, ntokens=17691.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3373.8, ups=0.19, wpb=17691.3, bsz=1024, num_updates=7270, lr=4.33014e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=44, gb_free=13.5, wall=46079
+2024-11-07 02:30:13 - progress_bar.py[line:274] - INFO: epoch 019:    245 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.52, ntokens=17732, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3357.7, ups=0.19, wpb=17732, bsz=1024, num_updates=7280, lr=4.32878e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=41, gb_free=13.6, wall=46132
+2024-11-07 02:31:06 - progress_bar.py[line:274] - INFO: epoch 019:    255 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.468, ntokens=17446.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3296.4, ups=0.19, wpb=17446.4, bsz=1024, num_updates=7290, lr=4.32742e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=46, gb_free=13.4, wall=46185
+2024-11-07 02:31:58 - progress_bar.py[line:274] - INFO: epoch 019:    265 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.533, ntokens=17797.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3435.8, ups=0.19, wpb=17797.7, bsz=1024, num_updates=7300, lr=4.32606e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=43, gb_free=13.5, wall=46237
+2024-11-07 02:32:50 - progress_bar.py[line:274] - INFO: epoch 019:    275 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.528, ntokens=17889.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3459.5, ups=0.19, wpb=17889.1, bsz=1024, num_updates=7310, lr=4.3247e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=34, gb_free=13.4, wall=46289
+2024-11-07 02:33:42 - progress_bar.py[line:274] - INFO: epoch 019:    285 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.464, ntokens=17466.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3328.1, ups=0.19, wpb=17466.2, bsz=1024, num_updates=7320, lr=4.32334e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=46341
+2024-11-07 02:34:34 - progress_bar.py[line:274] - INFO: epoch 019:    295 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.554, ntokens=17861.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3435.4, ups=0.19, wpb=17861.7, bsz=1024, num_updates=7330, lr=4.32198e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=46393
+2024-11-07 02:35:26 - progress_bar.py[line:274] - INFO: epoch 019:    305 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.614, ntokens=18372.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.12, wps=3565.3, ups=0.19, wpb=18372.3, bsz=1024, num_updates=7340, lr=4.32062e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=33, gb_free=13.4, wall=46445
+2024-11-07 02:36:18 - progress_bar.py[line:274] - INFO: epoch 019:    315 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.682, ntokens=18596.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3585.6, ups=0.19, wpb=18596.5, bsz=1024, num_updates=7350, lr=4.31926e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=33, gb_free=13.4, wall=46496
+2024-11-07 02:37:10 - progress_bar.py[line:274] - INFO: epoch 019:    325 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.59, ntokens=18042.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3470.1, ups=0.19, wpb=18042.2, bsz=1024, num_updates=7360, lr=4.3179e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=36, gb_free=13.4, wall=46548
+2024-11-07 02:38:01 - progress_bar.py[line:274] - INFO: epoch 019:    335 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.626, ntokens=18295.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.17, wps=3560, ups=0.19, wpb=18295.6, bsz=1024, num_updates=7370, lr=4.31654e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=40, gb_free=13.3, wall=46600
+2024-11-07 02:38:53 - progress_bar.py[line:274] - INFO: epoch 019:    345 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.586, ntokens=18163.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3505.3, ups=0.19, wpb=18163.3, bsz=1024, num_updates=7380, lr=4.31518e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=38, gb_free=13.3, wall=46652
+2024-11-07 02:39:45 - progress_bar.py[line:274] - INFO: epoch 019:    355 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.659, ntokens=18614.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3571.7, ups=0.19, wpb=18614.8, bsz=1024, num_updates=7390, lr=4.31382e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=36, gb_free=13.3, wall=46704
+2024-11-07 02:40:37 - progress_bar.py[line:274] - INFO: epoch 019:    365 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.593, ntokens=17982.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3434.7, ups=0.19, wpb=17982.7, bsz=1024, num_updates=7400, lr=4.31246e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=35, gb_free=13.3, wall=46756
+2024-11-07 02:41:30 - progress_bar.py[line:274] - INFO: epoch 019:    375 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.59, ntokens=18113.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3448.6, ups=0.19, wpb=18113.1, bsz=1024, num_updates=7410, lr=4.3111e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=44, gb_free=13.4, wall=46809
+2024-11-07 02:42:22 - progress_bar.py[line:274] - INFO: epoch 019:    385 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.583, ntokens=17905.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.99, wps=3443.6, ups=0.19, wpb=17905.6, bsz=1024, num_updates=7420, lr=4.30973e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=51, gb_free=13.4, wall=46861
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 18:40:08 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 19 @ 7424 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt2024-11-07 02:42:49 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 19 @ 7426 updates
+
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt
+
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt
-2024-11-03 18:40:08 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 02:42:49 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt': File exists: File exists: File exists
+
+
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 18:42:08 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-03 18:44:08 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 19 @ 7424 updates, score 0) (writing took 240.31966106701293 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 02:44:38 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 3 seek offset 150000
-slice_id 4 seek offset 200000
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+slice_id 2 seek offset 100000slice_id 6 seek offset 300000
+
+slice_id 7 seek offset 350000
+2024-11-07 02:46:30 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 19 @ 7426 updates, score 0) (writing took 220.57849960200838 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_19.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+slice_id 3 seek offset 150000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
 slice_id 5 seek offset 250000
-2024-11-03 18:45:07 - train.py[line:336] - INFO: end of epoch 19 (average epoch stats below)
-2024-11-03 18:45:07 - progress_bar.py[line:282] - INFO: epoch 019 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.223 | ntokens 17834.4 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 9.34 | wps 2761.5 | ups 0.15 | wpb 17834.4 | bsz 1023 | num_updates 7424 | lr 4.30919e-05 | gnorm 0.008 | clip 0 | loss_scale 16384 | train_wall 1362 | gb_free 13.4 | wall 47684
-2024-11-03 18:45:07 - trainer.py[line:639] - INFO: loading train data for epoch 20
+slice_id 4 seek offset 200000
+2024-11-07 02:47:24 - train.py[line:336] - INFO: end of epoch 19 (average epoch stats below)
+2024-11-07 02:47:24 - progress_bar.py[line:282] - INFO: epoch 019 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.543 | ntokens 17834.4 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.83 | wps 2791.9 | ups 0.16 | wpb 17834.4 | bsz 1023 | num_updates 7426 | lr 4.30892e-05 | gnorm 0.008 | clip 0 | loss_scale 65536 | train_wall 1541 | gb_free 13.3 | wall 47163
+2024-11-07 02:47:24 - trainer.py[line:639] - INFO: loading train data for epoch 20
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 18:47:52 - trainer.py[line:703] - INFO: begin training epoch 20
-2024-11-03 18:47:52 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 18:48:25 - progress_bar.py[line:274] - INFO: epoch 020:      6 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.18, ntokens=16942.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.07, wps=328.3, ups=0.02, wpb=16942.4, bsz=985.6, num_updates=7430, lr=4.30837e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=36, gb_free=13.6, wall=47883
-2024-11-03 18:49:19 - progress_bar.py[line:274] - INFO: epoch 020:     16 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.207, ntokens=17945.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.23, wps=3373.7, ups=0.19, wpb=17945.1, bsz=1024, num_updates=7440, lr=4.30701e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=47936
-2024-11-03 18:50:14 - progress_bar.py[line:274] - INFO: epoch 020:     26 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.115, ntokens=17409, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.67, wps=3151.3, ups=0.18, wpb=17409, bsz=1024, num_updates=7450, lr=4.30565e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=49, gb_free=13.5, wall=47991
-2024-11-03 18:51:09 - progress_bar.py[line:274] - INFO: epoch 020:     36 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.18, ntokens=17639.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.06, wps=3199.5, ups=0.18, wpb=17639.4, bsz=1024, num_updates=7460, lr=4.30429e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=55, gb_free=13.5, wall=48046
-2024-11-03 18:52:04 - progress_bar.py[line:274] - INFO: epoch 020:     46 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.152, ntokens=17799.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.89, wps=3226.7, ups=0.18, wpb=17799.5, bsz=1024, num_updates=7470, lr=4.30293e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=55, gb_free=13.6, wall=48101
-2024-11-03 18:52:59 - progress_bar.py[line:274] - INFO: epoch 020:     56 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.155, ntokens=17520.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.91, wps=3214.1, ups=0.18, wpb=17520.5, bsz=1024, num_updates=7480, lr=4.30157e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=54, gb_free=13.4, wall=48156
-2024-11-03 18:53:53 - progress_bar.py[line:274] - INFO: epoch 020:     66 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.223, ntokens=18075.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.34, wps=3309.2, ups=0.18, wpb=18075.1, bsz=1024, num_updates=7490, lr=4.30021e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=48210
-2024-11-03 18:54:45 - progress_bar.py[line:274] - INFO: epoch 020:     76 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.162, ntokens=17614, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.95, wps=3392.4, ups=0.19, wpb=17614, bsz=1024, num_updates=7500, lr=4.29885e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=48262
-2024-11-03 18:55:37 - progress_bar.py[line:274] - INFO: epoch 020:     86 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.141, ntokens=17611.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.82, wps=3403.9, ups=0.19, wpb=17611.6, bsz=1024, num_updates=7510, lr=4.29749e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=48314
-2024-11-03 18:56:29 - progress_bar.py[line:274] - INFO: epoch 020:     96 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.128, ntokens=17555.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.74, wps=3356.3, ups=0.19, wpb=17555.3, bsz=1024, num_updates=7520, lr=4.29613e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=48366
-2024-11-03 18:57:21 - progress_bar.py[line:274] - INFO: epoch 020:    106 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.15, ntokens=17617.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.88, wps=3429.9, ups=0.19, wpb=17617.2, bsz=1024, num_updates=7530, lr=4.29477e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=49, gb_free=13.4, wall=48418
-2024-11-03 18:58:13 - progress_bar.py[line:274] - INFO: epoch 020:    116 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.17, ntokens=17674.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9, wps=3361.6, ups=0.19, wpb=17674.1, bsz=1024, num_updates=7540, lr=4.29341e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=39, gb_free=13.5, wall=48470
-2024-11-03 18:59:06 - progress_bar.py[line:274] - INFO: epoch 020:    126 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.139, ntokens=17659.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.81, wps=3332.9, ups=0.19, wpb=17659.1, bsz=1024, num_updates=7550, lr=4.29205e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=48523
-2024-11-03 18:59:59 - progress_bar.py[line:274] - INFO: epoch 020:    136 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.12, ntokens=17323, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.69, wps=3278.1, ups=0.19, wpb=17323, bsz=1024, num_updates=7560, lr=4.29069e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=48576
-2024-11-03 19:00:52 - progress_bar.py[line:274] - INFO: epoch 020:    146 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.185, ntokens=17650.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.1, wps=3353.1, ups=0.19, wpb=17650.2, bsz=1024, num_updates=7570, lr=4.28933e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=48629
-2024-11-03 19:01:45 - progress_bar.py[line:274] - INFO: epoch 020:    156 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.168, ntokens=17778.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.99, wps=3351.7, ups=0.19, wpb=17778.3, bsz=1024, num_updates=7580, lr=4.28797e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=48682
-2024-11-03 19:02:37 - progress_bar.py[line:274] - INFO: epoch 020:    166 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.196, ntokens=17922.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.16, wps=3444.5, ups=0.19, wpb=17922.3, bsz=1024, num_updates=7590, lr=4.28661e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=48734
-2024-11-03 19:03:30 - progress_bar.py[line:274] - INFO: epoch 020:    176 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.125, ntokens=17492.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.72, wps=3310.4, ups=0.19, wpb=17492.3, bsz=1024, num_updates=7600, lr=4.28525e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=48787
-2024-11-03 19:04:23 - progress_bar.py[line:274] - INFO: epoch 020:    186 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.18, ntokens=17807.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.06, wps=3326.6, ups=0.19, wpb=17807.6, bsz=1024, num_updates=7610, lr=4.28389e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=33, gb_free=13.4, wall=48840
-2024-11-03 19:05:17 - progress_bar.py[line:274] - INFO: epoch 020:    196 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.146, ntokens=17567.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.85, wps=3289.1, ups=0.19, wpb=17567.1, bsz=1024, num_updates=7620, lr=4.28253e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=45, gb_free=13.4, wall=48894
-2024-11-03 19:06:10 - progress_bar.py[line:274] - INFO: epoch 020:    206 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.174, ntokens=17909.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.02, wps=3349.9, ups=0.19, wpb=17909.9, bsz=1024, num_updates=7630, lr=4.28117e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=48947
-2024-11-03 19:07:04 - progress_bar.py[line:274] - INFO: epoch 020:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.11, ntokens=17713.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.63, wps=3303.3, ups=0.19, wpb=17713.7, bsz=1024, num_updates=7640, lr=4.27981e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=49001
-2024-11-03 19:07:57 - progress_bar.py[line:274] - INFO: epoch 020:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.155, ntokens=17774.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.91, wps=3343.4, ups=0.19, wpb=17774.8, bsz=1024, num_updates=7650, lr=4.27845e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.6, wall=49054
-2024-11-03 19:08:49 - progress_bar.py[line:274] - INFO: epoch 020:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.154, ntokens=17426.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.9, wps=3359.5, ups=0.19, wpb=17426.9, bsz=1024, num_updates=7660, lr=4.27709e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=49106
-2024-11-03 19:09:41 - progress_bar.py[line:274] - INFO: epoch 020:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.18, ntokens=17576.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.07, wps=3350.6, ups=0.19, wpb=17576.4, bsz=1024, num_updates=7670, lr=4.27573e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=49158
-2024-11-03 19:10:33 - progress_bar.py[line:274] - INFO: epoch 020:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.15, ntokens=17755.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.88, wps=3422.8, ups=0.19, wpb=17755.1, bsz=1024, num_updates=7680, lr=4.27436e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=49210
-2024-11-03 19:11:26 - progress_bar.py[line:274] - INFO: epoch 020:    266 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.188, ntokens=17800.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.11, wps=3393.4, ups=0.19, wpb=17800.6, bsz=1024, num_updates=7690, lr=4.273e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=49263
-2024-11-03 19:12:18 - progress_bar.py[line:274] - INFO: epoch 020:    276 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.122, ntokens=17653, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.71, wps=3394.9, ups=0.19, wpb=17653, bsz=1024, num_updates=7700, lr=4.27164e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=49315
-2024-11-03 19:13:11 - progress_bar.py[line:274] - INFO: epoch 020:    286 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.176, ntokens=17800.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.04, wps=3306.2, ups=0.19, wpb=17800.4, bsz=1024, num_updates=7710, lr=4.27028e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=49368
-2024-11-03 19:14:05 - progress_bar.py[line:274] - INFO: epoch 020:    296 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.208, ntokens=17745, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.24, wps=3334.8, ups=0.19, wpb=17745, bsz=1024, num_updates=7720, lr=4.26892e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=49422
-2024-11-03 19:14:58 - progress_bar.py[line:274] - INFO: epoch 020:    306 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.269, ntokens=17993.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.64, wps=3350, ups=0.19, wpb=17993.9, bsz=1024, num_updates=7730, lr=4.26756e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=49475
-2024-11-03 19:15:52 - progress_bar.py[line:274] - INFO: epoch 020:    316 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.391, ntokens=18585.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.49, wps=3478.4, ups=0.19, wpb=18585.9, bsz=1024, num_updates=7740, lr=4.2662e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=49529
-2024-11-03 19:16:45 - progress_bar.py[line:274] - INFO: epoch 020:    326 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.311, ntokens=18323.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.93, wps=3454.9, ups=0.19, wpb=18323.3, bsz=1024, num_updates=7750, lr=4.26484e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.3, wall=49582
-2024-11-03 19:17:38 - progress_bar.py[line:274] - INFO: epoch 020:    336 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.333, ntokens=18430.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.08, wps=3447.3, ups=0.19, wpb=18430.8, bsz=1024, num_updates=7760, lr=4.26348e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=49635
-2024-11-03 19:18:32 - progress_bar.py[line:274] - INFO: epoch 020:    346 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.301, ntokens=18462.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.86, wps=3460.1, ups=0.19, wpb=18462.3, bsz=1024, num_updates=7770, lr=4.26212e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=49689
-2024-11-03 19:19:24 - progress_bar.py[line:274] - INFO: epoch 020:    356 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.28, ntokens=18197.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.71, wps=3481.6, ups=0.19, wpb=18197.6, bsz=1024, num_updates=7780, lr=4.26076e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=49741
-2024-11-03 19:20:17 - progress_bar.py[line:274] - INFO: epoch 020:    366 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.285, ntokens=18062.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.75, wps=3372.2, ups=0.19, wpb=18062.6, bsz=1024, num_updates=7790, lr=4.2594e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=41, gb_free=13.4, wall=49795
-2024-11-03 19:21:14 - progress_bar.py[line:274] - INFO: epoch 020:    376 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.273, ntokens=18087.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.67, wps=3203.4, ups=0.18, wpb=18087.4, bsz=1024, num_updates=7800, lr=4.25804e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=49851
-2024-11-03 19:22:10 - progress_bar.py[line:274] - INFO: epoch 020:    386 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.333, ntokens=18112.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.07, wps=3244, ups=0.18, wpb=18112.2, bsz=1024, num_updates=7810, lr=4.25668e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.2, wall=49907
+2024-11-07 02:50:04 - trainer.py[line:703] - INFO: begin training epoch 20
+2024-11-07 02:50:04 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 02:50:27 - progress_bar.py[line:274] - INFO: epoch 020:      4 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.547, ntokens=17192.9, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=354.3, ups=0.02, wpb=17192.9, bsz=985.6, num_updates=7430, lr=4.30837e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=40, gb_free=13.5, wall=47346
+2024-11-07 02:51:20 - progress_bar.py[line:274] - INFO: epoch 020:     14 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.512, ntokens=17757.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.7, wps=3359.4, ups=0.19, wpb=17757.2, bsz=1024, num_updates=7440, lr=4.30701e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=47399
+2024-11-07 02:52:12 - progress_bar.py[line:274] - INFO: epoch 020:     24 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.464, ntokens=17396.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3339.3, ups=0.19, wpb=17396.6, bsz=1024, num_updates=7450, lr=4.30565e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=47451
+2024-11-07 02:53:05 - progress_bar.py[line:274] - INFO: epoch 020:     34 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.5, ntokens=17766.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.66, wps=3372.4, ups=0.19, wpb=17766.5, bsz=1024, num_updates=7460, lr=4.30429e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=47504
+2024-11-07 02:53:57 - progress_bar.py[line:274] - INFO: epoch 020:     44 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.509, ntokens=17741, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.69, wps=3363, ups=0.19, wpb=17741, bsz=1024, num_updates=7470, lr=4.30293e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=47556
+2024-11-07 02:54:50 - progress_bar.py[line:274] - INFO: epoch 020:     54 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.485, ntokens=17645.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3330.8, ups=0.19, wpb=17645.5, bsz=1024, num_updates=7480, lr=4.30157e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=47609
+2024-11-07 02:55:43 - progress_bar.py[line:274] - INFO: epoch 020:     64 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.561, ntokens=17994.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3421.2, ups=0.19, wpb=17994.5, bsz=1024, num_updates=7490, lr=4.30021e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=47662
+2024-11-07 02:56:35 - progress_bar.py[line:274] - INFO: epoch 020:     74 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.532, ntokens=17682.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3374.9, ups=0.19, wpb=17682.7, bsz=1024, num_updates=7500, lr=4.29885e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=47714
+2024-11-07 02:57:28 - progress_bar.py[line:274] - INFO: epoch 020:     84 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.46, ntokens=17556.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3353.8, ups=0.19, wpb=17556.8, bsz=1024, num_updates=7510, lr=4.29749e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=47767
+2024-11-07 02:58:20 - progress_bar.py[line:274] - INFO: epoch 020:     94 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.516, ntokens=17654.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.72, wps=3350.3, ups=0.19, wpb=17654.1, bsz=1024, num_updates=7520, lr=4.29613e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=47819
+2024-11-07 02:59:13 - progress_bar.py[line:274] - INFO: epoch 020:    104 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.505, ntokens=17676.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.68, wps=3357.8, ups=0.19, wpb=17676.8, bsz=1024, num_updates=7530, lr=4.29477e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=47872
+2024-11-07 03:00:06 - progress_bar.py[line:274] - INFO: epoch 020:    114 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.513, ntokens=17431.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3295, ups=0.19, wpb=17431.4, bsz=1024, num_updates=7540, lr=4.29341e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=47925
+2024-11-07 03:00:33 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
+2024-11-07 03:01:05 - progress_bar.py[line:274] - INFO: epoch 020:    125 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.497, ntokens=17729.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3022.1, ups=0.17, wpb=17729.2, bsz=1024, num_updates=7550, lr=4.29205e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=33, gb_free=13.5, wall=47983
+2024-11-07 03:01:56 - progress_bar.py[line:274] - INFO: epoch 020:    135 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.481, ntokens=17258.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3328.2, ups=0.19, wpb=17258.7, bsz=1024, num_updates=7560, lr=4.29069e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=48035
+2024-11-07 03:02:49 - progress_bar.py[line:274] - INFO: epoch 020:    145 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.495, ntokens=17628.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3338.3, ups=0.19, wpb=17628.9, bsz=1024, num_updates=7570, lr=4.28933e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=48088
+2024-11-07 03:03:42 - progress_bar.py[line:274] - INFO: epoch 020:    155 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.523, ntokens=17754.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3353, ups=0.19, wpb=17754.2, bsz=1024, num_updates=7580, lr=4.28797e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=48141
+2024-11-07 03:04:35 - progress_bar.py[line:274] - INFO: epoch 020:    165 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.504, ntokens=17845.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.67, wps=3365.1, ups=0.19, wpb=17845.6, bsz=1024, num_updates=7590, lr=4.28661e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=48194
+2024-11-07 03:05:27 - progress_bar.py[line:274] - INFO: epoch 020:    175 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.496, ntokens=17604.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3378.6, ups=0.19, wpb=17604.2, bsz=1024, num_updates=7600, lr=4.28525e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=48246
+2024-11-07 03:06:20 - progress_bar.py[line:274] - INFO: epoch 020:    185 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.524, ntokens=17799.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3413, ups=0.19, wpb=17799.8, bsz=1024, num_updates=7610, lr=4.28389e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=48298
+2024-11-07 03:07:12 - progress_bar.py[line:274] - INFO: epoch 020:    195 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.477, ntokens=17513.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3358.3, ups=0.19, wpb=17513.9, bsz=1024, num_updates=7620, lr=4.28253e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=48351
+2024-11-07 03:08:05 - progress_bar.py[line:274] - INFO: epoch 020:    205 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.524, ntokens=17929.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3392.4, ups=0.19, wpb=17929.4, bsz=1024, num_updates=7630, lr=4.28117e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=48403
+2024-11-07 03:08:57 - progress_bar.py[line:274] - INFO: epoch 020:    215 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.497, ntokens=17787.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3385.8, ups=0.19, wpb=17787.4, bsz=1024, num_updates=7640, lr=4.27981e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=48456
+2024-11-07 03:09:50 - progress_bar.py[line:274] - INFO: epoch 020:    225 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.514, ntokens=17715.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3360.5, ups=0.19, wpb=17715.5, bsz=1024, num_updates=7650, lr=4.27845e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=48509
+2024-11-07 03:10:42 - progress_bar.py[line:274] - INFO: epoch 020:    235 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.493, ntokens=17534.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.63, wps=3351, ups=0.19, wpb=17534.3, bsz=1024, num_updates=7660, lr=4.27709e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=48561
+2024-11-07 03:11:35 - progress_bar.py[line:274] - INFO: epoch 020:    245 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.484, ntokens=17514.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3317.5, ups=0.19, wpb=17514.9, bsz=1024, num_updates=7670, lr=4.27573e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=48614
+2024-11-07 03:12:28 - progress_bar.py[line:274] - INFO: epoch 020:    255 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.518, ntokens=17731, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3339.4, ups=0.19, wpb=17731, bsz=1024, num_updates=7680, lr=4.27436e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=48667
+2024-11-07 03:13:20 - progress_bar.py[line:274] - INFO: epoch 020:    265 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.532, ntokens=17856.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3412.6, ups=0.19, wpb=17856.2, bsz=1024, num_updates=7690, lr=4.273e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=48719
+2024-11-07 03:14:13 - progress_bar.py[line:274] - INFO: epoch 020:    275 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.493, ntokens=17547.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.63, wps=3319.1, ups=0.19, wpb=17547.8, bsz=1024, num_updates=7700, lr=4.27164e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=48772
+2024-11-07 03:15:06 - progress_bar.py[line:274] - INFO: epoch 020:    285 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.499, ntokens=17758.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3379, ups=0.19, wpb=17758.7, bsz=1024, num_updates=7710, lr=4.27028e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=48825
+2024-11-07 03:15:58 - progress_bar.py[line:274] - INFO: epoch 020:    295 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17727.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3418.8, ups=0.19, wpb=17727.3, bsz=1024, num_updates=7720, lr=4.26892e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=48876
+2024-11-07 03:16:51 - progress_bar.py[line:274] - INFO: epoch 020:    305 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.578, ntokens=18039.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3387.8, ups=0.19, wpb=18039.3, bsz=1024, num_updates=7730, lr=4.26756e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=41, gb_free=13.4, wall=48930
+2024-11-07 03:17:45 - progress_bar.py[line:274] - INFO: epoch 020:    315 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.633, ntokens=18462.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3437.2, ups=0.19, wpb=18462.5, bsz=1024, num_updates=7740, lr=4.2662e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=48983
+2024-11-07 03:18:38 - progress_bar.py[line:274] - INFO: epoch 020:    325 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.634, ntokens=18349, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3450, ups=0.19, wpb=18349, bsz=1024, num_updates=7750, lr=4.26484e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=49037
+2024-11-07 03:19:30 - progress_bar.py[line:274] - INFO: epoch 020:    335 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.649, ntokens=18494.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3517, ups=0.19, wpb=18494.7, bsz=1024, num_updates=7760, lr=4.26348e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=52, gb_free=13.3, wall=49089
+2024-11-07 03:20:23 - progress_bar.py[line:274] - INFO: epoch 020:    345 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.606, ntokens=18378.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.09, wps=3507.7, ups=0.19, wpb=18378.6, bsz=1024, num_updates=7770, lr=4.26212e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=52, gb_free=13.3, wall=49142
+2024-11-07 03:21:14 - progress_bar.py[line:274] - INFO: epoch 020:    355 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.624, ntokens=18349, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3557.9, ups=0.19, wpb=18349, bsz=1024, num_updates=7780, lr=4.26076e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=51, gb_free=13.3, wall=49193
+2024-11-07 03:22:07 - progress_bar.py[line:274] - INFO: epoch 020:    365 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.579, ntokens=18115.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3455.7, ups=0.19, wpb=18115.1, bsz=1024, num_updates=7790, lr=4.2594e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=49246
+2024-11-07 03:22:59 - progress_bar.py[line:274] - INFO: epoch 020:    375 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.55, ntokens=17835.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.86, wps=3381.8, ups=0.19, wpb=17835.7, bsz=1024, num_updates=7800, lr=4.25804e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=49298
+2024-11-07 03:23:51 - progress_bar.py[line:274] - INFO: epoch 020:    385 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=18203.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3498.5, ups=0.19, wpb=18203.4, bsz=1024, num_updates=7810, lr=4.25668e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=52, gb_free=13.2, wall=49350
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 19:22:32 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 20 @ 7815 updates
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt
 
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt
-2024-11-03 19:22:32 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt': File exists: File exists
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 03:24:19 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 20 @ 7816 updates
+
+
+2024-11-07 03:24:19 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt': File exists
+: File exists: File exists
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 19:24:32 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 03:26:08 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 
-slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-2024-11-03 19:26:33 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 20 @ 7815 updates, score 0) (writing took 240.22212723101256 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+
+slice_id 6 seek offset 300000
+slice_id 7 seek offset 350000
 slice_id 3 seek offset 150000
+2024-11-07 03:28:00 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 20 @ 7816 updates, score 0) (writing took 220.5842493160162 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_20.pt
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
 slice_id 2 seek offset 100000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 1 seek offset 50000
-slice_id 6 seek offset 300000
-2024-11-03 19:27:32 - train.py[line:336] - INFO: end of epoch 20 (average epoch stats below)
-2024-11-03 19:27:32 - progress_bar.py[line:282] - INFO: epoch 020 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.197 | ntokens 17798.4 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 9.17 | wps 2734.9 | ups 0.15 | wpb 17798.4 | bsz 1023 | num_updates 7815 | lr 4.256e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1781 | gb_free 13.3 | wall 50229
-2024-11-03 19:27:32 - trainer.py[line:639] - INFO: loading train data for epoch 21
+slice_id 5 seek offset 250000
+slice_id 4 seek offset 200000
+2024-11-07 03:28:54 - train.py[line:336] - INFO: end of epoch 20 (average epoch stats below)
+2024-11-07 03:28:54 - progress_bar.py[line:282] - INFO: epoch 020 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.53 | ntokens 17797.1 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.78 | wps 2787.5 | ups 0.16 | wpb 17797.1 | bsz 1023 | num_updates 7816 | lr 4.25586e-05 | gnorm 0.008 | clip 0 | loss_scale 65536 | train_wall 1384 | gb_free 13.3 | wall 49653
+2024-11-07 03:28:54 - trainer.py[line:639] - INFO: loading train data for epoch 21
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 19:30:16 - trainer.py[line:703] - INFO: begin training epoch 21
-2024-11-03 19:30:16 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 19:30:44 - progress_bar.py[line:274] - INFO: epoch 021:      5 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.2, ntokens=17108.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.19, wps=332.4, ups=0.02, wpb=17108.1, bsz=985.6, num_updates=7820, lr=4.25532e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=29, gb_free=13.6, wall=50422
-2024-11-03 19:31:37 - progress_bar.py[line:274] - INFO: epoch 021:     15 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.145, ntokens=17750.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.85, wps=3357, ups=0.19, wpb=17750.7, bsz=1024, num_updates=7830, lr=4.25396e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=50474
-2024-11-03 19:32:30 - progress_bar.py[line:274] - INFO: epoch 021:     25 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.176, ntokens=17877.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.04, wps=3388, ups=0.19, wpb=17877.3, bsz=1024, num_updates=7840, lr=4.2526e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=50527
-2024-11-03 19:33:23 - progress_bar.py[line:274] - INFO: epoch 021:     35 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.206, ntokens=18081.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.23, wps=3402.6, ups=0.19, wpb=18081.1, bsz=1024, num_updates=7850, lr=4.25124e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=50580
-2024-11-03 19:34:16 - progress_bar.py[line:274] - INFO: epoch 021:     45 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.094, ntokens=17571, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.54, wps=3342.3, ups=0.19, wpb=17571, bsz=1024, num_updates=7860, lr=4.24988e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=50633
-2024-11-03 19:35:08 - progress_bar.py[line:274] - INFO: epoch 021:     55 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.12, ntokens=17481.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.69, wps=3359, ups=0.19, wpb=17481.1, bsz=1024, num_updates=7870, lr=4.24852e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=50685
-2024-11-03 19:36:02 - progress_bar.py[line:274] - INFO: epoch 021:     65 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.157, ntokens=18150.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.92, wps=3341.5, ups=0.18, wpb=18150.3, bsz=1024, num_updates=7880, lr=4.24716e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=36, gb_free=13.6, wall=50739
-2024-11-03 19:36:58 - progress_bar.py[line:274] - INFO: epoch 021:     75 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.178, ntokens=17652.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.05, wps=3139.8, ups=0.18, wpb=17652.4, bsz=1024, num_updates=7890, lr=4.2458e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=44, gb_free=13.4, wall=50795
-2024-11-03 19:37:54 - progress_bar.py[line:274] - INFO: epoch 021:     85 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.135, ntokens=17576.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.79, wps=3173.7, ups=0.18, wpb=17576.8, bsz=1024, num_updates=7900, lr=4.24444e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=50851
-2024-11-03 19:38:46 - progress_bar.py[line:274] - INFO: epoch 021:     95 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.133, ntokens=17762.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.77, wps=3399.4, ups=0.19, wpb=17762.6, bsz=1024, num_updates=7910, lr=4.24308e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=50903
-2024-11-03 19:39:39 - progress_bar.py[line:274] - INFO: epoch 021:    105 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.141, ntokens=17674.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.82, wps=3329.7, ups=0.19, wpb=17674.1, bsz=1024, num_updates=7920, lr=4.24172e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=50956
-2024-11-03 19:40:34 - progress_bar.py[line:274] - INFO: epoch 021:    115 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.172, ntokens=17753.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.01, wps=3250, ups=0.18, wpb=17753.4, bsz=1024, num_updates=7930, lr=4.24035e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=39, gb_free=13.5, wall=51011
-2024-11-03 19:41:26 - progress_bar.py[line:274] - INFO: epoch 021:    125 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.115, ntokens=17592.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.66, wps=3336.9, ups=0.19, wpb=17592.1, bsz=1024, num_updates=7940, lr=4.23899e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=41, gb_free=13.5, wall=51064
-2024-11-03 19:42:19 - progress_bar.py[line:274] - INFO: epoch 021:    135 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.103, ntokens=17513.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.59, wps=3354.9, ups=0.19, wpb=17513.5, bsz=1024, num_updates=7950, lr=4.23763e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=41, gb_free=13.5, wall=51116
-2024-11-03 19:43:11 - progress_bar.py[line:274] - INFO: epoch 021:    145 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.108, ntokens=17354.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.62, wps=3283.9, ups=0.19, wpb=17354.2, bsz=1024, num_updates=7960, lr=4.23627e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=33, gb_free=13.6, wall=51169
-2024-11-03 19:44:04 - progress_bar.py[line:274] - INFO: epoch 021:    155 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.149, ntokens=17677.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.87, wps=3344, ups=0.19, wpb=17677.9, bsz=1024, num_updates=7970, lr=4.23491e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=51221
-2024-11-03 19:44:57 - progress_bar.py[line:274] - INFO: epoch 021:    165 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.193, ntokens=18066.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.15, wps=3419.9, ups=0.19, wpb=18066.3, bsz=1024, num_updates=7980, lr=4.23355e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=51274
-2024-11-03 19:45:50 - progress_bar.py[line:274] - INFO: epoch 021:    175 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.154, ntokens=17774.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.9, wps=3365.8, ups=0.19, wpb=17774.6, bsz=1024, num_updates=7990, lr=4.23219e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=51327
-2024-11-03 19:46:43 - progress_bar.py[line:274] - INFO: epoch 021:    185 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.118, ntokens=17512.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.68, wps=3328.8, ups=0.19, wpb=17512.3, bsz=1024, num_updates=8000, lr=4.23083e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=51380
-2024-11-03 19:47:36 - progress_bar.py[line:274] - INFO: epoch 021:    195 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.138, ntokens=17658.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.8, wps=3325.5, ups=0.19, wpb=17658.6, bsz=1024, num_updates=8010, lr=4.22947e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=51433
-2024-11-03 19:48:29 - progress_bar.py[line:274] - INFO: epoch 021:    205 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.164, ntokens=17993.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.96, wps=3390, ups=0.19, wpb=17993.9, bsz=1024, num_updates=8020, lr=4.22811e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=51486
-2024-11-03 19:49:12 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-03 19:49:28 - progress_bar.py[line:274] - INFO: epoch 021:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.065, ntokens=17548.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.37, wps=2976.7, ups=0.17, wpb=17548.3, bsz=1024, num_updates=8030, lr=4.22675e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=33, gb_free=13.6, wall=51545
-2024-11-03 19:50:21 - progress_bar.py[line:274] - INFO: epoch 021:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.17, ntokens=17998.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9, wps=3398.6, ups=0.19, wpb=17998.7, bsz=1024, num_updates=8040, lr=4.22539e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=51598
-2024-11-03 19:51:14 - progress_bar.py[line:274] - INFO: epoch 021:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.207, ntokens=17803.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.24, wps=3355.4, ups=0.19, wpb=17803.5, bsz=1024, num_updates=8050, lr=4.22403e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=51651
-2024-11-03 19:52:06 - progress_bar.py[line:274] - INFO: epoch 021:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.152, ntokens=17631.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.89, wps=3356.9, ups=0.19, wpb=17631.3, bsz=1024, num_updates=8060, lr=4.22267e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=51703
-2024-11-03 19:52:58 - progress_bar.py[line:274] - INFO: epoch 021:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.149, ntokens=17899.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.87, wps=3446.7, ups=0.19, wpb=17899.6, bsz=1024, num_updates=8070, lr=4.22131e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=51755
-2024-11-03 19:53:51 - progress_bar.py[line:274] - INFO: epoch 021:    266 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.208, ntokens=18023.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.24, wps=3430.3, ups=0.19, wpb=18023.2, bsz=1024, num_updates=8080, lr=4.21995e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=51808
-2024-11-03 19:54:44 - progress_bar.py[line:274] - INFO: epoch 021:    276 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.148, ntokens=17946.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.87, wps=3391.1, ups=0.19, wpb=17946.6, bsz=1024, num_updates=8090, lr=4.21859e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=51861
-2024-11-03 19:55:37 - progress_bar.py[line:274] - INFO: epoch 021:    286 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.12, ntokens=17577.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.69, wps=3290.7, ups=0.19, wpb=17577.6, bsz=1024, num_updates=8100, lr=4.21723e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=51914
-2024-11-03 19:56:30 - progress_bar.py[line:274] - INFO: epoch 021:    296 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.223, ntokens=17754.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.34, wps=3361.9, ups=0.19, wpb=17754.3, bsz=1024, num_updates=8110, lr=4.21587e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=51967
-2024-11-03 19:57:22 - progress_bar.py[line:274] - INFO: epoch 021:    306 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.3, ntokens=18391.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.85, wps=3538.4, ups=0.19, wpb=18391.4, bsz=1024, num_updates=8120, lr=4.21451e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=52019
-2024-11-03 19:58:14 - progress_bar.py[line:274] - INFO: epoch 021:    316 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.323, ntokens=18262.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.01, wps=3503.4, ups=0.19, wpb=18262.5, bsz=1024, num_updates=8130, lr=4.21315e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=52071
-2024-11-03 19:59:06 - progress_bar.py[line:274] - INFO: epoch 021:    326 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.277, ntokens=18143.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.69, wps=3480.4, ups=0.19, wpb=18143.8, bsz=1024, num_updates=8140, lr=4.21179e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=52123
-2024-11-03 19:59:59 - progress_bar.py[line:274] - INFO: epoch 021:    336 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.229, ntokens=17891.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.38, wps=3368.5, ups=0.19, wpb=17891.8, bsz=1024, num_updates=8150, lr=4.21043e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=35, gb_free=13.5, wall=52176
-2024-11-03 20:00:51 - progress_bar.py[line:274] - INFO: epoch 021:    346 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.258, ntokens=18265.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.57, wps=3503.5, ups=0.19, wpb=18265.2, bsz=1024, num_updates=8160, lr=4.20907e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=44, gb_free=13.4, wall=52228
-2024-11-03 20:01:45 - progress_bar.py[line:274] - INFO: epoch 021:    356 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.321, ntokens=18459.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10, wps=3467.3, ups=0.19, wpb=18459.2, bsz=1024, num_updates=8170, lr=4.20771e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=51, gb_free=13.4, wall=52282
-2024-11-03 20:02:37 - progress_bar.py[line:274] - INFO: epoch 021:    366 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.258, ntokens=18071.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.57, wps=3465.4, ups=0.19, wpb=18071.5, bsz=1024, num_updates=8180, lr=4.20634e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=52334
-2024-11-03 20:03:30 - progress_bar.py[line:274] - INFO: epoch 021:    376 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.276, ntokens=18354.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.69, wps=3425.1, ups=0.19, wpb=18354.3, bsz=1024, num_updates=8190, lr=4.20498e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=52387
-2024-11-03 20:04:25 - progress_bar.py[line:274] - INFO: epoch 021:    386 / 391 loss=0.002, loss_v1=0, loss_v2=0, nll_loss=3.286, ntokens=17980.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.75, wps=3301.7, ups=0.18, wpb=17980.5, bsz=1024, num_updates=8200, lr=4.20362e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.3, wall=52442
+2024-11-07 03:31:33 - trainer.py[line:703] - INFO: begin training epoch 21
+2024-11-07 03:31:33 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 03:31:56 - progress_bar.py[line:274] - INFO: epoch 021:      4 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17231.8, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=355.5, ups=0.02, wpb=17231.8, bsz=985.6, num_updates=7820, lr=4.25532e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=40, gb_free=13.5, wall=49835
+2024-11-07 03:32:49 - progress_bar.py[line:274] - INFO: epoch 021:     14 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.518, ntokens=17722.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3372, ups=0.19, wpb=17722.3, bsz=1024, num_updates=7830, lr=4.25396e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=49888
+2024-11-07 03:33:42 - progress_bar.py[line:274] - INFO: epoch 021:     24 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17796.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3349.4, ups=0.19, wpb=17796.3, bsz=1024, num_updates=7840, lr=4.2526e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=32, gb_free=13.5, wall=49941
+2024-11-07 03:34:38 - progress_bar.py[line:274] - INFO: epoch 021:     34 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.587, ntokens=18193.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.01, wps=3257.7, ups=0.18, wpb=18193.1, bsz=1024, num_updates=7850, lr=4.25124e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=49997
+2024-11-07 03:35:33 - progress_bar.py[line:274] - INFO: epoch 021:     44 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.508, ntokens=17621.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.69, wps=3165, ups=0.18, wpb=17621.8, bsz=1024, num_updates=7860, lr=4.24988e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=50052
+2024-11-07 03:36:29 - progress_bar.py[line:274] - INFO: epoch 021:     54 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.487, ntokens=17431.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.61, wps=3132.1, ups=0.18, wpb=17431.5, bsz=1024, num_updates=7870, lr=4.24852e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=50108
+2024-11-07 03:37:24 - progress_bar.py[line:274] - INFO: epoch 021:     64 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.585, ntokens=18104.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3280.7, ups=0.18, wpb=18104.3, bsz=1024, num_updates=7880, lr=4.24716e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=50163
+2024-11-07 03:38:20 - progress_bar.py[line:274] - INFO: epoch 021:     74 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.523, ntokens=17667.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3182.6, ups=0.18, wpb=17667.7, bsz=1024, num_updates=7890, lr=4.2458e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=50219
+2024-11-07 03:39:15 - progress_bar.py[line:274] - INFO: epoch 021:     84 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.481, ntokens=17587.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3201.4, ups=0.18, wpb=17587.7, bsz=1024, num_updates=7900, lr=4.24444e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=50274
+2024-11-07 03:40:07 - progress_bar.py[line:274] - INFO: epoch 021:     94 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.529, ntokens=17788.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3406.4, ups=0.19, wpb=17788.8, bsz=1024, num_updates=7910, lr=4.24308e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=50326
+2024-11-07 03:40:59 - progress_bar.py[line:274] - INFO: epoch 021:    104 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.54, ntokens=17705.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.82, wps=3403.5, ups=0.19, wpb=17705.7, bsz=1024, num_updates=7920, lr=4.24172e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=50378
+2024-11-07 03:41:51 - progress_bar.py[line:274] - INFO: epoch 021:    114 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.558, ntokens=17748.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3431.4, ups=0.19, wpb=17748.9, bsz=1024, num_updates=7930, lr=4.24035e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=50430
+2024-11-07 03:42:43 - progress_bar.py[line:274] - INFO: epoch 021:    124 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.508, ntokens=17675.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.69, wps=3396.2, ups=0.19, wpb=17675.6, bsz=1024, num_updates=7940, lr=4.23899e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=51, gb_free=13.5, wall=50482
+2024-11-07 03:43:35 - progress_bar.py[line:274] - INFO: epoch 021:    134 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.488, ntokens=17377.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.61, wps=3333.6, ups=0.19, wpb=17377.8, bsz=1024, num_updates=7950, lr=4.23763e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=47, gb_free=13.4, wall=50534
+2024-11-07 03:44:27 - progress_bar.py[line:274] - INFO: epoch 021:    144 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.477, ntokens=17478.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3358.5, ups=0.19, wpb=17478.3, bsz=1024, num_updates=7960, lr=4.23627e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=47, gb_free=13.4, wall=50586
+2024-11-07 03:45:19 - progress_bar.py[line:274] - INFO: epoch 021:    154 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.495, ntokens=17516.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3386, ups=0.19, wpb=17516.1, bsz=1024, num_updates=7970, lr=4.23491e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=37, gb_free=13.3, wall=50637
+2024-11-07 03:46:11 - progress_bar.py[line:274] - INFO: epoch 021:    164 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.576, ntokens=18064.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3436.3, ups=0.19, wpb=18064.3, bsz=1024, num_updates=7980, lr=4.23355e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=33, gb_free=13.5, wall=50690
+2024-11-07 03:47:03 - progress_bar.py[line:274] - INFO: epoch 021:    174 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.529, ntokens=17810.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3405.6, ups=0.19, wpb=17810.5, bsz=1024, num_updates=7990, lr=4.23219e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=50742
+2024-11-07 03:47:56 - progress_bar.py[line:274] - INFO: epoch 021:    184 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.507, ntokens=17513.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.68, wps=3357.9, ups=0.19, wpb=17513.2, bsz=1024, num_updates=8000, lr=4.23083e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=50795
+2024-11-07 03:48:48 - progress_bar.py[line:274] - INFO: epoch 021:    194 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.515, ntokens=17642.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3356.3, ups=0.19, wpb=17642.6, bsz=1024, num_updates=8010, lr=4.22947e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=50847
+2024-11-07 03:49:41 - progress_bar.py[line:274] - INFO: epoch 021:    204 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.523, ntokens=17899, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3420.6, ups=0.19, wpb=17899, bsz=1024, num_updates=8020, lr=4.22811e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=50899
+2024-11-07 03:50:33 - progress_bar.py[line:274] - INFO: epoch 021:    214 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.496, ntokens=17795.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3411.4, ups=0.19, wpb=17795.4, bsz=1024, num_updates=8030, lr=4.22675e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=50952
+2024-11-07 03:51:25 - progress_bar.py[line:274] - INFO: epoch 021:    224 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17831.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3391.9, ups=0.19, wpb=17831.6, bsz=1024, num_updates=8040, lr=4.22539e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=51004
+2024-11-07 03:52:18 - progress_bar.py[line:274] - INFO: epoch 021:    234 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.56, ntokens=17935.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3428.2, ups=0.19, wpb=17935.1, bsz=1024, num_updates=8050, lr=4.22403e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=51056
+2024-11-07 03:53:10 - progress_bar.py[line:274] - INFO: epoch 021:    244 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17758.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3361.7, ups=0.19, wpb=17758.6, bsz=1024, num_updates=8060, lr=4.22267e-05, gnorm=0.011, clip=0, loss_scale=131072, train_wall=31, gb_free=13.4, wall=51109
+2024-11-07 03:53:58 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
+2024-11-07 03:54:08 - progress_bar.py[line:274] - INFO: epoch 021:    255 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.518, ntokens=17733.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3055.6, ups=0.17, wpb=17733.8, bsz=1024, num_updates=8070, lr=4.22131e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=33, gb_free=13.3, wall=51167
+2024-11-07 03:55:02 - progress_bar.py[line:274] - INFO: epoch 021:    265 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.565, ntokens=18087.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3402.8, ups=0.19, wpb=18087.9, bsz=1024, num_updates=8080, lr=4.21995e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=51220
+2024-11-07 03:55:57 - progress_bar.py[line:274] - INFO: epoch 021:    275 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.543, ntokens=17919, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3258.4, ups=0.18, wpb=17919, bsz=1024, num_updates=8090, lr=4.21859e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=39, gb_free=13.5, wall=51275
+2024-11-07 03:56:52 - progress_bar.py[line:274] - INFO: epoch 021:    285 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.484, ntokens=17539.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3190.4, ups=0.18, wpb=17539.5, bsz=1024, num_updates=8100, lr=4.21723e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=51330
+2024-11-07 03:57:44 - progress_bar.py[line:274] - INFO: epoch 021:    295 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.545, ntokens=17770.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3408.5, ups=0.19, wpb=17770.4, bsz=1024, num_updates=8110, lr=4.21587e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=51383
+2024-11-07 03:58:36 - progress_bar.py[line:274] - INFO: epoch 021:    305 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.612, ntokens=18401.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3502.4, ups=0.19, wpb=18401.5, bsz=1024, num_updates=8120, lr=4.21451e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=51435
+2024-11-07 03:59:29 - progress_bar.py[line:274] - INFO: epoch 021:    315 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.631, ntokens=18292.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.19, wps=3466.9, ups=0.19, wpb=18292.9, bsz=1024, num_updates=8130, lr=4.21315e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=51488
+2024-11-07 04:00:21 - progress_bar.py[line:274] - INFO: epoch 021:    325 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.598, ntokens=17986.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3445.7, ups=0.19, wpb=17986.6, bsz=1024, num_updates=8140, lr=4.21179e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=51540
+2024-11-07 04:01:14 - progress_bar.py[line:274] - INFO: epoch 021:    335 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.589, ntokens=18082.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3441.3, ups=0.19, wpb=18082.3, bsz=1024, num_updates=8150, lr=4.21043e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=51593
+2024-11-07 04:02:06 - progress_bar.py[line:274] - INFO: epoch 021:    345 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.563, ntokens=18036.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.91, wps=3459, ups=0.19, wpb=18036.5, bsz=1024, num_updates=8160, lr=4.20907e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=52, gb_free=13.3, wall=51645
+2024-11-07 04:02:58 - progress_bar.py[line:274] - INFO: epoch 021:    355 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.667, ntokens=18647.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3569.4, ups=0.19, wpb=18647.4, bsz=1024, num_updates=8170, lr=4.20771e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=51697
+2024-11-07 04:03:50 - progress_bar.py[line:274] - INFO: epoch 021:    365 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.595, ntokens=18084.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3469.2, ups=0.19, wpb=18084.3, bsz=1024, num_updates=8180, lr=4.20634e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=51749
+2024-11-07 04:04:42 - progress_bar.py[line:274] - INFO: epoch 021:    375 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=18269.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3516.1, ups=0.19, wpb=18269.1, bsz=1024, num_updates=8190, lr=4.20498e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=51801
+2024-11-07 04:05:35 - progress_bar.py[line:274] - INFO: epoch 021:    385 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.589, ntokens=18061.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3398.9, ups=0.19, wpb=18061.3, bsz=1024, num_updates=8200, lr=4.20362e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=51854
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 20:04:48 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 21 @ 8205 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
-2024-11-03 20:04:48 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt': File exists: File exists
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+
+2024-11-07 04:06:03 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 21 @ 8206 updates
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
+2024-11-07 04:06:03 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 20:06:48 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-2024-11-03 20:08:51 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 21 @ 8205 updates, score 0) (writing took 243.07547801098553 seconds)
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 04:07:54 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+2024-11-07 04:09:46 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 21 @ 8206 updates, score 0) (writing took 222.484497611993 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_21.pt
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-slice_id 2 seek offset 100000
 slice_id 1 seek offset 50000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 5 seek offset 250000
-2024-11-03 20:09:51 - train.py[line:336] - INFO: end of epoch 21 (average epoch stats below)
-2024-11-03 20:09:51 - progress_bar.py[line:282] - INFO: epoch 021 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.181 | ntokens 17831.7 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 9.07 | wps 2738.5 | ups 0.15 | wpb 17831.7 | bsz 1023 | num_updates 8205 | lr 4.20294e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1355 | gb_free 13.3 | wall 52768
-2024-11-03 20:09:51 - trainer.py[line:639] - INFO: loading train data for epoch 22
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
+slice_id 3 seek offset 150000
+slice_id 7 seek offset 350000
+slice_id 6 seek offset 300000
+slice_id 4 seek offset 200000
+2024-11-07 04:10:40 - train.py[line:336] - INFO: end of epoch 21 (average epoch stats below)
+2024-11-07 04:10:40 - progress_bar.py[line:282] - INFO: epoch 021 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.544 | ntokens 17834.7 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.83 | wps 2776.2 | ups 0.16 | wpb 17834.7 | bsz 1023 | num_updates 8206 | lr 4.20281e-05 | gnorm 0.009 | clip 0 | loss_scale 65536 | train_wall 1728 | gb_free 13.3 | wall 52159
+2024-11-07 04:10:40 - trainer.py[line:639] - INFO: loading train data for epoch 22
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 20:12:36 - trainer.py[line:703] - INFO: begin training epoch 22
-2024-11-03 20:12:36 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 20:13:05 - progress_bar.py[line:274] - INFO: epoch 022:      5 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.131, ntokens=16911.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.76, wps=325.1, ups=0.02, wpb=16911.7, bsz=985.6, num_updates=8210, lr=4.20226e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=39, gb_free=13.6, wall=52962
-2024-11-03 20:13:58 - progress_bar.py[line:274] - INFO: epoch 022:     15 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.145, ntokens=17637.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.85, wps=3329.4, ups=0.19, wpb=17637.2, bsz=1024, num_updates=8220, lr=4.2009e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=53015
-2024-11-03 20:14:51 - progress_bar.py[line:274] - INFO: epoch 022:     25 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.091, ntokens=17640.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.52, wps=3346.2, ups=0.19, wpb=17640.4, bsz=1024, num_updates=8230, lr=4.19954e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=53068
-2024-11-03 20:15:43 - progress_bar.py[line:274] - INFO: epoch 022:     35 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.203, ntokens=18168.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.21, wps=3444.2, ups=0.19, wpb=18168.5, bsz=1024, num_updates=8240, lr=4.19818e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=53121
-2024-11-03 20:16:36 - progress_bar.py[line:274] - INFO: epoch 022:     45 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.042, ntokens=17388.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.24, wps=3299.3, ups=0.19, wpb=17388.5, bsz=1024, num_updates=8250, lr=4.19682e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=53173
-2024-11-03 20:17:30 - progress_bar.py[line:274] - INFO: epoch 022:     55 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.141, ntokens=17753.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.82, wps=3305.1, ups=0.19, wpb=17753.5, bsz=1024, num_updates=8260, lr=4.19546e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=40, gb_free=13.4, wall=53227
-2024-11-03 20:18:24 - progress_bar.py[line:274] - INFO: epoch 022:     65 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.092, ntokens=17524.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.53, wps=3239.8, ups=0.18, wpb=17524.3, bsz=1024, num_updates=8270, lr=4.1941e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=51, gb_free=13.6, wall=53281
-2024-11-03 20:19:18 - progress_bar.py[line:274] - INFO: epoch 022:     75 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.182, ntokens=17798.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.07, wps=3292.1, ups=0.18, wpb=17798.3, bsz=1024, num_updates=8280, lr=4.19274e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=53335
-2024-11-03 20:20:12 - progress_bar.py[line:274] - INFO: epoch 022:     85 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.094, ntokens=17672.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.54, wps=3298.2, ups=0.19, wpb=17672.6, bsz=1024, num_updates=8290, lr=4.19138e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=53389
-2024-11-03 20:21:06 - progress_bar.py[line:274] - INFO: epoch 022:     95 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.136, ntokens=17805, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.79, wps=3291.4, ups=0.18, wpb=17805, bsz=1024, num_updates=8300, lr=4.19002e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=53443
-2024-11-03 20:21:59 - progress_bar.py[line:274] - INFO: epoch 022:    105 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.092, ntokens=17598.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.53, wps=3304.2, ups=0.19, wpb=17598.8, bsz=1024, num_updates=8310, lr=4.18866e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=53496
-2024-11-03 20:22:51 - progress_bar.py[line:274] - INFO: epoch 022:    115 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.139, ntokens=17426.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.81, wps=3322, ups=0.19, wpb=17426.2, bsz=1024, num_updates=8320, lr=4.1873e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=53549
-2024-11-03 20:23:45 - progress_bar.py[line:274] - INFO: epoch 022:    125 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.121, ntokens=17847.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.7, wps=3328.8, ups=0.19, wpb=17847.9, bsz=1024, num_updates=8330, lr=4.18594e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=53602
-2024-11-03 20:24:38 - progress_bar.py[line:274] - INFO: epoch 022:    135 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.119, ntokens=17650.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.69, wps=3316.2, ups=0.19, wpb=17650.2, bsz=1024, num_updates=8340, lr=4.18458e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=53655
-2024-11-03 20:25:30 - progress_bar.py[line:274] - INFO: epoch 022:    145 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.179, ntokens=18026, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.06, wps=3473.4, ups=0.19, wpb=18026, bsz=1024, num_updates=8350, lr=4.18322e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=53707
-2024-11-03 20:26:23 - progress_bar.py[line:274] - INFO: epoch 022:    155 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.108, ntokens=17386.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.62, wps=3301.3, ups=0.19, wpb=17386.6, bsz=1024, num_updates=8360, lr=4.18186e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.3, wall=53760
-2024-11-03 20:26:49 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
-2024-11-03 20:27:20 - progress_bar.py[line:274] - INFO: epoch 022:    166 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.109, ntokens=17773.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.63, wps=3090.5, ups=0.17, wpb=17773.5, bsz=1024, num_updates=8370, lr=4.1805e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=57, gb_free=13.4, wall=53817
-2024-11-03 20:28:13 - progress_bar.py[line:274] - INFO: epoch 022:    176 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.127, ntokens=17722.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.73, wps=3357.1, ups=0.19, wpb=17722.1, bsz=1024, num_updates=8380, lr=4.17914e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=51, gb_free=13.5, wall=53870
-2024-11-03 20:29:06 - progress_bar.py[line:274] - INFO: epoch 022:    186 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.108, ntokens=17540.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.62, wps=3298.2, ups=0.19, wpb=17540.7, bsz=1024, num_updates=8390, lr=4.17778e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=53923
-2024-11-03 20:29:58 - progress_bar.py[line:274] - INFO: epoch 022:    196 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.075, ntokens=17404.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.43, wps=3342.7, ups=0.19, wpb=17404.4, bsz=1024, num_updates=8400, lr=4.17642e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=53975
-2024-11-03 20:30:53 - progress_bar.py[line:274] - INFO: epoch 022:    206 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.138, ntokens=18069.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.8, wps=3321.9, ups=0.18, wpb=18069.1, bsz=1024, num_updates=8410, lr=4.17506e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=54, gb_free=13.5, wall=54030
-2024-11-03 20:31:47 - progress_bar.py[line:274] - INFO: epoch 022:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.082, ntokens=17754.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.47, wps=3271.8, ups=0.18, wpb=17754.7, bsz=1024, num_updates=8420, lr=4.1737e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=54, gb_free=13.6, wall=54084
-2024-11-03 20:32:40 - progress_bar.py[line:274] - INFO: epoch 022:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.105, ntokens=17843.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.61, wps=3352.3, ups=0.19, wpb=17843.9, bsz=1024, num_updates=8430, lr=4.17233e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=54137
-2024-11-03 20:33:33 - progress_bar.py[line:274] - INFO: epoch 022:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.183, ntokens=17669.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.08, wps=3325.7, ups=0.19, wpb=17669.2, bsz=1024, num_updates=8440, lr=4.17097e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=54190
-2024-11-03 20:34:26 - progress_bar.py[line:274] - INFO: epoch 022:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.094, ntokens=17428.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.54, wps=3311.8, ups=0.19, wpb=17428.5, bsz=1024, num_updates=8450, lr=4.16961e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=53, gb_free=13.6, wall=54243
-2024-11-03 20:35:19 - progress_bar.py[line:274] - INFO: epoch 022:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.125, ntokens=17725.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.73, wps=3377.4, ups=0.19, wpb=17725.8, bsz=1024, num_updates=8460, lr=4.16825e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=54296
-2024-11-03 20:36:12 - progress_bar.py[line:274] - INFO: epoch 022:    266 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.203, ntokens=17970.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.21, wps=3387.2, ups=0.19, wpb=17970.2, bsz=1024, num_updates=8470, lr=4.16689e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=54349
-2024-11-03 20:37:05 - progress_bar.py[line:274] - INFO: epoch 022:    276 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.106, ntokens=17910.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.61, wps=3370.7, ups=0.19, wpb=17910.6, bsz=1024, num_updates=8480, lr=4.16553e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=53, gb_free=13.5, wall=54402
-2024-11-03 20:37:57 - progress_bar.py[line:274] - INFO: epoch 022:    286 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.115, ntokens=17615.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.66, wps=3379.1, ups=0.19, wpb=17615.8, bsz=1024, num_updates=8490, lr=4.16417e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.6, wall=54454
-2024-11-03 20:38:49 - progress_bar.py[line:274] - INFO: epoch 022:    296 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.186, ntokens=17884.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.1, wps=3411.7, ups=0.19, wpb=17884.6, bsz=1024, num_updates=8500, lr=4.16281e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=54506
-2024-11-03 20:39:42 - progress_bar.py[line:274] - INFO: epoch 022:    306 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.279, ntokens=18290.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.7, wps=3465.9, ups=0.19, wpb=18290.4, bsz=1024, num_updates=8510, lr=4.16145e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=53, gb_free=13.4, wall=54559
-2024-11-03 20:40:34 - progress_bar.py[line:274] - INFO: epoch 022:    316 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.336, ntokens=18610.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=10.1, wps=3566, ups=0.19, wpb=18610.9, bsz=1024, num_updates=8520, lr=4.16009e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=54611
-2024-11-03 20:41:26 - progress_bar.py[line:274] - INFO: epoch 022:    326 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.268, ntokens=18122.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.63, wps=3502.5, ups=0.19, wpb=18122.4, bsz=1024, num_updates=8530, lr=4.15873e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.3, wall=54663
-2024-11-03 20:42:19 - progress_bar.py[line:274] - INFO: epoch 022:    336 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.226, ntokens=17916.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.36, wps=3395.6, ups=0.19, wpb=17916.8, bsz=1024, num_updates=8540, lr=4.15737e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=53, gb_free=13.4, wall=54716
-2024-11-03 20:43:11 - progress_bar.py[line:274] - INFO: epoch 022:    346 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.201, ntokens=18174.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.2, wps=3480.3, ups=0.19, wpb=18174.5, bsz=1024, num_updates=8550, lr=4.15601e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=54768
-2024-11-03 20:44:04 - progress_bar.py[line:274] - INFO: epoch 022:    356 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.317, ntokens=18489.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.97, wps=3469.5, ups=0.19, wpb=18489.2, bsz=1024, num_updates=8560, lr=4.15465e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.4, wall=54821
-2024-11-03 20:44:57 - progress_bar.py[line:274] - INFO: epoch 022:    366 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.232, ntokens=18151.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.4, wps=3435, ups=0.19, wpb=18151.8, bsz=1024, num_updates=8570, lr=4.15329e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.4, wall=54874
-2024-11-03 20:45:49 - progress_bar.py[line:274] - INFO: epoch 022:    376 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.266, ntokens=18509.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.62, wps=3569.8, ups=0.19, wpb=18509.6, bsz=1024, num_updates=8580, lr=4.15193e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=54926
-2024-11-03 20:46:42 - progress_bar.py[line:274] - INFO: epoch 022:    386 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.264, ntokens=17979, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.6, wps=3417, ups=0.19, wpb=17979, bsz=1024, num_updates=8590, lr=4.15057e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=53, gb_free=13.3, wall=54979
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-03 20:47:05 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 22 @ 8595 updates
+2024-11-07 04:13:19 - trainer.py[line:703] - INFO: begin training epoch 22
+2024-11-07 04:13:19 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 04:13:42 - progress_bar.py[line:274] - INFO: epoch 022:      4 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.508, ntokens=16992.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.69, wps=349.1, ups=0.02, wpb=16992.6, bsz=985.6, num_updates=8210, lr=4.20226e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=40, gb_free=13.5, wall=52341
+2024-11-07 04:14:34 - progress_bar.py[line:274] - INFO: epoch 022:     14 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.479, ntokens=17556, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3388.5, ups=0.19, wpb=17556, bsz=1024, num_updates=8220, lr=4.2009e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=52393
+2024-11-07 04:15:26 - progress_bar.py[line:274] - INFO: epoch 022:     24 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.495, ntokens=17691.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3389.7, ups=0.19, wpb=17691.6, bsz=1024, num_updates=8230, lr=4.19954e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=52445
+2024-11-07 04:16:18 - progress_bar.py[line:274] - INFO: epoch 022:     34 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.535, ntokens=18134, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3499.2, ups=0.19, wpb=18134, bsz=1024, num_updates=8240, lr=4.19818e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=52497
+2024-11-07 04:17:11 - progress_bar.py[line:274] - INFO: epoch 022:     44 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.436, ntokens=17393.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.41, wps=3298.3, ups=0.19, wpb=17393.3, bsz=1024, num_updates=8250, lr=4.19682e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=52550
+2024-11-07 04:17:47 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
+2024-11-07 04:18:08 - progress_bar.py[line:274] - INFO: epoch 022:     55 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.518, ntokens=17768.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3090.5, ups=0.17, wpb=17768.8, bsz=1024, num_updates=8260, lr=4.19546e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=33, gb_free=13.4, wall=52607
+2024-11-07 04:19:00 - progress_bar.py[line:274] - INFO: epoch 022:     65 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.484, ntokens=17524.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3401.7, ups=0.19, wpb=17524.3, bsz=1024, num_updates=8270, lr=4.1941e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=52659
+2024-11-07 04:19:52 - progress_bar.py[line:274] - INFO: epoch 022:     75 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=17798.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3389.6, ups=0.19, wpb=17798.3, bsz=1024, num_updates=8280, lr=4.19274e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=52711
+2024-11-07 04:20:45 - progress_bar.py[line:274] - INFO: epoch 022:     85 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.477, ntokens=17672.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3353.1, ups=0.19, wpb=17672.6, bsz=1024, num_updates=8290, lr=4.19138e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=52764
+2024-11-07 04:21:38 - progress_bar.py[line:274] - INFO: epoch 022:     95 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.531, ntokens=17805, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3366.3, ups=0.19, wpb=17805, bsz=1024, num_updates=8300, lr=4.19002e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=52817
+2024-11-07 04:22:30 - progress_bar.py[line:274] - INFO: epoch 022:    105 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.486, ntokens=17598.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3381.8, ups=0.19, wpb=17598.8, bsz=1024, num_updates=8310, lr=4.18866e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=52869
+2024-11-07 04:23:22 - progress_bar.py[line:274] - INFO: epoch 022:    115 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.517, ntokens=17426.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.72, wps=3305.8, ups=0.19, wpb=17426.2, bsz=1024, num_updates=8320, lr=4.1873e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=52921
+2024-11-07 04:24:17 - progress_bar.py[line:274] - INFO: epoch 022:    125 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.501, ntokens=17847.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.66, wps=3252.9, ups=0.18, wpb=17847.9, bsz=1024, num_updates=8330, lr=4.18594e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=37, gb_free=13.5, wall=52976
+2024-11-07 04:25:13 - progress_bar.py[line:274] - INFO: epoch 022:    135 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.519, ntokens=17650.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3180.7, ups=0.18, wpb=17650.2, bsz=1024, num_updates=8340, lr=4.18458e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=43, gb_free=13.4, wall=53032
+2024-11-07 04:26:09 - progress_bar.py[line:274] - INFO: epoch 022:    145 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.53, ntokens=18026, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.78, wps=3228.7, ups=0.18, wpb=18026, bsz=1024, num_updates=8350, lr=4.18322e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=43, gb_free=13.5, wall=53088
+2024-11-07 04:27:04 - progress_bar.py[line:274] - INFO: epoch 022:    155 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.454, ntokens=17386.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.48, wps=3165.2, ups=0.18, wpb=17386.6, bsz=1024, num_updates=8360, lr=4.18186e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=42, gb_free=13.3, wall=53143
+2024-11-07 04:28:00 - progress_bar.py[line:274] - INFO: epoch 022:    165 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.465, ntokens=17676.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3157.9, ups=0.18, wpb=17676.5, bsz=1024, num_updates=8370, lr=4.1805e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=46, gb_free=13.6, wall=53198
+2024-11-07 04:28:55 - progress_bar.py[line:274] - INFO: epoch 022:    175 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.52, ntokens=17877.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=3212.7, ups=0.18, wpb=17877.9, bsz=1024, num_updates=8380, lr=4.17914e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=46, gb_free=13.4, wall=53254
+2024-11-07 04:29:51 - progress_bar.py[line:274] - INFO: epoch 022:    185 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.475, ntokens=17543.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3150.2, ups=0.18, wpb=17543.7, bsz=1024, num_updates=8390, lr=4.17778e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=44, gb_free=13.6, wall=53310
+2024-11-07 04:30:47 - progress_bar.py[line:274] - INFO: epoch 022:    195 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.444, ntokens=17333.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.44, wps=3074.8, ups=0.18, wpb=17333.1, bsz=1024, num_updates=8400, lr=4.17642e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=42, gb_free=13.6, wall=53366
+2024-11-07 04:31:43 - progress_bar.py[line:274] - INFO: epoch 022:    205 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.538, ntokens=18074.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3266.7, ups=0.18, wpb=18074.9, bsz=1024, num_updates=8410, lr=4.17506e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=42, gb_free=13.5, wall=53422
+2024-11-07 04:32:45 - progress_bar.py[line:274] - INFO: epoch 022:    215 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.49, ntokens=17863.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.62, wps=2845.1, ups=0.16, wpb=17863.2, bsz=1024, num_updates=8420, lr=4.1737e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=53484
+2024-11-07 04:33:52 - progress_bar.py[line:274] - INFO: epoch 022:    225 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.495, ntokens=17708.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=2676.7, ups=0.15, wpb=17708.1, bsz=1024, num_updates=8430, lr=4.17233e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=66, gb_free=13.5, wall=53550
+2024-11-07 04:34:58 - progress_bar.py[line:274] - INFO: epoch 022:    235 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.524, ntokens=17835.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=2697.8, ups=0.15, wpb=17835.7, bsz=1024, num_updates=8440, lr=4.17097e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=66, gb_free=13.4, wall=53617
+2024-11-07 04:36:04 - progress_bar.py[line:274] - INFO: epoch 022:    245 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.449, ntokens=17332.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=2613.6, ups=0.15, wpb=17332.5, bsz=1024, num_updates=8450, lr=4.16961e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=66, gb_free=13.6, wall=53683
+2024-11-07 04:36:55 - progress_bar.py[line:274] - INFO: epoch 022:    255 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.509, ntokens=17796.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.69, wps=3469.6, ups=0.19, wpb=17796.6, bsz=1024, num_updates=8460, lr=4.16825e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=37, gb_free=13.3, wall=53734
+2024-11-07 04:37:48 - progress_bar.py[line:274] - INFO: epoch 022:    265 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.52, ntokens=17883.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=3403.8, ups=0.19, wpb=17883.6, bsz=1024, num_updates=8470, lr=4.16689e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=53787
+2024-11-07 04:38:41 - progress_bar.py[line:274] - INFO: epoch 022:    275 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.506, ntokens=17876.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.68, wps=3359.8, ups=0.19, wpb=17876.5, bsz=1024, num_updates=8480, lr=4.16553e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=53840
+2024-11-07 04:39:34 - progress_bar.py[line:274] - INFO: epoch 022:    285 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.485, ntokens=17737.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3355.6, ups=0.19, wpb=17737.5, bsz=1024, num_updates=8490, lr=4.16417e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=53893
+2024-11-07 04:40:26 - progress_bar.py[line:274] - INFO: epoch 022:    295 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.522, ntokens=17731.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=3405.2, ups=0.19, wpb=17731.4, bsz=1024, num_updates=8500, lr=4.16281e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=53945
+2024-11-07 04:41:22 - progress_bar.py[line:274] - INFO: epoch 022:    305 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=18356.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3287, ups=0.18, wpb=18356.2, bsz=1024, num_updates=8510, lr=4.16145e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=51, gb_free=13.5, wall=54001
+2024-11-07 04:42:15 - progress_bar.py[line:274] - INFO: epoch 022:    315 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.664, ntokens=18555.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3490.7, ups=0.19, wpb=18555.4, bsz=1024, num_updates=8520, lr=4.16009e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=54054
+2024-11-07 04:43:08 - progress_bar.py[line:274] - INFO: epoch 022:    325 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.612, ntokens=18175.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3445.2, ups=0.19, wpb=18175.4, bsz=1024, num_updates=8530, lr=4.15873e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=54107
+2024-11-07 04:44:01 - progress_bar.py[line:274] - INFO: epoch 022:    335 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.55, ntokens=17985.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.86, wps=3374.3, ups=0.19, wpb=17985.8, bsz=1024, num_updates=8540, lr=4.15737e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=53, gb_free=13.2, wall=54160
+2024-11-07 04:44:54 - progress_bar.py[line:274] - INFO: epoch 022:    345 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.546, ntokens=18040.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3383.9, ups=0.19, wpb=18040.3, bsz=1024, num_updates=8550, lr=4.15601e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=54213
+2024-11-07 04:45:47 - progress_bar.py[line:274] - INFO: epoch 022:    355 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.641, ntokens=18614.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3519.1, ups=0.19, wpb=18614.7, bsz=1024, num_updates=8560, lr=4.15465e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=54266
+2024-11-07 04:46:41 - progress_bar.py[line:274] - INFO: epoch 022:    365 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.591, ntokens=18154, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.03, wps=3402.2, ups=0.19, wpb=18154, bsz=1024, num_updates=8570, lr=4.15329e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=53, gb_free=13.3, wall=54320
+2024-11-07 04:47:34 - progress_bar.py[line:274] - INFO: epoch 022:    375 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.573, ntokens=18234.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3436.1, ups=0.19, wpb=18234.4, bsz=1024, num_updates=8580, lr=4.15193e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.3, wall=54373
+2024-11-07 04:48:27 - progress_bar.py[line:274] - INFO: epoch 022:    385 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.595, ntokens=18153.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3434.1, ups=0.19, wpb=18153.1, bsz=1024, num_updates=8590, lr=4.15057e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.2, wall=54425
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+2024-11-07 04:48:55 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 22 @ 8596 updates
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
+2024-11-07 04:48:55 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt': File exists: File exists
 
-2024-11-03 20:47:05 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt': File exists
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 20:48:54 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 04:50:45 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+slice_id 5 seek offset 250000
 slice_id 4 seek offset 200000
-2024-11-03 20:50:46 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 22 @ 8595 updates, score 0) (writing took 221.32107832099427 seconds)
+2024-11-07 04:52:38 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 22 @ 8596 updates, score 0) (writing took 223.02192327100784 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_22.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 1 seek offset 50000
-slice_id 5 seek offset 250000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
 slice_id 6 seek offset 300000
-2024-11-03 20:51:40 - train.py[line:336] - INFO: end of epoch 22 (average epoch stats below)
-2024-11-03 20:51:40 - progress_bar.py[line:282] - INFO: epoch 022 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.16 | ntokens 17821.6 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.94 | wps 2770 | ups 0.16 | wpb 17821.6 | bsz 1023 | num_updates 8595 | lr 4.14989e-05 | gnorm 0.007 | clip 0 | loss_scale 16384 | train_wall 1944 | gb_free 13.4 | wall 55277
-2024-11-03 20:51:40 - trainer.py[line:639] - INFO: loading train data for epoch 23
+slice_id 2 seek offset 100000
+slice_id 3 seek offset 150000
+slice_id 7 seek offset 350000
+2024-11-07 04:53:32 - train.py[line:336] - INFO: end of epoch 22 (average epoch stats below)
+2024-11-07 04:53:32 - progress_bar.py[line:282] - INFO: epoch 022 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.523 | ntokens 17823.5 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.75 | wps 2702.5 | ups 0.15 | wpb 17823.5 | bsz 1023 | num_updates 8596 | lr 4.14975e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1653 | gb_free 13.4 | wall 54731
+2024-11-07 04:53:32 - trainer.py[line:639] - INFO: loading train data for epoch 23
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 20:54:25 - trainer.py[line:703] - INFO: begin training epoch 23
-2024-11-03 20:54:25 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 20:54:56 - progress_bar.py[line:274] - INFO: epoch 023:      5 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.164, ntokens=17170, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.96, wps=347.5, ups=0.02, wpb=17170, bsz=985.6, num_updates=8600, lr=4.14921e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=38, gb_free=13.6, wall=55473
-2024-11-03 20:55:51 - progress_bar.py[line:274] - INFO: epoch 023:     15 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.123, ntokens=17684.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.71, wps=3200.3, ups=0.18, wpb=17684.8, bsz=1024, num_updates=8610, lr=4.14785e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=55528
-2024-11-03 20:56:45 - progress_bar.py[line:274] - INFO: epoch 023:     25 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.079, ntokens=17629.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.45, wps=3288.1, ups=0.19, wpb=17629.5, bsz=1024, num_updates=8620, lr=4.14649e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=55582
-2024-11-03 20:57:38 - progress_bar.py[line:274] - INFO: epoch 023:     35 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.109, ntokens=17771.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.63, wps=3338, ups=0.19, wpb=17771.4, bsz=1024, num_updates=8630, lr=4.14513e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=55635
-2024-11-03 20:58:35 - progress_bar.py[line:274] - INFO: epoch 023:     45 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.09, ntokens=17694.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.51, wps=3111.3, ups=0.18, wpb=17694.5, bsz=1024, num_updates=8640, lr=4.14377e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=54, gb_free=13.4, wall=55692
-2024-11-03 20:59:32 - progress_bar.py[line:274] - INFO: epoch 023:     55 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.158, ntokens=17870.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.93, wps=3131.6, ups=0.18, wpb=17870.9, bsz=1024, num_updates=8650, lr=4.14241e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=57, gb_free=13.4, wall=55749
-2024-11-03 21:00:29 - progress_bar.py[line:274] - INFO: epoch 023:     65 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.081, ntokens=17763.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.46, wps=3104.7, ups=0.17, wpb=17763.5, bsz=1024, num_updates=8660, lr=4.14105e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=57, gb_free=13.6, wall=55806
-2024-11-03 21:01:25 - progress_bar.py[line:274] - INFO: epoch 023:     75 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.158, ntokens=17776.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.93, wps=3157.5, ups=0.18, wpb=17776.5, bsz=1024, num_updates=8670, lr=4.13969e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=56, gb_free=13.5, wall=55862
-2024-11-03 21:02:22 - progress_bar.py[line:274] - INFO: epoch 023:     85 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.108, ntokens=17832.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.62, wps=3137.8, ups=0.18, wpb=17832.3, bsz=1024, num_updates=8680, lr=4.13833e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=55919
-2024-11-03 21:03:19 - progress_bar.py[line:274] - INFO: epoch 023:     95 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.064, ntokens=17475, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.36, wps=3084.7, ups=0.18, wpb=17475, bsz=1024, num_updates=8690, lr=4.13696e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=57, gb_free=13.6, wall=55976
-2024-11-03 21:04:13 - progress_bar.py[line:274] - INFO: epoch 023:    105 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.115, ntokens=17747, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.66, wps=3247.2, ups=0.18, wpb=17747, bsz=1024, num_updates=8700, lr=4.1356e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=55, gb_free=13.6, wall=56030
-2024-11-03 21:05:11 - progress_bar.py[line:274] - INFO: epoch 023:    115 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.133, ntokens=17569.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.77, wps=3064.1, ups=0.17, wpb=17569.5, bsz=1024, num_updates=8710, lr=4.13424e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=57, gb_free=13.4, wall=56088
-2024-11-03 21:06:08 - progress_bar.py[line:274] - INFO: epoch 023:    125 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.088, ntokens=17934.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.5, wps=3147.4, ups=0.18, wpb=17934.6, bsz=1024, num_updates=8720, lr=4.13288e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=56145
-2024-11-03 21:07:05 - progress_bar.py[line:274] - INFO: epoch 023:    135 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.108, ntokens=17570.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.62, wps=3079.7, ups=0.18, wpb=17570.7, bsz=1024, num_updates=8730, lr=4.13152e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=56202
-2024-11-03 21:08:01 - progress_bar.py[line:274] - INFO: epoch 023:    145 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.106, ntokens=17632.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.61, wps=3120.1, ups=0.18, wpb=17632.9, bsz=1024, num_updates=8740, lr=4.13016e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=56, gb_free=13.6, wall=56258
-2024-11-03 21:08:59 - progress_bar.py[line:274] - INFO: epoch 023:    155 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.12, ntokens=17780.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.69, wps=3101.4, ups=0.17, wpb=17780.1, bsz=1024, num_updates=8750, lr=4.1288e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=57, gb_free=13.4, wall=56316
-2024-11-03 21:09:56 - progress_bar.py[line:274] - INFO: epoch 023:    165 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.105, ntokens=17851.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.61, wps=3114.5, ups=0.17, wpb=17851.9, bsz=1024, num_updates=8760, lr=4.12744e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=56373
-2024-11-03 21:10:51 - progress_bar.py[line:274] - INFO: epoch 023:    175 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.103, ntokens=17664.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.59, wps=3220.3, ups=0.18, wpb=17664.8, bsz=1024, num_updates=8770, lr=4.12608e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=55, gb_free=13.5, wall=56428
-2024-11-03 21:11:45 - progress_bar.py[line:274] - INFO: epoch 023:    185 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.093, ntokens=17661.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.54, wps=3279.4, ups=0.19, wpb=17661.6, bsz=1024, num_updates=8780, lr=4.12472e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=54, gb_free=13.6, wall=56482
-2024-11-03 21:12:42 - progress_bar.py[line:274] - INFO: epoch 023:    195 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.101, ntokens=17641.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.58, wps=3095.7, ups=0.18, wpb=17641.9, bsz=1024, num_updates=8790, lr=4.12336e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=57, gb_free=13.6, wall=56539
-2024-11-03 21:13:38 - progress_bar.py[line:274] - INFO: epoch 023:    205 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.133, ntokens=18071.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.78, wps=3190.7, ups=0.18, wpb=18071.4, bsz=1024, num_updates=8800, lr=4.122e-05, gnorm=0.004, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=56595
-2024-11-03 21:14:35 - progress_bar.py[line:274] - INFO: epoch 023:    215 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.043, ntokens=17663.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.24, wps=3104, ups=0.18, wpb=17663.2, bsz=1024, num_updates=8810, lr=4.12064e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=56652
-2024-11-03 21:15:31 - progress_bar.py[line:274] - INFO: epoch 023:    225 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.04, ntokens=17553.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=3136.5, ups=0.18, wpb=17553.8, bsz=1024, num_updates=8820, lr=4.11928e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=56, gb_free=13.6, wall=56708
-2024-11-03 21:16:28 - progress_bar.py[line:274] - INFO: epoch 023:    235 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.153, ntokens=17760.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.89, wps=3133.8, ups=0.18, wpb=17760.6, bsz=1024, num_updates=8830, lr=4.11792e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=56765
-2024-11-03 21:17:23 - progress_bar.py[line:274] - INFO: epoch 023:    245 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.08, ntokens=17270.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.46, wps=3104.3, ups=0.18, wpb=17270.5, bsz=1024, num_updates=8840, lr=4.11656e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=53, gb_free=13.6, wall=56821
-2024-11-03 21:18:21 - progress_bar.py[line:274] - INFO: epoch 023:    255 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.096, ntokens=17768.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.55, wps=3095.1, ups=0.17, wpb=17768.7, bsz=1024, num_updates=8850, lr=4.1152e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=57, gb_free=13.3, wall=56878
-2024-11-03 21:19:15 - progress_bar.py[line:274] - INFO: epoch 023:    265 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.136, ntokens=17787.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.79, wps=3284.6, ups=0.18, wpb=17787.6, bsz=1024, num_updates=8860, lr=4.11384e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=54, gb_free=13.6, wall=56932
-2024-11-03 21:20:12 - progress_bar.py[line:274] - INFO: epoch 023:    275 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.107, ntokens=17935.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.62, wps=3145.3, ups=0.18, wpb=17935.7, bsz=1024, num_updates=8870, lr=4.11248e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=56989
-2024-11-03 21:21:09 - progress_bar.py[line:274] - INFO: epoch 023:    285 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.062, ntokens=17516.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.35, wps=3060.5, ups=0.17, wpb=17516.7, bsz=1024, num_updates=8880, lr=4.11112e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=57046
-2024-11-03 21:22:04 - progress_bar.py[line:274] - INFO: epoch 023:    295 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.116, ntokens=17710.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.67, wps=3207.1, ups=0.18, wpb=17710.8, bsz=1024, num_updates=8890, lr=4.10976e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=57102
-2024-11-03 21:23:01 - progress_bar.py[line:274] - INFO: epoch 023:    305 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.243, ntokens=18292.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.47, wps=3211.6, ups=0.18, wpb=18292.8, bsz=1024, num_updates=8900, lr=4.1084e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=57159
-2024-11-03 21:23:58 - progress_bar.py[line:274] - INFO: epoch 023:    315 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.319, ntokens=18495.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.98, wps=3250, ups=0.18, wpb=18495.3, bsz=1024, num_updates=8910, lr=4.10704e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=57, gb_free=13.3, wall=57215
-2024-11-03 21:24:55 - progress_bar.py[line:274] - INFO: epoch 023:    325 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.223, ntokens=18161.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.34, wps=3201.1, ups=0.18, wpb=18161.8, bsz=1024, num_updates=8920, lr=4.10568e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=57272
-2024-11-03 21:25:53 - progress_bar.py[line:274] - INFO: epoch 023:    335 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.265, ntokens=18349.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.62, wps=3194.3, ups=0.17, wpb=18349.6, bsz=1024, num_updates=8930, lr=4.10432e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.3, wall=57330
-2024-11-03 21:26:49 - progress_bar.py[line:274] - INFO: epoch 023:    345 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.218, ntokens=18223.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.3, wps=3202, ups=0.18, wpb=18223.2, bsz=1024, num_updates=8940, lr=4.10295e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=57387
-2024-11-03 21:27:46 - progress_bar.py[line:274] - INFO: epoch 023:    355 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.31, ntokens=18800.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.92, wps=3295.8, ups=0.18, wpb=18800.1, bsz=1024, num_updates=8950, lr=4.10159e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.3, wall=57444
-2024-11-03 21:28:42 - progress_bar.py[line:274] - INFO: epoch 023:    365 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.264, ntokens=18393.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.61, wps=3291.6, ups=0.18, wpb=18393.3, bsz=1024, num_updates=8960, lr=4.10023e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=57499
-2024-11-03 21:29:38 - progress_bar.py[line:274] - INFO: epoch 023:    375 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.195, ntokens=17908.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.16, wps=3211.9, ups=0.18, wpb=17908.8, bsz=1024, num_updates=8970, lr=4.09887e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=57555
-2024-11-03 21:30:36 - progress_bar.py[line:274] - INFO: epoch 023:    385 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.241, ntokens=18107.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.45, wps=3146.2, ups=0.17, wpb=18107.4, bsz=1024, num_updates=8980, lr=4.09751e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.3, wall=57613
+2024-11-07 04:56:15 - trainer.py[line:703] - INFO: begin training epoch 23
+2024-11-07 04:56:15 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 04:56:39 - progress_bar.py[line:274] - INFO: epoch 023:      4 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.535, ntokens=17311.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=351.8, ups=0.02, wpb=17311.7, bsz=985.6, num_updates=8600, lr=4.14921e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=40, gb_free=13.5, wall=54918
+2024-11-07 04:57:31 - progress_bar.py[line:274] - INFO: epoch 023:     14 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.47, ntokens=17606.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3344.3, ups=0.19, wpb=17606.5, bsz=1024, num_updates=8610, lr=4.14785e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=54970
+2024-11-07 04:58:23 - progress_bar.py[line:274] - INFO: epoch 023:     24 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.469, ntokens=17645.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3392.3, ups=0.19, wpb=17645.9, bsz=1024, num_updates=8620, lr=4.14649e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=55022
+2024-11-07 04:59:17 - progress_bar.py[line:274] - INFO: epoch 023:     34 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.479, ntokens=17783.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3294.8, ups=0.19, wpb=17783.2, bsz=1024, num_updates=8630, lr=4.14513e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=39, gb_free=13.4, wall=55076
+2024-11-07 05:00:12 - progress_bar.py[line:274] - INFO: epoch 023:     44 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.466, ntokens=17734.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3255.6, ups=0.18, wpb=17734.8, bsz=1024, num_updates=8640, lr=4.14377e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=55131
+2024-11-07 05:01:06 - progress_bar.py[line:274] - INFO: epoch 023:     54 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.497, ntokens=17833.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3261.2, ups=0.18, wpb=17833.4, bsz=1024, num_updates=8650, lr=4.14241e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=55185
+2024-11-07 05:02:01 - progress_bar.py[line:274] - INFO: epoch 023:     64 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.477, ntokens=17736, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3223.6, ups=0.18, wpb=17736, bsz=1024, num_updates=8660, lr=4.14105e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=55240
+2024-11-07 05:02:56 - progress_bar.py[line:274] - INFO: epoch 023:     74 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.517, ntokens=17766.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.72, wps=3244, ups=0.18, wpb=17766.2, bsz=1024, num_updates=8670, lr=4.13969e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=55295
+2024-11-07 05:03:49 - progress_bar.py[line:274] - INFO: epoch 023:     84 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.496, ntokens=17804, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3383.1, ups=0.19, wpb=17804, bsz=1024, num_updates=8680, lr=4.13833e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=55348
+2024-11-07 05:04:41 - progress_bar.py[line:274] - INFO: epoch 023:     94 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.469, ntokens=17586.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3392.7, ups=0.19, wpb=17586.2, bsz=1024, num_updates=8690, lr=4.13696e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=55400
+2024-11-07 05:05:35 - progress_bar.py[line:274] - INFO: epoch 023:    104 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.477, ntokens=17701.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3271.3, ups=0.18, wpb=17701.1, bsz=1024, num_updates=8700, lr=4.1356e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=55454
+2024-11-07 05:06:29 - progress_bar.py[line:274] - INFO: epoch 023:    114 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.505, ntokens=17579.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.68, wps=3247.4, ups=0.18, wpb=17579.7, bsz=1024, num_updates=8710, lr=4.13424e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=55508
+2024-11-07 05:07:23 - progress_bar.py[line:274] - INFO: epoch 023:    124 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.494, ntokens=17962.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3339.9, ups=0.19, wpb=17962.2, bsz=1024, num_updates=8720, lr=4.13288e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=55562
+2024-11-07 05:08:17 - progress_bar.py[line:274] - INFO: epoch 023:    134 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.495, ntokens=17532.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3225.2, ups=0.18, wpb=17532.8, bsz=1024, num_updates=8730, lr=4.13152e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=55616
+2024-11-07 05:09:11 - progress_bar.py[line:274] - INFO: epoch 023:    144 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.465, ntokens=17612.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3283.5, ups=0.19, wpb=17612.1, bsz=1024, num_updates=8740, lr=4.13016e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=55670
+2024-11-07 05:10:04 - progress_bar.py[line:274] - INFO: epoch 023:    154 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.49, ntokens=17721.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.62, wps=3309.8, ups=0.19, wpb=17721.7, bsz=1024, num_updates=8750, lr=4.1288e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=55723
+2024-11-07 05:10:56 - progress_bar.py[line:274] - INFO: epoch 023:    164 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.494, ntokens=17833.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.63, wps=3437.9, ups=0.19, wpb=17833.4, bsz=1024, num_updates=8760, lr=4.12744e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=55775
+2024-11-07 05:11:48 - progress_bar.py[line:274] - INFO: epoch 023:    174 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.499, ntokens=17770.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3433.9, ups=0.19, wpb=17770.4, bsz=1024, num_updates=8770, lr=4.12608e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=55827
+2024-11-07 05:12:39 - progress_bar.py[line:274] - INFO: epoch 023:    184 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.49, ntokens=17604.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.62, wps=3424.8, ups=0.19, wpb=17604.9, bsz=1024, num_updates=8780, lr=4.12472e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=51, gb_free=13.4, wall=55878
+2024-11-07 05:13:32 - progress_bar.py[line:274] - INFO: epoch 023:    194 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.48, ntokens=17626.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3343.6, ups=0.19, wpb=17626.3, bsz=1024, num_updates=8790, lr=4.12336e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=49, gb_free=13.5, wall=55931
+2024-11-07 05:14:23 - progress_bar.py[line:274] - INFO: epoch 023:    204 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.523, ntokens=17997.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3499.3, ups=0.19, wpb=17997.2, bsz=1024, num_updates=8800, lr=4.122e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=51, gb_free=13.6, wall=55982
+2024-11-07 05:15:15 - progress_bar.py[line:274] - INFO: epoch 023:    214 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.47, ntokens=17825.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3438.2, ups=0.19, wpb=17825.3, bsz=1024, num_updates=8810, lr=4.12064e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=46, gb_free=13.4, wall=56034
+2024-11-07 05:16:07 - progress_bar.py[line:274] - INFO: epoch 023:    224 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.452, ntokens=17427.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.47, wps=3345.9, ups=0.19, wpb=17427.6, bsz=1024, num_updates=8820, lr=4.11928e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=41, gb_free=13.5, wall=56086
+2024-11-07 05:16:59 - progress_bar.py[line:274] - INFO: epoch 023:    234 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.527, ntokens=17942.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3489.3, ups=0.19, wpb=17942.7, bsz=1024, num_updates=8830, lr=4.11792e-05, gnorm=0.013, clip=0, loss_scale=65536, train_wall=37, gb_free=13.4, wall=56138
+2024-11-07 05:17:51 - progress_bar.py[line:274] - INFO: epoch 023:    244 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.421, ntokens=17166.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3282.3, ups=0.19, wpb=17166.3, bsz=1024, num_updates=8840, lr=4.11656e-05, gnorm=0.013, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=56190
+2024-11-07 05:18:12 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
+2024-11-07 05:18:49 - progress_bar.py[line:274] - INFO: epoch 023:    255 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.506, ntokens=17789.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.68, wps=3059.2, ups=0.17, wpb=17789.9, bsz=1024, num_updates=8850, lr=4.1152e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=33, gb_free=13.3, wall=56248
+2024-11-07 05:19:43 - progress_bar.py[line:274] - INFO: epoch 023:    265 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.485, ntokens=17787.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3332, ups=0.19, wpb=17787.6, bsz=1024, num_updates=8860, lr=4.11384e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=36, gb_free=13.6, wall=56302
+2024-11-07 05:20:35 - progress_bar.py[line:274] - INFO: epoch 023:    275 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.521, ntokens=17935.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=3417.2, ups=0.19, wpb=17935.7, bsz=1024, num_updates=8870, lr=4.11248e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=47, gb_free=13.5, wall=56354
+2024-11-07 05:21:28 - progress_bar.py[line:274] - INFO: epoch 023:    285 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.452, ntokens=17516.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.47, wps=3323.4, ups=0.19, wpb=17516.7, bsz=1024, num_updates=8880, lr=4.11112e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=49, gb_free=13.5, wall=56407
+2024-11-07 05:22:20 - progress_bar.py[line:274] - INFO: epoch 023:    295 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.512, ntokens=17710.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.7, wps=3411.5, ups=0.19, wpb=17710.8, bsz=1024, num_updates=8890, lr=4.10976e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=48, gb_free=13.5, wall=56459
+2024-11-07 05:23:12 - progress_bar.py[line:274] - INFO: epoch 023:    305 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.568, ntokens=18292.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3525.1, ups=0.19, wpb=18292.8, bsz=1024, num_updates=8900, lr=4.1084e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=49, gb_free=13.4, wall=56511
+2024-11-07 05:24:04 - progress_bar.py[line:274] - INFO: epoch 023:    315 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=18495.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3513.3, ups=0.19, wpb=18495.3, bsz=1024, num_updates=8910, lr=4.10704e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=56563
+2024-11-07 05:24:56 - progress_bar.py[line:274] - INFO: epoch 023:    325 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.58, ntokens=18161.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.98, wps=3481.6, ups=0.19, wpb=18161.8, bsz=1024, num_updates=8920, lr=4.10568e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=56615
+2024-11-07 05:25:48 - progress_bar.py[line:274] - INFO: epoch 023:    335 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.589, ntokens=18349.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3566, ups=0.19, wpb=18349.6, bsz=1024, num_updates=8930, lr=4.10432e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=51, gb_free=13.3, wall=56667
+2024-11-07 05:26:40 - progress_bar.py[line:274] - INFO: epoch 023:    345 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.558, ntokens=18223.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.89, wps=3503.8, ups=0.19, wpb=18223.2, bsz=1024, num_updates=8940, lr=4.10295e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=56719
+2024-11-07 05:27:32 - progress_bar.py[line:274] - INFO: epoch 023:    355 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.654, ntokens=18800.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.29, wps=3575.1, ups=0.19, wpb=18800.1, bsz=1024, num_updates=8950, lr=4.10159e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.3, wall=56771
+2024-11-07 05:28:25 - progress_bar.py[line:274] - INFO: epoch 023:    365 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.609, ntokens=18393.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3518.2, ups=0.19, wpb=18393.3, bsz=1024, num_updates=8960, lr=4.10023e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=56824
+2024-11-07 05:29:17 - progress_bar.py[line:274] - INFO: epoch 023:    375 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.518, ntokens=17908.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3431.3, ups=0.19, wpb=17908.8, bsz=1024, num_updates=8970, lr=4.09887e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=56876
+2024-11-07 05:30:09 - progress_bar.py[line:274] - INFO: epoch 023:    385 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.566, ntokens=18107.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3469.9, ups=0.19, wpb=18107.4, bsz=1024, num_updates=8980, lr=4.09751e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=56928
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 21:31:06 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 23 @ 8986 updates
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt
+2024-11-07 05:30:37 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 23 @ 8986 updates
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt
-2024-11-03 21:31:06 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt': File exists: File exists
-
+2024-11-07 05:30:37 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt': File exists
+: File exists
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 21:32:53 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-
+2024-11-07 05:32:28 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 5 seek offset 250000
-2024-11-03 21:34:45 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 23 @ 8986 updates, score 0) (writing took 218.91785066499142 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-slice_id 3 seek offset 150000
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+slice_id 7 seek offset 350000
 slice_id 2 seek offset 100000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+2024-11-07 05:34:20 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 23 @ 8986 updates, score 0) (writing took 222.71613502601394 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_23.pt
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+
 slice_id 6 seek offset 300000
-2024-11-03 21:35:39 - train.py[line:336] - INFO: end of epoch 23 (average epoch stats below)
-2024-11-03 21:35:39 - progress_bar.py[line:282] - INFO: epoch 023 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.141 | ntokens 17829.2 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.82 | wps 2641.7 | ups 0.15 | wpb 17829.2 | bsz 1023 | num_updates 8986 | lr 4.0967e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 2107 | gb_free 13.3 | wall 57916
-2024-11-03 21:35:39 - trainer.py[line:639] - INFO: loading train data for epoch 24
+slice_id 3 seek offset 150000
+slice_id 1 seek offset 50000
+slice_id 4 seek offset 200000
+slice_id 5 seek offset 250000
+2024-11-07 05:35:22 - train.py[line:336] - INFO: end of epoch 23 (average epoch stats below)
+2024-11-07 05:35:22 - progress_bar.py[line:282] - INFO: epoch 023 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.511 | ntokens 17830.2 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.7 | wps 2770.9 | ups 0.16 | wpb 17830.2 | bsz 1023 | num_updates 8986 | lr 4.0967e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1874 | gb_free 13.3 | wall 57241
+2024-11-07 05:35:22 - trainer.py[line:639] - INFO: loading train data for epoch 24
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 21:38:24 - trainer.py[line:703] - INFO: begin training epoch 24
-2024-11-03 21:38:24 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 21:38:47 - progress_bar.py[line:274] - INFO: epoch 024:      4 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.168, ntokens=17100.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.99, wps=347.8, ups=0.02, wpb=17100.6, bsz=985.6, num_updates=8990, lr=4.09615e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=42, gb_free=13.6, wall=58105
-2024-11-03 21:39:41 - progress_bar.py[line:274] - INFO: epoch 024:     14 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.102, ntokens=17684.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.58, wps=3327.9, ups=0.19, wpb=17684.8, bsz=1024, num_updates=9000, lr=4.09479e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=34, gb_free=13.5, wall=58158
-2024-11-03 21:40:34 - progress_bar.py[line:274] - INFO: epoch 024:     24 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.112, ntokens=17837.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.65, wps=3359, ups=0.19, wpb=17837.6, bsz=1024, num_updates=9010, lr=4.09343e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=35, gb_free=13.5, wall=58211
-2024-11-03 21:41:31 - progress_bar.py[line:274] - INFO: epoch 024:     34 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.131, ntokens=17921.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.76, wps=3131, ups=0.17, wpb=17921.9, bsz=1024, num_updates=9020, lr=4.09207e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=58268
-2024-11-03 21:42:28 - progress_bar.py[line:274] - INFO: epoch 024:     44 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.073, ntokens=17808.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.42, wps=3142.3, ups=0.18, wpb=17808.8, bsz=1024, num_updates=9030, lr=4.09071e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=58325
-2024-11-03 21:43:25 - progress_bar.py[line:274] - INFO: epoch 024:     54 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.054, ntokens=17455.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.31, wps=3016.7, ups=0.17, wpb=17455.2, bsz=1024, num_updates=9040, lr=4.08935e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=58383
-2024-11-03 21:44:23 - progress_bar.py[line:274] - INFO: epoch 024:     64 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.097, ntokens=17838.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.56, wps=3096.5, ups=0.17, wpb=17838.2, bsz=1024, num_updates=9050, lr=4.08799e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=58440
-2024-11-03 21:45:20 - progress_bar.py[line:274] - INFO: epoch 024:     74 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.134, ntokens=17871.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.78, wps=3126.6, ups=0.17, wpb=17871.4, bsz=1024, num_updates=9060, lr=4.08663e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=58497
-2024-11-03 21:46:17 - progress_bar.py[line:274] - INFO: epoch 024:     84 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.103, ntokens=17824.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.59, wps=3117.4, ups=0.17, wpb=17824.7, bsz=1024, num_updates=9070, lr=4.08527e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=58554
-2024-11-03 21:47:15 - progress_bar.py[line:274] - INFO: epoch 024:     94 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.078, ntokens=17649.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.44, wps=3071.7, ups=0.17, wpb=17649.2, bsz=1024, num_updates=9080, lr=4.08391e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=58612
-2024-11-03 21:48:12 - progress_bar.py[line:274] - INFO: epoch 024:    104 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.106, ntokens=17786.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.61, wps=3119, ups=0.18, wpb=17786.2, bsz=1024, num_updates=9090, lr=4.08255e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=58669
-2024-11-03 21:49:10 - progress_bar.py[line:274] - INFO: epoch 024:    114 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.089, ntokens=17381.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.51, wps=3010.1, ups=0.17, wpb=17381.3, bsz=1024, num_updates=9100, lr=4.08119e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=58727
-2024-11-03 21:50:07 - progress_bar.py[line:274] - INFO: epoch 024:    124 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.066, ntokens=17658.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.38, wps=3075.6, ups=0.17, wpb=17658.3, bsz=1024, num_updates=9110, lr=4.07983e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=58784
-2024-11-03 21:51:02 - progress_bar.py[line:274] - INFO: epoch 024:    134 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.065, ntokens=17456.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.37, wps=3161.6, ups=0.18, wpb=17456.3, bsz=1024, num_updates=9120, lr=4.07847e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=58839
-2024-11-03 21:52:00 - progress_bar.py[line:274] - INFO: epoch 024:    144 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.119, ntokens=17872, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.69, wps=3108, ups=0.17, wpb=17872, bsz=1024, num_updates=9130, lr=4.07711e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=58897
-2024-11-03 21:52:56 - progress_bar.py[line:274] - INFO: epoch 024:    154 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.064, ntokens=17487.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.36, wps=3094.7, ups=0.18, wpb=17487.6, bsz=1024, num_updates=9140, lr=4.07575e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.3, wall=58953
-2024-11-03 21:53:50 - progress_bar.py[line:274] - INFO: epoch 024:    164 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.1, ntokens=17893.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.58, wps=3331.9, ups=0.19, wpb=17893.4, bsz=1024, num_updates=9150, lr=4.07439e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=49, gb_free=13.4, wall=59007
-2024-11-03 21:54:46 - progress_bar.py[line:274] - INFO: epoch 024:    174 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.073, ntokens=17663.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.42, wps=3156.7, ups=0.18, wpb=17663.8, bsz=1024, num_updates=9160, lr=4.07303e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=43, gb_free=13.4, wall=59063
-2024-11-03 21:55:41 - progress_bar.py[line:274] - INFO: epoch 024:    184 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.063, ntokens=17506.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.36, wps=3205.9, ups=0.18, wpb=17506.5, bsz=1024, num_updates=9170, lr=4.07167e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=59118
-2024-11-03 21:56:37 - progress_bar.py[line:274] - INFO: epoch 024:    194 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.035, ntokens=17482.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.2, wps=3077.6, ups=0.18, wpb=17482.1, bsz=1024, num_updates=9180, lr=4.07031e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=51, gb_free=13.5, wall=59174
-2024-11-03 21:57:35 - progress_bar.py[line:274] - INFO: epoch 024:    204 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.121, ntokens=18053.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.7, wps=3136.4, ups=0.17, wpb=18053.6, bsz=1024, num_updates=9190, lr=4.06894e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=59232
-2024-11-03 21:58:33 - progress_bar.py[line:274] - INFO: epoch 024:    214 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.087, ntokens=18087, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.5, wps=3129.1, ups=0.17, wpb=18087, bsz=1024, num_updates=9200, lr=4.06758e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=59290
-2024-11-03 21:59:30 - progress_bar.py[line:274] - INFO: epoch 024:    224 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.104, ntokens=18017.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.6, wps=3120.5, ups=0.17, wpb=18017.8, bsz=1024, num_updates=9210, lr=4.06622e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=59348
-2024-11-03 22:00:28 - progress_bar.py[line:274] - INFO: epoch 024:    234 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.13, ntokens=17766.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.76, wps=3083.1, ups=0.17, wpb=17766.6, bsz=1024, num_updates=9220, lr=4.06486e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=59405
-2024-11-03 22:01:23 - progress_bar.py[line:274] - INFO: epoch 024:    244 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.097, ntokens=17552.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.56, wps=3220.4, ups=0.18, wpb=17552.1, bsz=1024, num_updates=9230, lr=4.0635e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=59460
-2024-11-03 22:02:20 - progress_bar.py[line:274] - INFO: epoch 024:    254 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.025, ntokens=17570.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.14, wps=3049.9, ups=0.17, wpb=17570.6, bsz=1024, num_updates=9240, lr=4.06214e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=59517
-2024-11-03 22:03:18 - progress_bar.py[line:274] - INFO: epoch 024:    264 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.174, ntokens=18045.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.03, wps=3106.6, ups=0.17, wpb=18045.8, bsz=1024, num_updates=9250, lr=4.06078e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=59575
-2024-11-03 22:04:16 - progress_bar.py[line:274] - INFO: epoch 024:    274 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.052, ntokens=17679.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.29, wps=3080.9, ups=0.17, wpb=17679.3, bsz=1024, num_updates=9260, lr=4.05942e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=59633
-2024-11-03 22:05:13 - progress_bar.py[line:274] - INFO: epoch 024:    284 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.083, ntokens=17651.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.47, wps=3077, ups=0.17, wpb=17651.9, bsz=1024, num_updates=9270, lr=4.05806e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=59690
-2024-11-03 22:06:11 - progress_bar.py[line:274] - INFO: epoch 024:    294 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.112, ntokens=17685.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.65, wps=3058, ups=0.17, wpb=17685.5, bsz=1024, num_updates=9280, lr=4.0567e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=59748
-2024-11-03 22:07:09 - progress_bar.py[line:274] - INFO: epoch 024:    304 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.224, ntokens=18315.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.34, wps=3156.9, ups=0.17, wpb=18315.8, bsz=1024, num_updates=9290, lr=4.05534e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.3, wall=59806
-2024-11-03 22:08:07 - progress_bar.py[line:274] - INFO: epoch 024:    314 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.234, ntokens=18095.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.41, wps=3135.6, ups=0.17, wpb=18095.4, bsz=1024, num_updates=9300, lr=4.05398e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.3, wall=59864
-2024-11-03 22:09:04 - progress_bar.py[line:274] - INFO: epoch 024:    324 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.243, ntokens=18242.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.46, wps=3152.3, ups=0.17, wpb=18242.8, bsz=1024, num_updates=9310, lr=4.05262e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=59922
-2024-11-03 22:10:02 - progress_bar.py[line:274] - INFO: epoch 024:    334 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.299, ntokens=18663.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.84, wps=3245, ups=0.17, wpb=18663.3, bsz=1024, num_updates=9320, lr=4.05126e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=59979
-2024-11-03 22:11:00 - progress_bar.py[line:274] - INFO: epoch 024:    344 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.168, ntokens=18129.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.99, wps=3148.8, ups=0.17, wpb=18129.8, bsz=1024, num_updates=9330, lr=4.0499e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=60037
-2024-11-03 22:11:57 - progress_bar.py[line:274] - INFO: epoch 024:    354 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.258, ntokens=18563.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.57, wps=3228.2, ups=0.17, wpb=18563.8, bsz=1024, num_updates=9340, lr=4.04854e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=57, gb_free=13.2, wall=60094
-2024-11-03 22:12:55 - progress_bar.py[line:274] - INFO: epoch 024:    364 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.195, ntokens=18122.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.16, wps=3138.8, ups=0.17, wpb=18122.9, bsz=1024, num_updates=9350, lr=4.04718e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=60152
-2024-11-03 22:13:53 - progress_bar.py[line:274] - INFO: epoch 024:    374 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.207, ntokens=18056.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.23, wps=3121.6, ups=0.17, wpb=18056.8, bsz=1024, num_updates=9360, lr=4.04582e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=60210
-2024-11-03 22:14:50 - progress_bar.py[line:274] - INFO: epoch 024:    384 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.186, ntokens=17907.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.1, wps=3133, ups=0.17, wpb=17907.9, bsz=1024, num_updates=9370, lr=4.04446e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=60267
+2024-11-07 05:38:04 - trainer.py[line:703] - INFO: begin training epoch 24
+2024-11-07 05:38:04 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 05:38:29 - progress_bar.py[line:274] - INFO: epoch 024:      4 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.512, ntokens=17100.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=342.2, ups=0.02, wpb=17100.6, bsz=985.6, num_updates=8990, lr=4.09615e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=41, gb_free=13.6, wall=57428
+2024-11-07 05:39:22 - progress_bar.py[line:274] - INFO: epoch 024:     14 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.476, ntokens=17684.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3327, ups=0.19, wpb=17684.8, bsz=1024, num_updates=9000, lr=4.09479e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=57481
+2024-11-07 05:40:16 - progress_bar.py[line:274] - INFO: epoch 024:     24 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.506, ntokens=17837.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.68, wps=3322.6, ups=0.19, wpb=17837.6, bsz=1024, num_updates=9010, lr=4.09343e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=57535
+2024-11-07 05:41:09 - progress_bar.py[line:274] - INFO: epoch 024:     34 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.482, ntokens=17921.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3391.1, ups=0.19, wpb=17921.9, bsz=1024, num_updates=9020, lr=4.09207e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=57587
+2024-11-07 05:42:03 - progress_bar.py[line:274] - INFO: epoch 024:     44 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.488, ntokens=17808.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.61, wps=3270.4, ups=0.18, wpb=17808.8, bsz=1024, num_updates=9030, lr=4.09071e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=37, gb_free=13.6, wall=57642
+2024-11-07 05:43:00 - progress_bar.py[line:274] - INFO: epoch 024:     54 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.444, ntokens=17455.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.44, wps=3082.3, ups=0.18, wpb=17455.2, bsz=1024, num_updates=9040, lr=4.08935e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=45, gb_free=13.3, wall=57699
+2024-11-07 05:43:54 - progress_bar.py[line:274] - INFO: epoch 024:     64 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.504, ntokens=17838.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.67, wps=3298.3, ups=0.18, wpb=17838.2, bsz=1024, num_updates=9050, lr=4.08799e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=57753
+2024-11-07 05:44:49 - progress_bar.py[line:274] - INFO: epoch 024:     74 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.495, ntokens=17871.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3213, ups=0.18, wpb=17871.4, bsz=1024, num_updates=9060, lr=4.08663e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=57808
+2024-11-07 05:45:44 - progress_bar.py[line:274] - INFO: epoch 024:     84 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.47, ntokens=17824.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3251.5, ups=0.18, wpb=17824.7, bsz=1024, num_updates=9070, lr=4.08527e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=57863
+2024-11-07 05:46:39 - progress_bar.py[line:274] - INFO: epoch 024:     94 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.481, ntokens=17649.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3223.9, ups=0.18, wpb=17649.2, bsz=1024, num_updates=9080, lr=4.08391e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=57918
+2024-11-07 05:47:32 - progress_bar.py[line:274] - INFO: epoch 024:    104 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.481, ntokens=17786.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3348.7, ups=0.19, wpb=17786.2, bsz=1024, num_updates=9090, lr=4.08255e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=57971
+2024-11-07 05:48:24 - progress_bar.py[line:274] - INFO: epoch 024:    114 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.46, ntokens=17381.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3314.2, ups=0.19, wpb=17381.3, bsz=1024, num_updates=9100, lr=4.08119e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=58023
+2024-11-07 05:49:17 - progress_bar.py[line:274] - INFO: epoch 024:    124 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.467, ntokens=17658.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3384.7, ups=0.19, wpb=17658.3, bsz=1024, num_updates=9110, lr=4.07983e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=58076
+2024-11-07 05:50:10 - progress_bar.py[line:274] - INFO: epoch 024:    134 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.465, ntokens=17456.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3286.8, ups=0.19, wpb=17456.3, bsz=1024, num_updates=9120, lr=4.07847e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=58129
+2024-11-07 05:51:02 - progress_bar.py[line:274] - INFO: epoch 024:    144 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.5, ntokens=17872, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.66, wps=3445.7, ups=0.19, wpb=17872, bsz=1024, num_updates=9130, lr=4.07711e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=58181
+2024-11-07 05:51:54 - progress_bar.py[line:274] - INFO: epoch 024:    154 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.447, ntokens=17487.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.45, wps=3313.3, ups=0.19, wpb=17487.6, bsz=1024, num_updates=9140, lr=4.07575e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=51, gb_free=13.3, wall=58233
+2024-11-07 05:52:47 - progress_bar.py[line:274] - INFO: epoch 024:    164 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.485, ntokens=17893.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3411.8, ups=0.19, wpb=17893.4, bsz=1024, num_updates=9150, lr=4.07439e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=58286
+2024-11-07 05:53:39 - progress_bar.py[line:274] - INFO: epoch 024:    174 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.473, ntokens=17663.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.55, wps=3361.2, ups=0.19, wpb=17663.8, bsz=1024, num_updates=9160, lr=4.07303e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=50, gb_free=13.4, wall=58338
+2024-11-07 05:54:32 - progress_bar.py[line:274] - INFO: epoch 024:    184 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.472, ntokens=17506.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.55, wps=3309.5, ups=0.19, wpb=17506.5, bsz=1024, num_updates=9170, lr=4.07167e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=48, gb_free=13.4, wall=58391
+2024-11-07 05:55:25 - progress_bar.py[line:274] - INFO: epoch 024:    194 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.46, ntokens=17482.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3325.7, ups=0.19, wpb=17482.1, bsz=1024, num_updates=9180, lr=4.07031e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=49, gb_free=13.5, wall=58444
+2024-11-07 05:56:18 - progress_bar.py[line:274] - INFO: epoch 024:    204 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.524, ntokens=18053.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3418.2, ups=0.19, wpb=18053.6, bsz=1024, num_updates=9190, lr=4.06894e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=48, gb_free=13.6, wall=58497
+2024-11-07 05:57:11 - progress_bar.py[line:274] - INFO: epoch 024:    214 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.511, ntokens=18087, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.7, wps=3392.3, ups=0.19, wpb=18087, bsz=1024, num_updates=9200, lr=4.06758e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=58550
+2024-11-07 05:58:03 - progress_bar.py[line:274] - INFO: epoch 024:    224 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.527, ntokens=18017.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3443.2, ups=0.19, wpb=18017.8, bsz=1024, num_updates=9210, lr=4.06622e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=58602
+2024-11-07 05:58:55 - progress_bar.py[line:274] - INFO: epoch 024:    234 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.486, ntokens=17766.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3437.1, ups=0.19, wpb=17766.6, bsz=1024, num_updates=9220, lr=4.06486e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=58654
+2024-11-07 05:59:48 - progress_bar.py[line:274] - INFO: epoch 024:    244 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.474, ntokens=17552.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3338, ups=0.19, wpb=17552.1, bsz=1024, num_updates=9230, lr=4.0635e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=58707
+2024-11-07 06:00:40 - progress_bar.py[line:274] - INFO: epoch 024:    254 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.44, ntokens=17570.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3342.1, ups=0.19, wpb=17570.6, bsz=1024, num_updates=9240, lr=4.06214e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=58759
+2024-11-07 06:01:33 - progress_bar.py[line:274] - INFO: epoch 024:    264 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.539, ntokens=18045.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3449.2, ups=0.19, wpb=18045.8, bsz=1024, num_updates=9250, lr=4.06078e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=58811
+2024-11-07 06:02:24 - progress_bar.py[line:274] - INFO: epoch 024:    274 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.464, ntokens=17679.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3405.1, ups=0.19, wpb=17679.3, bsz=1024, num_updates=9260, lr=4.05942e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=58863
+2024-11-07 06:03:17 - progress_bar.py[line:274] - INFO: epoch 024:    284 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.48, ntokens=17651.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3342.3, ups=0.19, wpb=17651.9, bsz=1024, num_updates=9270, lr=4.05806e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=45, gb_free=13.6, wall=58916
+2024-11-07 06:04:10 - progress_bar.py[line:274] - INFO: epoch 024:    294 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.499, ntokens=17685.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3327.7, ups=0.19, wpb=17685.5, bsz=1024, num_updates=9280, lr=4.0567e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=43, gb_free=13.5, wall=58969
+2024-11-07 06:05:02 - progress_bar.py[line:274] - INFO: epoch 024:    304 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.556, ntokens=18315.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3534.6, ups=0.19, wpb=18315.8, bsz=1024, num_updates=9290, lr=4.05534e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=45, gb_free=13.3, wall=59021
+2024-11-07 06:05:55 - progress_bar.py[line:274] - INFO: epoch 024:    314 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.586, ntokens=18095.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3441.6, ups=0.19, wpb=18095.4, bsz=1024, num_updates=9300, lr=4.05398e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=51, gb_free=13.3, wall=59074
+2024-11-07 06:06:47 - progress_bar.py[line:274] - INFO: epoch 024:    324 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.566, ntokens=18242.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3521.2, ups=0.19, wpb=18242.8, bsz=1024, num_updates=9310, lr=4.05262e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=59125
+2024-11-07 06:07:39 - progress_bar.py[line:274] - INFO: epoch 024:    334 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.636, ntokens=18663.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3538.1, ups=0.19, wpb=18663.3, bsz=1024, num_updates=9320, lr=4.05126e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=59178
+2024-11-07 06:08:32 - progress_bar.py[line:274] - INFO: epoch 024:    344 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.547, ntokens=18129.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=3457.3, ups=0.19, wpb=18129.8, bsz=1024, num_updates=9330, lr=4.0499e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=59231
+2024-11-07 06:09:24 - progress_bar.py[line:274] - INFO: epoch 024:    354 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.608, ntokens=18563.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.1, wps=3578.5, ups=0.19, wpb=18563.8, bsz=1024, num_updates=9340, lr=4.04854e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.2, wall=59283
+2024-11-07 06:10:16 - progress_bar.py[line:274] - INFO: epoch 024:    364 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.553, ntokens=18122.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3436.8, ups=0.19, wpb=18122.9, bsz=1024, num_updates=9350, lr=4.04718e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=59335
+2024-11-07 06:11:09 - progress_bar.py[line:274] - INFO: epoch 024:    374 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.54, ntokens=18056.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.82, wps=3406.8, ups=0.19, wpb=18056.8, bsz=1024, num_updates=9360, lr=4.04582e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=59388
+2024-11-07 06:12:02 - progress_bar.py[line:274] - INFO: epoch 024:    384 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.538, ntokens=17907.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3433.1, ups=0.19, wpb=17907.9, bsz=1024, num_updates=9370, lr=4.04446e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=59440
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 22:15:26 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 24 @ 9377 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt
-2024-11-03 22:15:26 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt': File exists
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt2024-11-07 06:12:35 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 24 @ 9377 updates
+
+
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt
+2024-11-07 06:12:35 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt': File exists: File exists
+
 : File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 22:17:13 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+2024-11-07 06:14:20 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+
+
+slice_id 2 seek offset 100000
 slice_id 3 seek offset 150000
-2024-11-03 22:19:06 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 24 @ 9377 updates, score 0) (writing took 219.91394419397693 seconds)
+slice_id 7 seek offset 350000
+2024-11-07 06:16:12 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 24 @ 9377 updates, score 0) (writing took 217.80983646999812 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_24.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+slice_id 6 seek offset 300000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-03 22:20:01 - train.py[line:336] - INFO: end of epoch 24 (average epoch stats below)
-2024-11-03 22:20:01 - progress_bar.py[line:282] - INFO: epoch 024 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.125 | ntokens 17837.7 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.73 | wps 2620.6 | ups 0.15 | wpb 17837.7 | bsz 1023 | num_updates 9377 | lr 4.04351e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 2121 | gb_free 13.4 | wall 60578
-2024-11-03 22:20:01 - trainer.py[line:639] - INFO: loading train data for epoch 25
+slice_id 4 seek offset 200000
+slice_id 5 seek offset 250000
+2024-11-07 06:17:07 - train.py[line:336] - INFO: end of epoch 24 (average epoch stats below)
+2024-11-07 06:17:07 - progress_bar.py[line:282] - INFO: epoch 024 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.505 | ntokens 17837.7 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.68 | wps 2784.1 | ups 0.16 | wpb 17837.7 | bsz 1023 | num_updates 9377 | lr 4.04351e-05 | gnorm 0.007 | clip 0 | loss_scale 65536 | train_wall 1918 | gb_free 13.4 | wall 59746
+2024-11-07 06:17:07 - trainer.py[line:639] - INFO: loading train data for epoch 25
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 22:22:45 - trainer.py[line:703] - INFO: begin training epoch 25
-2024-11-03 22:22:45 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 22:23:03 - progress_bar.py[line:274] - INFO: epoch 025:      3 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.177, ntokens=17337.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.04, wps=351.4, ups=0.02, wpb=17337.4, bsz=985.6, num_updates=9380, lr=4.0431e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=45, gb_free=13.6, wall=60760
-2024-11-03 22:23:57 - progress_bar.py[line:274] - INFO: epoch 025:     13 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.079, ntokens=17519.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.45, wps=3251.4, ups=0.19, wpb=17519.7, bsz=1024, num_updates=9390, lr=4.04174e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=60814
-2024-11-03 22:24:53 - progress_bar.py[line:274] - INFO: epoch 025:     23 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.067, ntokens=17852.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.38, wps=3190.6, ups=0.18, wpb=17852.8, bsz=1024, num_updates=9400, lr=4.04038e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=43, gb_free=13.6, wall=60870
-2024-11-03 22:25:47 - progress_bar.py[line:274] - INFO: epoch 025:     33 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.087, ntokens=17961.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.5, wps=3303.3, ups=0.18, wpb=17961.6, bsz=1024, num_updates=9410, lr=4.03902e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=60924
-2024-11-03 22:26:44 - progress_bar.py[line:274] - INFO: epoch 025:     43 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.046, ntokens=17522.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.26, wps=3117.7, ups=0.18, wpb=17522.3, bsz=1024, num_updates=9420, lr=4.03766e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=60981
-2024-11-03 22:27:41 - progress_bar.py[line:274] - INFO: epoch 025:     53 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.03, ntokens=17646, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.17, wps=3098.7, ups=0.18, wpb=17646, bsz=1024, num_updates=9430, lr=4.0363e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=61038
-2024-11-03 22:28:36 - progress_bar.py[line:274] - INFO: epoch 025:     63 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.119, ntokens=18090.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.69, wps=3269.2, ups=0.18, wpb=18090.1, bsz=1024, num_updates=9440, lr=4.03493e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=61093
-2024-11-03 22:29:33 - progress_bar.py[line:274] - INFO: epoch 025:     73 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.08, ntokens=17496.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.45, wps=3081, ups=0.18, wpb=17496.1, bsz=1024, num_updates=9450, lr=4.03357e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=61150
-2024-11-03 22:30:29 - progress_bar.py[line:274] - INFO: epoch 025:     83 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.06, ntokens=17600.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.34, wps=3097.2, ups=0.18, wpb=17600.9, bsz=1024, num_updates=9460, lr=4.03221e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=42, gb_free=13.5, wall=61207
-2024-11-03 22:31:28 - progress_bar.py[line:274] - INFO: epoch 025:     93 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.048, ntokens=17612.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.27, wps=3031, ups=0.17, wpb=17612.7, bsz=1024, num_updates=9470, lr=4.03085e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=61265
-2024-11-03 22:32:22 - progress_bar.py[line:274] - INFO: epoch 025:    103 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.074, ntokens=17585.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.42, wps=3210, ups=0.18, wpb=17585.5, bsz=1024, num_updates=9480, lr=4.02949e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=39, gb_free=13.6, wall=61319
-2024-11-03 22:33:19 - progress_bar.py[line:274] - INFO: epoch 025:    113 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.072, ntokens=17500.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.41, wps=3100.2, ups=0.18, wpb=17500.3, bsz=1024, num_updates=9490, lr=4.02813e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=61376
-2024-11-03 22:34:16 - progress_bar.py[line:274] - INFO: epoch 025:    123 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.041, ntokens=17844.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=3133.2, ups=0.18, wpb=17844.5, bsz=1024, num_updates=9500, lr=4.02677e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=61433
-2024-11-03 22:35:13 - progress_bar.py[line:274] - INFO: epoch 025:    133 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.087, ntokens=17742.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.5, wps=3103.8, ups=0.17, wpb=17742.7, bsz=1024, num_updates=9510, lr=4.02541e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=61490
-2024-11-03 22:36:09 - progress_bar.py[line:274] - INFO: epoch 025:    143 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.089, ntokens=17698.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.51, wps=3133.5, ups=0.18, wpb=17698.8, bsz=1024, num_updates=9520, lr=4.02405e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=61547
-2024-11-03 22:37:06 - progress_bar.py[line:274] - INFO: epoch 025:    153 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.067, ntokens=17569.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.38, wps=3102.1, ups=0.18, wpb=17569.1, bsz=1024, num_updates=9530, lr=4.02269e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=61603
-2024-11-03 22:38:04 - progress_bar.py[line:274] - INFO: epoch 025:    163 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.025, ntokens=17429.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.14, wps=3026.3, ups=0.17, wpb=17429.7, bsz=1024, num_updates=9540, lr=4.02133e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=61661
-2024-11-03 22:39:01 - progress_bar.py[line:274] - INFO: epoch 025:    173 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.092, ntokens=17728.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.53, wps=3092.1, ups=0.17, wpb=17728.7, bsz=1024, num_updates=9550, lr=4.01997e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=61718
-2024-11-03 22:39:55 - progress_bar.py[line:274] - INFO: epoch 025:    183 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.044, ntokens=17519.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.25, wps=3226.6, ups=0.18, wpb=17519.8, bsz=1024, num_updates=9560, lr=4.01861e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=61772
-2024-11-03 22:40:52 - progress_bar.py[line:274] - INFO: epoch 025:    193 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.1, ntokens=17907.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.57, wps=3161.4, ups=0.18, wpb=17907.1, bsz=1024, num_updates=9570, lr=4.01725e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.3, wall=61829
-2024-11-03 22:41:50 - progress_bar.py[line:274] - INFO: epoch 025:    203 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.053, ntokens=17720.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.3, wps=3078, ups=0.17, wpb=17720.5, bsz=1024, num_updates=9580, lr=4.01589e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=61887
-2024-11-03 22:42:46 - progress_bar.py[line:274] - INFO: epoch 025:    213 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.044, ntokens=17981.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.24, wps=3203.1, ups=0.18, wpb=17981.7, bsz=1024, num_updates=9590, lr=4.01453e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=61943
-2024-11-03 22:43:40 - progress_bar.py[line:274] - INFO: epoch 025:    223 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.01, ntokens=17513.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.05, wps=3212.6, ups=0.18, wpb=17513.9, bsz=1024, num_updates=9600, lr=4.01317e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=61997
-2024-11-03 22:44:38 - progress_bar.py[line:274] - INFO: epoch 025:    233 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.066, ntokens=17692.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.38, wps=3059.5, ups=0.17, wpb=17692.8, bsz=1024, num_updates=9610, lr=4.01181e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=62055
-2024-11-03 22:45:36 - progress_bar.py[line:274] - INFO: epoch 025:    243 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.116, ntokens=17537, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.67, wps=3033.2, ups=0.17, wpb=17537, bsz=1024, num_updates=9620, lr=4.01045e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=58, gb_free=13.6, wall=62113
-2024-11-03 22:46:32 - progress_bar.py[line:274] - INFO: epoch 025:    253 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.023, ntokens=17396.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=3090.5, ups=0.18, wpb=17396.5, bsz=1024, num_updates=9630, lr=4.00909e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=62169
-2024-11-03 22:47:26 - progress_bar.py[line:274] - INFO: epoch 025:    263 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.068, ntokens=17555.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.38, wps=3273, ups=0.19, wpb=17555.8, bsz=1024, num_updates=9640, lr=4.00773e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=62223
-2024-11-03 22:48:23 - progress_bar.py[line:274] - INFO: epoch 025:    273 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.051, ntokens=17808.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.29, wps=3094, ups=0.17, wpb=17808.3, bsz=1024, num_updates=9650, lr=4.00637e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=62280
-2024-11-03 22:49:19 - progress_bar.py[line:274] - INFO: epoch 025:    283 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.026, ntokens=17301.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.15, wps=3082.5, ups=0.18, wpb=17301.5, bsz=1024, num_updates=9660, lr=4.00501e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=62337
-2024-11-03 22:50:15 - progress_bar.py[line:274] - INFO: epoch 025:    293 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.047, ntokens=17540.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.27, wps=3146.5, ups=0.18, wpb=17540.3, bsz=1024, num_updates=9670, lr=4.00365e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=62392
-2024-11-03 22:51:14 - progress_bar.py[line:274] - INFO: epoch 025:    303 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.195, ntokens=18161.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.16, wps=3101.3, ups=0.17, wpb=18161.7, bsz=1024, num_updates=9680, lr=4.00229e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=62451
-2024-11-03 22:52:10 - progress_bar.py[line:274] - INFO: epoch 025:    313 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.201, ntokens=18148.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.2, wps=3221.1, ups=0.18, wpb=18148.4, bsz=1024, num_updates=9690, lr=4.00093e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=62507
-2024-11-03 22:53:07 - progress_bar.py[line:274] - INFO: epoch 025:    323 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.204, ntokens=18133.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.21, wps=3182.1, ups=0.18, wpb=18133.7, bsz=1024, num_updates=9700, lr=3.99956e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=62564
-2024-11-03 22:54:05 - progress_bar.py[line:274] - INFO: epoch 025:    333 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.248, ntokens=18519.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.5, wps=3221.5, ups=0.17, wpb=18519.8, bsz=1024, num_updates=9710, lr=3.9982e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.1, wall=62622
-2024-11-03 22:54:57 - progress_bar.py[line:274] - INFO: epoch 025:    343 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.142, ntokens=18058.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.83, wps=3426.3, ups=0.19, wpb=18058.3, bsz=1024, num_updates=9720, lr=3.99684e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=62674
-2024-11-03 22:55:54 - progress_bar.py[line:274] - INFO: epoch 025:    353 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.24, ntokens=18580.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.45, wps=3279.1, ups=0.18, wpb=18580.1, bsz=1024, num_updates=9730, lr=3.99548e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=62731
-2024-11-03 22:56:52 - progress_bar.py[line:274] - INFO: epoch 025:    363 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.131, ntokens=17780.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.76, wps=3072.3, ups=0.17, wpb=17780.2, bsz=1024, num_updates=9740, lr=3.99412e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.3, wall=62789
-2024-11-03 22:57:48 - progress_bar.py[line:274] - INFO: epoch 025:    373 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.167, ntokens=18042.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.98, wps=3229.1, ups=0.18, wpb=18042.6, bsz=1024, num_updates=9750, lr=3.99276e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=62845
-2024-11-03 22:58:41 - progress_bar.py[line:274] - INFO: epoch 025:    383 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.178, ntokens=18214.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.05, wps=3396, ups=0.19, wpb=18214.3, bsz=1024, num_updates=9760, lr=3.9914e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=62898
+2024-11-07 06:19:46 - trainer.py[line:703] - INFO: begin training epoch 25
+2024-11-07 06:19:46 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 06:20:04 - progress_bar.py[line:274] - INFO: epoch 025:      3 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.527, ntokens=17337.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=359.4, ups=0.02, wpb=17337.4, bsz=985.6, num_updates=9380, lr=4.0431e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=42, gb_free=13.6, wall=59923
+2024-11-07 06:20:56 - progress_bar.py[line:274] - INFO: epoch 025:     13 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.458, ntokens=17519.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3357.4, ups=0.19, wpb=17519.7, bsz=1024, num_updates=9390, lr=4.04174e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=59975
+2024-11-07 06:21:50 - progress_bar.py[line:274] - INFO: epoch 025:     23 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.492, ntokens=17852.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.62, wps=3344.3, ups=0.19, wpb=17852.8, bsz=1024, num_updates=9400, lr=4.04038e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=60028
+2024-11-07 06:22:42 - progress_bar.py[line:274] - INFO: epoch 025:     33 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.47, ntokens=17961.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3439.1, ups=0.19, wpb=17961.6, bsz=1024, num_updates=9410, lr=4.03902e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=60081
+2024-11-07 06:23:34 - progress_bar.py[line:274] - INFO: epoch 025:     43 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.456, ntokens=17522.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.49, wps=3347, ups=0.19, wpb=17522.3, bsz=1024, num_updates=9420, lr=4.03766e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=60133
+2024-11-07 06:24:26 - progress_bar.py[line:274] - INFO: epoch 025:     53 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.444, ntokens=17646, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.44, wps=3393.5, ups=0.19, wpb=17646, bsz=1024, num_updates=9430, lr=4.0363e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=60185
+2024-11-07 06:25:18 - progress_bar.py[line:274] - INFO: epoch 025:     63 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.538, ntokens=18090.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3462.3, ups=0.19, wpb=18090.1, bsz=1024, num_updates=9440, lr=4.03493e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=60237
+2024-11-07 06:26:11 - progress_bar.py[line:274] - INFO: epoch 025:     73 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.453, ntokens=17496.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.48, wps=3314.3, ups=0.19, wpb=17496.1, bsz=1024, num_updates=9450, lr=4.03357e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=60290
+2024-11-07 06:27:05 - progress_bar.py[line:274] - INFO: epoch 025:     83 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.464, ntokens=17600.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3291.6, ups=0.19, wpb=17600.9, bsz=1024, num_updates=9460, lr=4.03221e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=60343
+2024-11-07 06:27:57 - progress_bar.py[line:274] - INFO: epoch 025:     93 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.462, ntokens=17612.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.51, wps=3361.7, ups=0.19, wpb=17612.7, bsz=1024, num_updates=9470, lr=4.03085e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=60396
+2024-11-07 06:28:50 - progress_bar.py[line:274] - INFO: epoch 025:    103 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.474, ntokens=17585.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3341.7, ups=0.19, wpb=17585.5, bsz=1024, num_updates=9480, lr=4.02949e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=60449
+2024-11-07 06:29:42 - progress_bar.py[line:274] - INFO: epoch 025:    113 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.474, ntokens=17500.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3315.6, ups=0.19, wpb=17500.3, bsz=1024, num_updates=9490, lr=4.02813e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=60501
+2024-11-07 06:30:37 - progress_bar.py[line:274] - INFO: epoch 025:    123 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.5, ntokens=17844.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.66, wps=3267.2, ups=0.18, wpb=17844.5, bsz=1024, num_updates=9500, lr=4.02677e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=60556
+2024-11-07 06:31:35 - progress_bar.py[line:274] - INFO: epoch 025:    133 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.5, ntokens=17742.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.66, wps=3083.9, ups=0.17, wpb=17742.7, bsz=1024, num_updates=9510, lr=4.02541e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=60613
+2024-11-07 06:32:29 - progress_bar.py[line:274] - INFO: epoch 025:    143 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.482, ntokens=17698.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3236.4, ups=0.18, wpb=17698.8, bsz=1024, num_updates=9520, lr=4.02405e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=60668
+2024-11-07 06:33:21 - progress_bar.py[line:274] - INFO: epoch 025:    153 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.462, ntokens=17569.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.51, wps=3381.2, ups=0.19, wpb=17569.1, bsz=1024, num_updates=9530, lr=4.02269e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=60720
+2024-11-07 06:34:14 - progress_bar.py[line:274] - INFO: epoch 025:    163 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.423, ntokens=17429.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3282.5, ups=0.19, wpb=17429.7, bsz=1024, num_updates=9540, lr=4.02133e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=60773
+2024-11-07 06:35:10 - progress_bar.py[line:274] - INFO: epoch 025:    173 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.484, ntokens=17728.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3185.3, ups=0.18, wpb=17728.7, bsz=1024, num_updates=9550, lr=4.01997e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=46, gb_free=13.4, wall=60829
+2024-11-07 06:36:05 - progress_bar.py[line:274] - INFO: epoch 025:    183 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.451, ntokens=17519.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.47, wps=3172.9, ups=0.18, wpb=17519.8, bsz=1024, num_updates=9560, lr=4.01861e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=60884
+2024-11-07 06:36:57 - progress_bar.py[line:274] - INFO: epoch 025:    193 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.514, ntokens=17907.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.71, wps=3442.9, ups=0.19, wpb=17907.1, bsz=1024, num_updates=9570, lr=4.01725e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=51, gb_free=13.3, wall=60936
+2024-11-07 06:37:49 - progress_bar.py[line:274] - INFO: epoch 025:    203 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.471, ntokens=17720.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3436.3, ups=0.19, wpb=17720.5, bsz=1024, num_updates=9580, lr=4.01589e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=42, gb_free=13.5, wall=60988
+2024-11-07 06:38:41 - progress_bar.py[line:274] - INFO: epoch 025:    213 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.483, ntokens=17981.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3459, ups=0.19, wpb=17981.7, bsz=1024, num_updates=9590, lr=4.01453e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=35, gb_free=13.6, wall=61040
+2024-11-07 06:39:34 - progress_bar.py[line:274] - INFO: epoch 025:    223 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.441, ntokens=17513.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3272.9, ups=0.19, wpb=17513.9, bsz=1024, num_updates=9600, lr=4.01317e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=61093
+2024-11-07 06:40:26 - progress_bar.py[line:274] - INFO: epoch 025:    233 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.467, ntokens=17692.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3405.4, ups=0.19, wpb=17692.8, bsz=1024, num_updates=9610, lr=4.01181e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=61145
+2024-11-07 06:41:19 - progress_bar.py[line:274] - INFO: epoch 025:    243 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.454, ntokens=17537, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.48, wps=3340.1, ups=0.19, wpb=17537, bsz=1024, num_updates=9620, lr=4.01045e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=61198
+2024-11-07 06:42:12 - progress_bar.py[line:274] - INFO: epoch 025:    253 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.439, ntokens=17396.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.42, wps=3247.4, ups=0.19, wpb=17396.5, bsz=1024, num_updates=9630, lr=4.00909e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=61251
+2024-11-07 06:43:04 - progress_bar.py[line:274] - INFO: epoch 025:    263 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.441, ntokens=17555.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3385.2, ups=0.19, wpb=17555.8, bsz=1024, num_updates=9640, lr=4.00773e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=61303
+2024-11-07 06:43:57 - progress_bar.py[line:274] - INFO: epoch 025:    273 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.462, ntokens=17808.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.51, wps=3392.6, ups=0.19, wpb=17808.3, bsz=1024, num_updates=9650, lr=4.00637e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=61356
+2024-11-07 06:44:49 - progress_bar.py[line:274] - INFO: epoch 025:    283 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.421, ntokens=17301.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3283.2, ups=0.19, wpb=17301.5, bsz=1024, num_updates=9660, lr=4.00501e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=61408
+2024-11-07 06:45:41 - progress_bar.py[line:274] - INFO: epoch 025:    293 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.461, ntokens=17540.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.51, wps=3380.7, ups=0.19, wpb=17540.3, bsz=1024, num_updates=9670, lr=4.00365e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=61460
+2024-11-07 06:46:32 - progress_bar.py[line:274] - INFO: epoch 025:    303 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.544, ntokens=18161.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3555.5, ups=0.2, wpb=18161.7, bsz=1024, num_updates=9680, lr=4.00229e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=61511
+2024-11-07 06:47:25 - progress_bar.py[line:274] - INFO: epoch 025:    313 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.586, ntokens=18148.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6, wps=3451.9, ups=0.19, wpb=18148.4, bsz=1024, num_updates=9690, lr=4.00093e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=34, gb_free=13.4, wall=61564
+2024-11-07 06:48:17 - progress_bar.py[line:274] - INFO: epoch 025:    323 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.566, ntokens=18133.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3477, ups=0.19, wpb=18133.7, bsz=1024, num_updates=9700, lr=3.99956e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=41, gb_free=13.4, wall=61616
+2024-11-07 06:49:09 - progress_bar.py[line:274] - INFO: epoch 025:    333 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.593, ntokens=18519.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.04, wps=3556.1, ups=0.19, wpb=18519.8, bsz=1024, num_updates=9710, lr=3.9982e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=45, gb_free=13.1, wall=61668
+2024-11-07 06:50:02 - progress_bar.py[line:274] - INFO: epoch 025:    343 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.533, ntokens=18058.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3438.2, ups=0.19, wpb=18058.3, bsz=1024, num_updates=9720, lr=3.99684e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=51, gb_free=13.5, wall=61721
+2024-11-07 06:50:54 - progress_bar.py[line:274] - INFO: epoch 025:    353 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.6, ntokens=18580.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3539.1, ups=0.19, wpb=18580.1, bsz=1024, num_updates=9730, lr=3.99548e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=61773
+2024-11-07 06:50:59 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
+2024-11-07 06:51:51 - progress_bar.py[line:274] - INFO: epoch 025:    364 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.51, ntokens=17803.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.7, wps=3112, ups=0.17, wpb=17803.6, bsz=1024, num_updates=9740, lr=3.99412e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=61830
+2024-11-07 06:52:44 - progress_bar.py[line:274] - INFO: epoch 025:    374 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.521, ntokens=17919.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=3411.5, ups=0.19, wpb=17919.6, bsz=1024, num_updates=9750, lr=3.99276e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=61883
+2024-11-07 06:53:36 - progress_bar.py[line:274] - INFO: epoch 025:    384 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.575, ntokens=18243.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.96, wps=3481.9, ups=0.19, wpb=18243.9, bsz=1024, num_updates=9760, lr=3.9914e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=52, gb_free=13.3, wall=61935
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-03 22:59:23 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 25 @ 9768 updates
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt
@@ -3724,3915 +3754,773 @@ cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt
-2024-11-03 22:59:23 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt': File exists: File exists
+2024-11-07 06:54:09 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 25 @ 9767 updates
+2024-11-07 06:54:09 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt': File exists: File exists: File exists
+
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 23:01:12 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 06:55:59 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+
+
+slice_id 6 seek offset 300000
+slice_id 7 seek offset 350000
 slice_id 3 seek offset 150000
-2024-11-03 23:03:04 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 25 @ 9768 updates, score 0) (writing took 220.6272238130041 seconds)
+2024-11-07 06:57:51 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 25 @ 9767 updates, score 0) (writing took 221.21820808498887 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_25.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-03 23:03:58 - train.py[line:336] - INFO: end of epoch 25 (average epoch stats below)
-2024-11-03 23:03:58 - progress_bar.py[line:282] - INFO: epoch 025 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.095 | ntokens 17760.7 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.55 | wps 2633 | ups 0.15 | wpb 17760.7 | bsz 1023 | num_updates 9768 | lr 3.99031e-05 | gnorm 0.007 | clip 0 | loss_scale 65536 | train_wall 2088 | gb_free 13.4 | wall 63215
-2024-11-03 23:03:58 - trainer.py[line:639] - INFO: loading train data for epoch 26
+slice_id 4 seek offset 200000
+slice_id 5 seek offset 250000
+2024-11-07 06:58:45 - train.py[line:336] - INFO: end of epoch 25 (average epoch stats below)
+2024-11-07 06:58:45 - progress_bar.py[line:282] - INFO: epoch 025 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.49 | ntokens 17758.8 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.62 | wps 2772.4 | ups 0.16 | wpb 17758.8 | bsz 1023 | num_updates 9767 | lr 3.99045e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 1419 | gb_free 13.4 | wall 62244
+2024-11-07 06:58:45 - trainer.py[line:639] - INFO: loading train data for epoch 26
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 23:06:42 - trainer.py[line:703] - INFO: begin training epoch 26
-2024-11-03 23:06:42 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 23:06:55 - progress_bar.py[line:274] - INFO: epoch 026:      2 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.179, ntokens=17324.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.06, wps=351.3, ups=0.02, wpb=17324.3, bsz=985.6, num_updates=9770, lr=3.99004e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=48, gb_free=13.5, wall=63392
-2024-11-03 23:07:47 - progress_bar.py[line:274] - INFO: epoch 026:     12 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.063, ntokens=17594.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.36, wps=3346.3, ups=0.19, wpb=17594.5, bsz=1024, num_updates=9780, lr=3.98868e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=63444
-2024-11-03 23:08:44 - progress_bar.py[line:274] - INFO: epoch 026:     22 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.083, ntokens=17875.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.48, wps=3139.1, ups=0.18, wpb=17875.7, bsz=1024, num_updates=9790, lr=3.98732e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=48, gb_free=13.5, wall=63501
-2024-11-03 23:09:39 - progress_bar.py[line:274] - INFO: epoch 026:     32 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.033, ntokens=17682.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.19, wps=3246.4, ups=0.18, wpb=17682.6, bsz=1024, num_updates=9800, lr=3.98596e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=63556
-2024-11-03 23:10:34 - progress_bar.py[line:274] - INFO: epoch 026:     42 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.073, ntokens=17850.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.41, wps=3189.5, ups=0.18, wpb=17850.6, bsz=1024, num_updates=9810, lr=3.9846e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=63612
-2024-11-03 23:11:29 - progress_bar.py[line:274] - INFO: epoch 026:     52 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.996, ntokens=17571.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.98, wps=3244.5, ups=0.18, wpb=17571.1, bsz=1024, num_updates=9820, lr=3.98324e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=63666
-2024-11-03 23:12:19 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-03 23:12:29 - progress_bar.py[line:274] - INFO: epoch 026:     63 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.094, ntokens=17934.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.54, wps=2960.1, ups=0.17, wpb=17934.6, bsz=1024, num_updates=9830, lr=3.98188e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=60, gb_free=13.4, wall=63726
-2024-11-03 23:13:24 - progress_bar.py[line:274] - INFO: epoch 026:     73 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.108, ntokens=17813.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.62, wps=3230.7, ups=0.18, wpb=17813.7, bsz=1024, num_updates=9840, lr=3.98052e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=63781
-2024-11-03 23:14:22 - progress_bar.py[line:274] - INFO: epoch 026:     83 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.044, ntokens=17806.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.25, wps=3065.5, ups=0.17, wpb=17806.8, bsz=1024, num_updates=9850, lr=3.97916e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=63840
-2024-11-03 23:15:18 - progress_bar.py[line:274] - INFO: epoch 026:     93 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.037, ntokens=17563.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.21, wps=3159.9, ups=0.18, wpb=17563.4, bsz=1024, num_updates=9860, lr=3.9778e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=63895
-2024-11-03 23:16:16 - progress_bar.py[line:274] - INFO: epoch 026:    103 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.049, ntokens=17612.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.27, wps=3057.1, ups=0.17, wpb=17612.2, bsz=1024, num_updates=9870, lr=3.97644e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.6, wall=63953
-2024-11-03 23:17:09 - progress_bar.py[line:274] - INFO: epoch 026:    113 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.031, ntokens=17468.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.18, wps=3260.3, ups=0.19, wpb=17468.5, bsz=1024, num_updates=9880, lr=3.97508e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=64006
-2024-11-03 23:18:04 - progress_bar.py[line:274] - INFO: epoch 026:    123 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.023, ntokens=17679, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=3248.1, ups=0.18, wpb=17679, bsz=1024, num_updates=9890, lr=3.97372e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=37, gb_free=13.5, wall=64061
-2024-11-03 23:19:00 - progress_bar.py[line:274] - INFO: epoch 026:    133 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.031, ntokens=17636.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.18, wps=3135.1, ups=0.18, wpb=17636.7, bsz=1024, num_updates=9900, lr=3.97236e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=64117
-2024-11-03 23:19:57 - progress_bar.py[line:274] - INFO: epoch 026:    143 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.039, ntokens=17510.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.22, wps=3063.2, ups=0.17, wpb=17510.7, bsz=1024, num_updates=9910, lr=3.971e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=64174
-2024-11-03 23:20:53 - progress_bar.py[line:274] - INFO: epoch 026:    153 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.06, ntokens=17683.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.34, wps=3187.8, ups=0.18, wpb=17683.3, bsz=1024, num_updates=9920, lr=3.96964e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=33, gb_free=13.6, wall=64230
-2024-11-03 23:21:50 - progress_bar.py[line:274] - INFO: epoch 026:    163 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.988, ntokens=17434.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.93, wps=3061.1, ups=0.18, wpb=17434.3, bsz=1024, num_updates=9930, lr=3.96828e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=47, gb_free=13.6, wall=64287
-2024-11-03 23:22:47 - progress_bar.py[line:274] - INFO: epoch 026:    173 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.117, ntokens=17981.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.67, wps=3117.4, ups=0.17, wpb=17981.3, bsz=1024, num_updates=9940, lr=3.96692e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=64344
-2024-11-03 23:23:44 - progress_bar.py[line:274] - INFO: epoch 026:    183 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.023, ntokens=17613, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=3108.5, ups=0.18, wpb=17613, bsz=1024, num_updates=9950, lr=3.96555e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=64401
-2024-11-03 23:24:39 - progress_bar.py[line:274] - INFO: epoch 026:    193 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.051, ntokens=17806.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.29, wps=3226.7, ups=0.18, wpb=17806.2, bsz=1024, num_updates=9960, lr=3.96419e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=36, gb_free=13.4, wall=64456
-2024-11-03 23:25:36 - progress_bar.py[line:274] - INFO: epoch 026:    203 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.071, ntokens=17994.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.41, wps=3157.4, ups=0.18, wpb=17994.9, bsz=1024, num_updates=9970, lr=3.96283e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=64513
-2024-11-03 23:26:33 - progress_bar.py[line:274] - INFO: epoch 026:    213 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.026, ntokens=17957.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.15, wps=3139.6, ups=0.17, wpb=17957.3, bsz=1024, num_updates=9980, lr=3.96147e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=64570
-2024-11-03 23:27:30 - progress_bar.py[line:274] - INFO: epoch 026:    223 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.04, ntokens=17828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=3129.6, ups=0.18, wpb=17828, bsz=1024, num_updates=9990, lr=3.96011e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=64627
-2024-11-03 23:28:27 - progress_bar.py[line:274] - INFO: epoch 026:    233 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.074, ntokens=17835, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.42, wps=3136.1, ups=0.18, wpb=17835, bsz=1024, num_updates=10000, lr=3.95875e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=64684
-loss_reg: 0.024009554402081426 loss_cls: 0.002913308097049594
-loss_reg: 0.029057670073514417 loss_cls: 0.003773762844502926
-loss_reg: 0.02387541431927078 loss_cls: 0.003633089829236269
-loss_reg: 0.00806202573976523 loss_cls: 0.0033063949085772038
-loss_reg: 0.017442249578840656 loss_cls: 0.0035335971042513847
-loss_reg: 0.01560563943595944 loss_cls: 0.0035322033800184727
-loss_reg: 0.017925309979497334 loss_cls: 0.003224762622267008
-loss_reg: 0.021475232385586303 loss_cls: 0.004139862954616547
-loss_reg: 0.019212442041587466 loss_cls: 0.004636294208467007
-loss_reg: 0.02298952895526123 loss_cls: 0.004337613470852375
-loss_reg: 0.015755922608533767 loss_cls: 0.003918623086065054
-loss_reg: 0.012574451285857047 loss_cls: 0.003612976288422942
-loss_reg: 0.01243543862523698 loss_cls: 0.004324307199567556
-loss_reg: 0.019054113554028397 loss_cls: 0.004323714878410101
-loss_reg: 0.023495244930363333 loss_cls: 0.0030855333898216486
-loss_reg: 0.018591099292568487 loss_cls: 0.00368236075155437
-loss_reg: 0.015502869817541943 loss_cls: 0.0027660492341965437
-loss_reg: 0.015859306394490232 loss_cls: 0.0034819957800209522
-loss_reg: 0.02555947013852586 loss_cls: 0.0027793420013040304
-loss_reg: 0.02555978784508782 loss_cls: 0.0034053330309689045
-loss_reg: 0.025877513539890526 loss_cls: 0.0037767000030726194
-loss_reg: 0.018013028815931337 loss_cls: 0.004463385324925184
-loss_reg: 0.02501034505346093 loss_cls: 0.00385320489294827
-loss_reg: 0.022984660974057054 loss_cls: 0.0030237166211009026
-loss_reg: 0.018586391480814524 loss_cls: 0.0035756428260356188
-loss_reg: 0.016171806526082532 loss_cls: 0.004293103236705065
-loss_reg: 0.025194489883594927 loss_cls: 0.00429863017052412
-loss_reg: 0.012264889743753945 loss_cls: 0.0037223382387310266
-loss_reg: 0.021478532558509067 loss_cls: 0.003373862477019429
-loss_reg: 0.026635029649260473 loss_cls: 0.004217386245727539
-loss_reg: 0.018073775244936044 loss_cls: 0.0029761327896267176
-loss_reg: 0.01453620047909866 loss_cls: 0.0032300434540957212
-loss_reg: 0.01707920333073589 loss_cls: 0.0034554998856037855
-loss_reg: 0.01799081701613118 loss_cls: 0.003075179411098361
-loss_reg: 0.020394696091736893 loss_cls: 0.003556930460035801
-loss_reg: 0.017730361855242165 loss_cls: 0.003194676013663411
-loss_reg: 0.017916731735793483 loss_cls: 0.004337878432124853
-loss_reg: 0.009584889259255422 loss_cls: 0.003500823164358735
-loss_reg: 0.01731719623851395 loss_cls: 0.003912357147783041
-loss_reg: 0.013972641310243129 loss_cls: 0.00400466239079833
-loss_reg: 0.025290286048570476 loss_cls: 0.003811389207839966
-loss_reg: 0.01556921987590833 loss_cls: 0.0031135277822613716
-loss_reg: 0.01558574645800347 loss_cls: 0.002564462134614587
-loss_reg: 0.014048913978864415 loss_cls: 0.003962301649153233
-loss_reg: 0.012099596795099362 loss_cls: 0.003758506616577506
-loss_reg: 0.019323280610311126 loss_cls: 0.004621682222932577
-loss_reg: 0.02095275200771042 loss_cls: 0.005151212215423584
-loss_reg: 0.016858057313358835 loss_cls: 0.005530933383852243
-loss_reg: 0.01693656313885504 loss_cls: 0.004467173013836145
-loss_reg: 0.013478701632697742 loss_cls: 0.0028530231211334467
-loss_reg: 0.02864147863257177 loss_cls: 0.0037131276912987232
-loss_reg: 0.015526413350842193 loss_cls: 0.004572258796542883
-loss_reg: 0.018220835975390596 loss_cls: 0.004257673397660255
-loss_reg: 0.013676547562148518 loss_cls: 0.0036317126359790564
-loss_reg: 0.030225945180876 loss_cls: 0.00419874070212245
-loss_reg: 0.023543093838013014 loss_cls: 0.0033695613965392113
-loss_reg: 0.02266093323222378 loss_cls: 0.004185986239463091
-loss_reg: 0.005029385124355595 loss_cls: 0.002705653663724661
-loss_reg: 0.011928302508846935 loss_cls: 0.0037097306922078133
-loss_reg: 0.005583927882468484 loss_cls: 0.0023218540009111166
-loss_reg: 0.005298348308984797 loss_cls: 0.002704889513552189
-loss_reg: 0.00870531574208817 loss_cls: 0.002889571012929082
-loss_reg: 0.005590101625463573 loss_cls: 0.0029470317531377077
-loss_reg: 0.008415580190929816 loss_cls: 0.004257140681147575
-2024-11-03 23:29:24 - progress_bar.py[line:274] - INFO: epoch 026:    243 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.082, ntokens=17575.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.47, wps=3098.3, ups=0.18, wpb=17575.2, bsz=1024, num_updates=10010, lr=3.95739e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=64741
-2024-11-03 23:30:21 - progress_bar.py[line:274] - INFO: epoch 026:    253 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.047, ntokens=17897.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.26, wps=3106.7, ups=0.17, wpb=17897.5, bsz=1024, num_updates=10020, lr=3.95603e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=64799
-2024-11-03 23:31:18 - progress_bar.py[line:274] - INFO: epoch 026:    263 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.113, ntokens=17882.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.65, wps=3176.4, ups=0.18, wpb=17882.1, bsz=1024, num_updates=10030, lr=3.95467e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=64855
-2024-11-03 23:32:15 - progress_bar.py[line:274] - INFO: epoch 026:    273 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.05, ntokens=18033.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.28, wps=3164.2, ups=0.18, wpb=18033.8, bsz=1024, num_updates=10040, lr=3.95331e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=64912
-2024-11-03 23:33:12 - progress_bar.py[line:274] - INFO: epoch 026:    283 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.071, ntokens=17667.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.4, wps=3106.4, ups=0.18, wpb=17667.5, bsz=1024, num_updates=10050, lr=3.95195e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=64969
-2024-11-03 23:34:08 - progress_bar.py[line:274] - INFO: epoch 026:    293 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.067, ntokens=17816.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.38, wps=3136.1, ups=0.18, wpb=17816.2, bsz=1024, num_updates=10060, lr=3.95059e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=65025
-2024-11-03 23:35:05 - progress_bar.py[line:274] - INFO: epoch 026:    303 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.208, ntokens=18317.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.24, wps=3261.6, ups=0.18, wpb=18317.7, bsz=1024, num_updates=10070, lr=3.94923e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=65082
-2024-11-03 23:36:00 - progress_bar.py[line:274] - INFO: epoch 026:    313 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.241, ntokens=18384.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.46, wps=3296.9, ups=0.18, wpb=18384.9, bsz=1024, num_updates=10080, lr=3.94787e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=65137
-2024-11-03 23:36:56 - progress_bar.py[line:274] - INFO: epoch 026:    323 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.144, ntokens=17889.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.84, wps=3199.2, ups=0.18, wpb=17889.5, bsz=1024, num_updates=10090, lr=3.94651e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=65193
-2024-11-03 23:37:53 - progress_bar.py[line:274] - INFO: epoch 026:    333 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.237, ntokens=18294.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.43, wps=3229.7, ups=0.18, wpb=18294.1, bsz=1024, num_updates=10100, lr=3.94515e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=65250
-2024-11-03 23:38:47 - progress_bar.py[line:274] - INFO: epoch 026:    343 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.082, ntokens=17845.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.47, wps=3279.4, ups=0.18, wpb=17845.6, bsz=1024, num_updates=10110, lr=3.94379e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=65304
-2024-11-03 23:39:44 - progress_bar.py[line:274] - INFO: epoch 026:    353 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.229, ntokens=18658.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.37, wps=3282.4, ups=0.18, wpb=18658.6, bsz=1024, num_updates=10120, lr=3.94243e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=46, gb_free=13.4, wall=65361
-2024-11-03 23:40:39 - progress_bar.py[line:274] - INFO: epoch 026:    363 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.169, ntokens=18331, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.99, wps=3329.9, ups=0.18, wpb=18331, bsz=1024, num_updates=10130, lr=3.94107e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=65416
-2024-11-03 23:41:32 - progress_bar.py[line:274] - INFO: epoch 026:    373 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.147, ntokens=17887.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.86, wps=3361.9, ups=0.19, wpb=17887.5, bsz=1024, num_updates=10140, lr=3.93971e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=49, gb_free=13.4, wall=65470
-2024-11-03 23:42:29 - progress_bar.py[line:274] - INFO: epoch 026:    383 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.14, ntokens=17957.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.82, wps=3174.1, ups=0.18, wpb=17957.5, bsz=1024, num_updates=10150, lr=3.93835e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=65526
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
-2024-11-03 23:43:11 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 26 @ 10158 updates
+2024-11-07 07:01:25 - trainer.py[line:703] - INFO: begin training epoch 26
+2024-11-07 07:01:25 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 07:01:42 - progress_bar.py[line:274] - INFO: epoch 026:      3 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.518, ntokens=17355.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=357, ups=0.02, wpb=17355.6, bsz=985.6, num_updates=9770, lr=3.99004e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=42, gb_free=13.4, wall=62421
+2024-11-07 07:02:35 - progress_bar.py[line:274] - INFO: epoch 026:     13 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.461, ntokens=17687.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.51, wps=3349.8, ups=0.19, wpb=17687.4, bsz=1024, num_updates=9780, lr=3.98868e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=62474
+2024-11-07 07:03:28 - progress_bar.py[line:274] - INFO: epoch 026:     23 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.46, ntokens=17679.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3369.5, ups=0.19, wpb=17679.3, bsz=1024, num_updates=9790, lr=3.98732e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=62527
+2024-11-07 07:04:20 - progress_bar.py[line:274] - INFO: epoch 026:     33 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.424, ntokens=17685.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3389.8, ups=0.19, wpb=17685.5, bsz=1024, num_updates=9800, lr=3.98596e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=62579
+2024-11-07 07:05:12 - progress_bar.py[line:274] - INFO: epoch 026:     43 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.489, ntokens=17888.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.62, wps=3432, ups=0.19, wpb=17888.7, bsz=1024, num_updates=9810, lr=3.9846e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=62631
+2024-11-07 07:06:05 - progress_bar.py[line:274] - INFO: epoch 026:     53 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.449, ntokens=17617.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3356, ups=0.19, wpb=17617.3, bsz=1024, num_updates=9820, lr=3.98324e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=62683
+2024-11-07 07:06:57 - progress_bar.py[line:274] - INFO: epoch 026:     63 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.48, ntokens=17884.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3410.6, ups=0.19, wpb=17884.5, bsz=1024, num_updates=9830, lr=3.98188e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=62736
+2024-11-07 07:07:50 - progress_bar.py[line:274] - INFO: epoch 026:     73 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.475, ntokens=17813.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3372, ups=0.19, wpb=17813.7, bsz=1024, num_updates=9840, lr=3.98052e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=62789
+2024-11-07 07:08:43 - progress_bar.py[line:274] - INFO: epoch 026:     83 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.469, ntokens=17806.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3315.3, ups=0.19, wpb=17806.8, bsz=1024, num_updates=9850, lr=3.97916e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=62842
+2024-11-07 07:09:39 - progress_bar.py[line:274] - INFO: epoch 026:     93 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.441, ntokens=17563.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3137.3, ups=0.18, wpb=17563.4, bsz=1024, num_updates=9860, lr=3.9778e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=48, gb_free=13.5, wall=62898
+2024-11-07 07:10:32 - progress_bar.py[line:274] - INFO: epoch 026:    103 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.46, ntokens=17612.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3384.9, ups=0.19, wpb=17612.2, bsz=1024, num_updates=9870, lr=3.97644e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=62950
+2024-11-07 07:11:23 - progress_bar.py[line:274] - INFO: epoch 026:    113 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.436, ntokens=17468.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.41, wps=3361.4, ups=0.19, wpb=17468.5, bsz=1024, num_updates=9880, lr=3.97508e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=63002
+2024-11-07 07:12:18 - progress_bar.py[line:274] - INFO: epoch 026:    123 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.435, ntokens=17679, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.41, wps=3229.8, ups=0.18, wpb=17679, bsz=1024, num_updates=9890, lr=3.97372e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=63057
+2024-11-07 07:13:14 - progress_bar.py[line:274] - INFO: epoch 026:    133 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.469, ntokens=17636.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3163.7, ups=0.18, wpb=17636.7, bsz=1024, num_updates=9900, lr=3.97236e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=63113
+2024-11-07 07:14:06 - progress_bar.py[line:274] - INFO: epoch 026:    143 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.441, ntokens=17510.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3336.3, ups=0.19, wpb=17510.7, bsz=1024, num_updates=9910, lr=3.971e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=63165
+2024-11-07 07:14:59 - progress_bar.py[line:274] - INFO: epoch 026:    153 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.435, ntokens=17683.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.41, wps=3334.4, ups=0.19, wpb=17683.3, bsz=1024, num_updates=9920, lr=3.96964e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=53, gb_free=13.6, wall=63218
+2024-11-07 07:15:52 - progress_bar.py[line:274] - INFO: epoch 026:    163 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.422, ntokens=17434.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3350.7, ups=0.19, wpb=17434.3, bsz=1024, num_updates=9930, lr=3.96828e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=63270
+2024-11-07 07:16:44 - progress_bar.py[line:274] - INFO: epoch 026:    173 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.486, ntokens=17981.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3445.7, ups=0.19, wpb=17981.3, bsz=1024, num_updates=9940, lr=3.96692e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=63323
+2024-11-07 07:17:38 - progress_bar.py[line:274] - INFO: epoch 026:    183 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.454, ntokens=17613, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.48, wps=3225.4, ups=0.18, wpb=17613, bsz=1024, num_updates=9950, lr=3.96555e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=63377
+2024-11-07 07:18:32 - progress_bar.py[line:274] - INFO: epoch 026:    193 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.466, ntokens=17806.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3330.5, ups=0.19, wpb=17806.2, bsz=1024, num_updates=9960, lr=3.96419e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=63431
+2024-11-07 07:19:24 - progress_bar.py[line:274] - INFO: epoch 026:    203 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.49, ntokens=17994.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.62, wps=3458.7, ups=0.19, wpb=17994.9, bsz=1024, num_updates=9970, lr=3.96283e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=63483
+2024-11-07 07:20:16 - progress_bar.py[line:274] - INFO: epoch 026:    213 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.46, ntokens=17957.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3428.7, ups=0.19, wpb=17957.3, bsz=1024, num_updates=9980, lr=3.96147e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=63535
+2024-11-07 07:21:11 - progress_bar.py[line:274] - INFO: epoch 026:    223 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.472, ntokens=17828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.55, wps=3278.7, ups=0.18, wpb=17828, bsz=1024, num_updates=9990, lr=3.96011e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=63589
+2024-11-07 07:22:03 - progress_bar.py[line:274] - INFO: epoch 026:    233 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.476, ntokens=17835, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3410, ups=0.19, wpb=17835, bsz=1024, num_updates=10000, lr=3.95875e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=63642
+loss_reg: 0.005072457349734271 loss_cls: 0.0029850576538592577
+loss_reg: 0.011574835184792593 loss_cls: 0.003301245393231511
+loss_reg: 0.01806650845530307 loss_cls: 0.0028240042738616467
+loss_reg: 0.012605174076533759 loss_cls: 0.003352748230099678
+loss_reg: 0.018765818374008952 loss_cls: 0.003354056039825082
+loss_reg: 0.006726486582906949 loss_cls: 0.0026976405642926693
+loss_reg: 0.023671384557910072 loss_cls: 0.003688663709908724
+loss_reg: 0.017193558134857616 loss_cls: 0.004349737428128719
+loss_reg: 0.020649800006708817 loss_cls: 0.004239168483763933
+loss_reg: 0.017635722727165994 loss_cls: 0.004073155578225851
+loss_reg: 0.015941459556420307 loss_cls: 0.00382440909743309
+loss_reg: 0.003092497793174779 loss_cls: 0.0025461153127253056
+loss_reg: 0.010132632506271406 loss_cls: 0.0031686476431787014
+loss_reg: 0.013024612628201763 loss_cls: 0.003592904657125473
+loss_reg: 0.018866784788445022 loss_cls: 0.002823794260621071
+loss_reg: 0.01610965090049986 loss_cls: 0.003522136015817523
+loss_reg: 0.020264873564619153 loss_cls: 0.0030317187774926424
+loss_reg: 0.012803599459763733 loss_cls: 0.0025839530862867832
+loss_reg: 0.016515906738524475 loss_cls: 0.0024925819598138332
+loss_reg: 0.0048059882839326246 loss_cls: 0.003466129070147872
+loss_reg: 0.009767073852577052 loss_cls: 0.0030016815289855003
+loss_reg: 0.015327750020412412 loss_cls: 0.004330474883317947
+loss_reg: 0.022377223969332238 loss_cls: 0.004033925477415323
+loss_reg: 0.019587593694097315 loss_cls: 0.002701906720176339
+loss_reg: 0.020255689766306834 loss_cls: 0.003601281438022852
+loss_reg: 0.014188317141174078 loss_cls: 0.004078863188624382
+loss_reg: 0.00400660483965813 loss_cls: 0.002314819023013115
+loss_reg: 0.01518792842477173 loss_cls: 0.0034420606680214405
+loss_reg: 0.02101943739668691 loss_cls: 0.004031266551464796
+loss_reg: 0.018474877659499324 loss_cls: 0.003058665432035923
+loss_reg: 0.010148546074884055 loss_cls: 0.0028529323171824217
+loss_reg: 0.014291745239295453 loss_cls: 0.0028770584613084793
+loss_reg: 0.012876515068350015 loss_cls: 0.0031052157282829285
+loss_reg: 0.002574398349857977 loss_cls: 0.0026204001624137163
+loss_reg: 0.010619121191212381 loss_cls: 0.0031315633095800877
+loss_reg: 0.011285099152285321 loss_cls: 0.003931376151740551
+loss_reg: 0.013464573752272928 loss_cls: 0.0040021175518631935
+loss_reg: 0.007731428875924249 loss_cls: 0.0033343788236379623
+loss_reg: 0.011449482697426768 loss_cls: 0.003489007242023945
+loss_reg: 0.014254637760358862 loss_cls: 0.0030181719921529293
+loss_reg: 0.005651881569730247 loss_cls: 0.002793073421344161
+loss_reg: 0.02171674839962207 loss_cls: 0.003650813363492489
+loss_reg: 0.012549379904845216 loss_cls: 0.003618222661316395
+loss_reg: 0.012603569846935652 loss_cls: 0.005365218501538038
+loss_reg: 0.009510106913334242 loss_cls: 0.003541261423379183
+loss_reg: 0.01579545287692194 loss_cls: 0.004597025457769632
+loss_reg: 0.014157939663628766 loss_cls: 0.0025651913601905107
+loss_reg: 0.011169865904173102 loss_cls: 0.002962418831884861
+loss_reg: 0.0037872292290399108 loss_cls: 0.0028251672629266977
+loss_reg: 0.019161269465257803 loss_cls: 0.004639727063477039
+loss_reg: 0.01323476787555892 loss_cls: 0.004171007312834263
+loss_reg: 0.024624168119753237 loss_cls: 0.003858195850625634
+loss_reg: 0.011137786480519232 loss_cls: 0.004299917723983526
+loss_reg: 0.009535568253668027 loss_cls: 0.0034535862505435944
+loss_reg: 0.021431152143581832 loss_cls: 0.0034742269199341536
+loss_reg: 0.011526694225963242 loss_cls: 0.004021471366286278
+loss_reg: 0.005006155194671892 loss_cls: 0.003943929448723793
+loss_reg: 0.018863963371590194 loss_cls: 0.003208993235602975
+loss_reg: 0.020406717043124866 loss_cls: 0.0031902615446597338
+loss_reg: 0.008540935542405419 loss_cls: 0.003262000158429146
+loss_reg: 0.016971099352354958 loss_cls: 0.0033526422921568155
+loss_reg: 0.012924091531637084 loss_cls: 0.0029463334940373898
+loss_reg: 0.013714326172997206 loss_cls: 0.004171769600361586
+loss_reg: 0.014725998202935991 loss_cls: 0.003666035132482648
+2024-11-07 07:22:56 - progress_bar.py[line:274] - INFO: epoch 026:    243 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.443, ntokens=17575.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.44, wps=3330.6, ups=0.19, wpb=17575.2, bsz=1024, num_updates=10010, lr=3.95739e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.6, wall=63695
+2024-11-07 07:23:47 - progress_bar.py[line:274] - INFO: epoch 026:    253 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.486, ntokens=17897.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3497.3, ups=0.2, wpb=17897.5, bsz=1024, num_updates=10020, lr=3.95603e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=51, gb_free=13.5, wall=63746
+2024-11-07 07:24:38 - progress_bar.py[line:274] - INFO: epoch 026:    263 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.484, ntokens=17882.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3494.6, ups=0.2, wpb=17882.1, bsz=1024, num_updates=10030, lr=3.95467e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=48, gb_free=13.5, wall=63797
+2024-11-07 07:25:30 - progress_bar.py[line:274] - INFO: epoch 026:    273 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.468, ntokens=18033.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3468.1, ups=0.19, wpb=18033.8, bsz=1024, num_updates=10040, lr=3.95331e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=44, gb_free=13.5, wall=63849
+2024-11-07 07:26:22 - progress_bar.py[line:274] - INFO: epoch 026:    283 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.447, ntokens=17667.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.45, wps=3386.7, ups=0.19, wpb=17667.5, bsz=1024, num_updates=10050, lr=3.95195e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=63901
+2024-11-07 07:27:15 - progress_bar.py[line:274] - INFO: epoch 026:    293 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.474, ntokens=17816.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3374.4, ups=0.19, wpb=17816.2, bsz=1024, num_updates=10060, lr=3.95059e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=63954
+2024-11-07 07:28:07 - progress_bar.py[line:274] - INFO: epoch 026:    303 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.574, ntokens=18317.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3492.7, ups=0.19, wpb=18317.7, bsz=1024, num_updates=10070, lr=3.94923e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=40, gb_free=13.4, wall=64006
+2024-11-07 07:29:00 - progress_bar.py[line:274] - INFO: epoch 026:    313 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.596, ntokens=18384.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3508.5, ups=0.19, wpb=18384.9, bsz=1024, num_updates=10080, lr=3.94787e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=49, gb_free=13.4, wall=64059
+2024-11-07 07:29:52 - progress_bar.py[line:274] - INFO: epoch 026:    323 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.534, ntokens=17889.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3410, ups=0.19, wpb=17889.5, bsz=1024, num_updates=10090, lr=3.94651e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=64111
+2024-11-07 07:30:44 - progress_bar.py[line:274] - INFO: epoch 026:    333 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.555, ntokens=18294.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3523.3, ups=0.19, wpb=18294.1, bsz=1024, num_updates=10100, lr=3.94515e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.2, wall=64163
+2024-11-07 07:31:37 - progress_bar.py[line:274] - INFO: epoch 026:    343 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.497, ntokens=17845.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3371.2, ups=0.19, wpb=17845.6, bsz=1024, num_updates=10110, lr=3.94379e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=64216
+2024-11-07 07:32:30 - progress_bar.py[line:274] - INFO: epoch 026:    353 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.59, ntokens=18658.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.02, wps=3528.9, ups=0.19, wpb=18658.6, bsz=1024, num_updates=10120, lr=3.94243e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=64269
+2024-11-07 07:33:26 - progress_bar.py[line:274] - INFO: epoch 026:    363 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.578, ntokens=18331, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.97, wps=3287.1, ups=0.18, wpb=18331, bsz=1024, num_updates=10130, lr=3.94107e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=56, gb_free=13.3, wall=64325
+2024-11-07 07:34:20 - progress_bar.py[line:274] - INFO: epoch 026:    373 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.501, ntokens=17887.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.66, wps=3313, ups=0.19, wpb=17887.5, bsz=1024, num_updates=10140, lr=3.93971e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=64379
+2024-11-07 07:35:12 - progress_bar.py[line:274] - INFO: epoch 026:    383 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.529, ntokens=17957.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3429.9, ups=0.19, wpb=17957.5, bsz=1024, num_updates=10150, lr=3.93835e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=64431
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt2024-11-07 07:35:51 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 26 @ 10158 updates
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
 
-2024-11-03 23:43:11 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
+2024-11-07 07:35:51 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt': File exists
+: File exists
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-03 23:44:56 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-03 23:46:47 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 26 @ 10158 updates, score 0) (writing took 216.6377034969919 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 07:37:38 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
+slice_id 6 seek offset 300000
+2024-11-07 07:39:30 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 26 @ 10158 updates, score 0) (writing took 218.71483593300218 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_26.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 slice_id 3 seek offset 150000
-slice_id 4 seek offset 200000
-slice_id 1 seek offset 50000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-03 23:47:42 - train.py[line:336] - INFO: end of epoch 26 (average epoch stats below)
-2024-11-03 23:47:42 - progress_bar.py[line:282] - INFO: epoch 026 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.086 | ntokens 17827.4 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.49 | wps 2649.9 | ups 0.15 | wpb 17827.4 | bsz 1023 | num_updates 10158 | lr 3.93726e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 1582 | gb_free 13.4 | wall 65839
-2024-11-03 23:47:42 - trainer.py[line:639] - INFO: loading train data for epoch 27
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+
+slice_id 7 seek offset 350000
+slice_id 1 seek offset 50000
+slice_id 5 seek offset 250000
+slice_id 4 seek offset 200000
+2024-11-07 07:40:24 - train.py[line:336] - INFO: end of epoch 26 (average epoch stats below)
+2024-11-07 07:40:24 - progress_bar.py[line:282] - INFO: epoch 026 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.482 | ntokens 17826.4 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.59 | wps 2788.8 | ups 0.16 | wpb 17826.4 | bsz 1023 | num_updates 10158 | lr 3.93726e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 1799 | gb_free 13.4 | wall 64743
+2024-11-07 07:40:24 - trainer.py[line:639] - INFO: loading train data for epoch 27
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-03 23:50:26 - trainer.py[line:703] - INFO: begin training epoch 27
-2024-11-03 23:50:26 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-03 23:50:39 - progress_bar.py[line:274] - INFO: epoch 027:      2 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.132, ntokens=17074.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.77, wps=348.6, ups=0.02, wpb=17074.3, bsz=985.6, num_updates=10160, lr=3.93699e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=66016
-2024-11-03 23:51:33 - progress_bar.py[line:274] - INFO: epoch 027:     12 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.979, ntokens=17256.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.88, wps=3183.2, ups=0.18, wpb=17256.5, bsz=1024, num_updates=10170, lr=3.93563e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=66070
-2024-11-03 23:52:29 - progress_bar.py[line:274] - INFO: epoch 027:     22 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.111, ntokens=18036.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.64, wps=3218.2, ups=0.18, wpb=18036.6, bsz=1024, num_updates=10180, lr=3.93427e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=66126
-2024-11-03 23:53:22 - progress_bar.py[line:274] - INFO: epoch 027:     32 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.024, ntokens=17872.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=3376.6, ups=0.19, wpb=17872.8, bsz=1024, num_updates=10190, lr=3.93291e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=66179
-2024-11-03 23:54:18 - progress_bar.py[line:274] - INFO: epoch 027:     42 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.069, ntokens=17899.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.39, wps=3216, ups=0.18, wpb=17899.1, bsz=1024, num_updates=10200, lr=3.93154e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=43, gb_free=13.6, wall=66235
-2024-11-03 23:55:11 - progress_bar.py[line:274] - INFO: epoch 027:     52 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.967, ntokens=17432.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.82, wps=3256.1, ups=0.19, wpb=17432.2, bsz=1024, num_updates=10210, lr=3.93018e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=66288
-2024-11-03 23:56:04 - progress_bar.py[line:274] - INFO: epoch 027:     62 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.104, ntokens=18065.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.6, wps=3396.9, ups=0.19, wpb=18065.2, bsz=1024, num_updates=10220, lr=3.92882e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=66341
-2024-11-03 23:57:00 - progress_bar.py[line:274] - INFO: epoch 027:     72 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.02, ntokens=17540.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.11, wps=3165.4, ups=0.18, wpb=17540.5, bsz=1024, num_updates=10230, lr=3.92746e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=66397
-2024-11-03 23:57:58 - progress_bar.py[line:274] - INFO: epoch 027:     82 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.02, ntokens=17496.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.11, wps=3023.8, ups=0.17, wpb=17496.7, bsz=1024, num_updates=10240, lr=3.9261e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=66455
-2024-11-03 23:58:54 - progress_bar.py[line:274] - INFO: epoch 027:     92 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.015, ntokens=17708.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.08, wps=3134.1, ups=0.18, wpb=17708.8, bsz=1024, num_updates=10250, lr=3.92474e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=66511
-2024-11-03 23:59:48 - progress_bar.py[line:274] - INFO: epoch 027:    102 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.093, ntokens=17696.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.53, wps=3301.3, ups=0.19, wpb=17696.1, bsz=1024, num_updates=10260, lr=3.92338e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=66565
-2024-11-04 00:00:44 - progress_bar.py[line:274] - INFO: epoch 027:    112 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.031, ntokens=17614.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.18, wps=3120.7, ups=0.18, wpb=17614.1, bsz=1024, num_updates=10270, lr=3.92202e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=66621
-2024-11-04 00:01:39 - progress_bar.py[line:274] - INFO: epoch 027:    122 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.022, ntokens=17784.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=3243.5, ups=0.18, wpb=17784.8, bsz=1024, num_updates=10280, lr=3.92066e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=66676
-2024-11-04 00:02:34 - progress_bar.py[line:274] - INFO: epoch 027:    132 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.993, ntokens=17416.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.96, wps=3175.5, ups=0.18, wpb=17416.7, bsz=1024, num_updates=10290, lr=3.9193e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=66731
-2024-11-04 00:03:27 - progress_bar.py[line:274] - INFO: epoch 027:    142 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.063, ntokens=17756.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.36, wps=3315.4, ups=0.19, wpb=17756.6, bsz=1024, num_updates=10300, lr=3.91794e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=66784
-2024-11-04 00:04:23 - progress_bar.py[line:274] - INFO: epoch 027:    152 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.09, ntokens=18058.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.51, wps=3240.5, ups=0.18, wpb=18058.9, bsz=1024, num_updates=10310, lr=3.91658e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=66840
-2024-11-04 00:05:19 - progress_bar.py[line:274] - INFO: epoch 027:    162 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.03, ntokens=17677.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.17, wps=3136.5, ups=0.18, wpb=17677.3, bsz=1024, num_updates=10320, lr=3.91522e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=66897
-2024-11-04 00:06:16 - progress_bar.py[line:274] - INFO: epoch 027:    172 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.015, ntokens=17586.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.08, wps=3113.3, ups=0.18, wpb=17586.1, bsz=1024, num_updates=10330, lr=3.91386e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=66953
-2024-11-04 00:07:12 - progress_bar.py[line:274] - INFO: epoch 027:    182 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.017, ntokens=17576.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.09, wps=3158.4, ups=0.18, wpb=17576.5, bsz=1024, num_updates=10340, lr=3.9125e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=67009
-2024-11-04 00:08:08 - progress_bar.py[line:274] - INFO: epoch 027:    192 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.017, ntokens=17667.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.09, wps=3139.5, ups=0.18, wpb=17667.5, bsz=1024, num_updates=10350, lr=3.91114e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=67065
-2024-11-04 00:09:04 - progress_bar.py[line:274] - INFO: epoch 027:    202 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.054, ntokens=17989.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.3, wps=3222.3, ups=0.18, wpb=17989.9, bsz=1024, num_updates=10360, lr=3.90978e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=67121
-2024-11-04 00:09:59 - progress_bar.py[line:274] - INFO: epoch 027:    212 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.014, ntokens=17932.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.08, wps=3237.8, ups=0.18, wpb=17932.7, bsz=1024, num_updates=10370, lr=3.90842e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=67176
-2024-11-04 00:10:55 - progress_bar.py[line:274] - INFO: epoch 027:    222 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.968, ntokens=17579, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.82, wps=3123.6, ups=0.18, wpb=17579, bsz=1024, num_updates=10380, lr=3.90706e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=67232
-2024-11-04 00:11:51 - progress_bar.py[line:274] - INFO: epoch 027:    232 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.036, ntokens=17732.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.2, wps=3177.2, ups=0.18, wpb=17732.6, bsz=1024, num_updates=10390, lr=3.9057e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=67288
-2024-11-04 00:12:47 - progress_bar.py[line:274] - INFO: epoch 027:    242 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.044, ntokens=17456.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.25, wps=3104.5, ups=0.18, wpb=17456.8, bsz=1024, num_updates=10400, lr=3.90434e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=67345
-2024-11-04 00:13:43 - progress_bar.py[line:274] - INFO: epoch 027:    252 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.021, ntokens=17653.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.11, wps=3164.6, ups=0.18, wpb=17653.5, bsz=1024, num_updates=10410, lr=3.90298e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=67400
-2024-11-04 00:14:39 - progress_bar.py[line:274] - INFO: epoch 027:    262 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.046, ntokens=17762.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.26, wps=3205.5, ups=0.18, wpb=17762.5, bsz=1024, num_updates=10420, lr=3.90162e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=67456
-2024-11-04 00:15:17 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-04 00:15:39 - progress_bar.py[line:274] - INFO: epoch 027:    273 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.055, ntokens=17998.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.31, wps=2957.2, ups=0.16, wpb=17998.8, bsz=1024, num_updates=10430, lr=3.90026e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=61, gb_free=13.5, wall=67517
-2024-11-04 00:16:35 - progress_bar.py[line:274] - INFO: epoch 027:    283 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.041, ntokens=17740.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=3175.1, ups=0.18, wpb=17740.1, bsz=1024, num_updates=10440, lr=3.8989e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=67572
-2024-11-04 00:17:31 - progress_bar.py[line:274] - INFO: epoch 027:    293 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.06, ntokens=17689.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.34, wps=3183.9, ups=0.18, wpb=17689.6, bsz=1024, num_updates=10450, lr=3.89753e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=67628
-2024-11-04 00:18:27 - progress_bar.py[line:274] - INFO: epoch 027:    303 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.113, ntokens=17971.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.65, wps=3197.4, ups=0.18, wpb=17971.5, bsz=1024, num_updates=10460, lr=3.89617e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=67684
-2024-11-04 00:19:23 - progress_bar.py[line:274] - INFO: epoch 027:    313 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.193, ntokens=18229.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.15, wps=3288.9, ups=0.18, wpb=18229.9, bsz=1024, num_updates=10470, lr=3.89481e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=67740
-2024-11-04 00:20:18 - progress_bar.py[line:274] - INFO: epoch 027:    323 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.152, ntokens=18031.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.89, wps=3250.5, ups=0.18, wpb=18031.1, bsz=1024, num_updates=10480, lr=3.89345e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=67795
-2024-11-04 00:21:14 - progress_bar.py[line:274] - INFO: epoch 027:    333 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.204, ntokens=18435.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.22, wps=3293.2, ups=0.18, wpb=18435.1, bsz=1024, num_updates=10490, lr=3.89209e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.1, wall=67851
-2024-11-04 00:22:09 - progress_bar.py[line:274] - INFO: epoch 027:    343 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.12, ntokens=17985.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.69, wps=3269.2, ups=0.18, wpb=17985.5, bsz=1024, num_updates=10500, lr=3.89073e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=67906
-2024-11-04 00:23:05 - progress_bar.py[line:274] - INFO: epoch 027:    353 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.177, ntokens=18472.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.05, wps=3280.5, ups=0.18, wpb=18472.1, bsz=1024, num_updates=10510, lr=3.88937e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=67962
-2024-11-04 00:24:01 - progress_bar.py[line:274] - INFO: epoch 027:    363 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.171, ntokens=18119.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.01, wps=3263.8, ups=0.18, wpb=18119.3, bsz=1024, num_updates=10520, lr=3.88801e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=68018
-2024-11-04 00:24:57 - progress_bar.py[line:274] - INFO: epoch 027:    373 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.137, ntokens=18171.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.8, wps=3255.1, ups=0.18, wpb=18171.5, bsz=1024, num_updates=10530, lr=3.88665e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=68074
-2024-11-04 00:25:52 - progress_bar.py[line:274] - INFO: epoch 027:    383 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.145, ntokens=18057, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.85, wps=3246, ups=0.18, wpb=18057, bsz=1024, num_updates=10540, lr=3.88529e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=68129
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 00:26:34 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 27 @ 10548 updates
+2024-11-07 07:43:05 - trainer.py[line:703] - INFO: begin training epoch 27
+2024-11-07 07:43:05 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 07:43:17 - progress_bar.py[line:274] - INFO: epoch 027:      2 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.475, ntokens=17074.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=351.8, ups=0.02, wpb=17074.3, bsz=985.6, num_updates=10160, lr=3.93699e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=45, gb_free=13.5, wall=64916
+2024-11-07 07:44:10 - progress_bar.py[line:274] - INFO: epoch 027:     12 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.422, ntokens=17256.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3257.3, ups=0.19, wpb=17256.5, bsz=1024, num_updates=10170, lr=3.93563e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=64969
+2024-11-07 07:44:42 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
+2024-11-07 07:45:08 - progress_bar.py[line:274] - INFO: epoch 027:     23 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.478, ntokens=18026.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3140.9, ups=0.17, wpb=18026.2, bsz=1024, num_updates=10180, lr=3.93427e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=33, gb_free=13.6, wall=65027
+2024-11-07 07:46:00 - progress_bar.py[line:274] - INFO: epoch 027:     33 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.444, ntokens=17900, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.44, wps=3437.9, ups=0.19, wpb=17900, bsz=1024, num_updates=10190, lr=3.93291e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=65079
+2024-11-07 07:46:52 - progress_bar.py[line:274] - INFO: epoch 027:     43 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.494, ntokens=17914.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.63, wps=3420, ups=0.19, wpb=17914.7, bsz=1024, num_updates=10200, lr=3.93154e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=65131
+2024-11-07 07:47:45 - progress_bar.py[line:274] - INFO: epoch 027:     53 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.402, ntokens=17459.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.28, wps=3342.4, ups=0.19, wpb=17459.7, bsz=1024, num_updates=10210, lr=3.93018e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=65183
+2024-11-07 07:48:36 - progress_bar.py[line:274] - INFO: epoch 027:     63 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.492, ntokens=18087.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.63, wps=3483, ups=0.19, wpb=18087.4, bsz=1024, num_updates=10220, lr=3.92882e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=65235
+2024-11-07 07:49:29 - progress_bar.py[line:274] - INFO: epoch 027:     73 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.425, ntokens=17419.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.37, wps=3314.9, ups=0.19, wpb=17419.5, bsz=1024, num_updates=10230, lr=3.92746e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=65288
+2024-11-07 07:50:22 - progress_bar.py[line:274] - INFO: epoch 027:     83 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.422, ntokens=17564.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3328.2, ups=0.19, wpb=17564.5, bsz=1024, num_updates=10240, lr=3.9261e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=65341
+2024-11-07 07:51:15 - progress_bar.py[line:274] - INFO: epoch 027:     93 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.44, ntokens=17657.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3324.2, ups=0.19, wpb=17657.3, bsz=1024, num_updates=10250, lr=3.92474e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=65394
+2024-11-07 07:52:08 - progress_bar.py[line:274] - INFO: epoch 027:    103 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.455, ntokens=17711.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.48, wps=3360.9, ups=0.19, wpb=17711.6, bsz=1024, num_updates=10260, lr=3.92338e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=65446
+2024-11-07 07:53:00 - progress_bar.py[line:274] - INFO: epoch 027:    113 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.472, ntokens=17576.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.55, wps=3348.4, ups=0.19, wpb=17576.1, bsz=1024, num_updates=10270, lr=3.92202e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=65499
+2024-11-07 07:53:53 - progress_bar.py[line:274] - INFO: epoch 027:    123 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.459, ntokens=17802.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3366.6, ups=0.19, wpb=17802.5, bsz=1024, num_updates=10280, lr=3.92066e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=65552
+2024-11-07 07:54:46 - progress_bar.py[line:274] - INFO: epoch 027:    133 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.445, ntokens=17535.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.45, wps=3307.8, ups=0.19, wpb=17535.4, bsz=1024, num_updates=10290, lr=3.9193e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=65605
+2024-11-07 07:55:39 - progress_bar.py[line:274] - INFO: epoch 027:    143 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.465, ntokens=17737.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3348.2, ups=0.19, wpb=17737.9, bsz=1024, num_updates=10300, lr=3.91794e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=65658
+2024-11-07 07:56:31 - progress_bar.py[line:274] - INFO: epoch 027:    153 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.484, ntokens=17932.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3422.3, ups=0.19, wpb=17932.7, bsz=1024, num_updates=10310, lr=3.91658e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=65710
+2024-11-07 07:57:23 - progress_bar.py[line:274] - INFO: epoch 027:    163 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.464, ntokens=17756.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3409.5, ups=0.19, wpb=17756.9, bsz=1024, num_updates=10320, lr=3.91522e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=65762
+2024-11-07 07:58:16 - progress_bar.py[line:274] - INFO: epoch 027:    173 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.449, ntokens=17557.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3329.9, ups=0.19, wpb=17557.6, bsz=1024, num_updates=10330, lr=3.91386e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=65815
+2024-11-07 07:59:09 - progress_bar.py[line:274] - INFO: epoch 027:    183 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.46, ntokens=17668.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3331, ups=0.19, wpb=17668.1, bsz=1024, num_updates=10340, lr=3.9125e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=65868
+2024-11-07 08:00:02 - progress_bar.py[line:274] - INFO: epoch 027:    193 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.448, ntokens=17543.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3335.6, ups=0.19, wpb=17543.2, bsz=1024, num_updates=10350, lr=3.91114e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=65921
+2024-11-07 08:00:56 - progress_bar.py[line:274] - INFO: epoch 027:    203 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.486, ntokens=17995.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3349.1, ups=0.19, wpb=17995.7, bsz=1024, num_updates=10360, lr=3.90978e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=36, gb_free=13.4, wall=65974
+2024-11-07 08:01:48 - progress_bar.py[line:274] - INFO: epoch 027:    213 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.458, ntokens=17953.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3418.4, ups=0.19, wpb=17953.5, bsz=1024, num_updates=10370, lr=3.90842e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=33, gb_free=13.5, wall=66027
+2024-11-07 08:02:40 - progress_bar.py[line:274] - INFO: epoch 027:    223 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.437, ntokens=17539.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.42, wps=3352.2, ups=0.19, wpb=17539.3, bsz=1024, num_updates=10380, lr=3.90706e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=66079
+2024-11-07 08:03:34 - progress_bar.py[line:274] - INFO: epoch 027:    233 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.434, ntokens=17737.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.41, wps=3330.1, ups=0.19, wpb=17737.5, bsz=1024, num_updates=10390, lr=3.9057e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=66133
+2024-11-07 08:04:26 - progress_bar.py[line:274] - INFO: epoch 027:    243 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.438, ntokens=17444.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.42, wps=3330.9, ups=0.19, wpb=17444.1, bsz=1024, num_updates=10400, lr=3.90434e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=66185
+2024-11-07 08:05:19 - progress_bar.py[line:274] - INFO: epoch 027:    253 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.449, ntokens=17704.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3371.3, ups=0.19, wpb=17704.8, bsz=1024, num_updates=10410, lr=3.90298e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=66237
+2024-11-07 08:06:11 - progress_bar.py[line:274] - INFO: epoch 027:    263 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.468, ntokens=17815, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3379.9, ups=0.19, wpb=17815, bsz=1024, num_updates=10420, lr=3.90162e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=66290
+2024-11-07 08:07:04 - progress_bar.py[line:274] - INFO: epoch 027:    273 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.467, ntokens=17890.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3383.3, ups=0.19, wpb=17890.8, bsz=1024, num_updates=10430, lr=3.90026e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=66343
+2024-11-07 08:07:57 - progress_bar.py[line:274] - INFO: epoch 027:    283 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.448, ntokens=17740.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3333.4, ups=0.19, wpb=17740.1, bsz=1024, num_updates=10440, lr=3.8989e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=66396
+2024-11-07 08:08:50 - progress_bar.py[line:274] - INFO: epoch 027:    293 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.458, ntokens=17689.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.49, wps=3365.7, ups=0.19, wpb=17689.6, bsz=1024, num_updates=10450, lr=3.89753e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=66449
+2024-11-07 08:09:42 - progress_bar.py[line:274] - INFO: epoch 027:    303 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.511, ntokens=17971.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.7, wps=3466.1, ups=0.19, wpb=17971.5, bsz=1024, num_updates=10460, lr=3.89617e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=66501
+2024-11-07 08:10:33 - progress_bar.py[line:274] - INFO: epoch 027:    313 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.568, ntokens=18229.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3523.5, ups=0.19, wpb=18229.9, bsz=1024, num_updates=10470, lr=3.89481e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=66552
+2024-11-07 08:11:25 - progress_bar.py[line:274] - INFO: epoch 027:    323 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.544, ntokens=18031.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3471.8, ups=0.19, wpb=18031.1, bsz=1024, num_updates=10480, lr=3.89345e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=66604
+2024-11-07 08:12:18 - progress_bar.py[line:274] - INFO: epoch 027:    333 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.572, ntokens=18435.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.95, wps=3520.8, ups=0.19, wpb=18435.1, bsz=1024, num_updates=10490, lr=3.89209e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=31, gb_free=13.1, wall=66657
+2024-11-07 08:13:10 - progress_bar.py[line:274] - INFO: epoch 027:    343 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.504, ntokens=17985.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.67, wps=3438.8, ups=0.19, wpb=17985.5, bsz=1024, num_updates=10500, lr=3.89073e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=66709
+2024-11-07 08:14:02 - progress_bar.py[line:274] - INFO: epoch 027:    353 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.568, ntokens=18472.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.93, wps=3562.8, ups=0.19, wpb=18472.1, bsz=1024, num_updates=10510, lr=3.88937e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=32, gb_free=13.4, wall=66761
+2024-11-07 08:14:54 - progress_bar.py[line:274] - INFO: epoch 027:    363 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.539, ntokens=18119.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.81, wps=3451.7, ups=0.19, wpb=18119.3, bsz=1024, num_updates=10520, lr=3.88801e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=31, gb_free=13.3, wall=66813
+2024-11-07 08:15:47 - progress_bar.py[line:274] - INFO: epoch 027:    373 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.537, ntokens=18171.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3479.5, ups=0.19, wpb=18171.5, bsz=1024, num_updates=10530, lr=3.88665e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=66866
+2024-11-07 08:16:40 - progress_bar.py[line:274] - INFO: epoch 027:    383 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.544, ntokens=18057, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.83, wps=3381.6, ups=0.19, wpb=18057, bsz=1024, num_updates=10540, lr=3.88529e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=35, gb_free=13.5, wall=66919
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+2024-11-07 08:17:20 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 27 @ 10548 updates
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
-
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
-2024-11-04 00:26:34 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt': File exists: File exists
+2024-11-07 08:17:20 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt': File exists
+: File exists: File exists: File exists
+
+: File exists
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 00:28:21 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 08:19:05 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 5 seek offset 250000
-2024-11-04 00:30:13 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 27 @ 10548 updates, score 0) (writing took 219.58905516800587 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 4 seek offset 200000
+slice_id 5 seek offset 250000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+
+
 slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
 slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-
+slice_id 2 seek offset 100000
+2024-11-07 08:20:57 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 27 @ 10548 updates, score 0) (writing took 217.7570783739793 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_27.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+slice_id 7 seek offset 350000
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-2024-11-04 00:31:08 - train.py[line:336] - INFO: end of epoch 27 (average epoch stats below)
-2024-11-04 00:31:08 - progress_bar.py[line:282] - INFO: epoch 027 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.068 | ntokens 17810.8 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.39 | wps 2665.6 | ups 0.15 | wpb 17810.8 | bsz 1023 | num_updates 10548 | lr 3.8842e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 2074 | gb_free 13.3 | wall 68445
-2024-11-04 00:31:08 - trainer.py[line:639] - INFO: loading train data for epoch 28
+2024-11-07 08:21:52 - train.py[line:336] - INFO: end of epoch 27 (average epoch stats below)
+2024-11-07 08:21:52 - progress_bar.py[line:282] - INFO: epoch 027 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.477 | ntokens 17809.3 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.57 | wps 2792.2 | ups 0.16 | wpb 17809.3 | bsz 1023 | num_updates 10548 | lr 3.8842e-05 | gnorm 0.008 | clip 0 | loss_scale 16384 | train_wall 1216 | gb_free 13.3 | wall 67231
+2024-11-07 08:21:52 - trainer.py[line:639] - INFO: loading train data for epoch 28
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-04 00:33:52 - trainer.py[line:703] - INFO: begin training epoch 28
-2024-11-04 00:33:52 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 00:34:05 - progress_bar.py[line:274] - INFO: epoch 028:      2 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.175, ntokens=17396.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=9.03, wps=353.4, ups=0.02, wpb=17396.6, bsz=985.6, num_updates=10550, lr=3.88393e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=47, gb_free=13.6, wall=68622
-2024-11-04 00:34:59 - progress_bar.py[line:274] - INFO: epoch 028:     12 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.99, ntokens=17404.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.94, wps=3223.9, ups=0.19, wpb=17404.9, bsz=1024, num_updates=10560, lr=3.88257e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=68676
-2024-11-04 00:35:53 - progress_bar.py[line:274] - INFO: epoch 028:     22 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.119, ntokens=18166.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.69, wps=3330, ups=0.18, wpb=18166.3, bsz=1024, num_updates=10570, lr=3.88121e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=44, gb_free=13.6, wall=68730
-2024-11-04 00:36:49 - progress_bar.py[line:274] - INFO: epoch 028:     32 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.929, ntokens=17575.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.62, wps=3137.6, ups=0.18, wpb=17575.4, bsz=1024, num_updates=10580, lr=3.87985e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=68786
-2024-11-04 00:37:46 - progress_bar.py[line:274] - INFO: epoch 028:     42 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.045, ntokens=17742.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.26, wps=3121.1, ups=0.18, wpb=17742.7, bsz=1024, num_updates=10590, lr=3.87849e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=68843
-2024-11-04 00:38:41 - progress_bar.py[line:274] - INFO: epoch 028:     52 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.942, ntokens=17453.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.68, wps=3169.4, ups=0.18, wpb=17453.4, bsz=1024, num_updates=10600, lr=3.87713e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=68898
-2024-11-04 00:39:38 - progress_bar.py[line:274] - INFO: epoch 028:     62 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.094, ntokens=18149, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.54, wps=3206.7, ups=0.18, wpb=18149, bsz=1024, num_updates=10610, lr=3.87577e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=68955
-2024-11-04 00:40:34 - progress_bar.py[line:274] - INFO: epoch 028:     72 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.071, ntokens=17813.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.4, wps=3161.2, ups=0.18, wpb=17813.9, bsz=1024, num_updates=10620, lr=3.87441e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=69011
-2024-11-04 00:41:29 - progress_bar.py[line:274] - INFO: epoch 028:     82 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.985, ntokens=17543.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.92, wps=3217.2, ups=0.18, wpb=17543.5, bsz=1024, num_updates=10630, lr=3.87305e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=69066
-2024-11-04 00:42:24 - progress_bar.py[line:274] - INFO: epoch 028:     92 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.005, ntokens=17686.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.03, wps=3163.4, ups=0.18, wpb=17686.1, bsz=1024, num_updates=10640, lr=3.87169e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=69122
-2024-11-04 00:43:21 - progress_bar.py[line:274] - INFO: epoch 028:    102 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.026, ntokens=17570.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.15, wps=3122, ups=0.18, wpb=17570.1, bsz=1024, num_updates=10650, lr=3.87033e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=69178
-2024-11-04 00:44:17 - progress_bar.py[line:274] - INFO: epoch 028:    112 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.994, ntokens=17301.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.97, wps=3067, ups=0.18, wpb=17301.4, bsz=1024, num_updates=10660, lr=3.86897e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=69234
-2024-11-04 00:45:14 - progress_bar.py[line:274] - INFO: epoch 028:    122 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.96, ntokens=17626.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.78, wps=3125.1, ups=0.18, wpb=17626.6, bsz=1024, num_updates=10670, lr=3.86761e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=69291
-2024-11-04 00:46:10 - progress_bar.py[line:274] - INFO: epoch 028:    132 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.026, ntokens=17526.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.14, wps=3113, ups=0.18, wpb=17526.5, bsz=1024, num_updates=10680, lr=3.86625e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=69347
-2024-11-04 00:47:06 - progress_bar.py[line:274] - INFO: epoch 028:    142 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.052, ntokens=17776.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.29, wps=3153.9, ups=0.18, wpb=17776.2, bsz=1024, num_updates=10690, lr=3.86489e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=69403
-2024-11-04 00:48:01 - progress_bar.py[line:274] - INFO: epoch 028:    152 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.018, ntokens=17754.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.1, wps=3220.2, ups=0.18, wpb=17754.5, bsz=1024, num_updates=10700, lr=3.86353e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=69458
-2024-11-04 00:48:56 - progress_bar.py[line:274] - INFO: epoch 028:    162 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.965, ntokens=17457.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.81, wps=3168, ups=0.18, wpb=17457.6, bsz=1024, num_updates=10710, lr=3.86216e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=69514
-2024-11-04 00:49:53 - progress_bar.py[line:274] - INFO: epoch 028:    172 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.053, ntokens=17778.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.3, wps=3154.5, ups=0.18, wpb=17778.6, bsz=1024, num_updates=10720, lr=3.8608e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=69570
-2024-11-04 00:50:48 - progress_bar.py[line:274] - INFO: epoch 028:    182 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.011, ntokens=17660.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.06, wps=3183.8, ups=0.18, wpb=17660.5, bsz=1024, num_updates=10730, lr=3.85944e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=69625
-2024-11-04 00:51:45 - progress_bar.py[line:274] - INFO: epoch 028:    192 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.028, ntokens=17879.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.15, wps=3172.3, ups=0.18, wpb=17879.5, bsz=1024, num_updates=10740, lr=3.85808e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=69682
-2024-11-04 00:52:41 - progress_bar.py[line:274] - INFO: epoch 028:    202 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.062, ntokens=18124, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.35, wps=3244, ups=0.18, wpb=18124, bsz=1024, num_updates=10750, lr=3.85672e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=69738
-2024-11-04 00:53:37 - progress_bar.py[line:274] - INFO: epoch 028:    212 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.037, ntokens=18060.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.21, wps=3209.4, ups=0.18, wpb=18060.9, bsz=1024, num_updates=10760, lr=3.85536e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=69794
-2024-11-04 00:54:32 - progress_bar.py[line:274] - INFO: epoch 028:    222 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.015, ntokens=17866.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.09, wps=3208, ups=0.18, wpb=17866.3, bsz=1024, num_updates=10770, lr=3.854e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=69850
-2024-11-04 00:55:28 - progress_bar.py[line:274] - INFO: epoch 028:    232 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.044, ntokens=18005, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.25, wps=3255.8, ups=0.18, wpb=18005, bsz=1024, num_updates=10780, lr=3.85264e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=69905
-2024-11-04 00:56:23 - progress_bar.py[line:274] - INFO: epoch 028:    242 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.079, ntokens=17661, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.45, wps=3220, ups=0.18, wpb=17661, bsz=1024, num_updates=10790, lr=3.85128e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=40, gb_free=13.5, wall=69960
-2024-11-04 00:57:17 - progress_bar.py[line:274] - INFO: epoch 028:    252 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.973, ntokens=17537.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.85, wps=3225.1, ups=0.18, wpb=17537.4, bsz=1024, num_updates=10800, lr=3.84992e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=38, gb_free=13.5, wall=70014
-2024-11-04 00:58:13 - progress_bar.py[line:274] - INFO: epoch 028:    262 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.021, ntokens=17623.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.12, wps=3143.2, ups=0.18, wpb=17623.5, bsz=1024, num_updates=10810, lr=3.84856e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=70070
-2024-11-04 00:59:08 - progress_bar.py[line:274] - INFO: epoch 028:    272 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.024, ntokens=17943.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=3259.8, ups=0.18, wpb=17943.7, bsz=1024, num_updates=10820, lr=3.8472e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=47, gb_free=13.5, wall=70125
-2024-11-04 01:00:03 - progress_bar.py[line:274] - INFO: epoch 028:    282 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.017, ntokens=17571.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.09, wps=3190.7, ups=0.18, wpb=17571.4, bsz=1024, num_updates=10830, lr=3.84584e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=35, gb_free=13.6, wall=70180
-2024-11-04 01:00:59 - progress_bar.py[line:274] - INFO: epoch 028:    292 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.021, ntokens=17632.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.12, wps=3174.6, ups=0.18, wpb=17632.5, bsz=1024, num_updates=10840, lr=3.84448e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=70236
-2024-11-04 01:01:54 - progress_bar.py[line:274] - INFO: epoch 028:    302 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.076, ntokens=17665.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.43, wps=3188.9, ups=0.18, wpb=17665.8, bsz=1024, num_updates=10850, lr=3.84312e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=70291
-2024-11-04 01:02:49 - progress_bar.py[line:274] - INFO: epoch 028:    312 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.227, ntokens=18662.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.36, wps=3375, ups=0.18, wpb=18662.2, bsz=1024, num_updates=10860, lr=3.84176e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=70347
-2024-11-04 01:03:45 - progress_bar.py[line:274] - INFO: epoch 028:    322 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.135, ntokens=18034.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.79, wps=3229.9, ups=0.18, wpb=18034.8, bsz=1024, num_updates=10870, lr=3.8404e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=70402
-2024-11-04 01:04:41 - progress_bar.py[line:274] - INFO: epoch 028:    332 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.178, ntokens=18251.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.05, wps=3265.1, ups=0.18, wpb=18251.7, bsz=1024, num_updates=10880, lr=3.83904e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=70458
-2024-11-04 01:05:36 - progress_bar.py[line:274] - INFO: epoch 028:    342 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.071, ntokens=17884.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.4, wps=3241.1, ups=0.18, wpb=17884.3, bsz=1024, num_updates=10890, lr=3.83768e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=70513
-2024-11-04 01:06:32 - progress_bar.py[line:274] - INFO: epoch 028:    352 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.236, ntokens=18828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.42, wps=3371.8, ups=0.18, wpb=18828, bsz=1024, num_updates=10900, lr=3.83632e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=70569
-2024-11-04 01:07:29 - progress_bar.py[line:274] - INFO: epoch 028:    362 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.141, ntokens=18233, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.82, wps=3237.5, ups=0.18, wpb=18233, bsz=1024, num_updates=10910, lr=3.83496e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=70626
-2024-11-04 01:08:24 - progress_bar.py[line:274] - INFO: epoch 028:    372 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.136, ntokens=17918.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.79, wps=3219, ups=0.18, wpb=17918.7, bsz=1024, num_updates=10920, lr=3.8336e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=70681
-2024-11-04 01:09:20 - progress_bar.py[line:274] - INFO: epoch 028:    382 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.134, ntokens=18099.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.78, wps=3225.2, ups=0.18, wpb=18099.7, bsz=1024, num_updates=10930, lr=3.83224e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=70737
+2024-11-07 08:24:34 - trainer.py[line:703] - INFO: begin training epoch 28
+2024-11-07 08:24:34 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 08:24:47 - progress_bar.py[line:274] - INFO: epoch 028:      2 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.522, ntokens=17396.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=357.4, ups=0.02, wpb=17396.6, bsz=985.6, num_updates=10550, lr=3.88393e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=41, gb_free=13.6, wall=67406
+2024-11-07 08:25:39 - progress_bar.py[line:274] - INFO: epoch 028:     12 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.44, ntokens=17404.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3307.4, ups=0.19, wpb=17404.9, bsz=1024, num_updates=10560, lr=3.88257e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=30, gb_free=13.3, wall=67458
+2024-11-07 08:26:32 - progress_bar.py[line:274] - INFO: epoch 028:     22 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.522, ntokens=18166.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=3448.1, ups=0.19, wpb=18166.3, bsz=1024, num_updates=10570, lr=3.88121e-05, gnorm=0.015, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=67511
+2024-11-07 08:27:24 - progress_bar.py[line:274] - INFO: epoch 028:     32 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.397, ntokens=17575.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.27, wps=3360.3, ups=0.19, wpb=17575.4, bsz=1024, num_updates=10580, lr=3.87985e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=67563
+2024-11-07 08:28:17 - progress_bar.py[line:274] - INFO: epoch 028:     42 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.445, ntokens=17742.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.44, wps=3389.9, ups=0.19, wpb=17742.7, bsz=1024, num_updates=10590, lr=3.87849e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=67616
+2024-11-07 08:29:09 - progress_bar.py[line:274] - INFO: epoch 028:     52 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.414, ntokens=17453.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3321.3, ups=0.19, wpb=17453.4, bsz=1024, num_updates=10600, lr=3.87713e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.3, wall=67668
+2024-11-07 08:30:02 - progress_bar.py[line:274] - INFO: epoch 028:     62 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.488, ntokens=18149, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.61, wps=3462.3, ups=0.19, wpb=18149, bsz=1024, num_updates=10610, lr=3.87577e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=67721
+2024-11-07 08:30:58 - progress_bar.py[line:274] - INFO: epoch 028:     72 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.477, ntokens=17813.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3186, ups=0.18, wpb=17813.9, bsz=1024, num_updates=10620, lr=3.87441e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=43, gb_free=13.5, wall=67777
+2024-11-07 08:31:51 - progress_bar.py[line:274] - INFO: epoch 028:     82 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.418, ntokens=17543.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.34, wps=3276.8, ups=0.19, wpb=17543.5, bsz=1024, num_updates=10630, lr=3.87305e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.4, wall=67830
+2024-11-07 08:32:43 - progress_bar.py[line:274] - INFO: epoch 028:     92 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.442, ntokens=17686.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3396.9, ups=0.19, wpb=17686.1, bsz=1024, num_updates=10640, lr=3.87169e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=67882
+2024-11-07 08:33:36 - progress_bar.py[line:274] - INFO: epoch 028:    102 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.433, ntokens=17570.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.4, wps=3358.6, ups=0.19, wpb=17570.1, bsz=1024, num_updates=10650, lr=3.87033e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=51, gb_free=13.4, wall=67934
+2024-11-07 08:34:28 - progress_bar.py[line:274] - INFO: epoch 028:    112 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.428, ntokens=17301.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.38, wps=3312.2, ups=0.19, wpb=17301.4, bsz=1024, num_updates=10660, lr=3.86897e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=46, gb_free=13.6, wall=67987
+2024-11-07 08:35:20 - progress_bar.py[line:274] - INFO: epoch 028:    122 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.432, ntokens=17626.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.4, wps=3391.3, ups=0.19, wpb=17626.6, bsz=1024, num_updates=10670, lr=3.86761e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=42, gb_free=13.5, wall=68039
+2024-11-07 08:36:12 - progress_bar.py[line:274] - INFO: epoch 028:    132 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.45, ntokens=17526.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3359.5, ups=0.19, wpb=17526.5, bsz=1024, num_updates=10680, lr=3.86625e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=36, gb_free=13.4, wall=68091
+2024-11-07 08:37:04 - progress_bar.py[line:274] - INFO: epoch 028:    142 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.461, ntokens=17776.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3409.9, ups=0.19, wpb=17776.2, bsz=1024, num_updates=10690, lr=3.86489e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=68143
+2024-11-07 08:37:57 - progress_bar.py[line:274] - INFO: epoch 028:    152 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.45, ntokens=17754.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3355.9, ups=0.19, wpb=17754.5, bsz=1024, num_updates=10700, lr=3.86353e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=68196
+2024-11-07 08:38:49 - progress_bar.py[line:274] - INFO: epoch 028:    162 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.408, ntokens=17457.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.31, wps=3333.5, ups=0.19, wpb=17457.6, bsz=1024, num_updates=10710, lr=3.86216e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=68248
+2024-11-07 08:39:42 - progress_bar.py[line:274] - INFO: epoch 028:    172 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.451, ntokens=17778.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.47, wps=3388.1, ups=0.19, wpb=17778.6, bsz=1024, num_updates=10720, lr=3.8608e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=68301
+2024-11-07 08:40:35 - progress_bar.py[line:274] - INFO: epoch 028:    182 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.451, ntokens=17660.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.47, wps=3325.3, ups=0.19, wpb=17660.5, bsz=1024, num_updates=10730, lr=3.85944e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=68354
+2024-11-07 08:41:27 - progress_bar.py[line:274] - INFO: epoch 028:    192 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.458, ntokens=17879.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3433.9, ups=0.19, wpb=17879.5, bsz=1024, num_updates=10740, lr=3.85808e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=68406
+2024-11-07 08:42:22 - progress_bar.py[line:274] - INFO: epoch 028:    202 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.48, ntokens=18124, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3296.5, ups=0.18, wpb=18124, bsz=1024, num_updates=10750, lr=3.85672e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=38, gb_free=13.5, wall=68461
+2024-11-07 08:43:15 - progress_bar.py[line:274] - INFO: epoch 028:    212 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.47, ntokens=18060.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3394.5, ups=0.19, wpb=18060.9, bsz=1024, num_updates=10760, lr=3.85536e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=68514
+2024-11-07 08:44:08 - progress_bar.py[line:274] - INFO: epoch 028:    222 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.46, ntokens=17866.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3403.2, ups=0.19, wpb=17866.3, bsz=1024, num_updates=10770, lr=3.854e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=68567
+2024-11-07 08:44:59 - progress_bar.py[line:274] - INFO: epoch 028:    232 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.48, ntokens=18005, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3489.6, ups=0.19, wpb=18005, bsz=1024, num_updates=10780, lr=3.85264e-05, gnorm=0.014, clip=0, loss_scale=32768, train_wall=50, gb_free=13.5, wall=68618
+2024-11-07 08:45:51 - progress_bar.py[line:274] - INFO: epoch 028:    242 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.461, ntokens=17661, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.51, wps=3395, ups=0.19, wpb=17661, bsz=1024, num_updates=10790, lr=3.85128e-05, gnorm=0.013, clip=0, loss_scale=32768, train_wall=39, gb_free=13.5, wall=68670
+2024-11-07 08:46:45 - progress_bar.py[line:274] - INFO: epoch 028:    252 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.43, ntokens=17537.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.39, wps=3298, ups=0.19, wpb=17537.4, bsz=1024, num_updates=10800, lr=3.84992e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=34, gb_free=13.5, wall=68723
+2024-11-07 08:47:38 - progress_bar.py[line:274] - INFO: epoch 028:    262 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.447, ntokens=17623.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.45, wps=3313.9, ups=0.19, wpb=17623.5, bsz=1024, num_updates=10810, lr=3.84856e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=68777
+2024-11-07 08:48:31 - progress_bar.py[line:274] - INFO: epoch 028:    272 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.482, ntokens=17943.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3384.3, ups=0.19, wpb=17943.7, bsz=1024, num_updates=10820, lr=3.8472e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=68830
+2024-11-07 08:49:25 - progress_bar.py[line:274] - INFO: epoch 028:    282 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.438, ntokens=17571.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.42, wps=3260.4, ups=0.19, wpb=17571.4, bsz=1024, num_updates=10830, lr=3.84584e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=68884
+2024-11-07 08:50:18 - progress_bar.py[line:274] - INFO: epoch 028:    292 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.441, ntokens=17632.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3298.2, ups=0.19, wpb=17632.5, bsz=1024, num_updates=10840, lr=3.84448e-05, gnorm=0.015, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=68937
+2024-11-07 08:51:12 - progress_bar.py[line:274] - INFO: epoch 028:    302 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.468, ntokens=17665.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3303.2, ups=0.19, wpb=17665.8, bsz=1024, num_updates=10850, lr=3.84312e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=68990
+2024-11-07 08:52:05 - progress_bar.py[line:274] - INFO: epoch 028:    312 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.615, ntokens=18662.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3523.1, ups=0.19, wpb=18662.2, bsz=1024, num_updates=10860, lr=3.84176e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=69043
+2024-11-07 08:52:58 - progress_bar.py[line:274] - INFO: epoch 028:    322 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.518, ntokens=18034.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3363.3, ups=0.19, wpb=18034.8, bsz=1024, num_updates=10870, lr=3.8404e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=69097
+2024-11-07 08:53:51 - progress_bar.py[line:274] - INFO: epoch 028:    332 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.556, ntokens=18251.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.88, wps=3482.7, ups=0.19, wpb=18251.7, bsz=1024, num_updates=10880, lr=3.83904e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=69149
+2024-11-07 08:54:43 - progress_bar.py[line:274] - INFO: epoch 028:    342 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.491, ntokens=17884.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.62, wps=3413.3, ups=0.19, wpb=17884.3, bsz=1024, num_updates=10890, lr=3.83768e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=69202
+2024-11-07 08:55:37 - progress_bar.py[line:274] - INFO: epoch 028:    352 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.6, ntokens=18828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3496.3, ups=0.19, wpb=18828, bsz=1024, num_updates=10900, lr=3.83632e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=37, gb_free=13.5, wall=69256
+2024-11-07 08:56:30 - progress_bar.py[line:274] - INFO: epoch 028:    362 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.55, ntokens=18233, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=3408.7, ups=0.19, wpb=18233, bsz=1024, num_updates=10910, lr=3.83496e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=50, gb_free=13.4, wall=69309
+2024-11-07 08:57:24 - progress_bar.py[line:274] - INFO: epoch 028:    372 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.495, ntokens=17918.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3365.7, ups=0.19, wpb=17918.7, bsz=1024, num_updates=10920, lr=3.8336e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=69362
+2024-11-07 08:58:18 - progress_bar.py[line:274] - INFO: epoch 028:    382 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.519, ntokens=18099.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.73, wps=3338, ups=0.18, wpb=18099.7, bsz=1024, num_updates=10930, lr=3.83224e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=69417
 reach the end of datafile, start a new reader
 reach the end of datafile, start a new reader
-2024-11-04 01:10:06 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 28 @ 10939 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
 
+2024-11-07 08:59:04 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 28 @ 10939 updates
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
-2024-11-04 01:10:06 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt': File exists: File exists
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
+2024-11-07 08:59:04 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt': File exists: File exists: File exists
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 01:11:52 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 09:00:55 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 
 slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-2024-11-04 01:13:44 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 28 @ 10939 updates, score 0) (writing took 217.67759520799154 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
 slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-
+slice_id 6 seek offset 300000
+2024-11-07 09:02:47 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 28 @ 10939 updates, score 0) (writing took 223.62149764501373 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_28.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
 slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-04 01:14:38 - train.py[line:336] - INFO: end of epoch 28 (average epoch stats below)
-2024-11-04 01:14:38 - progress_bar.py[line:282] - INFO: epoch 028 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.053 | ntokens 17809.2 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.3 | wps 2667.2 | ups 0.15 | wpb 17809.2 | bsz 1023 | num_updates 10939 | lr 3.83101e-05 | gnorm 0.008 | clip 0 | loss_scale 65536 | train_wall 2072 | gb_free 13.4 | wall 71056
-2024-11-04 01:14:38 - trainer.py[line:639] - INFO: loading train data for epoch 29
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+
+slice_id 1 seek offset 50000
+slice_id 5 seek offset 250000
+slice_id 4 seek offset 200000
+2024-11-07 09:03:42 - train.py[line:336] - INFO: end of epoch 28 (average epoch stats below)
+2024-11-07 09:03:42 - progress_bar.py[line:282] - INFO: epoch 028 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.471 | ntokens 17809.2 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.54 | wps 2774.5 | ups 0.16 | wpb 17809.2 | bsz 1023 | num_updates 10939 | lr 3.83101e-05 | gnorm 0.01 | clip 0 | loss_scale 32768 | train_wall 1475 | gb_free 13.4 | wall 69740
+2024-11-07 09:03:42 - trainer.py[line:639] - INFO: loading train data for epoch 29
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-04 01:17:23 - trainer.py[line:703] - INFO: begin training epoch 29
-2024-11-04 01:17:23 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 01:17:30 - progress_bar.py[line:274] - INFO: epoch 029:      1 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.128, ntokens=17202.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.74, wps=351.2, ups=0.02, wpb=17202.1, bsz=985.6, num_updates=10940, lr=3.83088e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=49, gb_free=13.5, wall=71227
-2024-11-04 01:18:25 - progress_bar.py[line:274] - INFO: epoch 029:     11 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.01, ntokens=17516.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.06, wps=3190.3, ups=0.18, wpb=17516.4, bsz=1024, num_updates=10950, lr=3.82952e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=71282
-2024-11-04 01:19:20 - progress_bar.py[line:274] - INFO: epoch 029:     21 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.068, ntokens=17804.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.39, wps=3270.8, ups=0.18, wpb=17804.2, bsz=1024, num_updates=10960, lr=3.82815e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=37, gb_free=13.4, wall=71337
-2024-11-04 01:20:16 - progress_bar.py[line:274] - INFO: epoch 029:     31 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.014, ntokens=17740.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.08, wps=3137.7, ups=0.18, wpb=17740.2, bsz=1024, num_updates=10970, lr=3.82679e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=71393
-2024-11-04 01:21:12 - progress_bar.py[line:274] - INFO: epoch 029:     41 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.977, ntokens=17555, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.87, wps=3141.3, ups=0.18, wpb=17555, bsz=1024, num_updates=10980, lr=3.82543e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=71449
-2024-11-04 01:22:07 - progress_bar.py[line:274] - INFO: epoch 029:     51 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.97, ntokens=17549.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.84, wps=3181, ups=0.18, wpb=17549.4, bsz=1024, num_updates=10990, lr=3.82407e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=71504
-2024-11-04 01:23:04 - progress_bar.py[line:274] - INFO: epoch 029:     61 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.072, ntokens=18057.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.41, wps=3184.4, ups=0.18, wpb=18057.1, bsz=1024, num_updates=11000, lr=3.82271e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=71561
-2024-11-04 01:24:01 - progress_bar.py[line:274] - INFO: epoch 029:     71 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.981, ntokens=17383, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.9, wps=3062.2, ups=0.18, wpb=17383, bsz=1024, num_updates=11010, lr=3.82135e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=71618
-2024-11-04 01:24:56 - progress_bar.py[line:274] - INFO: epoch 029:     81 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.006, ntokens=17884.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.03, wps=3206.6, ups=0.18, wpb=17884.6, bsz=1024, num_updates=11020, lr=3.81999e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=71673
-2024-11-04 01:25:51 - progress_bar.py[line:274] - INFO: epoch 029:     91 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.019, ntokens=17673.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.1, wps=3227, ups=0.18, wpb=17673.3, bsz=1024, num_updates=11030, lr=3.81863e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=41, gb_free=13.5, wall=71728
-2024-11-04 01:26:48 - progress_bar.py[line:274] - INFO: epoch 029:    101 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.984, ntokens=17616.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.91, wps=3123.2, ups=0.18, wpb=17616.7, bsz=1024, num_updates=11040, lr=3.81727e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=71785
-2024-11-04 01:27:43 - progress_bar.py[line:274] - INFO: epoch 029:    111 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.022, ntokens=17463.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.12, wps=3147.4, ups=0.18, wpb=17463.3, bsz=1024, num_updates=11050, lr=3.81591e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=71840
-2024-11-04 01:28:39 - progress_bar.py[line:274] - INFO: epoch 029:    121 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.976, ntokens=17894.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.87, wps=3169.2, ups=0.18, wpb=17894.6, bsz=1024, num_updates=11060, lr=3.81455e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=71897
-2024-11-04 01:29:36 - progress_bar.py[line:274] - INFO: epoch 029:    131 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.981, ntokens=17375.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3058.6, ups=0.18, wpb=17375.8, bsz=1024, num_updates=11070, lr=3.81319e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=71953
-2024-11-04 01:30:33 - progress_bar.py[line:274] - INFO: epoch 029:    141 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.038, ntokens=17884.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.21, wps=3140.9, ups=0.18, wpb=17884.9, bsz=1024, num_updates=11080, lr=3.81183e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=72010
-2024-11-04 01:31:29 - progress_bar.py[line:274] - INFO: epoch 029:    151 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.994, ntokens=17572.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.97, wps=3160.7, ups=0.18, wpb=17572.4, bsz=1024, num_updates=11090, lr=3.81047e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=72066
-2024-11-04 01:32:25 - progress_bar.py[line:274] - INFO: epoch 029:    161 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3, ntokens=17607.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8, wps=3129.4, ups=0.18, wpb=17607.5, bsz=1024, num_updates=11100, lr=3.80911e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=72122
-2024-11-04 01:33:21 - progress_bar.py[line:274] - INFO: epoch 029:    171 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.013, ntokens=17920.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.07, wps=3196, ups=0.18, wpb=17920.8, bsz=1024, num_updates=11110, lr=3.80775e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=72178
-2024-11-04 01:34:17 - progress_bar.py[line:274] - INFO: epoch 029:    181 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.935, ntokens=17297.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.65, wps=3100.2, ups=0.18, wpb=17297.5, bsz=1024, num_updates=11120, lr=3.80639e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=72234
-2024-11-04 01:35:13 - progress_bar.py[line:274] - INFO: epoch 029:    191 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.031, ntokens=17852.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.17, wps=3169.8, ups=0.18, wpb=17852.9, bsz=1024, num_updates=11130, lr=3.80503e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=72290
-2024-11-04 01:36:09 - progress_bar.py[line:274] - INFO: epoch 029:    201 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.042, ntokens=18025.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=3236.7, ups=0.18, wpb=18025.6, bsz=1024, num_updates=11140, lr=3.80367e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=72346
-2024-11-04 01:37:05 - progress_bar.py[line:274] - INFO: epoch 029:    211 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.968, ntokens=17768.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.83, wps=3169, ups=0.18, wpb=17768.8, bsz=1024, num_updates=11150, lr=3.80231e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=72402
-2024-11-04 01:38:01 - progress_bar.py[line:274] - INFO: epoch 029:    221 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.931, ntokens=17514.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.63, wps=3139, ups=0.18, wpb=17514.4, bsz=1024, num_updates=11160, lr=3.80095e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=72458
-2024-11-04 01:38:57 - progress_bar.py[line:274] - INFO: epoch 029:    231 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.968, ntokens=17518.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.83, wps=3122, ups=0.18, wpb=17518.4, bsz=1024, num_updates=11170, lr=3.79959e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=72514
-2024-11-04 01:39:53 - progress_bar.py[line:274] - INFO: epoch 029:    241 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.066, ntokens=17797.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.38, wps=3188.3, ups=0.18, wpb=17797.2, bsz=1024, num_updates=11180, lr=3.79823e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=72570
-2024-11-04 01:40:50 - progress_bar.py[line:274] - INFO: epoch 029:    251 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.043, ntokens=18007.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.24, wps=3166.3, ups=0.18, wpb=18007.7, bsz=1024, num_updates=11190, lr=3.79687e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=72627
-2024-11-04 01:41:45 - progress_bar.py[line:274] - INFO: epoch 029:    261 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.04, ntokens=17879.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=3247, ups=0.18, wpb=17879.9, bsz=1024, num_updates=11200, lr=3.79551e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=72682
-2024-11-04 01:42:40 - progress_bar.py[line:274] - INFO: epoch 029:    271 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.051, ntokens=17990.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.29, wps=3230.6, ups=0.18, wpb=17990.5, bsz=1024, num_updates=11210, lr=3.79414e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=72738
-2024-11-04 01:43:36 - progress_bar.py[line:274] - INFO: epoch 029:    281 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.979, ntokens=17629.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.88, wps=3190.2, ups=0.18, wpb=17629.6, bsz=1024, num_updates=11220, lr=3.79278e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=72793
-2024-11-04 01:44:32 - progress_bar.py[line:274] - INFO: epoch 029:    291 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.985, ntokens=17577.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.92, wps=3140.1, ups=0.18, wpb=17577.3, bsz=1024, num_updates=11230, lr=3.79142e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=72849
-2024-11-04 01:45:26 - progress_bar.py[line:274] - INFO: epoch 029:    301 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.112, ntokens=17999.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.65, wps=3282.1, ups=0.18, wpb=17999.2, bsz=1024, num_updates=11240, lr=3.79006e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=72904
-2024-11-04 01:46:24 - progress_bar.py[line:274] - INFO: epoch 029:    311 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.192, ntokens=18407.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.14, wps=3220.1, ups=0.17, wpb=18407.9, bsz=1024, num_updates=11250, lr=3.7887e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=72961
-2024-11-04 01:47:19 - progress_bar.py[line:274] - INFO: epoch 029:    321 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.162, ntokens=18311.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.95, wps=3293.4, ups=0.18, wpb=18311.4, bsz=1024, num_updates=11260, lr=3.78734e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=73016
-2024-11-04 01:48:14 - progress_bar.py[line:274] - INFO: epoch 029:    331 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.184, ntokens=18325.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.09, wps=3372, ups=0.18, wpb=18325.2, bsz=1024, num_updates=11270, lr=3.78598e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=54, gb_free=13.3, wall=73071
-2024-11-04 01:49:10 - progress_bar.py[line:274] - INFO: epoch 029:    341 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.053, ntokens=17980.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.3, wps=3207.8, ups=0.18, wpb=17980.4, bsz=1024, num_updates=11280, lr=3.78462e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.2, wall=73127
-2024-11-04 01:50:07 - progress_bar.py[line:274] - INFO: epoch 029:    351 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.221, ntokens=18772.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.33, wps=3290.8, ups=0.18, wpb=18772.3, bsz=1024, num_updates=11290, lr=3.78326e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=73184
-2024-11-04 01:51:03 - progress_bar.py[line:274] - INFO: epoch 029:    361 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.08, ntokens=17920.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.45, wps=3209.3, ups=0.18, wpb=17920.2, bsz=1024, num_updates=11300, lr=3.7819e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=73240
-2024-11-04 01:51:58 - progress_bar.py[line:274] - INFO: epoch 029:    371 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.158, ntokens=18299.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.92, wps=3272, ups=0.18, wpb=18299.1, bsz=1024, num_updates=11310, lr=3.78054e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=73296
-2024-11-04 01:52:56 - progress_bar.py[line:274] - INFO: epoch 029:    381 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.103, ntokens=18111.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.59, wps=3168.6, ups=0.17, wpb=18111.4, bsz=1024, num_updates=11320, lr=3.77918e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=73353
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 01:53:46 - progress_bar.py[line:274] - INFO: epoch 029:    391 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.162, ntokens=17626.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.95, wps=3471.7, ups=0.2, wpb=17626.1, bsz=985.6, num_updates=11330, lr=3.77782e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=51, gb_free=13.3, wall=73404
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt
-2024-11-04 01:53:46 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 29 @ 11330 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt
+2024-11-07 09:06:21 - trainer.py[line:703] - INFO: begin training epoch 29
+2024-11-07 09:06:21 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 09:06:28 - progress_bar.py[line:274] - INFO: epoch 029:      1 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.492, ntokens=17202.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.63, wps=350.8, ups=0.02, wpb=17202.1, bsz=985.6, num_updates=10940, lr=3.83088e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=49, gb_free=13.4, wall=69907
+2024-11-07 09:07:21 - progress_bar.py[line:274] - INFO: epoch 029:     11 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.423, ntokens=17516.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3342.8, ups=0.19, wpb=17516.4, bsz=1024, num_updates=10950, lr=3.82952e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=69959
+2024-11-07 09:08:13 - progress_bar.py[line:274] - INFO: epoch 029:     21 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.465, ntokens=17804.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3399.9, ups=0.19, wpb=17804.2, bsz=1024, num_updates=10960, lr=3.82815e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=70012
+2024-11-07 09:09:05 - progress_bar.py[line:274] - INFO: epoch 029:     31 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.443, ntokens=17740.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.44, wps=3391.3, ups=0.19, wpb=17740.2, bsz=1024, num_updates=10970, lr=3.82679e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=70064
+2024-11-07 09:09:58 - progress_bar.py[line:274] - INFO: epoch 029:     41 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.389, ntokens=17555, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.24, wps=3317.8, ups=0.19, wpb=17555, bsz=1024, num_updates=10980, lr=3.82543e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=70117
+2024-11-07 09:10:50 - progress_bar.py[line:274] - INFO: epoch 029:     51 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.423, ntokens=17549.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3357, ups=0.19, wpb=17549.4, bsz=1024, num_updates=10990, lr=3.82407e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=70169
+2024-11-07 09:11:43 - progress_bar.py[line:274] - INFO: epoch 029:     61 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.473, ntokens=18057.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.55, wps=3470.2, ups=0.19, wpb=18057.1, bsz=1024, num_updates=11000, lr=3.82271e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=70221
+2024-11-07 09:12:35 - progress_bar.py[line:274] - INFO: epoch 029:     71 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.389, ntokens=17383, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.24, wps=3308, ups=0.19, wpb=17383, bsz=1024, num_updates=11010, lr=3.82135e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=70274
+2024-11-07 09:13:28 - progress_bar.py[line:274] - INFO: epoch 029:     81 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.435, ntokens=17884.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.41, wps=3395.7, ups=0.19, wpb=17884.6, bsz=1024, num_updates=11020, lr=3.81999e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=70327
+2024-11-07 09:14:21 - progress_bar.py[line:274] - INFO: epoch 029:     91 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.432, ntokens=17673.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.4, wps=3340.9, ups=0.19, wpb=17673.3, bsz=1024, num_updates=11030, lr=3.81863e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=70380
+2024-11-07 09:15:14 - progress_bar.py[line:274] - INFO: epoch 029:    101 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.43, ntokens=17616.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.39, wps=3325.1, ups=0.19, wpb=17616.7, bsz=1024, num_updates=11040, lr=3.81727e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=70432
+2024-11-07 09:16:06 - progress_bar.py[line:274] - INFO: epoch 029:    111 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.437, ntokens=17463.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.42, wps=3345, ups=0.19, wpb=17463.3, bsz=1024, num_updates=11050, lr=3.81591e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=70485
+2024-11-07 09:16:59 - progress_bar.py[line:274] - INFO: epoch 029:    121 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.474, ntokens=17894.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3386.3, ups=0.19, wpb=17894.6, bsz=1024, num_updates=11060, lr=3.81455e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=70538
+2024-11-07 09:17:51 - progress_bar.py[line:274] - INFO: epoch 029:    131 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.403, ntokens=17375.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.29, wps=3324.1, ups=0.19, wpb=17375.8, bsz=1024, num_updates=11070, lr=3.81319e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=70590
+2024-11-07 09:18:43 - progress_bar.py[line:274] - INFO: epoch 029:    141 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.472, ntokens=17884.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.55, wps=3423.2, ups=0.19, wpb=17884.9, bsz=1024, num_updates=11080, lr=3.81183e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=70642
+2024-11-07 09:19:36 - progress_bar.py[line:274] - INFO: epoch 029:    151 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.424, ntokens=17572.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.37, wps=3348.8, ups=0.19, wpb=17572.4, bsz=1024, num_updates=11090, lr=3.81047e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=70695
+2024-11-07 09:20:28 - progress_bar.py[line:274] - INFO: epoch 029:    161 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.414, ntokens=17607.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3365.4, ups=0.19, wpb=17607.5, bsz=1024, num_updates=11100, lr=3.80911e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=70747
+2024-11-07 09:21:20 - progress_bar.py[line:274] - INFO: epoch 029:    171 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.444, ntokens=17920.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.44, wps=3442.3, ups=0.19, wpb=17920.8, bsz=1024, num_updates=11110, lr=3.80775e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=70799
+2024-11-07 09:22:13 - progress_bar.py[line:274] - INFO: epoch 029:    181 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.376, ntokens=17297.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.19, wps=3261.1, ups=0.19, wpb=17297.5, bsz=1024, num_updates=11120, lr=3.80639e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=70852
+2024-11-07 09:23:06 - progress_bar.py[line:274] - INFO: epoch 029:    191 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.45, ntokens=17852.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3353.5, ups=0.19, wpb=17852.9, bsz=1024, num_updates=11130, lr=3.80503e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=70905
+2024-11-07 09:23:58 - progress_bar.py[line:274] - INFO: epoch 029:    201 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.474, ntokens=18025.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3466.1, ups=0.19, wpb=18025.6, bsz=1024, num_updates=11140, lr=3.80367e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=70957
+2024-11-07 09:24:52 - progress_bar.py[line:274] - INFO: epoch 029:    211 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.418, ntokens=17768.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.34, wps=3333.3, ups=0.19, wpb=17768.8, bsz=1024, num_updates=11150, lr=3.80231e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=71011
+2024-11-07 09:25:44 - progress_bar.py[line:274] - INFO: epoch 029:    221 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.415, ntokens=17514.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3343.8, ups=0.19, wpb=17514.4, bsz=1024, num_updates=11160, lr=3.80095e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=71063
+2024-11-07 09:26:37 - progress_bar.py[line:274] - INFO: epoch 029:    231 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.393, ntokens=17518.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.25, wps=3305.9, ups=0.19, wpb=17518.4, bsz=1024, num_updates=11170, lr=3.79959e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=71116
+2024-11-07 09:27:32 - progress_bar.py[line:274] - INFO: epoch 029:    241 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.456, ntokens=17797.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.49, wps=3214.7, ups=0.18, wpb=17797.2, bsz=1024, num_updates=11180, lr=3.79823e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=40, gb_free=13.5, wall=71171
+2024-11-07 09:28:28 - progress_bar.py[line:274] - INFO: epoch 029:    251 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.473, ntokens=18007.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.55, wps=3246.4, ups=0.18, wpb=18007.7, bsz=1024, num_updates=11190, lr=3.79687e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=71227
+2024-11-07 09:29:23 - progress_bar.py[line:274] - INFO: epoch 029:    261 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.458, ntokens=17879.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.49, wps=3243.2, ups=0.18, wpb=17879.9, bsz=1024, num_updates=11200, lr=3.79551e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=71282
+2024-11-07 09:30:18 - progress_bar.py[line:274] - INFO: epoch 029:    271 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.479, ntokens=17990.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3279.6, ups=0.18, wpb=17990.5, bsz=1024, num_updates=11210, lr=3.79414e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=71337
+2024-11-07 09:31:13 - progress_bar.py[line:274] - INFO: epoch 029:    281 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.42, ntokens=17629.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.35, wps=3199.5, ups=0.18, wpb=17629.6, bsz=1024, num_updates=11220, lr=3.79278e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=71392
+2024-11-07 09:32:08 - progress_bar.py[line:274] - INFO: epoch 029:    291 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.414, ntokens=17577.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3185, ups=0.18, wpb=17577.3, bsz=1024, num_updates=11230, lr=3.79142e-05, gnorm=0.014, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=71447
+2024-11-07 09:33:03 - progress_bar.py[line:274] - INFO: epoch 029:    301 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.48, ntokens=17999.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.58, wps=3252.3, ups=0.18, wpb=17999.2, bsz=1024, num_updates=11240, lr=3.79006e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=71502
+2024-11-07 09:33:59 - progress_bar.py[line:274] - INFO: epoch 029:    311 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.564, ntokens=18407.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.92, wps=3334.8, ups=0.18, wpb=18407.9, bsz=1024, num_updates=11250, lr=3.7887e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=71558
+2024-11-07 09:34:52 - progress_bar.py[line:274] - INFO: epoch 029:    321 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.553, ntokens=18311.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.87, wps=3440.5, ups=0.19, wpb=18311.4, bsz=1024, num_updates=11260, lr=3.78734e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=71611
+2024-11-07 09:35:44 - progress_bar.py[line:274] - INFO: epoch 029:    331 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.545, ntokens=18325.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3497.9, ups=0.19, wpb=18325.2, bsz=1024, num_updates=11270, lr=3.78598e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.3, wall=71663
+2024-11-07 09:36:37 - progress_bar.py[line:274] - INFO: epoch 029:    341 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.498, ntokens=17980.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.65, wps=3418.5, ups=0.19, wpb=17980.4, bsz=1024, num_updates=11280, lr=3.78462e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=53, gb_free=13.2, wall=71716
+2024-11-07 09:37:30 - progress_bar.py[line:274] - INFO: epoch 029:    351 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.6, ntokens=18772.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.06, wps=3552.5, ups=0.19, wpb=18772.3, bsz=1024, num_updates=11290, lr=3.78326e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=71769
+2024-11-07 09:38:22 - progress_bar.py[line:274] - INFO: epoch 029:    361 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.482, ntokens=17920.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3444.6, ups=0.19, wpb=17920.2, bsz=1024, num_updates=11300, lr=3.7819e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=71821
+2024-11-07 09:39:14 - progress_bar.py[line:274] - INFO: epoch 029:    371 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.549, ntokens=18299.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=3522.1, ups=0.19, wpb=18299.1, bsz=1024, num_updates=11310, lr=3.78054e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.3, wall=71873
+2024-11-07 09:40:06 - progress_bar.py[line:274] - INFO: epoch 029:    381 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.508, ntokens=18111.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.69, wps=3472.7, ups=0.19, wpb=18111.4, bsz=1024, num_updates=11320, lr=3.77918e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=71925
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+2024-11-07 09:40:55 - progress_bar.py[line:274] - INFO: epoch 029:    391 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.525, ntokens=17626.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3575.7, ups=0.2, wpb=17626.1, bsz=985.6, num_updates=11330, lr=3.77782e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=49, gb_free=13.3, wall=71974
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt
+
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt
 
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt
-2024-11-04 01:53:46 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt': File exists: File exists
+2024-11-07 09:40:55 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 29 @ 11330 updates
+2024-11-07 09:40:55 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt': File exists: File exists
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+: File exists
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 01:55:35 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 09:42:45 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+
+
+slice_id 2 seek offset 100000
+slice_id 6 seek offset 300000
 slice_id 3 seek offset 150000
-2024-11-04 01:57:27 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 29 @ 11330 updates, score 0) (writing took 220.31115137902088 seconds)
+2024-11-07 09:44:37 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 29 @ 11330 updates, score 0) (writing took 221.3782702370081 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_29.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+
 slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-2024-11-04 01:58:21 - train.py[line:336] - INFO: end of epoch 29 (average epoch stats below)
-2024-11-04 01:58:21 - progress_bar.py[line:282] - INFO: epoch 029 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.042 | ntokens 17824.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.24 | wps 2657.4 | ups 0.15 | wpb 17824.9 | bsz 1023 | num_updates 11330 | lr 3.77782e-05 | gnorm 0.009 | clip 0 | loss_scale 65536 | train_wall 2111 | gb_free 13.3 | wall 73678
-2024-11-04 01:58:21 - trainer.py[line:639] - INFO: loading train data for epoch 30
+slice_id 4 seek offset 200000
+slice_id 5 seek offset 250000
+2024-11-07 09:45:31 - train.py[line:336] - INFO: end of epoch 29 (average epoch stats below)
+2024-11-07 09:45:31 - progress_bar.py[line:282] - INFO: epoch 029 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.46 | ntokens 17824.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.5 | wps 2777.4 | ups 0.16 | wpb 17824.9 | bsz 1023 | num_updates 11330 | lr 3.77782e-05 | gnorm 0.008 | clip 0 | loss_scale 65536 | train_wall 1543 | gb_free 13.3 | wall 72250
+2024-11-07 09:45:31 - trainer.py[line:639] - INFO: loading train data for epoch 30
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-04 02:01:05 - trainer.py[line:703] - INFO: begin training epoch 30
-2024-11-04 02:01:05 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 02:02:04 - progress_bar.py[line:274] - INFO: epoch 030:     10 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.969, ntokens=17508.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.83, wps=352, ups=0.02, wpb=17508.9, bsz=1024, num_updates=11340, lr=3.77646e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=73901
-2024-11-04 02:02:58 - progress_bar.py[line:274] - INFO: epoch 030:     20 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.032, ntokens=17975, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.18, wps=3290.5, ups=0.18, wpb=17975, bsz=1024, num_updates=11350, lr=3.7751e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=37, gb_free=13.5, wall=73955
-2024-11-04 02:03:51 - progress_bar.py[line:274] - INFO: epoch 030:     30 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.999, ntokens=17817.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.99, wps=3357.5, ups=0.19, wpb=17817.1, bsz=1024, num_updates=11360, lr=3.77374e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=74009
-2024-11-04 02:04:45 - progress_bar.py[line:274] - INFO: epoch 030:     40 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.023, ntokens=17907.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=3328.9, ups=0.19, wpb=17907.7, bsz=1024, num_updates=11370, lr=3.77238e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=74062
-2024-11-04 02:05:39 - progress_bar.py[line:274] - INFO: epoch 030:     50 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.981, ntokens=17546.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3273.3, ups=0.19, wpb=17546.3, bsz=1024, num_updates=11380, lr=3.77102e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=74116
-2024-11-04 02:06:33 - progress_bar.py[line:274] - INFO: epoch 030:     60 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.999, ntokens=17750.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8, wps=3288.5, ups=0.19, wpb=17750.5, bsz=1024, num_updates=11390, lr=3.76966e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=74170
-2024-11-04 02:07:26 - progress_bar.py[line:274] - INFO: epoch 030:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.982, ntokens=17664.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.9, wps=3292.4, ups=0.19, wpb=17664.4, bsz=1024, num_updates=11400, lr=3.7683e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=74224
-2024-11-04 02:08:20 - progress_bar.py[line:274] - INFO: epoch 030:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.996, ntokens=17858.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.98, wps=3362.1, ups=0.19, wpb=17858.1, bsz=1024, num_updates=11410, lr=3.76694e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=74277
-2024-11-04 02:09:13 - progress_bar.py[line:274] - INFO: epoch 030:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.041, ntokens=17927.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=3344.3, ups=0.19, wpb=17927.8, bsz=1024, num_updates=11420, lr=3.76558e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=74330
-2024-11-04 02:10:06 - progress_bar.py[line:274] - INFO: epoch 030:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.997, ntokens=17741.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.99, wps=3343.7, ups=0.19, wpb=17741.7, bsz=1024, num_updates=11430, lr=3.76422e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.6, wall=74383
-2024-11-04 02:10:59 - progress_bar.py[line:274] - INFO: epoch 030:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.949, ntokens=17461.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.72, wps=3308.4, ups=0.19, wpb=17461.5, bsz=1024, num_updates=11440, lr=3.76286e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=74436
-2024-11-04 02:11:51 - progress_bar.py[line:274] - INFO: epoch 030:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.993, ntokens=17672, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.96, wps=3385.7, ups=0.19, wpb=17672, bsz=1024, num_updates=11450, lr=3.7615e-05, gnorm=0.007, clip=0, loss_scale=131072, train_wall=52, gb_free=13.5, wall=74488
-2024-11-04 02:12:18 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
-2024-11-04 02:12:50 - progress_bar.py[line:274] - INFO: epoch 030:    131 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.938, ntokens=17477.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.66, wps=2986, ups=0.17, wpb=17477.8, bsz=1024, num_updates=11460, lr=3.76013e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=74547
-2024-11-04 02:13:43 - progress_bar.py[line:274] - INFO: epoch 030:    141 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.035, ntokens=17783.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.19, wps=3365.4, ups=0.19, wpb=17783.1, bsz=1024, num_updates=11470, lr=3.75877e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=74600
-2024-11-04 02:14:36 - progress_bar.py[line:274] - INFO: epoch 030:    151 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.952, ntokens=17526.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.74, wps=3290, ups=0.19, wpb=17526.4, bsz=1024, num_updates=11480, lr=3.75741e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=74653
-2024-11-04 02:15:29 - progress_bar.py[line:274] - INFO: epoch 030:    161 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.974, ntokens=17643.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.86, wps=3323.6, ups=0.19, wpb=17643.2, bsz=1024, num_updates=11490, lr=3.75605e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=74706
-2024-11-04 02:16:22 - progress_bar.py[line:274] - INFO: epoch 030:    171 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.953, ntokens=17490.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.75, wps=3298.5, ups=0.19, wpb=17490.9, bsz=1024, num_updates=11500, lr=3.75469e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=74759
-2024-11-04 02:17:15 - progress_bar.py[line:274] - INFO: epoch 030:    181 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.948, ntokens=17462.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.72, wps=3312.6, ups=0.19, wpb=17462.6, bsz=1024, num_updates=11510, lr=3.75333e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.6, wall=74812
-2024-11-04 02:18:08 - progress_bar.py[line:274] - INFO: epoch 030:    191 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.001, ntokens=17876.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8, wps=3362, ups=0.19, wpb=17876.5, bsz=1024, num_updates=11520, lr=3.75197e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=74865
-2024-11-04 02:19:01 - progress_bar.py[line:274] - INFO: epoch 030:    201 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.014, ntokens=17939.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.08, wps=3401.5, ups=0.19, wpb=17939.7, bsz=1024, num_updates=11530, lr=3.75061e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=74918
-2024-11-04 02:19:54 - progress_bar.py[line:274] - INFO: epoch 030:    211 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3, ntokens=18051.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8, wps=3358.2, ups=0.19, wpb=18051.6, bsz=1024, num_updates=11540, lr=3.74925e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=74971
-2024-11-04 02:20:48 - progress_bar.py[line:274] - INFO: epoch 030:    221 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.942, ntokens=17511.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.68, wps=3245.3, ups=0.19, wpb=17511.5, bsz=1024, num_updates=11550, lr=3.74789e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=75025
-2024-11-04 02:21:43 - progress_bar.py[line:274] - INFO: epoch 030:    231 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.002, ntokens=17942.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.01, wps=3263.3, ups=0.18, wpb=17942.3, bsz=1024, num_updates=11560, lr=3.74653e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=75080
-2024-11-04 02:22:35 - progress_bar.py[line:274] - INFO: epoch 030:    241 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.043, ntokens=17821.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.24, wps=3415.8, ups=0.19, wpb=17821.6, bsz=1024, num_updates=11570, lr=3.74517e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=75133
-2024-11-04 02:23:29 - progress_bar.py[line:274] - INFO: epoch 030:    251 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.037, ntokens=17866.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.21, wps=3369.3, ups=0.19, wpb=17866.5, bsz=1024, num_updates=11580, lr=3.74381e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.6, wall=75186
-2024-11-04 02:24:21 - progress_bar.py[line:274] - INFO: epoch 030:    261 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.997, ntokens=17816.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.98, wps=3377.9, ups=0.19, wpb=17816.5, bsz=1024, num_updates=11590, lr=3.74245e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=75238
-2024-11-04 02:25:15 - progress_bar.py[line:274] - INFO: epoch 030:    271 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.046, ntokens=18074.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.26, wps=3389, ups=0.19, wpb=18074.9, bsz=1024, num_updates=11600, lr=3.74109e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=75292
-2024-11-04 02:26:08 - progress_bar.py[line:274] - INFO: epoch 030:    281 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.974, ntokens=17670.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.86, wps=3307.5, ups=0.19, wpb=17670.3, bsz=1024, num_updates=11610, lr=3.73973e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=75345
-2024-11-04 02:27:01 - progress_bar.py[line:274] - INFO: epoch 030:    291 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3, ntokens=17763.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8, wps=3340.8, ups=0.19, wpb=17763.3, bsz=1024, num_updates=11620, lr=3.73837e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.6, wall=75398
-2024-11-04 02:27:55 - progress_bar.py[line:274] - INFO: epoch 030:    301 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.064, ntokens=17836.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.36, wps=3312.7, ups=0.19, wpb=17836.2, bsz=1024, num_updates=11630, lr=3.73701e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.3, wall=75452
-2024-11-04 02:28:49 - progress_bar.py[line:274] - INFO: epoch 030:    311 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.184, ntokens=18531.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.09, wps=3446.8, ups=0.19, wpb=18531.7, bsz=1024, num_updates=11640, lr=3.73565e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.3, wall=75506
-2024-11-04 02:29:42 - progress_bar.py[line:274] - INFO: epoch 030:    321 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.144, ntokens=18454.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.84, wps=3482.5, ups=0.19, wpb=18454.7, bsz=1024, num_updates=11650, lr=3.73429e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=45, gb_free=13.5, wall=75559
-2024-11-04 02:30:35 - progress_bar.py[line:274] - INFO: epoch 030:    331 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.164, ntokens=18332.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.97, wps=3440.9, ups=0.19, wpb=18332.7, bsz=1024, num_updates=11660, lr=3.73293e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=46, gb_free=13.3, wall=75612
-2024-11-04 02:31:29 - progress_bar.py[line:274] - INFO: epoch 030:    341 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.02, ntokens=17833.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.11, wps=3293.6, ups=0.18, wpb=17833.9, bsz=1024, num_updates=11670, lr=3.73157e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.1, wall=75666
-2024-11-04 02:32:24 - progress_bar.py[line:274] - INFO: epoch 030:    351 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.156, ntokens=18547.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.91, wps=3387.8, ups=0.18, wpb=18547.2, bsz=1024, num_updates=11680, lr=3.73021e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=75721
-2024-11-04 02:33:19 - progress_bar.py[line:274] - INFO: epoch 030:    361 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.095, ntokens=18181.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.54, wps=3285.5, ups=0.18, wpb=18181.1, bsz=1024, num_updates=11690, lr=3.72885e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=75776
-2024-11-04 02:34:14 - progress_bar.py[line:274] - INFO: epoch 030:    371 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.112, ntokens=18171, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.64, wps=3306.4, ups=0.18, wpb=18171, bsz=1024, num_updates=11700, lr=3.72749e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=75831
-2024-11-04 02:35:10 - progress_bar.py[line:274] - INFO: epoch 030:    381 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.091, ntokens=18020.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.52, wps=3258.2, ups=0.18, wpb=18020.8, bsz=1024, num_updates=11710, lr=3.72613e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=75887
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 02:36:01 - progress_bar.py[line:274] - INFO: epoch 030:    391 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.113, ntokens=17418.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.65, wps=3397.2, ups=0.2, wpb=17418.4, bsz=985.6, num_updates=11720, lr=3.72476e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=51, gb_free=13.4, wall=75938
-2024-11-04 02:36:01 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 30 @ 11720 updates
+2024-11-07 09:48:10 - trainer.py[line:703] - INFO: begin training epoch 30
+2024-11-07 09:48:10 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 09:49:04 - progress_bar.py[line:274] - INFO: epoch 030:     10 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.414, ntokens=17508.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=358, ups=0.02, wpb=17508.9, bsz=1024, num_updates=11340, lr=3.77646e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=72463
+2024-11-07 09:49:56 - progress_bar.py[line:274] - INFO: epoch 030:     20 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.448, ntokens=17975, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3452.2, ups=0.19, wpb=17975, bsz=1024, num_updates=11350, lr=3.7751e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=72515
+2024-11-07 09:50:48 - progress_bar.py[line:274] - INFO: epoch 030:     30 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.479, ntokens=17817.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3416.6, ups=0.19, wpb=17817.1, bsz=1024, num_updates=11360, lr=3.77374e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=72567
+2024-11-07 09:51:41 - progress_bar.py[line:274] - INFO: epoch 030:     40 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.449, ntokens=17907.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3391.1, ups=0.19, wpb=17907.7, bsz=1024, num_updates=11370, lr=3.77238e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=72620
+2024-11-07 09:52:33 - progress_bar.py[line:274] - INFO: epoch 030:     50 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.448, ntokens=17546.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3369, ups=0.19, wpb=17546.3, bsz=1024, num_updates=11380, lr=3.77102e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=72672
+2024-11-07 09:53:26 - progress_bar.py[line:274] - INFO: epoch 030:     60 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.448, ntokens=17750.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3399.5, ups=0.19, wpb=17750.5, bsz=1024, num_updates=11390, lr=3.76966e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=72724
+2024-11-07 09:54:19 - progress_bar.py[line:274] - INFO: epoch 030:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.433, ntokens=17664.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.4, wps=3315.7, ups=0.19, wpb=17664.4, bsz=1024, num_updates=11400, lr=3.7683e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=72778
+2024-11-07 09:55:11 - progress_bar.py[line:274] - INFO: epoch 030:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.457, ntokens=17858.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.49, wps=3395.6, ups=0.19, wpb=17858.1, bsz=1024, num_updates=11410, lr=3.76694e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=72830
+2024-11-07 09:56:04 - progress_bar.py[line:274] - INFO: epoch 030:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.476, ntokens=17927.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.56, wps=3420.6, ups=0.19, wpb=17927.8, bsz=1024, num_updates=11420, lr=3.76558e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=72883
+2024-11-07 09:56:56 - progress_bar.py[line:274] - INFO: epoch 030:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.425, ntokens=17741.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.37, wps=3404, ups=0.19, wpb=17741.7, bsz=1024, num_updates=11430, lr=3.76422e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=72935
+2024-11-07 09:57:49 - progress_bar.py[line:274] - INFO: epoch 030:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.415, ntokens=17461.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3285.2, ups=0.19, wpb=17461.5, bsz=1024, num_updates=11440, lr=3.76286e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=72988
+2024-11-07 09:58:41 - progress_bar.py[line:274] - INFO: epoch 030:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.447, ntokens=17672, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.45, wps=3411, ups=0.19, wpb=17672, bsz=1024, num_updates=11450, lr=3.7615e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=73040
+2024-11-07 09:59:33 - progress_bar.py[line:274] - INFO: epoch 030:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.408, ntokens=17322.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.31, wps=3292.8, ups=0.19, wpb=17322.1, bsz=1024, num_updates=11460, lr=3.76013e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=73092
+2024-11-07 10:00:26 - progress_bar.py[line:274] - INFO: epoch 030:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.46, ntokens=17827.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3394.2, ups=0.19, wpb=17827.5, bsz=1024, num_updates=11470, lr=3.75877e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=73145
+2024-11-07 10:01:18 - progress_bar.py[line:274] - INFO: epoch 030:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.424, ntokens=17529.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.37, wps=3361.2, ups=0.19, wpb=17529.5, bsz=1024, num_updates=11480, lr=3.75741e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=73197
+2024-11-07 10:02:11 - progress_bar.py[line:274] - INFO: epoch 030:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.411, ntokens=17633.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.32, wps=3318, ups=0.19, wpb=17633.3, bsz=1024, num_updates=11490, lr=3.75605e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=35, gb_free=13.5, wall=73250
+2024-11-07 10:03:04 - progress_bar.py[line:274] - INFO: epoch 030:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.391, ntokens=17604.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.25, wps=3315.4, ups=0.19, wpb=17604.3, bsz=1024, num_updates=11500, lr=3.75469e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=41, gb_free=13.5, wall=73303
+2024-11-07 10:03:57 - progress_bar.py[line:274] - INFO: epoch 030:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.393, ntokens=17389.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.25, wps=3283.4, ups=0.19, wpb=17389.1, bsz=1024, num_updates=11510, lr=3.75333e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=42, gb_free=13.5, wall=73356
+2024-11-07 10:04:51 - progress_bar.py[line:274] - INFO: epoch 030:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.449, ntokens=17807.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3346.5, ups=0.19, wpb=17807.9, bsz=1024, num_updates=11520, lr=3.75197e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=46, gb_free=13.3, wall=73409
+2024-11-07 10:05:43 - progress_bar.py[line:274] - INFO: epoch 030:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.452, ntokens=17864.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.47, wps=3410.6, ups=0.19, wpb=17864.4, bsz=1024, num_updates=11530, lr=3.75061e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=73462
+2024-11-07 10:06:36 - progress_bar.py[line:274] - INFO: epoch 030:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.465, ntokens=18128.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.52, wps=3449, ups=0.19, wpb=18128.8, bsz=1024, num_updates=11540, lr=3.74925e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=51, gb_free=13.3, wall=73514
+2024-11-07 10:07:30 - progress_bar.py[line:274] - INFO: epoch 030:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.415, ntokens=17542.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3250.3, ups=0.19, wpb=17542.9, bsz=1024, num_updates=11550, lr=3.74789e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=73568
+2024-11-07 10:08:22 - progress_bar.py[line:274] - INFO: epoch 030:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.448, ntokens=17802.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.46, wps=3391.3, ups=0.19, wpb=17802.4, bsz=1024, num_updates=11560, lr=3.74653e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=73621
+2024-11-07 10:09:15 - progress_bar.py[line:274] - INFO: epoch 030:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.469, ntokens=17979.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3392.4, ups=0.19, wpb=17979.3, bsz=1024, num_updates=11570, lr=3.74517e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=73674
+2024-11-07 10:10:07 - progress_bar.py[line:274] - INFO: epoch 030:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.425, ntokens=17716.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.37, wps=3388.1, ups=0.19, wpb=17716.2, bsz=1024, num_updates=11580, lr=3.74381e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.6, wall=73726
+2024-11-07 10:10:59 - progress_bar.py[line:274] - INFO: epoch 030:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.454, ntokens=17883, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.48, wps=3428.1, ups=0.19, wpb=17883, bsz=1024, num_updates=11590, lr=3.74245e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=73778
+2024-11-07 10:11:52 - progress_bar.py[line:274] - INFO: epoch 030:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.477, ntokens=18066.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3442.3, ups=0.19, wpb=18066.2, bsz=1024, num_updates=11600, lr=3.74109e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=46, gb_free=13.6, wall=73831
+2024-11-07 10:12:44 - progress_bar.py[line:274] - INFO: epoch 030:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.436, ntokens=17778.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.41, wps=3394.6, ups=0.19, wpb=17778.7, bsz=1024, num_updates=11610, lr=3.73973e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=36, gb_free=13.5, wall=73883
+2024-11-07 10:13:37 - progress_bar.py[line:274] - INFO: epoch 030:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.442, ntokens=17760, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.44, wps=3395.4, ups=0.19, wpb=17760, bsz=1024, num_updates=11620, lr=3.73837e-05, gnorm=0.015, clip=0, loss_scale=65536, train_wall=33, gb_free=13.6, wall=73936
+2024-11-07 10:14:29 - progress_bar.py[line:274] - INFO: epoch 030:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.442, ntokens=17715.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3374.4, ups=0.19, wpb=17715.3, bsz=1024, num_updates=11630, lr=3.73701e-05, gnorm=0.016, clip=0, loss_scale=65536, train_wall=33, gb_free=13.2, wall=73988
+2024-11-07 10:15:21 - progress_bar.py[line:274] - INFO: epoch 030:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.597, ntokens=18595.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.05, wps=3562.8, ups=0.19, wpb=18595.4, bsz=1024, num_updates=11640, lr=3.73565e-05, gnorm=0.014, clip=0, loss_scale=65536, train_wall=45, gb_free=13.4, wall=74040
+2024-11-07 10:16:14 - progress_bar.py[line:274] - INFO: epoch 030:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.57, ntokens=18435.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.94, wps=3493.2, ups=0.19, wpb=18435.3, bsz=1024, num_updates=11650, lr=3.73429e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=74093
+2024-11-07 10:17:06 - progress_bar.py[line:274] - INFO: epoch 030:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.527, ntokens=18251.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3487.2, ups=0.19, wpb=18251.2, bsz=1024, num_updates=11660, lr=3.73293e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=74145
+2024-11-07 10:17:59 - progress_bar.py[line:274] - INFO: epoch 030:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.486, ntokens=17939.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.6, wps=3385.7, ups=0.19, wpb=17939.5, bsz=1024, num_updates=11670, lr=3.73157e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.3, wall=74198
+2024-11-07 10:18:53 - progress_bar.py[line:274] - INFO: epoch 030:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.546, ntokens=18537.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3490.2, ups=0.19, wpb=18537.7, bsz=1024, num_updates=11680, lr=3.73021e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=53, gb_free=13.3, wall=74251
+2024-11-07 10:19:45 - progress_bar.py[line:274] - INFO: epoch 030:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.522, ntokens=18126.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.74, wps=3470.7, ups=0.19, wpb=18126.2, bsz=1024, num_updates=11690, lr=3.72885e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=52, gb_free=13.3, wall=74304
+2024-11-07 10:20:37 - progress_bar.py[line:274] - INFO: epoch 030:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.536, ntokens=18153, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.8, wps=3470, ups=0.19, wpb=18153, bsz=1024, num_updates=11700, lr=3.72749e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.2, wall=74356
+2024-11-07 10:21:30 - progress_bar.py[line:274] - INFO: epoch 030:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.493, ntokens=18118.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.63, wps=3434.7, ups=0.19, wpb=18118.7, bsz=1024, num_updates=11710, lr=3.72613e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=74409
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+2024-11-07 10:22:22 - progress_bar.py[line:274] - INFO: epoch 030:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.495, ntokens=18092, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3494.9, ups=0.19, wpb=18092, bsz=1024, num_updates=11720, lr=3.72476e-05, gnorm=0.007, clip=0, loss_scale=131072, train_wall=52, gb_free=13.3, wall=74460
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt2024-11-07 10:22:24 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 30 @ 11721 updates
 
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt
+
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt
-2024-11-04 02:36:01 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 10:22:24 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt': File exists
+: File exists: File exists
+
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 02:37:48 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 02:39:41 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 30 @ 11720 updates, score 0) (writing took 220.45478392901714 seconds)
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
+2024-11-07 10:24:13 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+
+
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+
+slice_id 3 seek offset 150000
+slice_id 7 seek offset 350000
+slice_id 6 seek offset 300000
+2024-11-07 10:26:05 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 30 @ 11721 updates, score 0) (writing took 221.2052170060051 seconds)
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_30.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
+slice_id 2 seek offset 100000
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
+
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 3 seek offset 150000
-slice_id 7 seek offset 350000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
+slice_id 1 seek offset 50000
 slice_id 5 seek offset 250000
 slice_id 4 seek offset 200000
-2024-11-04 02:40:38 - train.py[line:336] - INFO: end of epoch 30 (average epoch stats below)
-2024-11-04 02:40:38 - progress_bar.py[line:282] - INFO: epoch 030 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.025 | ntokens 17843 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.14 | wps 2743.3 | ups 0.15 | wpb 17843 | bsz 1023 | num_updates 11720 | lr 3.72476e-05 | gnorm 0.008 | clip 0 | loss_scale 65536 | train_wall 2030 | gb_free 13.4 | wall 76215
-2024-11-04 02:40:38 - trainer.py[line:639] - INFO: loading train data for epoch 31
+2024-11-07 10:26:59 - train.py[line:336] - INFO: end of epoch 30 (average epoch stats below)
+2024-11-07 10:26:59 - progress_bar.py[line:282] - INFO: epoch 030 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.461 | ntokens 17839.4 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.51 | wps 2803.2 | ups 0.16 | wpb 17839.4 | bsz 1023 | num_updates 11721 | lr 3.72463e-05 | gnorm 0.008 | clip 0 | loss_scale 131072 | train_wall 1601 | gb_free 13.4 | wall 74738
+2024-11-07 10:26:59 - trainer.py[line:639] - INFO: loading train data for epoch 31
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-04 02:43:24 - trainer.py[line:703] - INFO: begin training epoch 31
-2024-11-04 02:43:24 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 02:44:29 - progress_bar.py[line:274] - INFO: epoch 031:     10 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.969, ntokens=17588.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.83, wps=346.2, ups=0.02, wpb=17588.9, bsz=1024, num_updates=11730, lr=3.7234e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=45, gb_free=13.5, wall=76446
-2024-11-04 02:45:31 - progress_bar.py[line:274] - INFO: epoch 031:     20 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.978, ntokens=17632.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.88, wps=2827.4, ups=0.16, wpb=17632.2, bsz=1024, num_updates=11740, lr=3.72204e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=43, gb_free=13.5, wall=76508
-2024-11-04 02:45:49 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-04 02:46:39 - progress_bar.py[line:274] - INFO: epoch 031:     31 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.974, ntokens=17937, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.86, wps=2655.5, ups=0.15, wpb=17937, bsz=1024, num_updates=11750, lr=3.72068e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=47, gb_free=13.4, wall=76576
-2024-11-04 02:47:40 - progress_bar.py[line:274] - INFO: epoch 031:     41 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.024, ntokens=17925.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=2930.5, ups=0.16, wpb=17925.9, bsz=1024, num_updates=11760, lr=3.71932e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=45, gb_free=13.4, wall=76637
-2024-11-04 02:48:42 - progress_bar.py[line:274] - INFO: epoch 031:     51 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.915, ntokens=17440.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=2810.7, ups=0.16, wpb=17440.5, bsz=1024, num_updates=11770, lr=3.71796e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=45, gb_free=13.5, wall=76699
-2024-11-04 02:49:44 - progress_bar.py[line:274] - INFO: epoch 031:     61 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.037, ntokens=17909.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.21, wps=2899.6, ups=0.16, wpb=17909.7, bsz=1024, num_updates=11780, lr=3.7166e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=44, gb_free=13.5, wall=76761
-2024-11-04 02:50:46 - progress_bar.py[line:274] - INFO: epoch 031:     71 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.98, ntokens=17584.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=2832.5, ups=0.16, wpb=17584.9, bsz=1024, num_updates=11790, lr=3.71524e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=44, gb_free=13.6, wall=76823
-2024-11-04 02:51:48 - progress_bar.py[line:274] - INFO: epoch 031:     81 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.957, ntokens=17771.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.76, wps=2868.3, ups=0.16, wpb=17771.3, bsz=1024, num_updates=11800, lr=3.71388e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=44, gb_free=13.6, wall=76885
-2024-11-04 02:52:51 - progress_bar.py[line:274] - INFO: epoch 031:     91 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.974, ntokens=17557.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.86, wps=2771.5, ups=0.16, wpb=17557.1, bsz=1024, num_updates=11810, lr=3.71252e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=45, gb_free=13.5, wall=76948
-2024-11-04 02:53:57 - progress_bar.py[line:274] - INFO: epoch 031:    101 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.019, ntokens=17840.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.1, wps=2709.4, ups=0.15, wpb=17840.7, bsz=1024, num_updates=11820, lr=3.71116e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=64, gb_free=13.5, wall=77014
-2024-11-04 02:55:03 - progress_bar.py[line:274] - INFO: epoch 031:    111 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.987, ntokens=17559.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.93, wps=2648.8, ups=0.15, wpb=17559.4, bsz=1024, num_updates=11830, lr=3.7098e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=66, gb_free=13.4, wall=77080
-2024-11-04 02:56:10 - progress_bar.py[line:274] - INFO: epoch 031:    121 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.007, ntokens=18070.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.04, wps=2728.5, ups=0.15, wpb=18070.9, bsz=1024, num_updates=11840, lr=3.70844e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=66, gb_free=13.5, wall=77147
-2024-11-04 02:57:16 - progress_bar.py[line:274] - INFO: epoch 031:    131 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.893, ntokens=17120.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=2576.8, ups=0.15, wpb=17120.1, bsz=1024, num_updates=11850, lr=3.70708e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=66, gb_free=13.4, wall=77213
-2024-11-04 02:58:22 - progress_bar.py[line:274] - INFO: epoch 031:    141 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.998, ntokens=17741.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.99, wps=2672, ups=0.15, wpb=17741.7, bsz=1024, num_updates=11860, lr=3.70572e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=66, gb_free=13.4, wall=77279
-2024-11-04 02:59:29 - progress_bar.py[line:274] - INFO: epoch 031:    151 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.979, ntokens=17747.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=2663.7, ups=0.15, wpb=17747.1, bsz=1024, num_updates=11870, lr=3.70436e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=66, gb_free=13.5, wall=77346
-2024-11-04 03:00:36 - progress_bar.py[line:274] - INFO: epoch 031:    161 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.976, ntokens=17777.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.87, wps=2658.4, ups=0.15, wpb=17777.9, bsz=1024, num_updates=11880, lr=3.703e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=67, gb_free=13.5, wall=77413
-2024-11-04 03:01:43 - progress_bar.py[line:274] - INFO: epoch 031:    171 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.006, ntokens=17947.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.03, wps=2678.8, ups=0.15, wpb=17947.5, bsz=1024, num_updates=11890, lr=3.70164e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=67, gb_free=13.5, wall=77480
-2024-11-04 03:02:50 - progress_bar.py[line:274] - INFO: epoch 031:    181 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.97, ntokens=17684.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.83, wps=2655.7, ups=0.15, wpb=17684.6, bsz=1024, num_updates=11900, lr=3.70028e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=66, gb_free=13.5, wall=77547
-2024-11-04 03:03:56 - progress_bar.py[line:274] - INFO: epoch 031:    191 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.004, ntokens=18026, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.02, wps=2696.9, ups=0.15, wpb=18026, bsz=1024, num_updates=11910, lr=3.69892e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=67, gb_free=13.4, wall=77613
-2024-11-04 03:05:03 - progress_bar.py[line:274] - INFO: epoch 031:    201 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.988, ntokens=17866.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.93, wps=2675.3, ups=0.15, wpb=17866.2, bsz=1024, num_updates=11920, lr=3.69756e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=67, gb_free=13.5, wall=77680
-2024-11-04 03:06:10 - progress_bar.py[line:274] - INFO: epoch 031:    211 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.978, ntokens=17979.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.88, wps=2689.5, ups=0.15, wpb=17979.8, bsz=1024, num_updates=11930, lr=3.6962e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=67, gb_free=13.4, wall=77747
-2024-11-04 03:07:17 - progress_bar.py[line:274] - INFO: epoch 031:    221 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.911, ntokens=17405, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.52, wps=2608.7, ups=0.15, wpb=17405, bsz=1024, num_updates=11940, lr=3.69484e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=67, gb_free=13.5, wall=77814
-2024-11-04 03:08:11 - progress_bar.py[line:274] - INFO: epoch 031:    231 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.939, ntokens=17673, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.67, wps=3234.6, ups=0.18, wpb=17673, bsz=1024, num_updates=11950, lr=3.69348e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=36, gb_free=13.5, wall=77869
-2024-11-04 03:09:05 - progress_bar.py[line:274] - INFO: epoch 031:    241 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.993, ntokens=17506.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.96, wps=3278.3, ups=0.19, wpb=17506.9, bsz=1024, num_updates=11960, lr=3.69212e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=77922
-2024-11-04 03:10:00 - progress_bar.py[line:274] - INFO: epoch 031:    251 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.966, ntokens=17659.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.81, wps=3225.7, ups=0.18, wpb=17659.2, bsz=1024, num_updates=11970, lr=3.69075e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=77977
-2024-11-04 03:10:53 - progress_bar.py[line:274] - INFO: epoch 031:    261 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.975, ntokens=17775.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.86, wps=3305.5, ups=0.19, wpb=17775.3, bsz=1024, num_updates=11980, lr=3.68939e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=36, gb_free=13.4, wall=78030
-2024-11-04 03:11:49 - progress_bar.py[line:274] - INFO: epoch 031:    271 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.963, ntokens=17618.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.8, wps=3187.5, ups=0.18, wpb=17618.9, bsz=1024, num_updates=11990, lr=3.68803e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=51, gb_free=13.5, wall=78086
-2024-11-04 03:12:44 - progress_bar.py[line:274] - INFO: epoch 031:    281 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.951, ntokens=17610.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.73, wps=3192.3, ups=0.18, wpb=17610.4, bsz=1024, num_updates=12000, lr=3.68667e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=78141
-2024-11-04 03:13:38 - progress_bar.py[line:274] - INFO: epoch 031:    291 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.955, ntokens=17644.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.76, wps=3227.3, ups=0.18, wpb=17644.7, bsz=1024, num_updates=12010, lr=3.68531e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=78196
-2024-11-04 03:14:33 - progress_bar.py[line:274] - INFO: epoch 031:    301 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.076, ntokens=17910.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.43, wps=3268.5, ups=0.18, wpb=17910.3, bsz=1024, num_updates=12020, lr=3.68395e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=78250
-2024-11-04 03:15:28 - progress_bar.py[line:274] - INFO: epoch 031:    311 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.125, ntokens=18371.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.73, wps=3353.8, ups=0.18, wpb=18371.8, bsz=1024, num_updates=12030, lr=3.68259e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=78305
-2024-11-04 03:16:23 - progress_bar.py[line:274] - INFO: epoch 031:    321 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.106, ntokens=18264.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.61, wps=3329.5, ups=0.18, wpb=18264.9, bsz=1024, num_updates=12040, lr=3.68123e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=50, gb_free=13.5, wall=78360
-2024-11-04 03:17:18 - progress_bar.py[line:274] - INFO: epoch 031:    331 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.157, ntokens=18410.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.92, wps=3369.4, ups=0.18, wpb=18410.2, bsz=1024, num_updates=12050, lr=3.67987e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=78415
-2024-11-04 03:18:12 - progress_bar.py[line:274] - INFO: epoch 031:    341 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.08, ntokens=18413.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.46, wps=3362.6, ups=0.18, wpb=18413.7, bsz=1024, num_updates=12060, lr=3.67851e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=49, gb_free=13.2, wall=78469
-2024-11-04 03:19:07 - progress_bar.py[line:274] - INFO: epoch 031:    351 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.193, ntokens=18862.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.15, wps=3423.1, ups=0.18, wpb=18862.6, bsz=1024, num_updates=12070, lr=3.67715e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=78524
-2024-11-04 03:20:02 - progress_bar.py[line:274] - INFO: epoch 031:    361 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.123, ntokens=18453.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.71, wps=3396.9, ups=0.18, wpb=18453.3, bsz=1024, num_updates=12080, lr=3.67579e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=78579
-2024-11-04 03:20:57 - progress_bar.py[line:274] - INFO: epoch 031:    371 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.068, ntokens=17964.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.38, wps=3261.1, ups=0.18, wpb=17964.6, bsz=1024, num_updates=12090, lr=3.67443e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=78634
-2024-11-04 03:21:51 - progress_bar.py[line:274] - INFO: epoch 031:    381 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.04, ntokens=18008, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=3304.9, ups=0.18, wpb=18008, bsz=1024, num_updates=12100, lr=3.67307e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=78688
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 03:22:43 - progress_bar.py[line:274] - INFO: epoch 031:    391 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.096, ntokens=17335.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.55, wps=3374, ups=0.19, wpb=17335.5, bsz=985.6, num_updates=12110, lr=3.67171e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=51, gb_free=13.3, wall=78740
-2024-11-04 03:22:43 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 31 @ 12110 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
-
+2024-11-07 10:29:39 - trainer.py[line:703] - INFO: begin training epoch 31
+2024-11-07 10:29:39 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 10:30:29 - progress_bar.py[line:274] - INFO: epoch 031:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.428, ntokens=16963.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.38, wps=348.3, ups=0.02, wpb=16963.7, bsz=985.6, num_updates=11730, lr=3.7234e-05, gnorm=0.007, clip=0, loss_scale=131072, train_wall=29, gb_free=13.5, wall=74948
+2024-11-07 10:31:22 - progress_bar.py[line:274] - INFO: epoch 031:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.407, ntokens=17615.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.3, wps=3324.8, ups=0.19, wpb=17615.5, bsz=1024, num_updates=11740, lr=3.72204e-05, gnorm=0.007, clip=0, loss_scale=131072, train_wall=30, gb_free=13.6, wall=75000
+2024-11-07 10:32:13 - progress_bar.py[line:274] - INFO: epoch 031:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.403, ntokens=17741.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.29, wps=3419.9, ups=0.19, wpb=17741.9, bsz=1024, num_updates=11750, lr=3.72068e-05, gnorm=0.006, clip=0, loss_scale=131072, train_wall=31, gb_free=13.5, wall=75052
+2024-11-07 10:33:06 - progress_bar.py[line:274] - INFO: epoch 031:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.462, ntokens=18035.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.51, wps=3460.3, ups=0.19, wpb=18035.1, bsz=1024, num_updates=11760, lr=3.71932e-05, gnorm=0.005, clip=0, loss_scale=131072, train_wall=30, gb_free=13.4, wall=75104
+2024-11-07 10:33:58 - progress_bar.py[line:274] - INFO: epoch 031:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.423, ntokens=17580.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.36, wps=3373.4, ups=0.19, wpb=17580.7, bsz=1024, num_updates=11770, lr=3.71796e-05, gnorm=0.005, clip=0, loss_scale=131072, train_wall=30, gb_free=13.6, wall=75157
+2024-11-07 10:34:50 - progress_bar.py[line:274] - INFO: epoch 031:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.407, ntokens=17687.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.3, wps=3372.9, ups=0.19, wpb=17687.6, bsz=1024, num_updates=11780, lr=3.7166e-05, gnorm=0.007, clip=0, loss_scale=131072, train_wall=30, gb_free=13.6, wall=75209
+2024-11-07 10:35:27 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
+2024-11-07 10:35:48 - progress_bar.py[line:274] - INFO: epoch 031:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.414, ntokens=17693, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3080.6, ups=0.17, wpb=17693, bsz=1024, num_updates=11790, lr=3.71524e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=33, gb_free=13.5, wall=75266
+2024-11-07 10:36:40 - progress_bar.py[line:274] - INFO: epoch 031:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.43, ntokens=17903.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.39, wps=3413.9, ups=0.19, wpb=17903.6, bsz=1024, num_updates=11800, lr=3.71388e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=75319
+2024-11-07 10:37:33 - progress_bar.py[line:274] - INFO: epoch 031:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.401, ntokens=17483, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.28, wps=3331.7, ups=0.19, wpb=17483, bsz=1024, num_updates=11810, lr=3.71252e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=75371
+2024-11-07 10:38:25 - progress_bar.py[line:274] - INFO: epoch 031:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.434, ntokens=17808.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.4, wps=3392, ups=0.19, wpb=17808.8, bsz=1024, num_updates=11820, lr=3.71116e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=75424
+2024-11-07 10:39:17 - progress_bar.py[line:274] - INFO: epoch 031:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.42, ntokens=17559.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.35, wps=3346.4, ups=0.19, wpb=17559.9, bsz=1024, num_updates=11830, lr=3.7098e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=75476
+2024-11-07 10:40:10 - progress_bar.py[line:274] - INFO: epoch 031:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.483, ntokens=18054.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3436.3, ups=0.19, wpb=18054.3, bsz=1024, num_updates=11840, lr=3.70844e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=75529
+2024-11-07 10:41:02 - progress_bar.py[line:274] - INFO: epoch 031:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.373, ntokens=17210.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.18, wps=3280.4, ups=0.19, wpb=17210.9, bsz=1024, num_updates=11850, lr=3.70708e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=75581
+2024-11-07 10:41:55 - progress_bar.py[line:274] - INFO: epoch 031:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.447, ntokens=17825.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.45, wps=3396.5, ups=0.19, wpb=17825.7, bsz=1024, num_updates=11860, lr=3.70572e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=75634
+2024-11-07 10:42:48 - progress_bar.py[line:274] - INFO: epoch 031:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.43, ntokens=17656.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.39, wps=3352.6, ups=0.19, wpb=17656.6, bsz=1024, num_updates=11870, lr=3.70436e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=75687
+2024-11-07 10:43:40 - progress_bar.py[line:274] - INFO: epoch 031:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.399, ntokens=17654.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.27, wps=3390.8, ups=0.19, wpb=17654.4, bsz=1024, num_updates=11880, lr=3.703e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=75739
+2024-11-07 10:44:33 - progress_bar.py[line:274] - INFO: epoch 031:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.478, ntokens=18158.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3430.5, ups=0.19, wpb=18158.1, bsz=1024, num_updates=11890, lr=3.70164e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=75792
+2024-11-07 10:45:25 - progress_bar.py[line:274] - INFO: epoch 031:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.395, ntokens=17633.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.26, wps=3372.9, ups=0.19, wpb=17633.8, bsz=1024, num_updates=11900, lr=3.70028e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=75844
+2024-11-07 10:46:18 - progress_bar.py[line:274] - INFO: epoch 031:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.458, ntokens=17907.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.49, wps=3395, ups=0.19, wpb=17907.3, bsz=1024, num_updates=11910, lr=3.69892e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=75897
+2024-11-07 10:47:10 - progress_bar.py[line:274] - INFO: epoch 031:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.456, ntokens=17973.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.49, wps=3448.8, ups=0.19, wpb=17973.2, bsz=1024, num_updates=11920, lr=3.69756e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=75949
+2024-11-07 10:48:02 - progress_bar.py[line:274] - INFO: epoch 031:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.436, ntokens=17877.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.41, wps=3404.6, ups=0.19, wpb=17877.2, bsz=1024, num_updates=11930, lr=3.6962e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=76001
+2024-11-07 10:48:54 - progress_bar.py[line:274] - INFO: epoch 031:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.403, ntokens=17559.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.29, wps=3364.7, ups=0.19, wpb=17559.7, bsz=1024, num_updates=11940, lr=3.69484e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=76053
+2024-11-07 10:49:47 - progress_bar.py[line:274] - INFO: epoch 031:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.4, ntokens=17497.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.28, wps=3335.1, ups=0.19, wpb=17497.6, bsz=1024, num_updates=11950, lr=3.69348e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=76106
+2024-11-07 10:50:39 - progress_bar.py[line:274] - INFO: epoch 031:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.413, ntokens=17653.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3380.9, ups=0.19, wpb=17653.1, bsz=1024, num_updates=11960, lr=3.69212e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=76158
+2024-11-07 10:51:32 - progress_bar.py[line:274] - INFO: epoch 031:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.417, ntokens=17593.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.34, wps=3324.4, ups=0.19, wpb=17593.4, bsz=1024, num_updates=11970, lr=3.69075e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=76211
+2024-11-07 10:52:25 - progress_bar.py[line:274] - INFO: epoch 031:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.408, ntokens=17668.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.31, wps=3361.6, ups=0.19, wpb=17668.8, bsz=1024, num_updates=11980, lr=3.68939e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=76264
+2024-11-07 10:53:17 - progress_bar.py[line:274] - INFO: epoch 031:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.41, ntokens=17689.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.32, wps=3355.8, ups=0.19, wpb=17689.1, bsz=1024, num_updates=11990, lr=3.68803e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=76316
+2024-11-07 10:54:10 - progress_bar.py[line:274] - INFO: epoch 031:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.425, ntokens=17771, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.37, wps=3373.7, ups=0.19, wpb=17771, bsz=1024, num_updates=12000, lr=3.68667e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=76369
+2024-11-07 10:55:02 - progress_bar.py[line:274] - INFO: epoch 031:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.406, ntokens=17589.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.3, wps=3402.8, ups=0.19, wpb=17589.3, bsz=1024, num_updates=12010, lr=3.68531e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=76421
+2024-11-07 10:55:54 - progress_bar.py[line:274] - INFO: epoch 031:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.458, ntokens=17762.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.5, wps=3425.7, ups=0.19, wpb=17762.1, bsz=1024, num_updates=12020, lr=3.68395e-05, gnorm=0.013, clip=0, loss_scale=65536, train_wall=35, gb_free=13.1, wall=76472
+2024-11-07 10:56:46 - progress_bar.py[line:274] - INFO: epoch 031:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.542, ntokens=18367.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.82, wps=3512.9, ups=0.19, wpb=18367.6, bsz=1024, num_updates=12030, lr=3.68259e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=44, gb_free=13.3, wall=76525
+2024-11-07 10:57:39 - progress_bar.py[line:274] - INFO: epoch 031:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.524, ntokens=18272.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.75, wps=3454.8, ups=0.19, wpb=18272.4, bsz=1024, num_updates=12040, lr=3.68123e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=76578
+2024-11-07 10:58:31 - progress_bar.py[line:274] - INFO: epoch 031:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.545, ntokens=18470.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.84, wps=3512.3, ups=0.19, wpb=18470.2, bsz=1024, num_updates=12050, lr=3.67987e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=76630
+2024-11-07 10:59:24 - progress_bar.py[line:274] - INFO: epoch 031:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.526, ntokens=18340.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.76, wps=3456.4, ups=0.19, wpb=18340.8, bsz=1024, num_updates=12060, lr=3.67851e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.3, wall=76683
+2024-11-07 11:00:17 - progress_bar.py[line:274] - INFO: epoch 031:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.603, ntokens=18917.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.07, wps=3564.5, ups=0.19, wpb=18917.2, bsz=1024, num_updates=12070, lr=3.67715e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.3, wall=76736
+2024-11-07 11:01:10 - progress_bar.py[line:274] - INFO: epoch 031:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.548, ntokens=18471.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.85, wps=3506.6, ups=0.19, wpb=18471.1, bsz=1024, num_updates=12080, lr=3.67579e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.3, wall=76789
+2024-11-07 11:02:03 - progress_bar.py[line:274] - INFO: epoch 031:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.467, ntokens=17893.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3414.3, ups=0.19, wpb=17893.1, bsz=1024, num_updates=12090, lr=3.67443e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=52, gb_free=13.3, wall=76841
+2024-11-07 11:02:55 - progress_bar.py[line:274] - INFO: epoch 031:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.482, ntokens=18096.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3448.7, ups=0.19, wpb=18096.5, bsz=1024, num_updates=12100, lr=3.67307e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=76894
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+2024-11-07 11:03:47 - progress_bar.py[line:274] - INFO: epoch 031:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.478, ntokens=17942.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.57, wps=3424.9, ups=0.19, wpb=17942.9, bsz=1024, num_updates=12110, lr=3.67171e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=76946
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
 cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
-2024-11-04 03:22:43 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt': File exists: File exists
+2024-11-07 11:03:49 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 31 @ 12111 updates
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
+2024-11-07 11:03:49 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt': File exists: File exists
+
+: File exists
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 03:24:30 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 4 seek offset 200000
-2024-11-04 03:26:21 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 31 @ 12110 updates, score 0) (writing took 218.6949696369993 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-04 03:27:16 - train.py[line:336] - INFO: end of epoch 31 (average epoch stats below)
-2024-11-04 03:27:16 - progress_bar.py[line:282] - INFO: epoch 031 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 3.01 | ntokens 17835.8 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 8.05 | wps 2485.9 | ups 0.14 | wpb 17835.8 | bsz 1023 | num_updates 12110 | lr 3.67171e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 2068 | gb_free 13.3 | wall 79013
-2024-11-04 03:27:16 - trainer.py[line:639] - INFO: loading train data for epoch 32
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 03:30:00 - trainer.py[line:703] - INFO: begin training epoch 32
-2024-11-04 03:30:00 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 03:30:55 - progress_bar.py[line:274] - INFO: epoch 032:     10 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.924, ntokens=17284.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.59, wps=351, ups=0.02, wpb=17284.5, bsz=1024, num_updates=12120, lr=3.67035e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=79232
-2024-11-04 03:31:53 - progress_bar.py[line:274] - INFO: epoch 032:     20 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.952, ntokens=17515.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.74, wps=3035.7, ups=0.17, wpb=17515.7, bsz=1024, num_updates=12130, lr=3.66899e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=79290
-2024-11-04 03:32:45 - progress_bar.py[line:274] - INFO: epoch 032:     30 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.936, ntokens=17749.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.65, wps=3376.5, ups=0.19, wpb=17749.5, bsz=1024, num_updates=12140, lr=3.66763e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=79342
-2024-11-04 03:33:39 - progress_bar.py[line:274] - INFO: epoch 032:     40 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.978, ntokens=17903.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.88, wps=3327.8, ups=0.19, wpb=17903.7, bsz=1024, num_updates=12150, lr=3.66627e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=34, gb_free=13.5, wall=79396
-2024-11-04 03:34:34 - progress_bar.py[line:274] - INFO: epoch 032:     50 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.92, ntokens=17381.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.57, wps=3154.9, ups=0.18, wpb=17381.9, bsz=1024, num_updates=12160, lr=3.66491e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=50, gb_free=13.5, wall=79451
-2024-11-04 03:35:29 - progress_bar.py[line:274] - INFO: epoch 032:     60 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.973, ntokens=17665.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.85, wps=3256.3, ups=0.18, wpb=17665.6, bsz=1024, num_updates=12170, lr=3.66355e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=79506
-2024-11-04 03:36:22 - progress_bar.py[line:274] - INFO: epoch 032:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.953, ntokens=17530.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.74, wps=3267.1, ups=0.19, wpb=17530.1, bsz=1024, num_updates=12180, lr=3.66219e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=79559
-2024-11-04 03:37:17 - progress_bar.py[line:274] - INFO: epoch 032:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.977, ntokens=17856.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.87, wps=3258.2, ups=0.18, wpb=17856.3, bsz=1024, num_updates=12190, lr=3.66083e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=79614
-2024-11-04 03:38:11 - progress_bar.py[line:274] - INFO: epoch 032:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.975, ntokens=17776.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.86, wps=3270.4, ups=0.18, wpb=17776.5, bsz=1024, num_updates=12200, lr=3.65947e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=79668
-2024-11-04 03:39:06 - progress_bar.py[line:274] - INFO: epoch 032:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.908, ntokens=17360.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.51, wps=3183.9, ups=0.18, wpb=17360.2, bsz=1024, num_updates=12210, lr=3.65811e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=79723
-2024-11-04 03:40:01 - progress_bar.py[line:274] - INFO: epoch 032:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.962, ntokens=17451.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.79, wps=3182.8, ups=0.18, wpb=17451.7, bsz=1024, num_updates=12220, lr=3.65674e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=79778
-2024-11-04 03:40:55 - progress_bar.py[line:274] - INFO: epoch 032:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.961, ntokens=17704.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.79, wps=3281.2, ups=0.19, wpb=17704.2, bsz=1024, num_updates=12230, lr=3.65538e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=79832
-2024-11-04 03:41:49 - progress_bar.py[line:274] - INFO: epoch 032:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.9, ntokens=17305.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.46, wps=3160.7, ups=0.18, wpb=17305.5, bsz=1024, num_updates=12240, lr=3.65402e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=79887
-2024-11-04 03:42:44 - progress_bar.py[line:274] - INFO: epoch 032:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.01, ntokens=17884.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.06, wps=3268.7, ups=0.18, wpb=17884.7, bsz=1024, num_updates=12250, lr=3.65266e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=79941
-2024-11-04 03:43:39 - progress_bar.py[line:274] - INFO: epoch 032:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.965, ntokens=17702.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.81, wps=3241.3, ups=0.18, wpb=17702.2, bsz=1024, num_updates=12260, lr=3.6513e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=79996
-2024-11-04 03:44:34 - progress_bar.py[line:274] - INFO: epoch 032:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.947, ntokens=17552.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.71, wps=3191.4, ups=0.18, wpb=17552.5, bsz=1024, num_updates=12270, lr=3.64994e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=80051
-2024-11-04 03:45:29 - progress_bar.py[line:274] - INFO: epoch 032:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.981, ntokens=17996.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3266.2, ups=0.18, wpb=17996.1, bsz=1024, num_updates=12280, lr=3.64858e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=80106
-2024-11-04 03:46:23 - progress_bar.py[line:274] - INFO: epoch 032:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.938, ntokens=17419.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.66, wps=3188.4, ups=0.18, wpb=17419.2, bsz=1024, num_updates=12290, lr=3.64722e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=80161
-2024-11-04 03:47:18 - progress_bar.py[line:274] - INFO: epoch 032:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.969, ntokens=18015, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.83, wps=3277.3, ups=0.18, wpb=18015, bsz=1024, num_updates=12300, lr=3.64586e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=80216
-2024-11-04 03:48:13 - progress_bar.py[line:274] - INFO: epoch 032:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3, ntokens=17964.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8, wps=3288.7, ups=0.18, wpb=17964.3, bsz=1024, num_updates=12310, lr=3.6445e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=80270
-2024-11-04 03:49:08 - progress_bar.py[line:274] - INFO: epoch 032:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.949, ntokens=17876.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.72, wps=3252.7, ups=0.18, wpb=17876.3, bsz=1024, num_updates=12320, lr=3.64314e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=80325
-2024-11-04 03:50:02 - progress_bar.py[line:274] - INFO: epoch 032:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.955, ntokens=17902.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.75, wps=3303.2, ups=0.18, wpb=17902.1, bsz=1024, num_updates=12330, lr=3.64178e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=80379
-2024-11-04 03:50:56 - progress_bar.py[line:274] - INFO: epoch 032:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.929, ntokens=17641.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.62, wps=3265.5, ups=0.19, wpb=17641.2, bsz=1024, num_updates=12340, lr=3.64042e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=80433
-2024-11-04 03:51:52 - progress_bar.py[line:274] - INFO: epoch 032:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.98, ntokens=17689.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3201, ups=0.18, wpb=17689.8, bsz=1024, num_updates=12350, lr=3.63906e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=80489
-2024-11-04 03:52:46 - progress_bar.py[line:274] - INFO: epoch 032:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.942, ntokens=17330.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.69, wps=3160.7, ups=0.18, wpb=17330.9, bsz=1024, num_updates=12360, lr=3.6377e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=80543
-2024-11-04 03:53:41 - progress_bar.py[line:274] - INFO: epoch 032:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.963, ntokens=17867.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.8, wps=3293.5, ups=0.18, wpb=17867.4, bsz=1024, num_updates=12370, lr=3.63634e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=80598
-2024-11-04 03:54:35 - progress_bar.py[line:274] - INFO: epoch 032:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.034, ntokens=18150.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.19, wps=3311.3, ups=0.18, wpb=18150.8, bsz=1024, num_updates=12380, lr=3.63498e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=80653
-2024-11-04 03:55:30 - progress_bar.py[line:274] - INFO: epoch 032:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.986, ntokens=17897.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.92, wps=3285, ups=0.18, wpb=17897.8, bsz=1024, num_updates=12390, lr=3.63362e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=80707
-2024-11-04 03:56:24 - progress_bar.py[line:274] - INFO: epoch 032:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.92, ntokens=17552.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.57, wps=3227.5, ups=0.18, wpb=17552.6, bsz=1024, num_updates=12400, lr=3.63226e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=80761
-2024-11-04 03:57:19 - progress_bar.py[line:274] - INFO: epoch 032:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.056, ntokens=17944.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.32, wps=3270.2, ups=0.18, wpb=17944.4, bsz=1024, num_updates=12410, lr=3.6309e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.1, wall=80816
-2024-11-04 03:58:14 - progress_bar.py[line:274] - INFO: epoch 032:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.102, ntokens=18320.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.59, wps=3359.6, ups=0.18, wpb=18320.9, bsz=1024, num_updates=12420, lr=3.62954e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=80871
-2024-11-04 03:59:08 - progress_bar.py[line:274] - INFO: epoch 032:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.108, ntokens=18238.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.62, wps=3335.2, ups=0.18, wpb=18238.7, bsz=1024, num_updates=12430, lr=3.62818e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=80925
-2024-11-04 04:00:03 - progress_bar.py[line:274] - INFO: epoch 032:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.079, ntokens=18102, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.45, wps=3333.3, ups=0.18, wpb=18102, bsz=1024, num_updates=12440, lr=3.62682e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=80980
-2024-11-04 04:00:57 - progress_bar.py[line:274] - INFO: epoch 032:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.039, ntokens=18057.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.22, wps=3313.9, ups=0.18, wpb=18057.9, bsz=1024, num_updates=12450, lr=3.62546e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.3, wall=81034
-2024-11-04 04:01:52 - progress_bar.py[line:274] - INFO: epoch 032:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.141, ntokens=18681.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.82, wps=3435.6, ups=0.18, wpb=18681.4, bsz=1024, num_updates=12460, lr=3.6241e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.3, wall=81089
-2024-11-04 04:02:45 - progress_bar.py[line:274] - INFO: epoch 032:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.097, ntokens=18267.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.55, wps=3392.3, ups=0.19, wpb=18267.2, bsz=1024, num_updates=12470, lr=3.62273e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.3, wall=81143
-2024-11-04 04:03:39 - progress_bar.py[line:274] - INFO: epoch 032:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.095, ntokens=18177.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.55, wps=3374.3, ups=0.19, wpb=18177.6, bsz=1024, num_updates=12480, lr=3.62137e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.1, wall=81196
-2024-11-04 04:04:34 - progress_bar.py[line:274] - INFO: epoch 032:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.044, ntokens=18084.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.25, wps=3327.4, ups=0.18, wpb=18084.9, bsz=1024, num_updates=12490, lr=3.62001e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=81251
-2024-11-04 04:05:23 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 04:05:30 - progress_bar.py[line:274] - INFO: epoch 032:    391 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.084, ntokens=17229.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=8.48, wps=3042.3, ups=0.18, wpb=17229.7, bsz=985.6, num_updates=12500, lr=3.61865e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=81307
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt2024-11-04 04:05:30 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 32 @ 12500 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
-
-2024-11-04 04:05:30 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt': File exists
-: File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 04:07:17 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 7 seek offset 350000
-2024-11-04 04:09:09 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 32 @ 12500 updates, score 0) (writing took 218.91895410299185 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 3 seek offset 150000
-slice_id 6 seek offset 300000
-slice_id 1 seek offset 50000
-2024-11-04 04:10:04 - train.py[line:336] - INFO: end of epoch 32 (average epoch stats below)
-2024-11-04 04:10:04 - progress_bar.py[line:282] - INFO: epoch 032 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.992 | ntokens 17795.7 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.95 | wps 2702.9 | ups 0.15 | wpb 17795.7 | bsz 1023 | num_updates 12500 | lr 3.61865e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 2028 | gb_free 13.4 | wall 81581
-2024-11-04 04:10:04 - trainer.py[line:639] - INFO: loading train data for epoch 33
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 04:12:49 - trainer.py[line:703] - INFO: begin training epoch 33
-2024-11-04 04:12:49 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 04:13:50 - progress_bar.py[line:274] - INFO: epoch 033:     10 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.926, ntokens=17580.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=352, ups=0.02, wpb=17580.1, bsz=1024, num_updates=12510, lr=3.61729e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=81807
-2024-11-04 04:14:48 - progress_bar.py[line:274] - INFO: epoch 033:     20 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.993, ntokens=17848.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.96, wps=3057.8, ups=0.17, wpb=17848.2, bsz=1024, num_updates=12520, lr=3.61593e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=81865
-2024-11-04 04:15:43 - progress_bar.py[line:274] - INFO: epoch 033:     30 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.913, ntokens=17735.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.53, wps=3254.1, ups=0.18, wpb=17735.2, bsz=1024, num_updates=12530, lr=3.61457e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=81920
-2024-11-04 04:16:41 - progress_bar.py[line:274] - INFO: epoch 033:     40 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.998, ntokens=18046.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.99, wps=3075.2, ups=0.17, wpb=18046.9, bsz=1024, num_updates=12540, lr=3.61321e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=81978
-2024-11-04 04:17:37 - progress_bar.py[line:274] - INFO: epoch 033:     50 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.948, ntokens=17702.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.72, wps=3175.4, ups=0.18, wpb=17702.9, bsz=1024, num_updates=12550, lr=3.61185e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=82034
-2024-11-04 04:18:30 - progress_bar.py[line:274] - INFO: epoch 033:     60 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.027, ntokens=18242.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.15, wps=3453.8, ups=0.19, wpb=18242.5, bsz=1024, num_updates=12560, lr=3.61049e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=82087
-2024-11-04 04:19:23 - progress_bar.py[line:274] - INFO: epoch 033:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.962, ntokens=17737.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.79, wps=3321.8, ups=0.19, wpb=17737.5, bsz=1024, num_updates=12570, lr=3.60913e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=82140
-2024-11-04 04:20:19 - progress_bar.py[line:274] - INFO: epoch 033:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.916, ntokens=17700.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.55, wps=3181.7, ups=0.18, wpb=17700.6, bsz=1024, num_updates=12580, lr=3.60777e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=47, gb_free=13.4, wall=82196
-2024-11-04 04:21:14 - progress_bar.py[line:274] - INFO: epoch 033:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.988, ntokens=17847.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.93, wps=3219.9, ups=0.18, wpb=17847.1, bsz=1024, num_updates=12590, lr=3.60641e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=82251
-2024-11-04 04:22:12 - progress_bar.py[line:274] - INFO: epoch 033:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.977, ntokens=17881.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.87, wps=3116.9, ups=0.17, wpb=17881.6, bsz=1024, num_updates=12600, lr=3.60505e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=82309
-2024-11-04 04:23:08 - progress_bar.py[line:274] - INFO: epoch 033:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.923, ntokens=17490.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.58, wps=3107.3, ups=0.18, wpb=17490.3, bsz=1024, num_updates=12610, lr=3.60369e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=82365
-2024-11-04 04:24:04 - progress_bar.py[line:274] - INFO: epoch 033:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.943, ntokens=17797.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.69, wps=3203.2, ups=0.18, wpb=17797.4, bsz=1024, num_updates=12620, lr=3.60233e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=82421
-2024-11-04 04:24:57 - progress_bar.py[line:274] - INFO: epoch 033:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.938, ntokens=17470.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.66, wps=3259.1, ups=0.19, wpb=17470.8, bsz=1024, num_updates=12630, lr=3.60097e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=82474
-2024-11-04 04:25:50 - progress_bar.py[line:274] - INFO: epoch 033:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3, ntokens=18089.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8, wps=3393.3, ups=0.19, wpb=18089.8, bsz=1024, num_updates=12640, lr=3.59961e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=53, gb_free=13.6, wall=82528
-2024-11-04 04:26:44 - progress_bar.py[line:274] - INFO: epoch 033:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.924, ntokens=17504.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.59, wps=3293.9, ups=0.19, wpb=17504.7, bsz=1024, num_updates=12650, lr=3.59825e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=82581
-2024-11-04 04:27:37 - progress_bar.py[line:274] - INFO: epoch 033:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.966, ntokens=17852.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.81, wps=3323.5, ups=0.19, wpb=17852.4, bsz=1024, num_updates=12660, lr=3.59689e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=82634
-2024-11-04 04:28:32 - progress_bar.py[line:274] - INFO: epoch 033:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.958, ntokens=17919.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.77, wps=3278.9, ups=0.18, wpb=17919.3, bsz=1024, num_updates=12670, lr=3.59553e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=82689
-2024-11-04 04:29:27 - progress_bar.py[line:274] - INFO: epoch 033:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.946, ntokens=17658, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.7, wps=3211.1, ups=0.18, wpb=17658, bsz=1024, num_updates=12680, lr=3.59417e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=82744
-2024-11-04 04:30:19 - progress_bar.py[line:274] - INFO: epoch 033:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.941, ntokens=17719.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.68, wps=3382.8, ups=0.19, wpb=17719.9, bsz=1024, num_updates=12690, lr=3.59281e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=82796
-2024-11-04 04:31:13 - progress_bar.py[line:274] - INFO: epoch 033:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.982, ntokens=18060.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.9, wps=3346.9, ups=0.19, wpb=18060.4, bsz=1024, num_updates=12700, lr=3.59145e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.3, wall=82850
-2024-11-04 04:32:07 - progress_bar.py[line:274] - INFO: epoch 033:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.957, ntokens=17881.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.77, wps=3321.7, ups=0.19, wpb=17881.3, bsz=1024, num_updates=12710, lr=3.59009e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.3, wall=82904
-2024-11-04 04:33:01 - progress_bar.py[line:274] - INFO: epoch 033:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.919, ntokens=17712.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.56, wps=3303.4, ups=0.19, wpb=17712.2, bsz=1024, num_updates=12720, lr=3.58873e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=82958
-2024-11-04 04:33:55 - progress_bar.py[line:274] - INFO: epoch 033:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=17690.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.49, wps=3278.4, ups=0.19, wpb=17690.4, bsz=1024, num_updates=12730, lr=3.58736e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=83012
-2024-11-04 04:34:49 - progress_bar.py[line:274] - INFO: epoch 033:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.01, ntokens=17836.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.06, wps=3295.8, ups=0.18, wpb=17836.1, bsz=1024, num_updates=12740, lr=3.586e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=83066
-2024-11-04 04:35:43 - progress_bar.py[line:274] - INFO: epoch 033:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.022, ntokens=18040.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=3329.4, ups=0.18, wpb=18040.9, bsz=1024, num_updates=12750, lr=3.58464e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=83120
-2024-11-04 04:36:37 - progress_bar.py[line:274] - INFO: epoch 033:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.944, ntokens=17773.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.69, wps=3296.9, ups=0.19, wpb=17773.4, bsz=1024, num_updates=12760, lr=3.58328e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=83174
-2024-11-04 04:37:30 - progress_bar.py[line:274] - INFO: epoch 033:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.947, ntokens=17701.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.71, wps=3317.5, ups=0.19, wpb=17701.5, bsz=1024, num_updates=12770, lr=3.58192e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=53, gb_free=13.6, wall=83227
-2024-11-04 04:38:24 - progress_bar.py[line:274] - INFO: epoch 033:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.935, ntokens=17691.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.65, wps=3292.2, ups=0.19, wpb=17691.5, bsz=1024, num_updates=12780, lr=3.58056e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=83281
-2024-11-04 04:39:18 - progress_bar.py[line:274] - INFO: epoch 033:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.916, ntokens=17418.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.55, wps=3235, ups=0.19, wpb=17418.5, bsz=1024, num_updates=12790, lr=3.5792e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=83335
-2024-11-04 04:40:13 - progress_bar.py[line:274] - INFO: epoch 033:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.996, ntokens=17655.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.98, wps=3199.3, ups=0.18, wpb=17655.5, bsz=1024, num_updates=12800, lr=3.57784e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=83390
-2024-11-04 04:41:09 - progress_bar.py[line:274] - INFO: epoch 033:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.11, ntokens=18380.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.63, wps=3299.2, ups=0.18, wpb=18380.1, bsz=1024, num_updates=12810, lr=3.57648e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=83446
-2024-11-04 04:42:02 - progress_bar.py[line:274] - INFO: epoch 033:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.101, ntokens=18380.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.58, wps=3423.4, ups=0.19, wpb=18380.8, bsz=1024, num_updates=12820, lr=3.57512e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=83500
-2024-11-04 04:42:58 - progress_bar.py[line:274] - INFO: epoch 033:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.086, ntokens=18071.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.49, wps=3277.9, ups=0.18, wpb=18071.2, bsz=1024, num_updates=12830, lr=3.57376e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=83555
-2024-11-04 04:43:52 - progress_bar.py[line:274] - INFO: epoch 033:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.028, ntokens=18067.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.16, wps=3306.7, ups=0.18, wpb=18067.8, bsz=1024, num_updates=12840, lr=3.5724e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=83609
-2024-11-04 04:44:48 - progress_bar.py[line:274] - INFO: epoch 033:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.14, ntokens=18678.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.82, wps=3369.2, ups=0.18, wpb=18678.6, bsz=1024, num_updates=12850, lr=3.57104e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=83665
-2024-11-04 04:45:43 - progress_bar.py[line:274] - INFO: epoch 033:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.054, ntokens=17996.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.3, wps=3269.7, ups=0.18, wpb=17996.3, bsz=1024, num_updates=12860, lr=3.56968e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=83720
-2024-11-04 04:46:37 - progress_bar.py[line:274] - INFO: epoch 033:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.041, ntokens=18022.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=3319.2, ups=0.18, wpb=18022.5, bsz=1024, num_updates=12870, lr=3.56832e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=54, gb_free=13.3, wall=83774
-2024-11-04 04:47:32 - progress_bar.py[line:274] - INFO: epoch 033:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.024, ntokens=18000.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.14, wps=3275.5, ups=0.18, wpb=18000.4, bsz=1024, num_updates=12880, lr=3.56696e-05, gnorm=0.012, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=83829
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 04:48:26 - progress_bar.py[line:274] - INFO: epoch 033:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.07, ntokens=17974.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.4, wps=3306.4, ups=0.18, wpb=17974.9, bsz=1024, num_updates=12890, lr=3.5656e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=83883
-2024-11-04 04:48:28 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 33 @ 12891 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.pt
-2024-11-04 04:48:28 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.pt': File exists: File exists
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 04:50:20 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 7 seek offset 350000
-2024-11-04 04:52:11 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 33 @ 12891 updates, score 0) (writing took 223.11510192300193 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_33.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-slice_id 1 seek offset 50000
-slice_id 3 seek offset 150000
-2024-11-04 04:53:06 - train.py[line:336] - INFO: end of epoch 33 (average epoch stats below)
-2024-11-04 04:53:06 - progress_bar.py[line:282] - INFO: epoch 033 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.984 | ntokens 17850.2 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.91 | wps 2702.9 | ups 0.15 | wpb 17850.2 | bsz 1023 | num_updates 12891 | lr 3.56546e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 2025 | gb_free 13.3 | wall 84163
-2024-11-04 04:53:06 - trainer.py[line:639] - INFO: loading train data for epoch 34
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 04:55:52 - trainer.py[line:703] - INFO: begin training epoch 34
-2024-11-04 04:55:52 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 04:56:43 - progress_bar.py[line:274] - INFO: epoch 034:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.892, ntokens=16704.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.42, wps=336, ups=0.02, wpb=16704.1, bsz=985.6, num_updates=12900, lr=3.56424e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=29, gb_free=13.5, wall=84381
-2024-11-04 04:57:41 - progress_bar.py[line:274] - INFO: epoch 034:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.928, ntokens=17709.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.61, wps=3065.6, ups=0.17, wpb=17709.1, bsz=1024, num_updates=12910, lr=3.56288e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=84438
-2024-11-04 04:58:35 - progress_bar.py[line:274] - INFO: epoch 034:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.918, ntokens=17685.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.56, wps=3277.6, ups=0.19, wpb=17685.1, bsz=1024, num_updates=12920, lr=3.56152e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=84492
-2024-11-04 04:59:31 - progress_bar.py[line:274] - INFO: epoch 034:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.991, ntokens=18141.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.95, wps=3280.2, ups=0.18, wpb=18141.2, bsz=1024, num_updates=12930, lr=3.56016e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=84548
-2024-11-04 05:00:26 - progress_bar.py[line:274] - INFO: epoch 034:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.914, ntokens=17554.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=3149.6, ups=0.18, wpb=17554.9, bsz=1024, num_updates=12940, lr=3.5588e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=84603
-2024-11-04 05:01:19 - progress_bar.py[line:274] - INFO: epoch 034:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.985, ntokens=17968.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.92, wps=3420.4, ups=0.19, wpb=17968.9, bsz=1024, num_updates=12950, lr=3.55744e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=84656
-2024-11-04 05:02:11 - progress_bar.py[line:274] - INFO: epoch 034:     69 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.981, ntokens=17933.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3442.6, ups=0.19, wpb=17933.5, bsz=1024, num_updates=12960, lr=3.55608e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.6, wall=84708
-2024-11-04 05:03:06 - progress_bar.py[line:274] - INFO: epoch 034:     79 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.935, ntokens=17715.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.65, wps=3206.2, ups=0.18, wpb=17715.1, bsz=1024, num_updates=12970, lr=3.55472e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=84763
-2024-11-04 05:04:02 - progress_bar.py[line:274] - INFO: epoch 034:     89 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.925, ntokens=17465.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.59, wps=3110, ups=0.18, wpb=17465.5, bsz=1024, num_updates=12980, lr=3.55335e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=84819
-2024-11-04 05:04:59 - progress_bar.py[line:274] - INFO: epoch 034:     99 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.888, ntokens=17546.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.4, wps=3104.6, ups=0.18, wpb=17546.7, bsz=1024, num_updates=12990, lr=3.55199e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=84876
-2024-11-04 05:05:55 - progress_bar.py[line:274] - INFO: epoch 034:    109 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.914, ntokens=17413.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=3088.3, ups=0.18, wpb=17413.6, bsz=1024, num_updates=13000, lr=3.55063e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=84932
-2024-11-04 05:06:52 - progress_bar.py[line:274] - INFO: epoch 034:    119 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.927, ntokens=17632.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=3112.4, ups=0.18, wpb=17632.7, bsz=1024, num_updates=13010, lr=3.54927e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=84989
-2024-11-04 05:07:48 - progress_bar.py[line:274] - INFO: epoch 034:    129 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.915, ntokens=17573.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=3147, ups=0.18, wpb=17573.8, bsz=1024, num_updates=13020, lr=3.54791e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=85045
-2024-11-04 05:08:43 - progress_bar.py[line:274] - INFO: epoch 034:    139 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.989, ntokens=17975.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.94, wps=3258.6, ups=0.18, wpb=17975.5, bsz=1024, num_updates=13030, lr=3.54655e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=85100
-2024-11-04 05:09:38 - progress_bar.py[line:274] - INFO: epoch 034:    149 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.942, ntokens=17750.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.68, wps=3204.3, ups=0.18, wpb=17750.3, bsz=1024, num_updates=13040, lr=3.54519e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=85155
-2024-11-04 05:10:34 - progress_bar.py[line:274] - INFO: epoch 034:    159 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.942, ntokens=17720.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.69, wps=3199.1, ups=0.18, wpb=17720.6, bsz=1024, num_updates=13050, lr=3.54383e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=85211
-2024-11-04 05:11:29 - progress_bar.py[line:274] - INFO: epoch 034:    169 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.895, ntokens=17571.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.44, wps=3170.5, ups=0.18, wpb=17571.7, bsz=1024, num_updates=13060, lr=3.54247e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=85266
-2024-11-04 05:12:25 - progress_bar.py[line:274] - INFO: epoch 034:    179 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.869, ntokens=17226.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.31, wps=3100, ups=0.18, wpb=17226.8, bsz=1024, num_updates=13070, lr=3.54111e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=85322
-2024-11-04 05:13:20 - progress_bar.py[line:274] - INFO: epoch 034:    189 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.002, ntokens=18117.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.01, wps=3274.3, ups=0.18, wpb=18117.2, bsz=1024, num_updates=13080, lr=3.53975e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=85377
-2024-11-04 05:14:16 - progress_bar.py[line:274] - INFO: epoch 034:    199 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.98, ntokens=17978.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3229.1, ups=0.18, wpb=17978.3, bsz=1024, num_updates=13090, lr=3.53839e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=85433
-2024-11-04 05:15:11 - progress_bar.py[line:274] - INFO: epoch 034:    209 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.926, ntokens=17892.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=3208.5, ups=0.18, wpb=17892.5, bsz=1024, num_updates=13100, lr=3.53703e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=85489
-2024-11-04 05:16:08 - progress_bar.py[line:274] - INFO: epoch 034:    219 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.935, ntokens=17961.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.65, wps=3199.8, ups=0.18, wpb=17961.9, bsz=1024, num_updates=13110, lr=3.53567e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=85545
-2024-11-04 05:17:03 - progress_bar.py[line:274] - INFO: epoch 034:    229 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.907, ntokens=17758.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3175.4, ups=0.18, wpb=17758.9, bsz=1024, num_updates=13120, lr=3.53431e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=85601
-2024-11-04 05:18:02 - progress_bar.py[line:274] - INFO: epoch 034:    239 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.935, ntokens=17508.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.65, wps=3010.3, ups=0.17, wpb=17508.8, bsz=1024, num_updates=13130, lr=3.53295e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=85659
-2024-11-04 05:18:56 - progress_bar.py[line:274] - INFO: epoch 034:    249 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.932, ntokens=17406, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.63, wps=3177.9, ups=0.18, wpb=17406, bsz=1024, num_updates=13140, lr=3.53159e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=85714
-2024-11-04 05:19:51 - progress_bar.py[line:274] - INFO: epoch 034:    259 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.929, ntokens=17783.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.61, wps=3269.3, ups=0.18, wpb=17783.9, bsz=1024, num_updates=13150, lr=3.53023e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=85768
-2024-11-04 05:20:47 - progress_bar.py[line:274] - INFO: epoch 034:    269 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.976, ntokens=17875.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.87, wps=3198.6, ups=0.18, wpb=17875.5, bsz=1024, num_updates=13160, lr=3.52887e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=85824
-2024-11-04 05:21:42 - progress_bar.py[line:274] - INFO: epoch 034:    279 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=17716.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3184, ups=0.18, wpb=17716.1, bsz=1024, num_updates=13170, lr=3.52751e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=85879
-2024-11-04 05:22:39 - progress_bar.py[line:274] - INFO: epoch 034:    289 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.914, ntokens=17455.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=3103.9, ups=0.18, wpb=17455.4, bsz=1024, num_updates=13180, lr=3.52615e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=85936
-2024-11-04 05:23:35 - progress_bar.py[line:274] - INFO: epoch 034:    299 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.001, ntokens=17881.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.01, wps=3194.1, ups=0.18, wpb=17881.5, bsz=1024, num_updates=13190, lr=3.52479e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=85992
-2024-11-04 05:24:31 - progress_bar.py[line:274] - INFO: epoch 034:    309 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.178, ntokens=18779.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=9.05, wps=3312.1, ups=0.18, wpb=18779.2, bsz=1024, num_updates=13200, lr=3.52343e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=86048
-2024-11-04 05:25:28 - progress_bar.py[line:274] - INFO: epoch 034:    319 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.022, ntokens=18113.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.12, wps=3175.3, ups=0.18, wpb=18113.8, bsz=1024, num_updates=13210, lr=3.52207e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=86105
-2024-11-04 05:26:21 - progress_bar.py[line:274] - INFO: epoch 034:    329 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.093, ntokens=18280.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.53, wps=3484.3, ups=0.19, wpb=18280.7, bsz=1024, num_updates=13220, lr=3.52071e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.2, wall=86158
-2024-11-04 05:27:18 - progress_bar.py[line:274] - INFO: epoch 034:    339 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.048, ntokens=18273.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.27, wps=3216.6, ups=0.18, wpb=18273.6, bsz=1024, num_updates=13230, lr=3.51934e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=86215
-2024-11-04 05:28:13 - progress_bar.py[line:274] - INFO: epoch 034:    349 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.085, ntokens=18583.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.48, wps=3367, ups=0.18, wpb=18583.6, bsz=1024, num_updates=13240, lr=3.51798e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=86270
-2024-11-04 05:29:08 - progress_bar.py[line:274] - INFO: epoch 034:    359 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.096, ntokens=18417.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.55, wps=3324, ups=0.18, wpb=18417.3, bsz=1024, num_updates=13250, lr=3.51662e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=86325
-2024-11-04 05:30:01 - progress_bar.py[line:274] - INFO: epoch 034:    369 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.084, ntokens=18249, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.48, wps=3446.7, ups=0.19, wpb=18249, bsz=1024, num_updates=13260, lr=3.51526e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=86378
-2024-11-04 05:30:59 - progress_bar.py[line:274] - INFO: epoch 034:    379 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.001, ntokens=18119.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.01, wps=3122.4, ups=0.17, wpb=18119.4, bsz=1024, num_updates=13270, lr=3.5139e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=86436
-2024-11-04 05:31:56 - progress_bar.py[line:274] - INFO: epoch 034:    389 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.039, ntokens=17709, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.22, wps=3115.3, ups=0.18, wpb=17709, bsz=1024, num_updates=13280, lr=3.51254e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=86493
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_34.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_34.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_34.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_34.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_34.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_34.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_34.pt
-2024-11-04 05:32:04 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 34 @ 13282 updates
-2024-11-04 05:32:04 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 05:33:55 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 05:35:50 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 34 @ 13282 updates, score 0) (writing took 225.80116559099406 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_34.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 4 seek offset 200000
-slice_id 3 seek offset 150000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-04 05:36:45 - train.py[line:336] - INFO: end of epoch 34 (average epoch stats below)
-2024-11-04 05:36:45 - progress_bar.py[line:282] - INFO: epoch 034 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.97 | ntokens 17829.6 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.83 | wps 2661.3 | ups 0.15 | wpb 17829.6 | bsz 1023 | num_updates 13282 | lr 3.51227e-05 | gnorm 0.008 | clip 0 | loss_scale 65536 | train_wall 2091 | gb_free 13.3 | wall 86782
-2024-11-04 05:36:45 - trainer.py[line:639] - INFO: loading train data for epoch 35
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 05:39:31 - trainer.py[line:703] - INFO: begin training epoch 35
-2024-11-04 05:39:31 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 05:40:19 - progress_bar.py[line:274] - INFO: epoch 035:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.956, ntokens=17172.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.76, wps=341.2, ups=0.02, wpb=17172.5, bsz=985.6, num_updates=13290, lr=3.51118e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=32, gb_free=13.5, wall=86996
-2024-11-04 05:41:18 - progress_bar.py[line:274] - INFO: epoch 035:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.903, ntokens=17661.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.48, wps=3004, ups=0.17, wpb=17661.5, bsz=1024, num_updates=13300, lr=3.50982e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=87055
-2024-11-04 05:42:17 - progress_bar.py[line:274] - INFO: epoch 035:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.88, ntokens=17361.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.36, wps=2965.4, ups=0.17, wpb=17361.2, bsz=1024, num_updates=13310, lr=3.50846e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=87114
-2024-11-04 05:43:15 - progress_bar.py[line:274] - INFO: epoch 035:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=17683.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3023.9, ups=0.17, wpb=17683.2, bsz=1024, num_updates=13320, lr=3.5071e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=87172
-2024-11-04 05:44:12 - progress_bar.py[line:274] - INFO: epoch 035:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.926, ntokens=17821.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=3118.7, ups=0.17, wpb=17821.2, bsz=1024, num_updates=13330, lr=3.50574e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=87229
-2024-11-04 05:45:05 - progress_bar.py[line:274] - INFO: epoch 035:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.937, ntokens=17989.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.66, wps=3396.1, ups=0.19, wpb=17989.6, bsz=1024, num_updates=13340, lr=3.50438e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=87282
-2024-11-04 05:46:02 - progress_bar.py[line:274] - INFO: epoch 035:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.949, ntokens=17701.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.72, wps=3119.5, ups=0.18, wpb=17701.9, bsz=1024, num_updates=13350, lr=3.50302e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=50, gb_free=13.5, wall=87339
-2024-11-04 05:46:59 - progress_bar.py[line:274] - INFO: epoch 035:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.955, ntokens=17882, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.76, wps=3125.5, ups=0.17, wpb=17882, bsz=1024, num_updates=13360, lr=3.50166e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=87396
-2024-11-04 05:47:56 - progress_bar.py[line:274] - INFO: epoch 035:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.894, ntokens=17581.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=3084.7, ups=0.18, wpb=17581.1, bsz=1024, num_updates=13370, lr=3.5003e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=87453
-2024-11-04 05:48:53 - progress_bar.py[line:274] - INFO: epoch 035:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.885, ntokens=17461.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3079, ups=0.18, wpb=17461.5, bsz=1024, num_updates=13380, lr=3.49894e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=87510
-2024-11-04 05:49:50 - progress_bar.py[line:274] - INFO: epoch 035:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.909, ntokens=17567.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.51, wps=3096.4, ups=0.18, wpb=17567.2, bsz=1024, num_updates=13390, lr=3.49758e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=87567
-2024-11-04 05:50:47 - progress_bar.py[line:274] - INFO: epoch 035:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.93, ntokens=17625, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.62, wps=3077.9, ups=0.17, wpb=17625, bsz=1024, num_updates=13400, lr=3.49622e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=87624
-2024-11-04 05:51:44 - progress_bar.py[line:274] - INFO: epoch 035:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.889, ntokens=17541.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.41, wps=3090.7, ups=0.18, wpb=17541.7, bsz=1024, num_updates=13410, lr=3.49486e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=87681
-2024-11-04 05:52:40 - progress_bar.py[line:274] - INFO: epoch 035:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.894, ntokens=17456.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=3074.3, ups=0.18, wpb=17456.1, bsz=1024, num_updates=13420, lr=3.4935e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=87738
-2024-11-04 05:53:38 - progress_bar.py[line:274] - INFO: epoch 035:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.914, ntokens=17661.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=3070.7, ups=0.17, wpb=17661.4, bsz=1024, num_updates=13430, lr=3.49214e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=87795
-2024-11-04 05:54:35 - progress_bar.py[line:274] - INFO: epoch 035:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.942, ntokens=17637.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.68, wps=3096, ups=0.18, wpb=17637.8, bsz=1024, num_updates=13440, lr=3.49078e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=87852
-2024-11-04 05:55:32 - progress_bar.py[line:274] - INFO: epoch 035:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=17857.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.49, wps=3113, ups=0.17, wpb=17857.6, bsz=1024, num_updates=13450, lr=3.48942e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=87909
-2024-11-04 05:56:29 - progress_bar.py[line:274] - INFO: epoch 035:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.918, ntokens=17601.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.56, wps=3081.1, ups=0.18, wpb=17601.1, bsz=1024, num_updates=13460, lr=3.48806e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=87967
-2024-11-04 05:57:26 - progress_bar.py[line:274] - INFO: epoch 035:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.913, ntokens=17777.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.53, wps=3160.6, ups=0.18, wpb=17777.2, bsz=1024, num_updates=13470, lr=3.4867e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=88023
-2024-11-04 05:58:21 - progress_bar.py[line:274] - INFO: epoch 035:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.93, ntokens=17637.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.62, wps=3160.5, ups=0.18, wpb=17637.9, bsz=1024, num_updates=13480, lr=3.48533e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=88079
-2024-11-04 05:59:18 - progress_bar.py[line:274] - INFO: epoch 035:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.911, ntokens=17892.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.52, wps=3141.9, ups=0.18, wpb=17892.4, bsz=1024, num_updates=13490, lr=3.48397e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=88136
-2024-11-04 06:00:15 - progress_bar.py[line:274] - INFO: epoch 035:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.859, ntokens=17476.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.26, wps=3063.5, ups=0.18, wpb=17476.4, bsz=1024, num_updates=13500, lr=3.48261e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=88193
-2024-11-04 06:01:12 - progress_bar.py[line:274] - INFO: epoch 035:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.896, ntokens=17773.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.44, wps=3127.8, ups=0.18, wpb=17773.7, bsz=1024, num_updates=13510, lr=3.48125e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=88249
-2024-11-04 06:02:10 - progress_bar.py[line:274] - INFO: epoch 035:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.919, ntokens=17591.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.57, wps=3067.9, ups=0.17, wpb=17591.3, bsz=1024, num_updates=13520, lr=3.47989e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=88307
-2024-11-04 06:02:55 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
-2024-11-04 06:03:12 - progress_bar.py[line:274] - INFO: epoch 035:    249 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.944, ntokens=17515.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.69, wps=2786.2, ups=0.16, wpb=17515.7, bsz=1024, num_updates=13530, lr=3.47853e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=63, gb_free=13.5, wall=88370
-2024-11-04 06:04:09 - progress_bar.py[line:274] - INFO: epoch 035:    259 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.876, ntokens=17564.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.34, wps=3090.9, ups=0.18, wpb=17564.3, bsz=1024, num_updates=13540, lr=3.47717e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=88426
-2024-11-04 06:05:06 - progress_bar.py[line:274] - INFO: epoch 035:    269 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.958, ntokens=17765, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.77, wps=3111.8, ups=0.18, wpb=17765, bsz=1024, num_updates=13550, lr=3.47581e-05, gnorm=0.013, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=88484
-2024-11-04 06:06:04 - progress_bar.py[line:274] - INFO: epoch 035:    279 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.909, ntokens=17785, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.51, wps=3085.5, ups=0.17, wpb=17785, bsz=1024, num_updates=13560, lr=3.47445e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=88541
-2024-11-04 06:07:01 - progress_bar.py[line:274] - INFO: epoch 035:    289 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.934, ntokens=17749, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.64, wps=3112.5, ups=0.18, wpb=17749, bsz=1024, num_updates=13570, lr=3.47309e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=88598
-2024-11-04 06:07:58 - progress_bar.py[line:274] - INFO: epoch 035:    299 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.934, ntokens=17473.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.64, wps=3078.2, ups=0.18, wpb=17473.2, bsz=1024, num_updates=13580, lr=3.47173e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=88655
-2024-11-04 06:08:54 - progress_bar.py[line:274] - INFO: epoch 035:    309 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.088, ntokens=18392.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.5, wps=3281.7, ups=0.18, wpb=18392.9, bsz=1024, num_updates=13590, lr=3.47037e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=88711
-2024-11-04 06:09:51 - progress_bar.py[line:274] - INFO: epoch 035:    319 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.065, ntokens=18468.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.37, wps=3254.1, ups=0.18, wpb=18468.9, bsz=1024, num_updates=13600, lr=3.46901e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=88768
-2024-11-04 06:10:47 - progress_bar.py[line:274] - INFO: epoch 035:    329 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.073, ntokens=18210.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.41, wps=3217.7, ups=0.18, wpb=18210.8, bsz=1024, num_updates=13610, lr=3.46765e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.3, wall=88824
-2024-11-04 06:11:45 - progress_bar.py[line:274] - INFO: epoch 035:    339 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.035, ntokens=18380.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.2, wps=3193, ups=0.17, wpb=18380.4, bsz=1024, num_updates=13620, lr=3.46629e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=88882
-2024-11-04 06:12:40 - progress_bar.py[line:274] - INFO: epoch 035:    349 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.027, ntokens=18209.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.15, wps=3323.1, ups=0.18, wpb=18209.9, bsz=1024, num_updates=13630, lr=3.46493e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=88937
-2024-11-04 06:13:35 - progress_bar.py[line:274] - INFO: epoch 035:    359 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.112, ntokens=18735.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.64, wps=3388.3, ups=0.18, wpb=18735.3, bsz=1024, num_updates=13640, lr=3.46357e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=88992
-2024-11-04 06:14:31 - progress_bar.py[line:274] - INFO: epoch 035:    369 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.087, ntokens=18382.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.5, wps=3281.7, ups=0.18, wpb=18382.5, bsz=1024, num_updates=13650, lr=3.46221e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=89048
-2024-11-04 06:15:25 - progress_bar.py[line:274] - INFO: epoch 035:    379 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.058, ntokens=18458.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.33, wps=3389.3, ups=0.18, wpb=18458.8, bsz=1024, num_updates=13660, lr=3.46085e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=89102
-2024-11-04 06:16:22 - progress_bar.py[line:274] - INFO: epoch 035:    389 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.055, ntokens=18067.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.31, wps=3185.8, ups=0.18, wpb=18067.3, bsz=1024, num_updates=13670, lr=3.45949e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=89159
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_35.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_35.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_35.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_35.pt
-
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_35.pt2024-11-04 06:16:30 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 35 @ 13672 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_35.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_35.pt
-2024-11-04 06:16:30 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 06:18:23 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 06:20:17 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 35 @ 13672 updates, score 0) (writing took 227.8677660920075 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_35.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 5 seek offset 250000
-slice_id 4 seek offset 200000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 3 seek offset 150000
-2024-11-04 06:21:13 - train.py[line:336] - INFO: end of epoch 35 (average epoch stats below)
-2024-11-04 06:21:13 - progress_bar.py[line:282] - INFO: epoch 035 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.951 | ntokens 17804.2 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.73 | wps 2603.3 | ups 0.15 | wpb 17804.2 | bsz 1023 | num_updates 13672 | lr 3.45922e-05 | gnorm 0.008 | clip 0 | loss_scale 65536 | train_wall 2049 | gb_free 13.4 | wall 89450
-2024-11-04 06:21:13 - trainer.py[line:639] - INFO: loading train data for epoch 36
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 06:23:59 - trainer.py[line:703] - INFO: begin training epoch 36
-2024-11-04 06:23:59 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 06:24:46 - progress_bar.py[line:274] - INFO: epoch 036:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.958, ntokens=17184.9, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.77, wps=340.8, ups=0.02, wpb=17184.9, bsz=985.6, num_updates=13680, lr=3.45813e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=32, gb_free=13.5, wall=89663
-2024-11-04 06:25:44 - progress_bar.py[line:274] - INFO: epoch 036:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.947, ntokens=17967.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.71, wps=3141.2, ups=0.17, wpb=17967.8, bsz=1024, num_updates=13690, lr=3.45677e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=89721
-2024-11-04 06:26:41 - progress_bar.py[line:274] - INFO: epoch 036:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.889, ntokens=17519, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.41, wps=3025.4, ups=0.17, wpb=17519, bsz=1024, num_updates=13700, lr=3.45541e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=89779
-2024-11-04 06:27:39 - progress_bar.py[line:274] - INFO: epoch 036:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.954, ntokens=17997.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.75, wps=3120, ups=0.17, wpb=17997.8, bsz=1024, num_updates=13710, lr=3.45405e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=89836
-2024-11-04 06:28:35 - progress_bar.py[line:274] - INFO: epoch 036:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.852, ntokens=17345, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=3081.5, ups=0.18, wpb=17345, bsz=1024, num_updates=13720, lr=3.45269e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=89892
-2024-11-04 06:29:33 - progress_bar.py[line:274] - INFO: epoch 036:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.919, ntokens=17776.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.56, wps=3098.9, ups=0.17, wpb=17776.3, bsz=1024, num_updates=13730, lr=3.45133e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=89950
-2024-11-04 06:30:30 - progress_bar.py[line:274] - INFO: epoch 036:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.928, ntokens=17725.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.61, wps=3119, ups=0.18, wpb=17725.7, bsz=1024, num_updates=13740, lr=3.44996e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=90007
-2024-11-04 06:31:27 - progress_bar.py[line:274] - INFO: epoch 036:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.913, ntokens=17647.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.53, wps=3059.5, ups=0.17, wpb=17647.3, bsz=1024, num_updates=13750, lr=3.4486e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=90064
-2024-11-04 06:32:25 - progress_bar.py[line:274] - INFO: epoch 036:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.903, ntokens=17577.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.48, wps=3034.4, ups=0.17, wpb=17577.4, bsz=1024, num_updates=13760, lr=3.44724e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=90122
-2024-11-04 06:33:23 - progress_bar.py[line:274] - INFO: epoch 036:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.913, ntokens=17812.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.53, wps=3098.8, ups=0.17, wpb=17812.4, bsz=1024, num_updates=13770, lr=3.44588e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=90180
-2024-11-04 06:34:21 - progress_bar.py[line:274] - INFO: epoch 036:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.869, ntokens=17305.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.31, wps=2981.8, ups=0.17, wpb=17305.2, bsz=1024, num_updates=13780, lr=3.44452e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=90238
-2024-11-04 06:35:18 - progress_bar.py[line:274] - INFO: epoch 036:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.909, ntokens=17619.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.51, wps=3058.8, ups=0.17, wpb=17619.2, bsz=1024, num_updates=13790, lr=3.44316e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=90295
-2024-11-04 06:36:16 - progress_bar.py[line:274] - INFO: epoch 036:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.893, ntokens=17615.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=3074.2, ups=0.17, wpb=17615.7, bsz=1024, num_updates=13800, lr=3.4418e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=90353
-2024-11-04 06:37:13 - progress_bar.py[line:274] - INFO: epoch 036:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.879, ntokens=17557.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.36, wps=3084.5, ups=0.18, wpb=17557.6, bsz=1024, num_updates=13810, lr=3.44044e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=90410
-2024-11-04 06:38:10 - progress_bar.py[line:274] - INFO: epoch 036:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=17695.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.49, wps=3072.3, ups=0.17, wpb=17695.3, bsz=1024, num_updates=13820, lr=3.43908e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=90467
-2024-11-04 06:39:08 - progress_bar.py[line:274] - INFO: epoch 036:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.882, ntokens=17413.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.37, wps=3011.6, ups=0.17, wpb=17413.1, bsz=1024, num_updates=13830, lr=3.43772e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=90525
-2024-11-04 06:40:05 - progress_bar.py[line:274] - INFO: epoch 036:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.9, ntokens=17911, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.46, wps=3145.6, ups=0.18, wpb=17911, bsz=1024, num_updates=13840, lr=3.43636e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=90582
-2024-11-04 06:41:02 - progress_bar.py[line:274] - INFO: epoch 036:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.885, ntokens=17453.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3067.5, ups=0.18, wpb=17453.7, bsz=1024, num_updates=13850, lr=3.435e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=44, gb_free=13.6, wall=90639
-2024-11-04 06:41:59 - progress_bar.py[line:274] - INFO: epoch 036:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.937, ntokens=17964.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.66, wps=3151.5, ups=0.18, wpb=17964.6, bsz=1024, num_updates=13860, lr=3.43364e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=90696
-2024-11-04 06:42:57 - progress_bar.py[line:274] - INFO: epoch 036:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.946, ntokens=17889.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.7, wps=3056.9, ups=0.17, wpb=17889.4, bsz=1024, num_updates=13870, lr=3.43228e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=90754
-2024-11-04 06:43:56 - progress_bar.py[line:274] - INFO: epoch 036:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.9, ntokens=17948.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.46, wps=3081.8, ups=0.17, wpb=17948.4, bsz=1024, num_updates=13880, lr=3.43092e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=90813
-2024-11-04 06:44:52 - progress_bar.py[line:274] - INFO: epoch 036:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.877, ntokens=17743, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.35, wps=3124, ups=0.18, wpb=17743, bsz=1024, num_updates=13890, lr=3.42956e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=90869
-2024-11-04 06:45:50 - progress_bar.py[line:274] - INFO: epoch 036:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.92, ntokens=17918.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.57, wps=3110.8, ups=0.17, wpb=17918.6, bsz=1024, num_updates=13900, lr=3.4282e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=58, gb_free=13.6, wall=90927
-2024-11-04 06:46:45 - progress_bar.py[line:274] - INFO: epoch 036:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.885, ntokens=17710.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3220.3, ups=0.18, wpb=17710.2, bsz=1024, num_updates=13910, lr=3.42684e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=90982
-2024-11-04 06:47:40 - progress_bar.py[line:274] - INFO: epoch 036:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.961, ntokens=17709.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.79, wps=3224.1, ups=0.18, wpb=17709.9, bsz=1024, num_updates=13920, lr=3.42548e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=36, gb_free=13.4, wall=91037
-2024-11-04 06:48:37 - progress_bar.py[line:274] - INFO: epoch 036:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.921, ntokens=17898.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.57, wps=3113, ups=0.17, wpb=17898.5, bsz=1024, num_updates=13930, lr=3.42412e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=91094
-2024-11-04 06:49:34 - progress_bar.py[line:274] - INFO: epoch 036:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.998, ntokens=18246.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.99, wps=3197.4, ups=0.18, wpb=18246.7, bsz=1024, num_updates=13940, lr=3.42276e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=91152
-2024-11-04 06:50:30 - progress_bar.py[line:274] - INFO: epoch 036:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.868, ntokens=17596.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.3, wps=3169.3, ups=0.18, wpb=17596.9, bsz=1024, num_updates=13950, lr=3.4214e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=91207
-2024-11-04 06:51:26 - progress_bar.py[line:274] - INFO: epoch 036:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.867, ntokens=17513.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.3, wps=3134.9, ups=0.18, wpb=17513.2, bsz=1024, num_updates=13960, lr=3.42004e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=91263
-2024-11-04 06:52:20 - progress_bar.py[line:274] - INFO: epoch 036:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.024, ntokens=18031.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.13, wps=3319.1, ups=0.18, wpb=18031.7, bsz=1024, num_updates=13970, lr=3.41868e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=91317
-2024-11-04 06:53:15 - progress_bar.py[line:274] - INFO: epoch 036:    308 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.018, ntokens=18160, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.1, wps=3297.3, ups=0.18, wpb=18160, bsz=1024, num_updates=13980, lr=3.41732e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=33, gb_free=13.1, wall=91372
-2024-11-04 06:54:11 - progress_bar.py[line:274] - INFO: epoch 036:    318 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.063, ntokens=18385.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.36, wps=3279.5, ups=0.18, wpb=18385.1, bsz=1024, num_updates=13990, lr=3.41595e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.3, wall=91428
-2024-11-04 06:55:08 - progress_bar.py[line:274] - INFO: epoch 036:    328 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.995, ntokens=18076.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.97, wps=3203.1, ups=0.18, wpb=18076.9, bsz=1024, num_updates=14000, lr=3.41459e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=91485
-2024-11-04 06:56:04 - progress_bar.py[line:274] - INFO: epoch 036:    338 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.022, ntokens=18034.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.12, wps=3183.2, ups=0.18, wpb=18034.2, bsz=1024, num_updates=14010, lr=3.41323e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=91542
-2024-11-04 06:57:02 - progress_bar.py[line:274] - INFO: epoch 036:    348 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.021, ntokens=18474.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.12, wps=3224.9, ups=0.17, wpb=18474.9, bsz=1024, num_updates=14020, lr=3.41187e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.3, wall=91599
-2024-11-04 06:57:59 - progress_bar.py[line:274] - INFO: epoch 036:    358 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.066, ntokens=18227.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.38, wps=3182.7, ups=0.17, wpb=18227.2, bsz=1024, num_updates=14030, lr=3.41051e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.1, wall=91656
-2024-11-04 06:58:56 - progress_bar.py[line:274] - INFO: epoch 036:    368 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.059, ntokens=18422.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.34, wps=3245.9, ups=0.18, wpb=18422.2, bsz=1024, num_updates=14040, lr=3.40915e-05, gnorm=0.007, clip=0, loss_scale=131072, train_wall=57, gb_free=13.3, wall=91713
-2024-11-04 06:59:52 - progress_bar.py[line:274] - INFO: epoch 036:    378 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.059, ntokens=18407.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.33, wps=3258.9, ups=0.18, wpb=18407.3, bsz=1024, num_updates=14050, lr=3.40779e-05, gnorm=0.009, clip=0, loss_scale=131072, train_wall=56, gb_free=13.4, wall=91769
-2024-11-04 07:00:50 - progress_bar.py[line:274] - INFO: epoch 036:    388 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.045, ntokens=18179.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.25, wps=3165.8, ups=0.17, wpb=18179.6, bsz=1024, num_updates=14060, lr=3.40643e-05, gnorm=0.008, clip=0, loss_scale=131072, train_wall=57, gb_free=13.5, wall=91827
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_36.pt
-2024-11-04 07:01:03 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 36 @ 14063 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_36.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_36.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_36.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_36.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_36.pt
-2024-11-04 07:01:03 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_36.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 07:02:59 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 07:04:56 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 36 @ 14063 updates, score 0) (writing took 232.85288309198222 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_36.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-slice_id 7 seek offset 350000
-slice_id 5 seek offset 250000
-slice_id 3 seek offset 150000
-slice_id 4 seek offset 200000
-2024-11-04 07:05:52 - train.py[line:336] - INFO: end of epoch 36 (average epoch stats below)
-2024-11-04 07:05:52 - progress_bar.py[line:282] - INFO: epoch 036 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.944 | ntokens 17840.8 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.69 | wps 2603.6 | ups 0.15 | wpb 17840.8 | bsz 1023 | num_updates 14063 | lr 3.40602e-05 | gnorm 0.007 | clip 0 | loss_scale 131072 | train_wall 1590 | gb_free 13.3 | wall 92129
-2024-11-04 07:05:52 - trainer.py[line:639] - INFO: loading train data for epoch 37
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 07:08:36 - trainer.py[line:703] - INFO: begin training epoch 37
-2024-11-04 07:08:36 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 07:09:01 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
-2024-11-04 07:09:23 - progress_bar.py[line:274] - INFO: epoch 037:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.903, ntokens=17002.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.48, wps=331.1, ups=0.02, wpb=17002.3, bsz=985.6, num_updates=14070, lr=3.40507e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=38, gb_free=13.5, wall=92340
-2024-11-04 07:10:20 - progress_bar.py[line:274] - INFO: epoch 037:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.927, ntokens=17934.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=3171.9, ups=0.18, wpb=17934.4, bsz=1024, num_updates=14080, lr=3.40371e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=92397
-2024-11-04 07:11:17 - progress_bar.py[line:274] - INFO: epoch 037:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.902, ntokens=17637.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.47, wps=3050.8, ups=0.17, wpb=17637.8, bsz=1024, num_updates=14090, lr=3.40235e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=44, gb_free=13.4, wall=92455
-2024-11-04 07:12:14 - progress_bar.py[line:274] - INFO: epoch 037:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.916, ntokens=17808.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.55, wps=3143.9, ups=0.18, wpb=17808.3, bsz=1024, num_updates=14100, lr=3.40099e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=92511
-2024-11-04 07:13:12 - progress_bar.py[line:274] - INFO: epoch 037:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.897, ntokens=17754.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.45, wps=3060.3, ups=0.17, wpb=17754.3, bsz=1024, num_updates=14110, lr=3.39963e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=92569
-2024-11-04 07:14:11 - progress_bar.py[line:274] - INFO: epoch 037:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.909, ntokens=17912, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.51, wps=3065.6, ups=0.17, wpb=17912, bsz=1024, num_updates=14120, lr=3.39827e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=92628
-2024-11-04 07:15:09 - progress_bar.py[line:274] - INFO: epoch 037:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=17597.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3019.4, ups=0.17, wpb=17597.6, bsz=1024, num_updates=14130, lr=3.39691e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=92686
-2024-11-04 07:16:05 - progress_bar.py[line:274] - INFO: epoch 037:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.91, ntokens=17784.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.52, wps=3138.1, ups=0.18, wpb=17784.4, bsz=1024, num_updates=14140, lr=3.39555e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=92743
-2024-11-04 07:17:01 - progress_bar.py[line:274] - INFO: epoch 037:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.887, ntokens=17630.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.4, wps=3172.1, ups=0.18, wpb=17630.3, bsz=1024, num_updates=14150, lr=3.39419e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=92798
-2024-11-04 07:17:58 - progress_bar.py[line:274] - INFO: epoch 037:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.872, ntokens=17638.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.32, wps=3080.8, ups=0.17, wpb=17638.3, bsz=1024, num_updates=14160, lr=3.39283e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=92855
-2024-11-04 07:18:56 - progress_bar.py[line:274] - INFO: epoch 037:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.922, ntokens=17783.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.58, wps=3090.9, ups=0.17, wpb=17783.5, bsz=1024, num_updates=14170, lr=3.39147e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=92913
-2024-11-04 07:19:54 - progress_bar.py[line:274] - INFO: epoch 037:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.93, ntokens=17746.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.62, wps=3036.3, ups=0.17, wpb=17746.7, bsz=1024, num_updates=14180, lr=3.39011e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=92971
-2024-11-04 07:20:52 - progress_bar.py[line:274] - INFO: epoch 037:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.837, ntokens=17347.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.14, wps=3021.4, ups=0.17, wpb=17347.6, bsz=1024, num_updates=14190, lr=3.38875e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=93029
-2024-11-04 07:21:49 - progress_bar.py[line:274] - INFO: epoch 037:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.859, ntokens=17377.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.25, wps=3036.5, ups=0.17, wpb=17377.4, bsz=1024, num_updates=14200, lr=3.38739e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=93086
-2024-11-04 07:22:47 - progress_bar.py[line:274] - INFO: epoch 037:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.896, ntokens=17692.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.44, wps=3061.9, ups=0.17, wpb=17692.7, bsz=1024, num_updates=14210, lr=3.38603e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=93144
-2024-11-04 07:23:45 - progress_bar.py[line:274] - INFO: epoch 037:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.882, ntokens=17476.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.37, wps=3024.1, ups=0.17, wpb=17476.5, bsz=1024, num_updates=14220, lr=3.38467e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=93202
-2024-11-04 07:24:43 - progress_bar.py[line:274] - INFO: epoch 037:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.919, ntokens=18222.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.57, wps=3136.7, ups=0.17, wpb=18222.4, bsz=1024, num_updates=14230, lr=3.38331e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=93260
-2024-11-04 07:25:39 - progress_bar.py[line:274] - INFO: epoch 037:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.887, ntokens=17506.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3079.7, ups=0.18, wpb=17506.8, bsz=1024, num_updates=14240, lr=3.38194e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=93317
-2024-11-04 07:26:37 - progress_bar.py[line:274] - INFO: epoch 037:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.886, ntokens=17774.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3082.8, ups=0.17, wpb=17774.1, bsz=1024, num_updates=14250, lr=3.38058e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=93374
-2024-11-04 07:27:35 - progress_bar.py[line:274] - INFO: epoch 037:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.917, ntokens=17795.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.55, wps=3071.4, ups=0.17, wpb=17795.2, bsz=1024, num_updates=14260, lr=3.37922e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=93432
-2024-11-04 07:28:33 - progress_bar.py[line:274] - INFO: epoch 037:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.926, ntokens=18077.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=3124.3, ups=0.17, wpb=18077.9, bsz=1024, num_updates=14270, lr=3.37786e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=58, gb_free=13.6, wall=93490
-2024-11-04 07:29:31 - progress_bar.py[line:274] - INFO: epoch 037:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.865, ntokens=17844.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.29, wps=3095.2, ups=0.17, wpb=17844.1, bsz=1024, num_updates=14280, lr=3.3765e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.6, wall=93548
-2024-11-04 07:30:28 - progress_bar.py[line:274] - INFO: epoch 037:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.908, ntokens=17908.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.51, wps=3097.2, ups=0.17, wpb=17908.7, bsz=1024, num_updates=14290, lr=3.37514e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=93605
-2024-11-04 07:31:26 - progress_bar.py[line:274] - INFO: epoch 037:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.848, ntokens=17463.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.2, wps=3020.4, ups=0.17, wpb=17463.5, bsz=1024, num_updates=14300, lr=3.37378e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=93663
-2024-11-04 07:32:24 - progress_bar.py[line:274] - INFO: epoch 037:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.925, ntokens=17467.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=3028.7, ups=0.17, wpb=17467.4, bsz=1024, num_updates=14310, lr=3.37242e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=93721
-2024-11-04 07:33:21 - progress_bar.py[line:274] - INFO: epoch 037:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.874, ntokens=17751, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.33, wps=3121.7, ups=0.18, wpb=17751, bsz=1024, num_updates=14320, lr=3.37106e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=93778
-2024-11-04 07:34:18 - progress_bar.py[line:274] - INFO: epoch 037:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.96, ntokens=17994.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.78, wps=3127.9, ups=0.17, wpb=17994.4, bsz=1024, num_updates=14330, lr=3.3697e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=93835
-2024-11-04 07:35:16 - progress_bar.py[line:274] - INFO: epoch 037:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.868, ntokens=17765.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.3, wps=3086.6, ups=0.17, wpb=17765.3, bsz=1024, num_updates=14340, lr=3.36834e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=93893
-2024-11-04 07:36:14 - progress_bar.py[line:274] - INFO: epoch 037:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.927, ntokens=17865.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=3079, ups=0.17, wpb=17865.7, bsz=1024, num_updates=14350, lr=3.36698e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=93951
-2024-11-04 07:37:12 - progress_bar.py[line:274] - INFO: epoch 037:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.962, ntokens=17814.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.79, wps=3064.1, ups=0.17, wpb=17814.8, bsz=1024, num_updates=14360, lr=3.36562e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.3, wall=94009
-2024-11-04 07:38:10 - progress_bar.py[line:274] - INFO: epoch 037:    308 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.052, ntokens=18343.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.3, wps=3182.3, ups=0.17, wpb=18343.6, bsz=1024, num_updates=14370, lr=3.36426e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=58, gb_free=13.2, wall=94067
-2024-11-04 07:38:45 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-04 07:39:14 - progress_bar.py[line:274] - INFO: epoch 037:    319 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.004, ntokens=18224.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.02, wps=2845.5, ups=0.16, wpb=18224.7, bsz=1024, num_updates=14380, lr=3.3629e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=64, gb_free=13.5, wall=94131
-2024-11-04 07:40:11 - progress_bar.py[line:274] - INFO: epoch 037:    329 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.002, ntokens=18003.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.01, wps=3160.3, ups=0.18, wpb=18003.3, bsz=1024, num_updates=14390, lr=3.36154e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=94188
-2024-11-04 07:41:08 - progress_bar.py[line:274] - INFO: epoch 037:    339 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.964, ntokens=17925.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.8, wps=3143.3, ups=0.18, wpb=17925.3, bsz=1024, num_updates=14400, lr=3.36018e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=94245
-2024-11-04 07:42:06 - progress_bar.py[line:274] - INFO: epoch 037:    349 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.086, ntokens=18781.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.49, wps=3246.1, ups=0.17, wpb=18781.4, bsz=1024, num_updates=14410, lr=3.35882e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.3, wall=94303
-2024-11-04 07:43:04 - progress_bar.py[line:274] - INFO: epoch 037:    359 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.084, ntokens=18708.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.48, wps=3219.5, ups=0.17, wpb=18708.8, bsz=1024, num_updates=14420, lr=3.35746e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=58, gb_free=13.3, wall=94361
-2024-11-04 07:44:01 - progress_bar.py[line:274] - INFO: epoch 037:    369 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.036, ntokens=18160.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.2, wps=3174.9, ups=0.17, wpb=18160.4, bsz=1024, num_updates=14430, lr=3.3561e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=94418
-2024-11-04 07:44:58 - progress_bar.py[line:274] - INFO: epoch 037:    379 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.98, ntokens=18089.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3148.4, ups=0.17, wpb=18089.6, bsz=1024, num_updates=14440, lr=3.35474e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=94475
-2024-11-04 07:45:56 - progress_bar.py[line:274] - INFO: epoch 037:    389 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.032, ntokens=17984.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.18, wps=3137, ups=0.17, wpb=17984.4, bsz=1024, num_updates=14450, lr=3.35338e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=94533
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_37.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_37.pt2024-11-04 07:46:03 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 37 @ 14452 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_37.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_37.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_37.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_37.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_37.pt
-2024-11-04 07:46:03 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 07:47:59 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 07:49:55 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 37 @ 14452 updates, score 0) (writing took 232.07835571299074 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_37.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 4 seek offset 200000
-slice_id 7 seek offset 350000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-04 07:50:52 - train.py[line:336] - INFO: end of epoch 37 (average epoch stats below)
-2024-11-04 07:50:52 - progress_bar.py[line:282] - INFO: epoch 037 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.93 | ntokens 17831.6 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.62 | wps 2569.1 | ups 0.14 | wpb 17831.6 | bsz 1023 | num_updates 14452 | lr 3.3531e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 2181 | gb_free 13.3 | wall 94829
-2024-11-04 07:50:52 - trainer.py[line:639] - INFO: loading train data for epoch 38
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 07:53:37 - trainer.py[line:703] - INFO: begin training epoch 38
-2024-11-04 07:53:37 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 07:54:24 - progress_bar.py[line:274] - INFO: epoch 038:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.907, ntokens=17074.9, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=335.8, ups=0.02, wpb=17074.9, bsz=985.6, num_updates=14460, lr=3.35202e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=95041
-2024-11-04 07:55:20 - progress_bar.py[line:274] - INFO: epoch 038:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.855, ntokens=17422.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.24, wps=3134.7, ups=0.18, wpb=17422.7, bsz=1024, num_updates=14470, lr=3.35066e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=95097
-2024-11-04 07:56:17 - progress_bar.py[line:274] - INFO: epoch 038:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.889, ntokens=17630.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.41, wps=3072.7, ups=0.17, wpb=17630.4, bsz=1024, num_updates=14480, lr=3.3493e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=41, gb_free=13.4, wall=95154
-2024-11-04 07:57:15 - progress_bar.py[line:274] - INFO: epoch 038:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.927, ntokens=18019.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=3103.7, ups=0.17, wpb=18019.4, bsz=1024, num_updates=14490, lr=3.34793e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=95212
-2024-11-04 07:58:10 - progress_bar.py[line:274] - INFO: epoch 038:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.89, ntokens=17647.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.41, wps=3219, ups=0.18, wpb=17647.4, bsz=1024, num_updates=14500, lr=3.34657e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=95267
-2024-11-04 07:59:04 - progress_bar.py[line:274] - INFO: epoch 038:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=17951.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3295.7, ups=0.18, wpb=17951.1, bsz=1024, num_updates=14510, lr=3.34521e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=95322
-2024-11-04 08:00:01 - progress_bar.py[line:274] - INFO: epoch 038:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.914, ntokens=17813, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=3127.3, ups=0.18, wpb=17813, bsz=1024, num_updates=14520, lr=3.34385e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=95379
-2024-11-04 08:00:59 - progress_bar.py[line:274] - INFO: epoch 038:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.867, ntokens=17431.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.29, wps=3015.4, ups=0.17, wpb=17431.4, bsz=1024, num_updates=14530, lr=3.34249e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=95436
-2024-11-04 08:01:56 - progress_bar.py[line:274] - INFO: epoch 038:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.879, ntokens=17672.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.36, wps=3138.7, ups=0.18, wpb=17672.4, bsz=1024, num_updates=14540, lr=3.34113e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=95493
-2024-11-04 08:02:52 - progress_bar.py[line:274] - INFO: epoch 038:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.865, ntokens=17588.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.29, wps=3125.8, ups=0.18, wpb=17588.5, bsz=1024, num_updates=14550, lr=3.33977e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=95549
-2024-11-04 08:03:48 - progress_bar.py[line:274] - INFO: epoch 038:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.879, ntokens=17620.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.36, wps=3156.1, ups=0.18, wpb=17620.9, bsz=1024, num_updates=14560, lr=3.33841e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=95605
-2024-11-04 08:04:47 - progress_bar.py[line:274] - INFO: epoch 038:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.942, ntokens=17988.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.69, wps=3050.3, ups=0.17, wpb=17988.4, bsz=1024, num_updates=14570, lr=3.33705e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=59, gb_free=13.6, wall=95664
-2024-11-04 08:05:43 - progress_bar.py[line:274] - INFO: epoch 038:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.844, ntokens=17458.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.18, wps=3084.1, ups=0.18, wpb=17458.2, bsz=1024, num_updates=14580, lr=3.33569e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=95720
-2024-11-04 08:06:40 - progress_bar.py[line:274] - INFO: epoch 038:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.86, ntokens=17409.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.26, wps=3077.6, ups=0.18, wpb=17409.5, bsz=1024, num_updates=14590, lr=3.33433e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=95777
-2024-11-04 08:07:38 - progress_bar.py[line:274] - INFO: epoch 038:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.892, ntokens=17854.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.42, wps=3049.5, ups=0.17, wpb=17854.4, bsz=1024, num_updates=14600, lr=3.33297e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=95835
-2024-11-04 08:08:37 - progress_bar.py[line:274] - INFO: epoch 038:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.908, ntokens=17696.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3021.1, ups=0.17, wpb=17696.1, bsz=1024, num_updates=14610, lr=3.33161e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=95894
-2024-11-04 08:09:35 - progress_bar.py[line:274] - INFO: epoch 038:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.818, ntokens=17509.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.05, wps=3017.7, ups=0.17, wpb=17509.5, bsz=1024, num_updates=14620, lr=3.33025e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=95952
-2024-11-04 08:10:32 - progress_bar.py[line:274] - INFO: epoch 038:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.914, ntokens=17718, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=3132.7, ups=0.18, wpb=17718, bsz=1024, num_updates=14630, lr=3.32889e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=96009
-2024-11-04 08:11:29 - progress_bar.py[line:274] - INFO: epoch 038:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.86, ntokens=17666.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.26, wps=3064.4, ups=0.17, wpb=17666.1, bsz=1024, num_updates=14640, lr=3.32753e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=96066
-2024-11-04 08:12:27 - progress_bar.py[line:274] - INFO: epoch 038:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.896, ntokens=17675.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.44, wps=3053.4, ups=0.17, wpb=17675.1, bsz=1024, num_updates=14650, lr=3.32617e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=96124
-2024-11-04 08:13:25 - progress_bar.py[line:274] - INFO: epoch 038:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.901, ntokens=18029.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.47, wps=3099.3, ups=0.17, wpb=18029.5, bsz=1024, num_updates=14660, lr=3.32481e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.6, wall=96182
-2024-11-04 08:14:22 - progress_bar.py[line:274] - INFO: epoch 038:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.881, ntokens=17934, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.37, wps=3159.7, ups=0.18, wpb=17934, bsz=1024, num_updates=14670, lr=3.32345e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=96239
-2024-11-04 08:15:21 - progress_bar.py[line:274] - INFO: epoch 038:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.9, ntokens=17860.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.46, wps=3013.6, ups=0.17, wpb=17860.9, bsz=1024, num_updates=14680, lr=3.32209e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=59, gb_free=13.6, wall=96298
-2024-11-04 08:16:20 - progress_bar.py[line:274] - INFO: epoch 038:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.849, ntokens=17480.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.21, wps=2987, ups=0.17, wpb=17480.7, bsz=1024, num_updates=14690, lr=3.32073e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=96357
-2024-11-04 08:17:19 - progress_bar.py[line:274] - INFO: epoch 038:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.957, ntokens=17829, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.76, wps=3003.8, ups=0.17, wpb=17829, bsz=1024, num_updates=14700, lr=3.31937e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=59, gb_free=13.5, wall=96416
-2024-11-04 08:18:18 - progress_bar.py[line:274] - INFO: epoch 038:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.843, ntokens=17528.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.17, wps=2976.3, ups=0.17, wpb=17528.7, bsz=1024, num_updates=14710, lr=3.31801e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=59, gb_free=13.6, wall=96475
-2024-11-04 08:19:17 - progress_bar.py[line:274] - INFO: epoch 038:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.988, ntokens=18305.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.93, wps=3086.6, ups=0.17, wpb=18305.6, bsz=1024, num_updates=14720, lr=3.31665e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=59, gb_free=13.4, wall=96534
-2024-11-04 08:20:16 - progress_bar.py[line:274] - INFO: epoch 038:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.844, ntokens=17638.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.18, wps=3020.1, ups=0.17, wpb=17638.9, bsz=1024, num_updates=14730, lr=3.31529e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=96593
-2024-11-04 08:21:14 - progress_bar.py[line:274] - INFO: epoch 038:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.884, ntokens=17707.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.38, wps=3037.1, ups=0.17, wpb=17707.9, bsz=1024, num_updates=14740, lr=3.31393e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=58, gb_free=13.6, wall=96651
-2024-11-04 08:22:13 - progress_bar.py[line:274] - INFO: epoch 038:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.959, ntokens=17788.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.77, wps=3039.1, ups=0.17, wpb=17788.8, bsz=1024, num_updates=14750, lr=3.31256e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=58, gb_free=13.3, wall=96710
-2024-11-04 08:23:10 - progress_bar.py[line:274] - INFO: epoch 038:    308 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.001, ntokens=18114.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.01, wps=3160, ups=0.17, wpb=18114.7, bsz=1024, num_updates=14760, lr=3.3112e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.1, wall=96767
-2024-11-04 08:24:09 - progress_bar.py[line:274] - INFO: epoch 038:    318 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.079, ntokens=18703.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.45, wps=3162.4, ups=0.17, wpb=18703.1, bsz=1024, num_updates=14770, lr=3.30984e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=59, gb_free=13.4, wall=96826
-2024-11-04 08:25:08 - progress_bar.py[line:274] - INFO: epoch 038:    328 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.015, ntokens=18160.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.08, wps=3087.8, ups=0.17, wpb=18160.6, bsz=1024, num_updates=14780, lr=3.30848e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=59, gb_free=13.4, wall=96885
-2024-11-04 08:26:06 - progress_bar.py[line:274] - INFO: epoch 038:    338 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.002, ntokens=18230.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.01, wps=3126.9, ups=0.17, wpb=18230.7, bsz=1024, num_updates=14790, lr=3.30712e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=96943
-2024-11-04 08:27:05 - progress_bar.py[line:274] - INFO: epoch 038:    348 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.983, ntokens=18258.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.91, wps=3123, ups=0.17, wpb=18258.2, bsz=1024, num_updates=14800, lr=3.30576e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.2, wall=97002
-2024-11-04 08:28:03 - progress_bar.py[line:274] - INFO: epoch 038:    358 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.014, ntokens=18153.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.08, wps=3113.8, ups=0.17, wpb=18153.5, bsz=1024, num_updates=14810, lr=3.3044e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=58, gb_free=13.3, wall=97060
-2024-11-04 08:29:01 - progress_bar.py[line:274] - INFO: epoch 038:    368 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.972, ntokens=18007.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.85, wps=3091.9, ups=0.17, wpb=18007.9, bsz=1024, num_updates=14820, lr=3.30304e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=97118
-2024-11-04 08:29:59 - progress_bar.py[line:274] - INFO: epoch 038:    378 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.044, ntokens=18469.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.25, wps=3168, ups=0.17, wpb=18469.6, bsz=1024, num_updates=14830, lr=3.30168e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=97177
-2024-11-04 08:30:58 - progress_bar.py[line:274] - INFO: epoch 038:    388 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.009, ntokens=18012.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.05, wps=3100.1, ups=0.17, wpb=18012.9, bsz=1024, num_updates=14840, lr=3.30032e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=97235
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt
-2024-11-04 08:31:11 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 38 @ 14843 updates
-2024-11-04 08:31:11 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt': File exists: File exists
-: File exists: File exists
-: File exists: File exists
-
-
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 08:32:29 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 5 seek offset 250000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-2024-11-04 08:34:54 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 38 @ 14843 updates, score 0) (writing took 222.6358955600008 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_38.pt
-2024-11-04 08:35:51 - train.py[line:336] - INFO: end of epoch 38 (average epoch stats below)
-2024-11-04 08:35:51 - progress_bar.py[line:282] - INFO: epoch 038 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.92 | ntokens 17822.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.57 | wps 2581.7 | ups 0.14 | wpb 17822.9 | bsz 1023 | num_updates 14843 | lr 3.29991e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 2185 | gb_free 13.4 | wall 97528
-2024-11-04 08:35:51 - trainer.py[line:639] - INFO: loading train data for epoch 39
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 08:38:35 - trainer.py[line:703] - INFO: begin training epoch 39
-2024-11-04 08:38:35 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 08:39:17 - progress_bar.py[line:274] - INFO: epoch 039:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.881, ntokens=16841.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.37, wps=337, ups=0.02, wpb=16841.4, bsz=985.6, num_updates=14850, lr=3.29896e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=35, gb_free=13.6, wall=97734
-2024-11-04 08:40:14 - progress_bar.py[line:274] - INFO: epoch 039:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.904, ntokens=17862.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.49, wps=3139.4, ups=0.18, wpb=17862.7, bsz=1024, num_updates=14860, lr=3.2976e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=97791
-2024-11-04 08:41:12 - progress_bar.py[line:274] - INFO: epoch 039:     27 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.869, ntokens=17693.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.31, wps=3056.2, ups=0.17, wpb=17693.6, bsz=1024, num_updates=14870, lr=3.29624e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=97849
-2024-11-04 08:42:10 - progress_bar.py[line:274] - INFO: epoch 039:     37 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.863, ntokens=17718.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.28, wps=3057.6, ups=0.17, wpb=17718.1, bsz=1024, num_updates=14880, lr=3.29488e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=97907
-2024-11-04 08:43:10 - progress_bar.py[line:274] - INFO: epoch 039:     47 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.874, ntokens=17789.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.33, wps=2977.1, ups=0.17, wpb=17789.6, bsz=1024, num_updates=14890, lr=3.29352e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=97967
-2024-11-04 08:44:09 - progress_bar.py[line:274] - INFO: epoch 039:     57 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.887, ntokens=17808.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.4, wps=3009.9, ups=0.17, wpb=17808.8, bsz=1024, num_updates=14900, lr=3.29216e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=98026
-2024-11-04 08:45:08 - progress_bar.py[line:274] - INFO: epoch 039:     67 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.888, ntokens=17758, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.4, wps=2994.6, ups=0.17, wpb=17758, bsz=1024, num_updates=14910, lr=3.2908e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=98085
-2024-11-04 08:46:07 - progress_bar.py[line:274] - INFO: epoch 039:     77 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.893, ntokens=17545.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=2998.3, ups=0.17, wpb=17545.2, bsz=1024, num_updates=14920, lr=3.28944e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=98144
-2024-11-04 08:47:06 - progress_bar.py[line:274] - INFO: epoch 039:     87 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.902, ntokens=18097.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.47, wps=3048.2, ups=0.17, wpb=18097.7, bsz=1024, num_updates=14930, lr=3.28808e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=59, gb_free=13.6, wall=98203
-2024-11-04 08:48:06 - progress_bar.py[line:274] - INFO: epoch 039:     97 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.883, ntokens=17655.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.38, wps=2970.4, ups=0.17, wpb=17655.7, bsz=1024, num_updates=14940, lr=3.28672e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=98263
-2024-11-04 08:49:05 - progress_bar.py[line:274] - INFO: epoch 039:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.851, ntokens=17563.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=2963.7, ups=0.17, wpb=17563.1, bsz=1024, num_updates=14950, lr=3.28536e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=98322
-2024-11-04 08:50:02 - progress_bar.py[line:274] - INFO: epoch 039:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.892, ntokens=17578.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.42, wps=3058.7, ups=0.17, wpb=17578.7, bsz=1024, num_updates=14960, lr=3.284e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=98379
-2024-11-04 08:51:00 - progress_bar.py[line:274] - INFO: epoch 039:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.841, ntokens=17581.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.16, wps=3071.6, ups=0.17, wpb=17581.2, bsz=1024, num_updates=14970, lr=3.28264e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=98437
-2024-11-04 08:52:00 - progress_bar.py[line:274] - INFO: epoch 039:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.839, ntokens=17325, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.16, wps=2869.3, ups=0.17, wpb=17325, bsz=1024, num_updates=14980, lr=3.28128e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=60, gb_free=13.4, wall=98497
-2024-11-04 08:52:59 - progress_bar.py[line:274] - INFO: epoch 039:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.834, ntokens=17533.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.13, wps=2956.7, ups=0.17, wpb=17533.3, bsz=1024, num_updates=14990, lr=3.27992e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=98556
-2024-11-04 08:53:59 - progress_bar.py[line:274] - INFO: epoch 039:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.878, ntokens=17621.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.35, wps=2961.7, ups=0.17, wpb=17621.6, bsz=1024, num_updates=15000, lr=3.27855e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=59, gb_free=13.3, wall=98616
-loss_reg: 0.022489707375504427 loss_cls: 0.0037145938258618116
-loss_reg: 0.020479881678892368 loss_cls: 0.0035311677493155003
-loss_reg: 0.019566915853196823 loss_cls: 0.0032145848963409662
-loss_reg: 0.024252895851289844 loss_cls: 0.0028018562588840723
-loss_reg: 0.012218595280604923 loss_cls: 0.0030794732738286257
-loss_reg: 0.01878549122022521 loss_cls: 0.004403817467391491
-loss_reg: 0.015412860420974363 loss_cls: 0.004115327261388302
-loss_reg: 0.012826356650304242 loss_cls: 0.003744600573554635
-loss_reg: 0.0179330507178062 loss_cls: 0.003950426355004311
-loss_reg: 0.02522817684462563 loss_cls: 0.004372778348624706
-loss_reg: 0.012821645471139044 loss_cls: 0.0036960486322641373
-loss_reg: 0.0036446600209144283 loss_cls: 0.002990167587995529
-loss_reg: 0.012468713587122976 loss_cls: 0.003164804307743907
-loss_reg: 0.013050057274432552 loss_cls: 0.0043092528358101845
-loss_reg: 0.018061494477229975 loss_cls: 0.005887353792786598
-loss_reg: 0.01724495037814284 loss_cls: 0.004608602728694677
-loss_reg: 0.01573110250900114 loss_cls: 0.004620472900569439
-loss_reg: 0.021653501004773777 loss_cls: 0.005481445696204901
-loss_reg: 0.008959242038211492 loss_cls: 0.0022930142004042864
-loss_reg: 0.019322942547227037 loss_cls: 0.004358791280537844
-loss_reg: 0.01655451504623551 loss_cls: 0.003272206522524357
-loss_reg: 0.02001351704159523 loss_cls: 0.0042025251314044
-loss_reg: 0.02812723258918618 loss_cls: 0.004204269964247942
-loss_reg: 0.019852204257210643 loss_cls: 0.0031688171438872814
-loss_reg: 0.026284393144472944 loss_cls: 0.004068693146109581
-loss_reg: 0.004417368992598918 loss_cls: 0.002648159395903349
-loss_reg: 0.01708550017994872 loss_cls: 0.003161108586937189
-loss_reg: 0.01683942653124127 loss_cls: 0.0040612551383674145
-loss_reg: 0.01896271913814085 loss_cls: 0.0032649580389261246
-loss_reg: 0.025323178053957028 loss_cls: 0.005571174900978804
-loss_reg: 0.02394503594034332 loss_cls: 0.0033227521926164627
-loss_reg: 0.012690375550339805 loss_cls: 0.0041217077523469925
-loss_reg: 0.006078530136475983 loss_cls: 0.0036127029452472925
-loss_reg: 0.009224171905204855 loss_cls: 0.004407211672514677
-loss_reg: 0.01638828917236973 loss_cls: 0.004747590981423855
-loss_reg: 0.014920500101360435 loss_cls: 0.003098452929407358
-loss_reg: 0.02285641959761048 loss_cls: 0.0036814233753830194
-loss_reg: 0.020145187671824126 loss_cls: 0.003332133637741208
-loss_reg: 0.020325480740323504 loss_cls: 0.0028119853232055902
-loss_reg: 0.003477647806757177 loss_cls: 0.002593879820778966
-loss_reg: 0.020699173204379983 loss_cls: 0.0032971403561532497
-loss_reg: 0.02371374213833738 loss_cls: 0.003730672411620617
-loss_reg: 0.0172969827482039 loss_cls: 0.0036503856536000967
-loss_reg: 0.016845064627541238 loss_cls: 0.0026125365402549505
-loss_reg: 0.015021847238858787 loss_cls: 0.0035752488765865564
-loss_reg: 0.019370728271252614 loss_cls: 0.0045785666443407536
-loss_reg: 0.0010698365169560797 loss_cls: 0.0023451263550668955
-loss_reg: 0.017303453250626637 loss_cls: 0.0036169816739857197
-loss_reg: 0.016653075511752144 loss_cls: 0.00327519653365016
-loss_reg: 0.022812050874520578 loss_cls: 0.004489551763981581
-loss_reg: 0.02035942960522474 loss_cls: 0.004157643299549818
-loss_reg: 0.020381741908481125 loss_cls: 0.006989181973040104
-loss_reg: 0.019632900479246755 loss_cls: 0.00477798655629158
-loss_reg: 0.004918221626073684 loss_cls: 0.0032078996300697327
-loss_reg: 0.019856647682055382 loss_cls: 0.005071687512099743
-loss_reg: 0.01297326689879031 loss_cls: 0.004098125733435154
-loss_reg: 0.013503760869007276 loss_cls: 0.003383094444870949
-loss_reg: 0.013243404101372126 loss_cls: 0.0036976949777454138
-loss_reg: 0.012900582320984098 loss_cls: 0.004157336428761482
-loss_reg: 0.013856036150127357 loss_cls: 0.0036738752387464046
-loss_reg: 0.023474244714066353 loss_cls: 0.004362253472208977
-loss_reg: 0.019598157950787225 loss_cls: 0.0035400791093707085
-loss_reg: 0.01784134748451058 loss_cls: 0.004287663381546736
-loss_reg: 0.018615654445847226 loss_cls: 0.003918620757758617
-2024-11-04 08:54:58 - progress_bar.py[line:274] - INFO: epoch 039:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.851, ntokens=17796.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=3005.7, ups=0.17, wpb=17796.2, bsz=1024, num_updates=15010, lr=3.27719e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=59, gb_free=13.4, wall=98675
-2024-11-04 08:55:57 - progress_bar.py[line:274] - INFO: epoch 039:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=17267.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=2932.6, ups=0.17, wpb=17267.7, bsz=1024, num_updates=15020, lr=3.27583e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=59, gb_free=13.4, wall=98734
-2024-11-04 08:56:56 - progress_bar.py[line:274] - INFO: epoch 039:    187 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.849, ntokens=17661.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.2, wps=2995.4, ups=0.17, wpb=17661.7, bsz=1024, num_updates=15030, lr=3.27447e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=98793
-2024-11-04 08:57:55 - progress_bar.py[line:274] - INFO: epoch 039:    197 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.886, ntokens=17764, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=2993, ups=0.17, wpb=17764, bsz=1024, num_updates=15040, lr=3.27311e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=59, gb_free=13.4, wall=98852
-2024-11-04 08:58:54 - progress_bar.py[line:274] - INFO: epoch 039:    207 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=18128.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.49, wps=3064.7, ups=0.17, wpb=18128.1, bsz=1024, num_updates=15050, lr=3.27175e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=98911
-2024-11-04 08:59:52 - progress_bar.py[line:274] - INFO: epoch 039:    217 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.859, ntokens=17759.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.25, wps=3051.2, ups=0.17, wpb=17759.2, bsz=1024, num_updates=15060, lr=3.27039e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=98970
-2024-11-04 09:00:51 - progress_bar.py[line:274] - INFO: epoch 039:    227 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.885, ntokens=18029.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3058.2, ups=0.17, wpb=18029.3, bsz=1024, num_updates=15070, lr=3.26903e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=59, gb_free=13.4, wall=99029
-2024-11-04 09:01:46 - progress_bar.py[line:274] - INFO: epoch 039:    237 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.861, ntokens=17546.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.27, wps=3216, ups=0.18, wpb=17546.6, bsz=1024, num_updates=15080, lr=3.26767e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=51, gb_free=13.6, wall=99083
-2024-11-04 09:02:44 - progress_bar.py[line:274] - INFO: epoch 039:    247 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.936, ntokens=17787.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.65, wps=3074.2, ups=0.17, wpb=17787.1, bsz=1024, num_updates=15090, lr=3.26631e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=36, gb_free=13.6, wall=99141
-2024-11-04 09:03:40 - progress_bar.py[line:274] - INFO: epoch 039:    257 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.905, ntokens=17987.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.49, wps=3192, ups=0.18, wpb=17987.7, bsz=1024, num_updates=15100, lr=3.26495e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=34, gb_free=13.5, wall=99197
-2024-11-04 09:04:37 - progress_bar.py[line:274] - INFO: epoch 039:    267 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.934, ntokens=18063.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.64, wps=3153.6, ups=0.17, wpb=18063.9, bsz=1024, num_updates=15110, lr=3.26359e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=99255
-2024-11-04 09:05:35 - progress_bar.py[line:274] - INFO: epoch 039:    277 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.861, ntokens=17670.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.27, wps=3070.9, ups=0.17, wpb=17670.2, bsz=1024, num_updates=15120, lr=3.26223e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=99312
-2024-11-04 09:06:34 - progress_bar.py[line:274] - INFO: epoch 039:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.866, ntokens=17714.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.29, wps=2986.5, ups=0.17, wpb=17714.8, bsz=1024, num_updates=15130, lr=3.26087e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=39, gb_free=13.6, wall=99371
-2024-11-04 09:07:34 - progress_bar.py[line:274] - INFO: epoch 039:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.948, ntokens=17807.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.72, wps=2991.3, ups=0.17, wpb=17807.9, bsz=1024, num_updates=15140, lr=3.25951e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=99431
-2024-11-04 09:08:33 - progress_bar.py[line:274] - INFO: epoch 039:    307 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.076, ntokens=18550.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.43, wps=3125.2, ups=0.17, wpb=18550.6, bsz=1024, num_updates=15150, lr=3.25815e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=59, gb_free=13.2, wall=99490
-2024-11-04 09:09:33 - progress_bar.py[line:274] - INFO: epoch 039:    317 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.064, ntokens=18762.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.36, wps=3158.4, ups=0.17, wpb=18762.2, bsz=1024, num_updates=15160, lr=3.25679e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=59, gb_free=13.4, wall=99550
-2024-11-04 09:10:32 - progress_bar.py[line:274] - INFO: epoch 039:    327 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.011, ntokens=18182.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.06, wps=3069.4, ups=0.17, wpb=18182.5, bsz=1024, num_updates=15170, lr=3.25543e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=59, gb_free=13.4, wall=99609
-2024-11-04 09:11:31 - progress_bar.py[line:274] - INFO: epoch 039:    337 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.974, ntokens=18009, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.86, wps=3027.7, ups=0.17, wpb=18009, bsz=1024, num_updates=15180, lr=3.25407e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=59, gb_free=13.4, wall=99668
-2024-11-04 09:12:31 - progress_bar.py[line:274] - INFO: epoch 039:    347 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.953, ntokens=18178.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.75, wps=3055.6, ups=0.17, wpb=18178.6, bsz=1024, num_updates=15190, lr=3.25271e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=59, gb_free=13.4, wall=99728
-2024-11-04 09:13:30 - progress_bar.py[line:274] - INFO: epoch 039:    357 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.973, ntokens=18140.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.85, wps=3078.7, ups=0.17, wpb=18140.8, bsz=1024, num_updates=15200, lr=3.25135e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=99787
-2024-11-04 09:14:28 - progress_bar.py[line:274] - INFO: epoch 039:    367 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.019, ntokens=18211.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.1, wps=3117.9, ups=0.17, wpb=18211.3, bsz=1024, num_updates=15210, lr=3.24999e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.3, wall=99845
-2024-11-04 09:15:26 - progress_bar.py[line:274] - INFO: epoch 039:    377 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.936, ntokens=18073.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.66, wps=3152, ups=0.17, wpb=18073.7, bsz=1024, num_updates=15220, lr=3.24863e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.3, wall=99903
-2024-11-04 09:16:24 - progress_bar.py[line:274] - INFO: epoch 039:    387 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.98, ntokens=17809.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3046.2, ups=0.17, wpb=17809.1, bsz=1024, num_updates=15230, lr=3.24727e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=99961
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.pt2024-11-04 09:16:44 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 39 @ 15234 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.pt
-
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.pt
-2024-11-04 09:16:44 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.pt': File exists
-: File exists: File exists: File exists
-
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 09:18:34 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-
-
-slice_id 5 seek offset 250000
-slice_id 3 seek offset 150000
-slice_id 4 seek offset 200000
-2024-11-04 09:20:31 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 39 @ 15234 updates, score 0) (writing took 227.20593089499744 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_39.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-2024-11-04 09:21:28 - train.py[line:336] - INFO: end of epoch 39 (average epoch stats below)
-2024-11-04 09:21:28 - progress_bar.py[line:282] - INFO: epoch 039 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.908 | ntokens 17822.4 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.51 | wps 2546.3 | ups 0.14 | wpb 17822.4 | bsz 1023 | num_updates 15234 | lr 3.24672e-05 | gnorm 0.007 | clip 0 | loss_scale 65536 | train_wall 2054 | gb_free 13.3 | wall 100265
-2024-11-04 09:21:28 - trainer.py[line:639] - INFO: loading train data for epoch 40
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 09:24:12 - trainer.py[line:703] - INFO: begin training epoch 40
-2024-11-04 09:24:12 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 09:24:48 - progress_bar.py[line:274] - INFO: epoch 040:      6 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.912, ntokens=17267.2, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.52, wps=342.3, ups=0.02, wpb=17267.2, bsz=985.6, num_updates=15240, lr=3.24591e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=38, gb_free=13.5, wall=100466
-2024-11-04 09:25:47 - progress_bar.py[line:274] - INFO: epoch 040:     16 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.953, ntokens=18165.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.75, wps=3125, ups=0.17, wpb=18165.7, bsz=1024, num_updates=15250, lr=3.24454e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=100524
-2024-11-04 09:26:44 - progress_bar.py[line:274] - INFO: epoch 040:     26 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.822, ntokens=17501.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.07, wps=3028.6, ups=0.17, wpb=17501.4, bsz=1024, num_updates=15260, lr=3.24318e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=100581
-2024-11-04 09:27:42 - progress_bar.py[line:274] - INFO: epoch 040:     36 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.889, ntokens=17864.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.41, wps=3080.4, ups=0.17, wpb=17864.4, bsz=1024, num_updates=15270, lr=3.24182e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=100639
-2024-11-04 09:28:40 - progress_bar.py[line:274] - INFO: epoch 040:     46 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.854, ntokens=17597.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.23, wps=3055.4, ups=0.17, wpb=17597.7, bsz=1024, num_updates=15280, lr=3.24046e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=100697
-2024-11-04 09:29:39 - progress_bar.py[line:274] - INFO: epoch 040:     56 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.872, ntokens=17799, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.32, wps=2995.2, ups=0.17, wpb=17799, bsz=1024, num_updates=15290, lr=3.2391e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=36, gb_free=13.5, wall=100757
-2024-11-04 09:30:39 - progress_bar.py[line:274] - INFO: epoch 040:     66 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.886, ntokens=17831.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=2999.5, ups=0.17, wpb=17831.8, bsz=1024, num_updates=15300, lr=3.23774e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=100816
-2024-11-04 09:31:39 - progress_bar.py[line:274] - INFO: epoch 040:     76 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.905, ntokens=17666.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.49, wps=2952.4, ups=0.17, wpb=17666.8, bsz=1024, num_updates=15310, lr=3.23638e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=60, gb_free=13.5, wall=100876
-2024-11-04 09:32:38 - progress_bar.py[line:274] - INFO: epoch 040:     86 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.859, ntokens=17828.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.25, wps=3024.2, ups=0.17, wpb=17828.7, bsz=1024, num_updates=15320, lr=3.23502e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=100935
-2024-11-04 09:33:37 - progress_bar.py[line:274] - INFO: epoch 040:     96 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.873, ntokens=17709.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.32, wps=3000.5, ups=0.17, wpb=17709.2, bsz=1024, num_updates=15330, lr=3.23366e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=59, gb_free=13.6, wall=100994
-2024-11-04 09:33:43 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-04 09:34:42 - progress_bar.py[line:274] - INFO: epoch 040:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.876, ntokens=17817.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.34, wps=2742.5, ups=0.15, wpb=17817.7, bsz=1024, num_updates=15340, lr=3.2323e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=65, gb_free=13.5, wall=101059
-2024-11-04 09:35:41 - progress_bar.py[line:274] - INFO: epoch 040:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.918, ntokens=17956.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.56, wps=3001.3, ups=0.17, wpb=17956.2, bsz=1024, num_updates=15350, lr=3.23094e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=60, gb_free=13.5, wall=101119
-2024-11-04 09:36:40 - progress_bar.py[line:274] - INFO: epoch 040:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.806, ntokens=17406.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=2950.3, ups=0.17, wpb=17406.3, bsz=1024, num_updates=15360, lr=3.22958e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=59, gb_free=13.5, wall=101178
-2024-11-04 09:37:39 - progress_bar.py[line:274] - INFO: epoch 040:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.899, ntokens=17737.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.46, wps=3015.3, ups=0.17, wpb=17737.4, bsz=1024, num_updates=15370, lr=3.22822e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=59, gb_free=13.4, wall=101236
-2024-11-04 09:38:39 - progress_bar.py[line:274] - INFO: epoch 040:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.848, ntokens=17608.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.2, wps=2973.3, ups=0.17, wpb=17608.9, bsz=1024, num_updates=15380, lr=3.22686e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=59, gb_free=13.4, wall=101296
-2024-11-04 09:39:38 - progress_bar.py[line:274] - INFO: epoch 040:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.869, ntokens=17638.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.31, wps=2970.6, ups=0.17, wpb=17638.2, bsz=1024, num_updates=15390, lr=3.2255e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=59, gb_free=13.6, wall=101355
-2024-11-04 09:40:37 - progress_bar.py[line:274] - INFO: epoch 040:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.868, ntokens=18004.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.3, wps=3051.5, ups=0.17, wpb=18004.2, bsz=1024, num_updates=15400, lr=3.22414e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=59, gb_free=13.5, wall=101414
-2024-11-04 09:41:35 - progress_bar.py[line:274] - INFO: epoch 040:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.893, ntokens=17596.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=3041.4, ups=0.17, wpb=17596.1, bsz=1024, num_updates=15410, lr=3.22278e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=58, gb_free=13.4, wall=101472
-2024-11-04 09:42:01 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
-2024-11-04 09:42:36 - progress_bar.py[line:274] - INFO: epoch 040:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.875, ntokens=17972.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.34, wps=2920.9, ups=0.16, wpb=17972.6, bsz=1024, num_updates=15420, lr=3.22142e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=48, gb_free=13.4, wall=101533
-2024-11-04 09:43:34 - progress_bar.py[line:274] - INFO: epoch 040:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.895, ntokens=17845.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.44, wps=3076.1, ups=0.17, wpb=17845.7, bsz=1024, num_updates=15430, lr=3.22006e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=43, gb_free=13.5, wall=101591
-2024-11-04 09:44:31 - progress_bar.py[line:274] - INFO: epoch 040:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.838, ntokens=17818.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=3148, ups=0.18, wpb=17818.2, bsz=1024, num_updates=15440, lr=3.2187e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=37, gb_free=13.6, wall=101648
-2024-11-04 09:45:29 - progress_bar.py[line:274] - INFO: epoch 040:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.882, ntokens=18035.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.37, wps=3103, ups=0.17, wpb=18035.5, bsz=1024, num_updates=15450, lr=3.21734e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=101706
-2024-11-04 09:46:28 - progress_bar.py[line:274] - INFO: epoch 040:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.885, ntokens=17852.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3022.4, ups=0.17, wpb=17852.1, bsz=1024, num_updates=15460, lr=3.21598e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=34, gb_free=13.6, wall=101765
-2024-11-04 09:47:27 - progress_bar.py[line:274] - INFO: epoch 040:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.868, ntokens=17705.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.3, wps=2984.9, ups=0.17, wpb=17705.9, bsz=1024, num_updates=15470, lr=3.21462e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=47, gb_free=13.5, wall=101825
-2024-11-04 09:48:26 - progress_bar.py[line:274] - INFO: epoch 040:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.883, ntokens=17433.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.38, wps=2952.3, ups=0.17, wpb=17433.3, bsz=1024, num_updates=15480, lr=3.21326e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=101884
-2024-11-04 09:49:25 - progress_bar.py[line:274] - INFO: epoch 040:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.872, ntokens=17796.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.32, wps=3047.5, ups=0.17, wpb=17796.1, bsz=1024, num_updates=15490, lr=3.2119e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=58, gb_free=13.6, wall=101942
-2024-11-04 09:50:24 - progress_bar.py[line:274] - INFO: epoch 040:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.945, ntokens=18094.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.7, wps=3056.2, ups=0.17, wpb=18094.3, bsz=1024, num_updates=15500, lr=3.21053e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=59, gb_free=13.5, wall=102001
-2024-11-04 09:51:23 - progress_bar.py[line:274] - INFO: epoch 040:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.823, ntokens=17577.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.07, wps=2985.4, ups=0.17, wpb=17577.9, bsz=1024, num_updates=15510, lr=3.20917e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=59, gb_free=13.5, wall=102060
-2024-11-04 09:52:22 - progress_bar.py[line:274] - INFO: epoch 040:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.862, ntokens=17609.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.27, wps=2974.7, ups=0.17, wpb=17609.4, bsz=1024, num_updates=15520, lr=3.20781e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=59, gb_free=13.6, wall=102119
-2024-11-04 09:53:21 - progress_bar.py[line:274] - INFO: epoch 040:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.91, ntokens=17645.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.52, wps=2986.9, ups=0.17, wpb=17645.5, bsz=1024, num_updates=15530, lr=3.20645e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=59, gb_free=13.4, wall=102178
-2024-11-04 09:54:20 - progress_bar.py[line:274] - INFO: epoch 040:    308 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.016, ntokens=18400, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.09, wps=3109.8, ups=0.17, wpb=18400, bsz=1024, num_updates=15540, lr=3.20509e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=59, gb_free=13.3, wall=102237
-2024-11-04 09:55:16 - progress_bar.py[line:274] - INFO: epoch 040:    318 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.993, ntokens=18248, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.96, wps=3287.2, ups=0.18, wpb=18248, bsz=1024, num_updates=15550, lr=3.20373e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=55, gb_free=13.4, wall=102293
-2024-11-04 09:56:14 - progress_bar.py[line:274] - INFO: epoch 040:    328 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.978, ntokens=18113.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.88, wps=3109.3, ups=0.17, wpb=18113.1, bsz=1024, num_updates=15560, lr=3.20237e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=58, gb_free=13.3, wall=102351
-2024-11-04 09:57:13 - progress_bar.py[line:274] - INFO: epoch 040:    338 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.942, ntokens=17933.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.69, wps=3053.3, ups=0.17, wpb=17933.2, bsz=1024, num_updates=15570, lr=3.20101e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=59, gb_free=13.5, wall=102410
-2024-11-04 09:58:08 - progress_bar.py[line:274] - INFO: epoch 040:    348 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.983, ntokens=18427.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.9, wps=3369.9, ups=0.18, wpb=18427.9, bsz=1024, num_updates=15580, lr=3.19965e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=50, gb_free=13.3, wall=102465
-2024-11-04 09:59:08 - progress_bar.py[line:274] - INFO: epoch 040:    358 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.018, ntokens=18411.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.1, wps=3065.9, ups=0.17, wpb=18411.2, bsz=1024, num_updates=15590, lr=3.19829e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=57, gb_free=13.2, wall=102525
-2024-11-04 10:00:07 - progress_bar.py[line:274] - INFO: epoch 040:    368 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.968, ntokens=18107.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.82, wps=3057.1, ups=0.17, wpb=18107.1, bsz=1024, num_updates=15600, lr=3.19693e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=59, gb_free=13.4, wall=102584
-2024-11-04 10:01:06 - progress_bar.py[line:274] - INFO: epoch 040:    378 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.934, ntokens=18141.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.64, wps=3070.6, ups=0.17, wpb=18141.5, bsz=1024, num_updates=15610, lr=3.19557e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=59, gb_free=13.5, wall=102643
-2024-11-04 10:02:04 - progress_bar.py[line:274] - INFO: epoch 040:    388 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.985, ntokens=18003.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.92, wps=3082.8, ups=0.17, wpb=18003.7, bsz=1024, num_updates=15620, lr=3.19421e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=58, gb_free=13.4, wall=102701
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_40.pt
-2024-11-04 10:02:18 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 40 @ 15623 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_40.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_40.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_40.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_40.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_40.pt
-2024-11-04 10:02:18 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_40.pt
-cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_40.pt': File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 10:04:13 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-2024-11-04 10:06:10 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 40 @ 15623 updates, score 0) (writing took 232.1807588050142 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_40.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-
-slice_id 1 seek offset 50000
-slice_id 6 seek offset 300000
-slice_id 7 seek offset 350000
-slice_id 3 seek offset 150000
-slice_id 5 seek offset 250000
-slice_id 4 seek offset 200000
-2024-11-04 10:07:07 - train.py[line:336] - INFO: end of epoch 40 (average epoch stats below)
-2024-11-04 10:07:07 - progress_bar.py[line:282] - INFO: epoch 040 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.901 | ntokens 17842.7 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.47 | wps 2533.9 | ups 0.14 | wpb 17842.7 | bsz 1023 | num_updates 15623 | lr 3.1938e-05 | gnorm 0.007 | clip 0 | loss_scale 16384 | train_wall 2004 | gb_free 13.3 | wall 103004
-2024-11-04 10:07:07 - trainer.py[line:639] - INFO: loading train data for epoch 41
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 10:09:52 - trainer.py[line:703] - INFO: begin training epoch 41
-2024-11-04 10:09:52 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 10:10:34 - progress_bar.py[line:274] - INFO: epoch 041:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.877, ntokens=16941.9, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.35, wps=332.2, ups=0.02, wpb=16941.9, bsz=985.6, num_updates=15630, lr=3.19285e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=35, gb_free=13.6, wall=103211
-2024-11-04 10:11:31 - progress_bar.py[line:274] - INFO: epoch 041:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.886, ntokens=17792.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3121.9, ups=0.18, wpb=17792.2, bsz=1024, num_updates=15640, lr=3.19149e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=103268
-2024-11-04 10:12:28 - progress_bar.py[line:274] - INFO: epoch 041:     27 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.835, ntokens=17536.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.14, wps=3080.1, ups=0.18, wpb=17536.6, bsz=1024, num_updates=15650, lr=3.19013e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=103325
-2024-11-04 10:13:30 - progress_bar.py[line:274] - INFO: epoch 041:     37 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.885, ntokens=17910.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=2905.4, ups=0.16, wpb=17910.7, bsz=1024, num_updates=15660, lr=3.18877e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=103387
-2024-11-04 10:14:34 - progress_bar.py[line:274] - INFO: epoch 041:     47 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.845, ntokens=17617.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.19, wps=2765.1, ups=0.16, wpb=17617.6, bsz=1024, num_updates=15670, lr=3.18741e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=32, gb_free=13.6, wall=103451
-2024-11-04 10:15:38 - progress_bar.py[line:274] - INFO: epoch 041:     57 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.841, ntokens=17764.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.16, wps=2757.7, ups=0.16, wpb=17764.5, bsz=1024, num_updates=15680, lr=3.18605e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=103515
-2024-11-04 10:16:43 - progress_bar.py[line:274] - INFO: epoch 041:     67 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.868, ntokens=17748.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.3, wps=2736.3, ups=0.15, wpb=17748.3, bsz=1024, num_updates=15690, lr=3.18469e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=103580
-2024-11-04 10:17:49 - progress_bar.py[line:274] - INFO: epoch 041:     77 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.854, ntokens=17567.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.23, wps=2652, ups=0.15, wpb=17567.6, bsz=1024, num_updates=15700, lr=3.18333e-05, gnorm=0.004, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=103646
-2024-11-04 10:18:54 - progress_bar.py[line:274] - INFO: epoch 041:     87 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.834, ntokens=17711, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.13, wps=2738.3, ups=0.15, wpb=17711, bsz=1024, num_updates=15710, lr=3.18197e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=32, gb_free=13.6, wall=103711
-2024-11-04 10:19:58 - progress_bar.py[line:274] - INFO: epoch 041:     97 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.858, ntokens=17749.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.25, wps=2749.1, ups=0.15, wpb=17749.7, bsz=1024, num_updates=15720, lr=3.18061e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=103775
-2024-11-04 10:21:01 - progress_bar.py[line:274] - INFO: epoch 041:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=17568.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=2795.7, ups=0.16, wpb=17568.8, bsz=1024, num_updates=15730, lr=3.17925e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=103838
-2024-11-04 10:22:02 - progress_bar.py[line:274] - INFO: epoch 041:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.858, ntokens=17555.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.25, wps=2873.5, ups=0.16, wpb=17555.2, bsz=1024, num_updates=15740, lr=3.17789e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=103899
-2024-11-04 10:22:59 - progress_bar.py[line:274] - INFO: epoch 041:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.803, ntokens=17363.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.98, wps=3084.3, ups=0.18, wpb=17363.5, bsz=1024, num_updates=15750, lr=3.17653e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=103956
-2024-11-04 10:23:57 - progress_bar.py[line:274] - INFO: epoch 041:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.846, ntokens=17581.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.19, wps=2989.5, ups=0.17, wpb=17581.6, bsz=1024, num_updates=15760, lr=3.17516e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=104014
-2024-11-04 10:24:55 - progress_bar.py[line:274] - INFO: epoch 041:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.874, ntokens=17927.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.33, wps=3105.1, ups=0.17, wpb=17927.8, bsz=1024, num_updates=15770, lr=3.1738e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=104072
-2024-11-04 10:25:53 - progress_bar.py[line:274] - INFO: epoch 041:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.875, ntokens=17683.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.33, wps=3077.2, ups=0.17, wpb=17683.9, bsz=1024, num_updates=15780, lr=3.17244e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=104130
-2024-11-04 10:26:51 - progress_bar.py[line:274] - INFO: epoch 041:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.88, ntokens=18157.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.36, wps=3133.5, ups=0.17, wpb=18157.6, bsz=1024, num_updates=15790, lr=3.17108e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=104188
-2024-11-04 10:27:49 - progress_bar.py[line:274] - INFO: epoch 041:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.804, ntokens=17173.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.98, wps=2926.7, ups=0.17, wpb=17173.3, bsz=1024, num_updates=15800, lr=3.16972e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.3, wall=104246
-2024-11-04 10:28:47 - progress_bar.py[line:274] - INFO: epoch 041:    187 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.816, ntokens=17651.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.04, wps=3052.2, ups=0.17, wpb=17651.2, bsz=1024, num_updates=15810, lr=3.16836e-05, gnorm=0.004, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=104304
-2024-11-04 10:29:42 - progress_bar.py[line:274] - INFO: epoch 041:    197 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.847, ntokens=17560.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.19, wps=3172.2, ups=0.18, wpb=17560.6, bsz=1024, num_updates=15820, lr=3.167e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=104360
-2024-11-04 10:30:36 - progress_bar.py[line:274] - INFO: epoch 041:    207 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.867, ntokens=18082.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.3, wps=3371.7, ups=0.19, wpb=18082.4, bsz=1024, num_updates=15830, lr=3.16564e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=104413
-2024-11-04 10:31:32 - progress_bar.py[line:274] - INFO: epoch 041:    217 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.838, ntokens=17735.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=3155.2, ups=0.18, wpb=17735.4, bsz=1024, num_updates=15840, lr=3.16428e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=104469
-2024-11-04 10:32:31 - progress_bar.py[line:274] - INFO: epoch 041:    227 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.814, ntokens=17655.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=2994.6, ups=0.17, wpb=17655.5, bsz=1024, num_updates=15850, lr=3.16292e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=32, gb_free=13.6, wall=104528
-2024-11-04 10:33:32 - progress_bar.py[line:274] - INFO: epoch 041:    237 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.835, ntokens=17574.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.13, wps=2876.8, ups=0.16, wpb=17574.5, bsz=1024, num_updates=15860, lr=3.16156e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=32, gb_free=13.6, wall=104589
-2024-11-04 10:34:30 - progress_bar.py[line:274] - INFO: epoch 041:    247 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.874, ntokens=17622.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.33, wps=3046.8, ups=0.17, wpb=17622.2, bsz=1024, num_updates=15870, lr=3.1602e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=104647
-2024-11-04 10:35:28 - progress_bar.py[line:274] - INFO: epoch 041:    257 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.883, ntokens=17990.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.38, wps=3107.4, ups=0.17, wpb=17990.1, bsz=1024, num_updates=15880, lr=3.15884e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=104705
-2024-11-04 10:36:25 - progress_bar.py[line:274] - INFO: epoch 041:    267 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.923, ntokens=18290.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.58, wps=3187.7, ups=0.17, wpb=18290.4, bsz=1024, num_updates=15890, lr=3.15748e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=104763
-2024-11-04 10:37:23 - progress_bar.py[line:274] - INFO: epoch 041:    277 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.806, ntokens=17435.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=3009.3, ups=0.17, wpb=17435.6, bsz=1024, num_updates=15900, lr=3.15612e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=104820
-2024-11-04 10:38:22 - progress_bar.py[line:274] - INFO: epoch 041:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.836, ntokens=17734.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.14, wps=3028.8, ups=0.17, wpb=17734.7, bsz=1024, num_updates=15910, lr=3.15476e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=104879
-2024-11-04 10:39:19 - progress_bar.py[line:274] - INFO: epoch 041:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=17660.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3117.5, ups=0.18, wpb=17660.1, bsz=1024, num_updates=15920, lr=3.1534e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=104936
-2024-11-04 10:40:15 - progress_bar.py[line:274] - INFO: epoch 041:    307 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.024, ntokens=18532.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.14, wps=3254.2, ups=0.18, wpb=18532.5, bsz=1024, num_updates=15930, lr=3.15204e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=104993
-2024-11-04 10:41:11 - progress_bar.py[line:274] - INFO: epoch 041:    317 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.938, ntokens=17972.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.66, wps=3235, ups=0.18, wpb=17972.7, bsz=1024, num_updates=15940, lr=3.15068e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=105048
-2024-11-04 10:42:08 - progress_bar.py[line:274] - INFO: epoch 041:    327 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.015, ntokens=18400, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.08, wps=3240.7, ups=0.18, wpb=18400, bsz=1024, num_updates=15950, lr=3.14932e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=105105
-2024-11-04 10:43:01 - progress_bar.py[line:274] - INFO: epoch 041:    337 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.951, ntokens=18030.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.73, wps=3366.2, ups=0.19, wpb=18030.8, bsz=1024, num_updates=15960, lr=3.14796e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=105159
-2024-11-04 10:43:57 - progress_bar.py[line:274] - INFO: epoch 041:    347 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.013, ntokens=18566.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.07, wps=3314.3, ups=0.18, wpb=18566.7, bsz=1024, num_updates=15970, lr=3.1466e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=43, gb_free=13.4, wall=105215
-2024-11-04 10:44:55 - progress_bar.py[line:274] - INFO: epoch 041:    357 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.977, ntokens=18271.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.88, wps=3190.3, ups=0.17, wpb=18271.7, bsz=1024, num_updates=15980, lr=3.14524e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=105272
-2024-11-04 10:45:51 - progress_bar.py[line:274] - INFO: epoch 041:    367 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.98, ntokens=18285.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3220, ups=0.18, wpb=18285.4, bsz=1024, num_updates=15990, lr=3.14388e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.3, wall=105329
-2024-11-04 10:46:47 - progress_bar.py[line:274] - INFO: epoch 041:    377 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.938, ntokens=18271.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.66, wps=3318.6, ups=0.18, wpb=18271.3, bsz=1024, num_updates=16000, lr=3.14252e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=51, gb_free=13.3, wall=105384
-2024-11-04 10:47:43 - progress_bar.py[line:274] - INFO: epoch 041:    387 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.925, ntokens=17573.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.6, wps=3102.6, ups=0.18, wpb=17573.2, bsz=1024, num_updates=16010, lr=3.14115e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=32, gb_free=13.3, wall=105440
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt
-2024-11-04 10:48:02 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 41 @ 16014 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt
-2024-11-04 10:48:02 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt': File exists: File exists: File exists
-: File exists
-: File exists
-: File exists
-
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 10:49:28 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 10:52:04 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 41 @ 16014 updates, score 0) (writing took 241.8641978999949 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_41.pt
-2024-11-04 10:53:06 - train.py[line:336] - INFO: end of epoch 41 (average epoch stats below)
-2024-11-04 10:53:06 - progress_bar.py[line:282] - INFO: epoch 041 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.882 | ntokens 17803.5 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.37 | wps 2523 | ups 0.14 | wpb 17803.5 | bsz 1023 | num_updates 16014 | lr 3.14061e-05 | gnorm 0.006 | clip 0 | loss_scale 32768 | train_wall 1300 | gb_free 13.4 | wall 105763
-2024-11-04 10:53:06 - trainer.py[line:639] - INFO: loading train data for epoch 42
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 10:58:22 - trainer.py[line:703] - INFO: begin training epoch 42
-2024-11-04 10:58:22 - train.py[line:297] - INFO: Start iterating over samples
-slice_id 3 seek offset 150000
-2024-11-04 10:59:01 - progress_bar.py[line:274] - INFO: epoch 042:      6 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.844, ntokens=16972.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.18, wps=250.4, ups=0.01, wpb=16972.5, bsz=985.6, num_updates=16020, lr=3.13979e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=34, gb_free=13.5, wall=106118
-2024-11-04 11:00:03 - progress_bar.py[line:274] - INFO: epoch 042:     16 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.903, ntokens=17938.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.48, wps=2880.6, ups=0.16, wpb=17938.9, bsz=1024, num_updates=16030, lr=3.13843e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=106180
-2024-11-04 11:01:02 - progress_bar.py[line:274] - INFO: epoch 042:     26 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.776, ntokens=17357.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=2959, ups=0.17, wpb=17357.2, bsz=1024, num_updates=16040, lr=3.13707e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=106239
-2024-11-04 11:02:00 - progress_bar.py[line:274] - INFO: epoch 042:     36 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.857, ntokens=17757.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.25, wps=3076.9, ups=0.17, wpb=17757.7, bsz=1024, num_updates=16050, lr=3.13571e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=106297
-2024-11-04 11:02:58 - progress_bar.py[line:274] - INFO: epoch 042:     46 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.8, ntokens=17410.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.96, wps=2992.5, ups=0.17, wpb=17410.2, bsz=1024, num_updates=16060, lr=3.13435e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=106355
-2024-11-04 11:03:56 - progress_bar.py[line:274] - INFO: epoch 042:     56 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.812, ntokens=17683.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=3053.3, ups=0.17, wpb=17683.4, bsz=1024, num_updates=16070, lr=3.13299e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=106413
-2024-11-04 11:04:51 - progress_bar.py[line:274] - INFO: epoch 042:     66 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.885, ntokens=17977.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.38, wps=3252.2, ups=0.18, wpb=17977.7, bsz=1024, num_updates=16080, lr=3.13163e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=106468
-2024-11-04 11:05:47 - progress_bar.py[line:274] - INFO: epoch 042:     76 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.824, ntokens=17375.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.08, wps=3109.6, ups=0.18, wpb=17375.7, bsz=1024, num_updates=16090, lr=3.13027e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=106524
-2024-11-04 11:06:44 - progress_bar.py[line:274] - INFO: epoch 042:     86 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.812, ntokens=17643.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=3113, ups=0.18, wpb=17643.3, bsz=1024, num_updates=16100, lr=3.12891e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=106581
-2024-11-04 11:07:41 - progress_bar.py[line:274] - INFO: epoch 042:     96 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.865, ntokens=17782.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.28, wps=3068.3, ups=0.17, wpb=17782.3, bsz=1024, num_updates=16110, lr=3.12755e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=106639
-2024-11-04 11:08:37 - progress_bar.py[line:274] - INFO: epoch 042:    106 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.848, ntokens=17733.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.2, wps=3183.3, ups=0.18, wpb=17733.7, bsz=1024, num_updates=16120, lr=3.12619e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=106694
-2024-11-04 11:09:31 - progress_bar.py[line:274] - INFO: epoch 042:    116 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.857, ntokens=17655.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.24, wps=3253.5, ups=0.18, wpb=17655.6, bsz=1024, num_updates=16130, lr=3.12483e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=106749
-2024-11-04 11:10:25 - progress_bar.py[line:274] - INFO: epoch 042:    126 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.812, ntokens=17605.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=3275, ups=0.19, wpb=17605.7, bsz=1024, num_updates=16140, lr=3.12347e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=33, gb_free=13.6, wall=106802
-2024-11-04 11:11:23 - progress_bar.py[line:274] - INFO: epoch 042:    136 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.846, ntokens=17598.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.19, wps=3068.2, ups=0.17, wpb=17598.8, bsz=1024, num_updates=16150, lr=3.12211e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=106860
-2024-11-04 11:12:22 - progress_bar.py[line:274] - INFO: epoch 042:    146 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.817, ntokens=17606.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.05, wps=2941.3, ups=0.17, wpb=17606.6, bsz=1024, num_updates=16160, lr=3.12075e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.6, wall=106920
-2024-11-04 11:13:20 - progress_bar.py[line:274] - INFO: epoch 042:    156 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.856, ntokens=17784.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.24, wps=3111, ups=0.17, wpb=17784.5, bsz=1024, num_updates=16170, lr=3.11939e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=106977
-2024-11-04 11:14:18 - progress_bar.py[line:274] - INFO: epoch 042:    166 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.791, ntokens=17560.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=2990.1, ups=0.17, wpb=17560.2, bsz=1024, num_updates=16180, lr=3.11803e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.3, wall=107035
-2024-11-04 11:15:21 - progress_bar.py[line:274] - INFO: epoch 042:    176 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.836, ntokens=17450.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.14, wps=2789.2, ups=0.16, wpb=17450.9, bsz=1024, num_updates=16190, lr=3.11667e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=107098
-2024-11-04 11:16:19 - progress_bar.py[line:274] - INFO: epoch 042:    186 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.873, ntokens=17897.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.33, wps=3078, ups=0.17, wpb=17897.8, bsz=1024, num_updates=16200, lr=3.11531e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=107156
-2024-11-04 11:17:16 - progress_bar.py[line:274] - INFO: epoch 042:    196 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.872, ntokens=17913.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.32, wps=3139.1, ups=0.18, wpb=17913.4, bsz=1024, num_updates=16210, lr=3.11395e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=107213
-2024-11-04 11:18:13 - progress_bar.py[line:274] - INFO: epoch 042:    206 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.892, ntokens=18070, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.42, wps=3165.4, ups=0.18, wpb=18070, bsz=1024, num_updates=16220, lr=3.11259e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=107270
-2024-11-04 11:19:11 - progress_bar.py[line:274] - INFO: epoch 042:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.836, ntokens=17943, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.14, wps=3113.5, ups=0.17, wpb=17943, bsz=1024, num_updates=16230, lr=3.11123e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=34, gb_free=13.6, wall=107328
-2024-11-04 11:20:11 - progress_bar.py[line:274] - INFO: epoch 042:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.806, ntokens=17525.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=2922.9, ups=0.17, wpb=17525.5, bsz=1024, num_updates=16240, lr=3.10987e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=39, gb_free=13.6, wall=107388
-2024-11-04 11:21:08 - progress_bar.py[line:274] - INFO: epoch 042:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.847, ntokens=17691.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.19, wps=3095.2, ups=0.17, wpb=17691.7, bsz=1024, num_updates=16250, lr=3.10851e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=34, gb_free=13.5, wall=107445
-2024-11-04 11:22:04 - progress_bar.py[line:274] - INFO: epoch 042:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.831, ntokens=17361.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3076.7, ups=0.18, wpb=17361.1, bsz=1024, num_updates=16260, lr=3.10714e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=107501
-2024-11-04 11:23:01 - progress_bar.py[line:274] - INFO: epoch 042:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.852, ntokens=17858.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=3140.3, ups=0.18, wpb=17858.4, bsz=1024, num_updates=16270, lr=3.10578e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=33, gb_free=13.4, wall=107558
-2024-11-04 11:23:58 - progress_bar.py[line:274] - INFO: epoch 042:    266 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.9, ntokens=18023.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.47, wps=3168.3, ups=0.18, wpb=18023.6, bsz=1024, num_updates=16280, lr=3.10442e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=45, gb_free=13.5, wall=107615
-2024-11-04 11:24:53 - progress_bar.py[line:274] - INFO: epoch 042:    276 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17529.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3176.4, ups=0.18, wpb=17529.4, bsz=1024, num_updates=16290, lr=3.10306e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=40, gb_free=13.5, wall=107670
-2024-11-04 11:25:50 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
-2024-11-04 11:25:56 - progress_bar.py[line:274] - INFO: epoch 042:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.826, ntokens=17637.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.09, wps=2811.9, ups=0.16, wpb=17637.4, bsz=1024, num_updates=16300, lr=3.1017e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=34, gb_free=13.6, wall=107733
-2024-11-04 11:26:54 - progress_bar.py[line:274] - INFO: epoch 042:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.913, ntokens=17767.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.53, wps=3065.4, ups=0.17, wpb=17767.6, bsz=1024, num_updates=16310, lr=3.10034e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=107791
-2024-11-04 11:27:00 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 8192.0
-2024-11-04 11:27:59 - progress_bar.py[line:274] - INFO: epoch 042:    308 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.041, ntokens=18664.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.23, wps=2867.9, ups=0.15, wpb=18664.5, bsz=1024, num_updates=16320, lr=3.09898e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=35, gb_free=13.3, wall=107856
-2024-11-04 11:28:56 - progress_bar.py[line:274] - INFO: epoch 042:    318 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.979, ntokens=18342, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=3236.5, ups=0.18, wpb=18342, bsz=1024, num_updates=16330, lr=3.09762e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=31, gb_free=13.4, wall=107913
-2024-11-04 11:29:54 - progress_bar.py[line:274] - INFO: epoch 042:    328 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.962, ntokens=18140.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.79, wps=3118.4, ups=0.17, wpb=18140.1, bsz=1024, num_updates=16340, lr=3.09626e-05, gnorm=0.005, clip=0, loss_scale=8192, train_wall=32, gb_free=13.3, wall=107971
-2024-11-04 11:30:51 - progress_bar.py[line:274] - INFO: epoch 042:    338 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.941, ntokens=17943.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.68, wps=3167.8, ups=0.18, wpb=17943.2, bsz=1024, num_updates=16350, lr=3.0949e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=31, gb_free=13.4, wall=108028
-2024-11-04 11:31:46 - progress_bar.py[line:274] - INFO: epoch 042:    348 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.972, ntokens=18648.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.84, wps=3358.6, ups=0.18, wpb=18648.8, bsz=1024, num_updates=16360, lr=3.09354e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=31, gb_free=13.2, wall=108083
-2024-11-04 11:32:41 - progress_bar.py[line:274] - INFO: epoch 042:    358 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.002, ntokens=18315.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.01, wps=3320.9, ups=0.18, wpb=18315.2, bsz=1024, num_updates=16370, lr=3.09218e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=31, gb_free=13.3, wall=108138
-2024-11-04 11:33:41 - progress_bar.py[line:274] - INFO: epoch 042:    368 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.947, ntokens=18329.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.71, wps=3053.9, ups=0.17, wpb=18329.2, bsz=1024, num_updates=16380, lr=3.09082e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=108198
-2024-11-04 11:34:44 - progress_bar.py[line:274] - INFO: epoch 042:    378 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.94, ntokens=18319.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.68, wps=2925.9, ups=0.16, wpb=18319.2, bsz=1024, num_updates=16390, lr=3.08946e-05, gnorm=0.005, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=108261
-2024-11-04 11:35:46 - progress_bar.py[line:274] - INFO: epoch 042:    388 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.944, ntokens=17834.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.7, wps=2883.2, ups=0.16, wpb=17834.7, bsz=1024, num_updates=16400, lr=3.0881e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=31, gb_free=13.5, wall=108323
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_42.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_42.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_42.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_42.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_42.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_42.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_42.pt
-2024-11-04 11:36:00 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 42 @ 16403 updates
-2024-11-04 11:36:00 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 11:37:46 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 11:39:41 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 42 @ 16403 updates, score 0) (writing took 221.13993188500172 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_42.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 1 seek offset 50000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-2024-11-04 11:40:36 - train.py[line:336] - INFO: end of epoch 42 (average epoch stats below)
-2024-11-04 11:40:36 - progress_bar.py[line:282] - INFO: epoch 042 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.873 | ntokens 17813.5 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.32 | wps 2431.5 | ups 0.14 | wpb 17813.5 | bsz 1023 | num_updates 16403 | lr 3.08769e-05 | gnorm 0.006 | clip 0 | loss_scale 8192 | train_wall 1266 | gb_free 13.3 | wall 108613
-2024-11-04 11:40:36 - trainer.py[line:639] - INFO: loading train data for epoch 43
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 11:43:56 - trainer.py[line:703] - INFO: begin training epoch 43
-2024-11-04 11:43:56 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 11:44:42 - progress_bar.py[line:274] - INFO: epoch 043:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.856, ntokens=17098.8, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.24, wps=319.1, ups=0.02, wpb=17098.8, bsz=985.6, num_updates=16410, lr=3.08674e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=30, gb_free=13.6, wall=108859
-2024-11-04 11:45:47 - progress_bar.py[line:274] - INFO: epoch 043:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.839, ntokens=17758.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=2710.7, ups=0.15, wpb=17758.7, bsz=1024, num_updates=16420, lr=3.08538e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=108924
-2024-11-04 11:46:50 - progress_bar.py[line:274] - INFO: epoch 043:     27 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.821, ntokens=17603.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.06, wps=2793.5, ups=0.16, wpb=17603.4, bsz=1024, num_updates=16430, lr=3.08402e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=33, gb_free=13.6, wall=108987
-2024-11-04 11:47:51 - progress_bar.py[line:274] - INFO: epoch 043:     37 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.848, ntokens=17881.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.2, wps=2927.1, ups=0.16, wpb=17881.4, bsz=1024, num_updates=16440, lr=3.08266e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=109048
-2024-11-04 11:48:51 - progress_bar.py[line:274] - INFO: epoch 043:     47 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.842, ntokens=17780.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.17, wps=2950.9, ups=0.17, wpb=17780.6, bsz=1024, num_updates=16450, lr=3.0813e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=109109
-2024-11-04 11:49:51 - progress_bar.py[line:274] - INFO: epoch 043:     57 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.862, ntokens=17853.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.27, wps=3007.5, ups=0.17, wpb=17853.2, bsz=1024, num_updates=16460, lr=3.07994e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=31, gb_free=13.4, wall=109168
-2024-11-04 11:50:50 - progress_bar.py[line:274] - INFO: epoch 043:     67 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.811, ntokens=17595.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=2990.4, ups=0.17, wpb=17595.6, bsz=1024, num_updates=16470, lr=3.07858e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=31, gb_free=13.5, wall=109227
-2024-11-04 11:51:50 - progress_bar.py[line:274] - INFO: epoch 043:     77 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.845, ntokens=17580.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.18, wps=2926.7, ups=0.17, wpb=17580.6, bsz=1024, num_updates=16480, lr=3.07722e-05, gnorm=0.005, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=109287
-2024-11-04 11:52:51 - progress_bar.py[line:274] - INFO: epoch 043:     87 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.845, ntokens=17907.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.18, wps=2902.1, ups=0.16, wpb=17907.3, bsz=1024, num_updates=16490, lr=3.07586e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.6, wall=109349
-2024-11-04 11:53:56 - progress_bar.py[line:274] - INFO: epoch 043:     97 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.843, ntokens=17741.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.18, wps=2760.1, ups=0.16, wpb=17741.1, bsz=1024, num_updates=16500, lr=3.0745e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=33, gb_free=13.4, wall=109413
-2024-11-04 11:54:55 - progress_bar.py[line:274] - INFO: epoch 043:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17423, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=2931.3, ups=0.17, wpb=17423, bsz=1024, num_updates=16510, lr=3.07313e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=32, gb_free=13.6, wall=109472
-2024-11-04 11:55:53 - progress_bar.py[line:274] - INFO: epoch 043:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.855, ntokens=17680.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.23, wps=3079.7, ups=0.17, wpb=17680.7, bsz=1024, num_updates=16520, lr=3.07177e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=31, gb_free=13.5, wall=109530
-2024-11-04 11:56:51 - progress_bar.py[line:274] - INFO: epoch 043:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.8, ntokens=17577.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.97, wps=3008.9, ups=0.17, wpb=17577.4, bsz=1024, num_updates=16530, lr=3.07041e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=109588
-2024-11-04 11:57:51 - progress_bar.py[line:274] - INFO: epoch 043:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.848, ntokens=17711.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.2, wps=2929.1, ups=0.17, wpb=17711.1, bsz=1024, num_updates=16540, lr=3.06905e-05, gnorm=0.005, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=109649
-2024-11-04 11:58:50 - progress_bar.py[line:274] - INFO: epoch 043:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.816, ntokens=17602.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.04, wps=3003, ups=0.17, wpb=17602.3, bsz=1024, num_updates=16550, lr=3.06769e-05, gnorm=0.008, clip=0, loss_scale=8192, train_wall=31, gb_free=13.4, wall=109707
-2024-11-04 11:59:51 - progress_bar.py[line:274] - INFO: epoch 043:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.83, ntokens=17550.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=2895.6, ups=0.16, wpb=17550.6, bsz=1024, num_updates=16560, lr=3.06633e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=109768
-2024-11-04 12:00:50 - progress_bar.py[line:274] - INFO: epoch 043:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.787, ntokens=17730.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.9, wps=2974.5, ups=0.17, wpb=17730.2, bsz=1024, num_updates=16570, lr=3.06497e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=31, gb_free=13.4, wall=109827
-2024-11-04 12:01:51 - progress_bar.py[line:274] - INFO: epoch 043:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.837, ntokens=17538, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=2906.1, ups=0.17, wpb=17538, bsz=1024, num_updates=16580, lr=3.06361e-05, gnorm=0.005, clip=0, loss_scale=8192, train_wall=32, gb_free=13.3, wall=109888
-2024-11-04 12:02:51 - progress_bar.py[line:274] - INFO: epoch 043:    187 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.821, ntokens=17725.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.07, wps=2915, ups=0.16, wpb=17725.7, bsz=1024, num_updates=16590, lr=3.06225e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.6, wall=109949
-2024-11-04 12:03:51 - progress_bar.py[line:274] - INFO: epoch 043:    197 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.874, ntokens=17997.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.33, wps=3042.3, ups=0.17, wpb=17997.7, bsz=1024, num_updates=16600, lr=3.06089e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=110008
-2024-11-04 12:04:50 - progress_bar.py[line:274] - INFO: epoch 043:    207 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.846, ntokens=18050.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.19, wps=3037.9, ups=0.17, wpb=18050.6, bsz=1024, num_updates=16610, lr=3.05953e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.6, wall=110067
-2024-11-04 12:05:51 - progress_bar.py[line:274] - INFO: epoch 043:    217 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.81, ntokens=17763.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=2908, ups=0.16, wpb=17763.9, bsz=1024, num_updates=16620, lr=3.05817e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=110128
-2024-11-04 12:06:54 - progress_bar.py[line:274] - INFO: epoch 043:    227 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.814, ntokens=17783.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=2821, ups=0.16, wpb=17783.9, bsz=1024, num_updates=16630, lr=3.05681e-05, gnorm=0.005, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=110191
-2024-11-04 12:08:02 - progress_bar.py[line:274] - INFO: epoch 043:    237 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.801, ntokens=17412.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.97, wps=2576.9, ups=0.15, wpb=17412.9, bsz=1024, num_updates=16640, lr=3.05545e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=33, gb_free=13.6, wall=110259
-2024-11-04 12:09:03 - progress_bar.py[line:274] - INFO: epoch 043:    247 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.826, ntokens=17431, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.09, wps=2850.3, ups=0.16, wpb=17431, bsz=1024, num_updates=16650, lr=3.05409e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=110320
-2024-11-04 12:10:05 - progress_bar.py[line:274] - INFO: epoch 043:    257 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.831, ntokens=17811.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=2862.3, ups=0.16, wpb=17811.3, bsz=1024, num_updates=16660, lr=3.05273e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=110382
-2024-11-04 12:11:07 - progress_bar.py[line:274] - INFO: epoch 043:    267 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.874, ntokens=18097.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.33, wps=2934.9, ups=0.16, wpb=18097.7, bsz=1024, num_updates=16670, lr=3.05137e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=110444
-2024-11-04 12:12:10 - progress_bar.py[line:274] - INFO: epoch 043:    277 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.806, ntokens=17411.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=2751.7, ups=0.16, wpb=17411.4, bsz=1024, num_updates=16680, lr=3.05001e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=110507
-2024-11-04 12:13:09 - progress_bar.py[line:274] - INFO: epoch 043:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.768, ntokens=17409.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.81, wps=2937.3, ups=0.17, wpb=17409.7, bsz=1024, num_updates=16690, lr=3.04865e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=31, gb_free=13.6, wall=110566
-2024-11-04 12:14:10 - progress_bar.py[line:274] - INFO: epoch 043:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.898, ntokens=17759.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.46, wps=2934.9, ups=0.17, wpb=17759.6, bsz=1024, num_updates=16700, lr=3.04729e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=110627
-2024-11-04 12:15:10 - progress_bar.py[line:274] - INFO: epoch 043:    307 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.974, ntokens=18326.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.86, wps=3060.9, ups=0.17, wpb=18326.4, bsz=1024, num_updates=16710, lr=3.04593e-05, gnorm=0.005, clip=0, loss_scale=8192, train_wall=32, gb_free=13.3, wall=110687
-2024-11-04 12:16:07 - progress_bar.py[line:274] - INFO: epoch 043:    317 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.011, ntokens=18589.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.06, wps=3224.4, ups=0.17, wpb=18589.1, bsz=1024, num_updates=16720, lr=3.04457e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=31, gb_free=13.4, wall=110744
-2024-11-04 12:17:10 - progress_bar.py[line:274] - INFO: epoch 043:    327 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.979, ntokens=18337, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.89, wps=2911.1, ups=0.16, wpb=18337, bsz=1024, num_updates=16730, lr=3.04321e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=110807
-2024-11-04 12:18:12 - progress_bar.py[line:274] - INFO: epoch 043:    337 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.964, ntokens=18261.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.8, wps=2952.3, ups=0.16, wpb=18261.2, bsz=1024, num_updates=16740, lr=3.04185e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=32, gb_free=13.5, wall=110869
-2024-11-04 12:19:14 - progress_bar.py[line:274] - INFO: epoch 043:    347 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.95, ntokens=18460, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.72, wps=2992, ups=0.16, wpb=18460, bsz=1024, num_updates=16750, lr=3.04049e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=110931
-2024-11-04 12:20:12 - progress_bar.py[line:274] - INFO: epoch 043:    357 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.909, ntokens=17956.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.51, wps=3066.7, ups=0.17, wpb=17956.9, bsz=1024, num_updates=16760, lr=3.03912e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=110990
-2024-11-04 12:21:12 - progress_bar.py[line:274] - INFO: epoch 043:    367 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.954, ntokens=18267.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.75, wps=3045.2, ups=0.17, wpb=18267.1, bsz=1024, num_updates=16770, lr=3.03776e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=32, gb_free=13.3, wall=111050
-2024-11-04 12:22:12 - progress_bar.py[line:274] - INFO: epoch 043:    377 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.931, ntokens=18225.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.62, wps=3066.5, ups=0.17, wpb=18225.3, bsz=1024, num_updates=16780, lr=3.0364e-05, gnorm=0.006, clip=0, loss_scale=8192, train_wall=32, gb_free=13.4, wall=111109
-2024-11-04 12:23:10 - progress_bar.py[line:274] - INFO: epoch 043:    387 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.907, ntokens=17821.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3048.8, ups=0.17, wpb=17821.2, bsz=1024, num_updates=16790, lr=3.03504e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=34, gb_free=13.3, wall=111167
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt': File exists
-: File exists: File exists
-
-2024-11-04 12:23:32 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 43 @ 16794 updates
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 12:23:32 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 12:25:23 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-04 12:27:18 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 43 @ 16794 updates, score 0) (writing took 225.50506996101467 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_43.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-2024-11-04 12:28:13 - train.py[line:336] - INFO: end of epoch 43 (average epoch stats below)
-2024-11-04 12:28:13 - progress_bar.py[line:282] - INFO: epoch 043 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.86 | ntokens 17806.3 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.26 | wps 2436.8 | ups 0.14 | wpb 17806.3 | bsz 1023 | num_updates 16794 | lr 3.0345e-05 | gnorm 0.007 | clip 0 | loss_scale 8192 | train_wall 1255 | gb_free 13.4 | wall 111470
-2024-11-04 12:28:13 - trainer.py[line:639] - INFO: loading train data for epoch 44
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 12:30:59 - trainer.py[line:703] - INFO: begin training epoch 44
-2024-11-04 12:30:59 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 12:31:34 - progress_bar.py[line:274] - INFO: epoch 044:      6 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.811, ntokens=16794, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=333.6, ups=0.02, wpb=16794, bsz=985.6, num_updates=16800, lr=3.03368e-05, gnorm=0.007, clip=0, loss_scale=8192, train_wall=38, gb_free=13.5, wall=111671
-2024-11-04 12:32:31 - progress_bar.py[line:274] - INFO: epoch 044:     16 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.848, ntokens=17796.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.2, wps=3130.4, ups=0.18, wpb=17796.4, bsz=1024, num_updates=16810, lr=3.03232e-05, gnorm=0.009, clip=0, loss_scale=8192, train_wall=31, gb_free=13.4, wall=111728
-2024-11-04 12:33:26 - progress_bar.py[line:274] - INFO: epoch 044:     26 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17400, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3117.1, ups=0.18, wpb=17400, bsz=1024, num_updates=16820, lr=3.03096e-05, gnorm=0.011, clip=0, loss_scale=8192, train_wall=31, gb_free=13.5, wall=111784
-2024-11-04 12:34:25 - progress_bar.py[line:274] - INFO: epoch 044:     36 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.875, ntokens=18082.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.34, wps=3098.3, ups=0.17, wpb=18082.8, bsz=1024, num_updates=16830, lr=3.0296e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=111842
-2024-11-04 12:35:24 - progress_bar.py[line:274] - INFO: epoch 044:     46 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.848, ntokens=17936.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.2, wps=3043.2, ups=0.17, wpb=17936.1, bsz=1024, num_updates=16840, lr=3.02824e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=111901
-2024-11-04 12:36:20 - progress_bar.py[line:274] - INFO: epoch 044:     56 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.857, ntokens=18091.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.25, wps=3247.1, ups=0.18, wpb=18091.6, bsz=1024, num_updates=16850, lr=3.02688e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=42, gb_free=13.4, wall=111957
-2024-11-04 12:37:19 - progress_bar.py[line:274] - INFO: epoch 044:     66 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.862, ntokens=17970.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.27, wps=3012.7, ups=0.17, wpb=17970.8, bsz=1024, num_updates=16860, lr=3.02552e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=35, gb_free=13.4, wall=112016
-2024-11-04 12:38:21 - progress_bar.py[line:274] - INFO: epoch 044:     76 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.813, ntokens=17517.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=2854, ups=0.16, wpb=17517.3, bsz=1024, num_updates=16870, lr=3.02416e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=112078
-2024-11-04 12:39:19 - progress_bar.py[line:274] - INFO: epoch 044:     86 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.829, ntokens=17903, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3049.4, ups=0.17, wpb=17903, bsz=1024, num_updates=16880, lr=3.0228e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=112136
-2024-11-04 12:40:22 - progress_bar.py[line:274] - INFO: epoch 044:     96 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.816, ntokens=17713.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.04, wps=2819.1, ups=0.16, wpb=17713.1, bsz=1024, num_updates=16890, lr=3.02144e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=112199
-2024-11-04 12:41:25 - progress_bar.py[line:274] - INFO: epoch 044:    106 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.791, ntokens=17344.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=2770.8, ups=0.16, wpb=17344.8, bsz=1024, num_updates=16900, lr=3.02008e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=32, gb_free=13.3, wall=112262
-2024-11-04 12:42:25 - progress_bar.py[line:274] - INFO: epoch 044:    116 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.88, ntokens=17954.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.36, wps=2995.7, ups=0.17, wpb=17954.9, bsz=1024, num_updates=16910, lr=3.01872e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=112322
-2024-11-04 12:43:22 - progress_bar.py[line:274] - INFO: epoch 044:    126 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.809, ntokens=17678.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=3060.2, ups=0.17, wpb=17678.1, bsz=1024, num_updates=16920, lr=3.01736e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=112379
-2024-11-04 12:44:19 - progress_bar.py[line:274] - INFO: epoch 044:    136 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.813, ntokens=17515.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=3101.7, ups=0.18, wpb=17515.1, bsz=1024, num_updates=16930, lr=3.016e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=112436
-2024-11-04 12:45:16 - progress_bar.py[line:274] - INFO: epoch 044:    146 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.814, ntokens=17641.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=3084.3, ups=0.17, wpb=17641.6, bsz=1024, num_updates=16940, lr=3.01464e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=112493
-2024-11-04 12:46:15 - progress_bar.py[line:274] - INFO: epoch 044:    156 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.84, ntokens=17639.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.16, wps=2999.4, ups=0.17, wpb=17639.9, bsz=1024, num_updates=16950, lr=3.01328e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=112552
-2024-11-04 12:47:13 - progress_bar.py[line:274] - INFO: epoch 044:    166 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.841, ntokens=18099.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.17, wps=3102.8, ups=0.17, wpb=18099.6, bsz=1024, num_updates=16960, lr=3.01192e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=112610
-2024-11-04 12:48:11 - progress_bar.py[line:274] - INFO: epoch 044:    176 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.783, ntokens=17335.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3005.8, ups=0.17, wpb=17335.6, bsz=1024, num_updates=16970, lr=3.01056e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=112668
-2024-11-04 12:49:12 - progress_bar.py[line:274] - INFO: epoch 044:    186 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.813, ntokens=17710.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=2886.2, ups=0.16, wpb=17710.2, bsz=1024, num_updates=16980, lr=3.0092e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=33, gb_free=13.4, wall=112729
-2024-11-04 12:50:18 - progress_bar.py[line:274] - INFO: epoch 044:    196 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.813, ntokens=17649, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=2691, ups=0.15, wpb=17649, bsz=1024, num_updates=16990, lr=3.00784e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=33, gb_free=13.5, wall=112795
-2024-11-04 12:51:17 - progress_bar.py[line:274] - INFO: epoch 044:    206 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.837, ntokens=17935.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=3038.5, ups=0.17, wpb=17935.7, bsz=1024, num_updates=17000, lr=3.00648e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=112854
-2024-11-04 12:52:13 - progress_bar.py[line:274] - INFO: epoch 044:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.809, ntokens=17836.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=3154.8, ups=0.18, wpb=17836.9, bsz=1024, num_updates=17010, lr=3.00512e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=112911
-2024-11-04 12:53:10 - progress_bar.py[line:274] - INFO: epoch 044:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.821, ntokens=17762.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.06, wps=3120.3, ups=0.18, wpb=17762.3, bsz=1024, num_updates=17020, lr=3.00375e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=32, gb_free=13.6, wall=112967
-2024-11-04 12:54:08 - progress_bar.py[line:274] - INFO: epoch 044:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.815, ntokens=17671, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.04, wps=3050.8, ups=0.17, wpb=17671, bsz=1024, num_updates=17030, lr=3.00239e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=113025
-2024-11-04 12:55:05 - progress_bar.py[line:274] - INFO: epoch 044:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.812, ntokens=17463.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=3050.3, ups=0.17, wpb=17463.7, bsz=1024, num_updates=17040, lr=3.00103e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=32, gb_free=13.6, wall=113083
-2024-11-04 12:56:01 - progress_bar.py[line:274] - INFO: epoch 044:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=17845.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3205.4, ups=0.18, wpb=17845.6, bsz=1024, num_updates=17050, lr=2.99967e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=113138
-2024-11-04 12:56:56 - progress_bar.py[line:274] - INFO: epoch 044:    266 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.875, ntokens=18082.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.34, wps=3282.1, ups=0.18, wpb=18082.2, bsz=1024, num_updates=17060, lr=2.99831e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=113193
-2024-11-04 12:57:52 - progress_bar.py[line:274] - INFO: epoch 044:    276 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=17960, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3205.4, ups=0.18, wpb=17960, bsz=1024, num_updates=17070, lr=2.99695e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=113249
-2024-11-04 12:58:50 - progress_bar.py[line:274] - INFO: epoch 044:    286 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.791, ntokens=17458.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=2999.9, ups=0.17, wpb=17458.6, bsz=1024, num_updates=17080, lr=2.99559e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=32, gb_free=13.6, wall=113308
-2024-11-04 12:59:47 - progress_bar.py[line:274] - INFO: epoch 044:    296 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.829, ntokens=17502.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3112.9, ups=0.18, wpb=17502.8, bsz=1024, num_updates=17090, lr=2.99423e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=113364
-2024-11-04 13:00:46 - progress_bar.py[line:274] - INFO: epoch 044:    306 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.986, ntokens=18364.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.92, wps=3122.2, ups=0.17, wpb=18364.8, bsz=1024, num_updates=17100, lr=2.99287e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=32, gb_free=13.4, wall=113423
-2024-11-04 13:01:43 - progress_bar.py[line:274] - INFO: epoch 044:    316 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.921, ntokens=18032.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.57, wps=3121.5, ups=0.17, wpb=18032.8, bsz=1024, num_updates=17110, lr=2.99151e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=32, gb_free=13.4, wall=113480
-2024-11-04 13:02:41 - progress_bar.py[line:274] - INFO: epoch 044:    326 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.924, ntokens=18100.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.59, wps=3139, ups=0.17, wpb=18100.7, bsz=1024, num_updates=17120, lr=2.99015e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=32, gb_free=13.3, wall=113538
-2024-11-04 13:03:36 - progress_bar.py[line:274] - INFO: epoch 044:    336 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.983, ntokens=18435.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.91, wps=3324.7, ups=0.18, wpb=18435.4, bsz=1024, num_updates=17130, lr=2.98879e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=113594
-2024-11-04 13:04:31 - progress_bar.py[line:274] - INFO: epoch 044:    346 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.897, ntokens=18143.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.45, wps=3334.7, ups=0.18, wpb=18143.5, bsz=1024, num_updates=17140, lr=2.98743e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=34, gb_free=13.4, wall=113648
-2024-11-04 13:05:28 - progress_bar.py[line:274] - INFO: epoch 044:    356 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.956, ntokens=18321.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.76, wps=3227.1, ups=0.18, wpb=18321.1, bsz=1024, num_updates=17150, lr=2.98607e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=55, gb_free=13.4, wall=113705
-2024-11-04 13:06:24 - progress_bar.py[line:274] - INFO: epoch 044:    366 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.894, ntokens=17902.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=3197.8, ups=0.18, wpb=17902.6, bsz=1024, num_updates=17160, lr=2.98471e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=56, gb_free=13.4, wall=113761
-2024-11-04 13:07:20 - progress_bar.py[line:274] - INFO: epoch 044:    376 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.89, ntokens=18169.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.41, wps=3232.8, ups=0.18, wpb=18169.4, bsz=1024, num_updates=17170, lr=2.98335e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=56, gb_free=13.4, wall=113817
-2024-11-04 13:08:16 - progress_bar.py[line:274] - INFO: epoch 044:    386 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.927, ntokens=17941.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.61, wps=3199.5, ups=0.18, wpb=17941.9, bsz=1024, num_updates=17180, lr=2.98199e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=56, gb_free=13.4, wall=113873
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_44.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_44.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_44.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_44.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_44.pt2024-11-04 13:08:40 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 44 @ 17185 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_44.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_44.pt2024-11-04 13:08:40 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 13:10:32 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 13:12:26 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 44 @ 17185 updates, score 0) (writing took 226.44486235198565 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_44.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 6 seek offset 300000
-slice_id 3 seek offset 150000
-slice_id 5 seek offset 250000
-slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-2024-11-04 13:13:22 - train.py[line:336] - INFO: end of epoch 44 (average epoch stats below)
-2024-11-04 13:13:22 - progress_bar.py[line:282] - INFO: epoch 044 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.851 | ntokens 17814.5 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.22 | wps 2571.9 | ups 0.14 | wpb 17814.5 | bsz 1023 | num_updates 17185 | lr 2.98131e-05 | gnorm 0.008 | clip 0 | loss_scale 16384 | train_wall 1354 | gb_free 13.3 | wall 114179
-2024-11-04 13:13:22 - trainer.py[line:639] - INFO: loading train data for epoch 45
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 13:16:06 - trainer.py[line:703] - INFO: begin training epoch 45
-2024-11-04 13:16:06 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 13:16:39 - progress_bar.py[line:274] - INFO: epoch 045:      5 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.855, ntokens=17138.9, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.24, wps=340.8, ups=0.02, wpb=17138.9, bsz=985.6, num_updates=17190, lr=2.98063e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=40, gb_free=13.6, wall=114376
-2024-11-04 13:17:43 - progress_bar.py[line:274] - INFO: epoch 045:     15 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.83, ntokens=17794.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=2764.8, ups=0.16, wpb=17794.4, bsz=1024, num_updates=17200, lr=2.97927e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=32, gb_free=13.5, wall=114440
-2024-11-04 13:18:43 - progress_bar.py[line:274] - INFO: epoch 045:     25 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.833, ntokens=17717.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=2952.3, ups=0.17, wpb=17717.5, bsz=1024, num_updates=17210, lr=2.97791e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=32, gb_free=13.6, wall=114500
-2024-11-04 13:19:39 - progress_bar.py[line:274] - INFO: epoch 045:     35 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.807, ntokens=17763.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3162.4, ups=0.18, wpb=17763.5, bsz=1024, num_updates=17220, lr=2.97655e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=114556
-2024-11-04 13:20:37 - progress_bar.py[line:274] - INFO: epoch 045:     45 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.77, ntokens=17488.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=3049.7, ups=0.17, wpb=17488.6, bsz=1024, num_updates=17230, lr=2.97519e-05, gnorm=0.013, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=114614
-2024-11-04 13:21:34 - progress_bar.py[line:274] - INFO: epoch 045:     55 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.808, ntokens=17827.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3125.9, ups=0.18, wpb=17827.6, bsz=1024, num_updates=17240, lr=2.97383e-05, gnorm=0.012, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=114671
-2024-11-04 13:22:30 - progress_bar.py[line:274] - INFO: epoch 045:     65 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.845, ntokens=17882.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.18, wps=3158.7, ups=0.18, wpb=17882.1, bsz=1024, num_updates=17250, lr=2.97247e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=114727
-2024-11-04 13:23:30 - progress_bar.py[line:274] - INFO: epoch 045:     75 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.799, ntokens=17520.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.96, wps=2932.4, ups=0.17, wpb=17520.8, bsz=1024, num_updates=17260, lr=2.97111e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=114787
-2024-11-04 13:24:27 - progress_bar.py[line:274] - INFO: epoch 045:     85 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.833, ntokens=17891.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.13, wps=3151, ups=0.18, wpb=17891.1, bsz=1024, num_updates=17270, lr=2.96974e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=114844
-2024-11-04 13:25:24 - progress_bar.py[line:274] - INFO: epoch 045:     95 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.818, ntokens=17731.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.05, wps=3107.6, ups=0.18, wpb=17731.8, bsz=1024, num_updates=17280, lr=2.96838e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=114901
-2024-11-04 13:26:20 - progress_bar.py[line:274] - INFO: epoch 045:    105 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.779, ntokens=17506.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.86, wps=3125, ups=0.18, wpb=17506.2, bsz=1024, num_updates=17290, lr=2.96702e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=114957
-2024-11-04 13:27:18 - progress_bar.py[line:274] - INFO: epoch 045:    115 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.851, ntokens=17655.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.21, wps=3066.3, ups=0.17, wpb=17655.9, bsz=1024, num_updates=17300, lr=2.96566e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.4, wall=115015
-2024-11-04 13:28:13 - progress_bar.py[line:274] - INFO: epoch 045:    125 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.779, ntokens=17640.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.86, wps=3185.5, ups=0.18, wpb=17640.7, bsz=1024, num_updates=17310, lr=2.9643e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=115070
-2024-11-04 13:29:08 - progress_bar.py[line:274] - INFO: epoch 045:    135 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=17342.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3126.3, ups=0.18, wpb=17342.4, bsz=1024, num_updates=17320, lr=2.96294e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=115126
-2024-11-04 13:30:06 - progress_bar.py[line:274] - INFO: epoch 045:    145 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.793, ntokens=17690.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.93, wps=3086, ups=0.17, wpb=17690.5, bsz=1024, num_updates=17330, lr=2.96158e-05, gnorm=0.011, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=115183
-2024-11-04 13:31:03 - progress_bar.py[line:274] - INFO: epoch 045:    155 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.823, ntokens=17701.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.08, wps=3115.5, ups=0.18, wpb=17701.5, bsz=1024, num_updates=17340, lr=2.96022e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=115240
-2024-11-04 13:31:59 - progress_bar.py[line:274] - INFO: epoch 045:    165 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.807, ntokens=17918.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3186.3, ups=0.18, wpb=17918.6, bsz=1024, num_updates=17350, lr=2.95886e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=115296
-2024-11-04 13:32:56 - progress_bar.py[line:274] - INFO: epoch 045:    175 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.822, ntokens=17707.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.07, wps=3119.5, ups=0.18, wpb=17707.3, bsz=1024, num_updates=17360, lr=2.9575e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=34, gb_free=13.5, wall=115353
-2024-11-04 13:33:50 - progress_bar.py[line:274] - INFO: epoch 045:    185 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.772, ntokens=17550.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=3223.5, ups=0.18, wpb=17550.4, bsz=1024, num_updates=17370, lr=2.95614e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=50, gb_free=13.5, wall=115407
-2024-11-04 13:34:45 - progress_bar.py[line:274] - INFO: epoch 045:    195 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.82, ntokens=17594.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.06, wps=3173.8, ups=0.18, wpb=17594.4, bsz=1024, num_updates=17380, lr=2.95478e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=115463
-2024-11-04 13:35:40 - progress_bar.py[line:274] - INFO: epoch 045:    205 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.823, ntokens=17984.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.07, wps=3291.1, ups=0.18, wpb=17984.8, bsz=1024, num_updates=17390, lr=2.95342e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=41, gb_free=13.5, wall=115517
-2024-11-04 13:36:39 - progress_bar.py[line:274] - INFO: epoch 045:    215 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.801, ntokens=17896.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.97, wps=3058, ups=0.17, wpb=17896.6, bsz=1024, num_updates=17400, lr=2.95206e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=115576
-2024-11-04 13:37:35 - progress_bar.py[line:274] - INFO: epoch 045:    225 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.748, ntokens=17353.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.72, wps=3062.5, ups=0.18, wpb=17353.7, bsz=1024, num_updates=17410, lr=2.9507e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=115632
-2024-11-04 13:38:29 - progress_bar.py[line:274] - INFO: epoch 045:    235 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.812, ntokens=17815.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=3328.6, ups=0.19, wpb=17815.1, bsz=1024, num_updates=17420, lr=2.94934e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=115686
-2024-11-04 13:39:24 - progress_bar.py[line:274] - INFO: epoch 045:    245 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.799, ntokens=17329.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.96, wps=3140.9, ups=0.18, wpb=17329.4, bsz=1024, num_updates=17430, lr=2.94798e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=115741
-2024-11-04 13:40:24 - progress_bar.py[line:274] - INFO: epoch 045:    255 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.81, ntokens=17748.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=2975.9, ups=0.17, wpb=17748.5, bsz=1024, num_updates=17440, lr=2.94662e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=115801
-2024-11-04 13:41:21 - progress_bar.py[line:274] - INFO: epoch 045:    265 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.866, ntokens=18048.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.29, wps=3150.3, ups=0.17, wpb=18048.7, bsz=1024, num_updates=17450, lr=2.94526e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=115858
-2024-11-04 13:42:14 - progress_bar.py[line:274] - INFO: epoch 045:    275 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.803, ntokens=17696.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.98, wps=3333.7, ups=0.19, wpb=17696.7, bsz=1024, num_updates=17460, lr=2.9439e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=115911
-2024-11-04 13:43:11 - progress_bar.py[line:274] - INFO: epoch 045:    285 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17625.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3115.4, ups=0.18, wpb=17625.1, bsz=1024, num_updates=17470, lr=2.94254e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=40, gb_free=13.5, wall=115968
-2024-11-04 13:44:05 - progress_bar.py[line:274] - INFO: epoch 045:    295 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=17516.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3231.2, ups=0.18, wpb=17516.6, bsz=1024, num_updates=17480, lr=2.94118e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=51, gb_free=13.5, wall=116022
-2024-11-04 13:45:01 - progress_bar.py[line:274] - INFO: epoch 045:    305 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.915, ntokens=18170, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=3251.9, ups=0.18, wpb=18170, bsz=1024, num_updates=17490, lr=2.93982e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=116078
-2024-11-04 13:45:57 - progress_bar.py[line:274] - INFO: epoch 045:    315 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.97, ntokens=18373.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.84, wps=3288.7, ups=0.18, wpb=18373.9, bsz=1024, num_updates=17500, lr=2.93846e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=116134
-2024-11-04 13:46:56 - progress_bar.py[line:274] - INFO: epoch 045:    325 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.902, ntokens=17967.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.47, wps=2999.6, ups=0.17, wpb=17967.8, bsz=1024, num_updates=17510, lr=2.9371e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=116194
-2024-11-04 13:47:55 - progress_bar.py[line:274] - INFO: epoch 045:    335 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.901, ntokens=17918.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.47, wps=3084.6, ups=0.17, wpb=17918.3, bsz=1024, num_updates=17520, lr=2.93573e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=116252
-2024-11-04 13:48:53 - progress_bar.py[line:274] - INFO: epoch 045:    345 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.904, ntokens=18309.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.49, wps=3130, ups=0.17, wpb=18309.1, bsz=1024, num_updates=17530, lr=2.93437e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=116310
-2024-11-04 13:49:51 - progress_bar.py[line:274] - INFO: epoch 045:    355 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.015, ntokens=18814.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.08, wps=3222.9, ups=0.17, wpb=18814.7, bsz=1024, num_updates=17540, lr=2.93301e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=116368
-2024-11-04 13:50:48 - progress_bar.py[line:274] - INFO: epoch 045:    365 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.941, ntokens=18346.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.68, wps=3220, ups=0.18, wpb=18346.4, bsz=1024, num_updates=17550, lr=2.93165e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=116425
-2024-11-04 13:51:45 - progress_bar.py[line:274] - INFO: epoch 045:    375 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.843, ntokens=17769.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.17, wps=3110.5, ups=0.18, wpb=17769.5, bsz=1024, num_updates=17560, lr=2.93029e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=116483
-2024-11-04 13:52:45 - progress_bar.py[line:274] - INFO: epoch 045:    385 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.893, ntokens=17867.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=3008.3, ups=0.17, wpb=17867.6, bsz=1024, num_updates=17570, lr=2.92893e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=116542
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_45.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_45.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_45.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_45.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_45.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_45.pt
-2024-11-04 13:53:15 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 45 @ 17576 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_45.pt
-2024-11-04 13:53:15 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 13:55:02 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 13:56:56 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 45 @ 17576 updates, score 0) (writing took 221.52597233001143 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_45.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-2024-11-04 13:57:51 - train.py[line:336] - INFO: end of epoch 45 (average epoch stats below)
-2024-11-04 13:57:51 - progress_bar.py[line:282] - INFO: epoch 045 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.837 | ntokens 17794.7 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.15 | wps 2606.5 | ups 0.15 | wpb 17794.7 | bsz 1023 | num_updates 17576 | lr 2.92812e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1297 | gb_free 13.3 | wall 116848
-2024-11-04 13:57:51 - trainer.py[line:639] - INFO: loading train data for epoch 46
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 14:00:47 - trainer.py[line:703] - INFO: begin training epoch 46
-2024-11-04 14:00:47 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 14:01:12 - progress_bar.py[line:274] - INFO: epoch 046:      4 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.928, ntokens=17615.9, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.61, wps=347.3, ups=0.02, wpb=17615.9, bsz=985.6, num_updates=17580, lr=2.92757e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=117049
-2024-11-04 14:02:11 - progress_bar.py[line:274] - INFO: epoch 046:     14 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.808, ntokens=17652.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=2992.5, ups=0.17, wpb=17652.4, bsz=1024, num_updates=17590, lr=2.92621e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=117108
-2024-11-04 14:03:12 - progress_bar.py[line:274] - INFO: epoch 046:     24 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.79, ntokens=17491.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=2867.5, ups=0.16, wpb=17491.3, bsz=1024, num_updates=17600, lr=2.92485e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=117169
-2024-11-04 14:04:12 - progress_bar.py[line:274] - INFO: epoch 046:     34 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.769, ntokens=17667, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=2957.3, ups=0.17, wpb=17667, bsz=1024, num_updates=17610, lr=2.92349e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=117229
-2024-11-04 14:05:15 - progress_bar.py[line:274] - INFO: epoch 046:     44 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.795, ntokens=17691.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.94, wps=2791.6, ups=0.16, wpb=17691.7, bsz=1024, num_updates=17620, lr=2.92213e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=32, gb_free=13.6, wall=117292
-2024-11-04 14:06:21 - progress_bar.py[line:274] - INFO: epoch 046:     54 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.792, ntokens=17817.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.93, wps=2709.6, ups=0.15, wpb=17817.9, bsz=1024, num_updates=17630, lr=2.92077e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=117358
-2024-11-04 14:07:27 - progress_bar.py[line:274] - INFO: epoch 046:     64 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.87, ntokens=18003.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.31, wps=2740.6, ups=0.15, wpb=18003.6, bsz=1024, num_updates=17640, lr=2.91941e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=117424
-2024-11-04 14:08:32 - progress_bar.py[line:274] - INFO: epoch 046:     74 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.79, ntokens=17571.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=2688.7, ups=0.15, wpb=17571.6, bsz=1024, num_updates=17650, lr=2.91805e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=117489
-2024-11-04 14:09:33 - progress_bar.py[line:274] - INFO: epoch 046:     84 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.837, ntokens=17943.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=2916.6, ups=0.16, wpb=17943.1, bsz=1024, num_updates=17660, lr=2.91669e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=32, gb_free=13.6, wall=117551
-2024-11-04 14:10:32 - progress_bar.py[line:274] - INFO: epoch 046:     94 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.791, ntokens=17752.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=3017.4, ups=0.17, wpb=17752.2, bsz=1024, num_updates=17670, lr=2.91533e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=32, gb_free=13.6, wall=117609
-2024-11-04 14:11:31 - progress_bar.py[line:274] - INFO: epoch 046:    104 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.817, ntokens=17676.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.05, wps=3022.5, ups=0.17, wpb=17676.6, bsz=1024, num_updates=17680, lr=2.91397e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=117668
-2024-11-04 14:12:27 - progress_bar.py[line:274] - INFO: epoch 046:    114 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.789, ntokens=17495.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.91, wps=3099.5, ups=0.18, wpb=17495.6, bsz=1024, num_updates=17690, lr=2.91261e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=117724
-2024-11-04 14:13:27 - progress_bar.py[line:274] - INFO: epoch 046:    124 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.853, ntokens=18079.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.23, wps=3035, ups=0.17, wpb=18079.7, bsz=1024, num_updates=17700, lr=2.91125e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=117784
-2024-11-04 14:14:31 - progress_bar.py[line:274] - INFO: epoch 046:    134 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.736, ntokens=17095.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.66, wps=2651.1, ups=0.16, wpb=17095.2, bsz=1024, num_updates=17710, lr=2.90989e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=117848
-2024-11-04 14:15:36 - progress_bar.py[line:274] - INFO: epoch 046:    144 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=17969.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=2776.1, ups=0.15, wpb=17969.4, bsz=1024, num_updates=17720, lr=2.90853e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=117913
-2024-11-04 14:16:40 - progress_bar.py[line:274] - INFO: epoch 046:    154 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17642.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=2743.7, ups=0.16, wpb=17642.6, bsz=1024, num_updates=17730, lr=2.90717e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=33, gb_free=13.4, wall=117977
-2024-11-04 14:17:47 - progress_bar.py[line:274] - INFO: epoch 046:    164 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.81, ntokens=17811, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=2674.2, ups=0.15, wpb=17811, bsz=1024, num_updates=17740, lr=2.90581e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=33, gb_free=13.4, wall=118044
-2024-11-04 14:18:44 - progress_bar.py[line:274] - INFO: epoch 046:    174 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.828, ntokens=17730.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.1, wps=3086.1, ups=0.17, wpb=17730.9, bsz=1024, num_updates=17750, lr=2.90445e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=118102
-2024-11-04 14:19:44 - progress_bar.py[line:274] - INFO: epoch 046:    184 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.813, ntokens=17859.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=2987.9, ups=0.17, wpb=17859.4, bsz=1024, num_updates=17760, lr=2.90309e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=118161
-2024-11-04 14:20:44 - progress_bar.py[line:274] - INFO: epoch 046:    194 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.817, ntokens=17766.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.05, wps=2992.2, ups=0.17, wpb=17766.1, bsz=1024, num_updates=17770, lr=2.90172e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=118221
-2024-11-04 14:21:44 - progress_bar.py[line:274] - INFO: epoch 046:    204 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.835, ntokens=18038.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.14, wps=3004.1, ups=0.17, wpb=18038.2, bsz=1024, num_updates=17780, lr=2.90036e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=32, gb_free=13.6, wall=118281
-2024-11-04 14:22:41 - progress_bar.py[line:274] - INFO: epoch 046:    214 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.777, ntokens=17787.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=3076.7, ups=0.17, wpb=17787.7, bsz=1024, num_updates=17790, lr=2.899e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=118339
-2024-11-04 14:23:42 - progress_bar.py[line:274] - INFO: epoch 046:    224 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.8, ntokens=17929.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.96, wps=2975, ups=0.17, wpb=17929.5, bsz=1024, num_updates=17800, lr=2.89764e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=118399
-2024-11-04 14:24:41 - progress_bar.py[line:274] - INFO: epoch 046:    234 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.808, ntokens=17759, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=2994.2, ups=0.17, wpb=17759, bsz=1024, num_updates=17810, lr=2.89628e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=118458
-2024-11-04 14:25:41 - progress_bar.py[line:274] - INFO: epoch 046:    244 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.752, ntokens=17288.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.74, wps=2882.1, ups=0.17, wpb=17288.3, bsz=1024, num_updates=17820, lr=2.89492e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=118518
-2024-11-04 14:26:39 - progress_bar.py[line:274] - INFO: epoch 046:    254 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.856, ntokens=17893.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.24, wps=3092.1, ups=0.17, wpb=17893.6, bsz=1024, num_updates=17830, lr=2.89356e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=118576
-2024-11-04 14:27:37 - progress_bar.py[line:274] - INFO: epoch 046:    264 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.83, ntokens=17919.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3064.9, ups=0.17, wpb=17919.9, bsz=1024, num_updates=17840, lr=2.8922e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=118634
-2024-11-04 14:28:40 - progress_bar.py[line:274] - INFO: epoch 046:    274 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.777, ntokens=17545, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.86, wps=2808.3, ups=0.16, wpb=17545, bsz=1024, num_updates=17850, lr=2.89084e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=32, gb_free=13.5, wall=118697
-2024-11-04 14:29:41 - progress_bar.py[line:274] - INFO: epoch 046:    284 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.806, ntokens=17819.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=2928.1, ups=0.16, wpb=17819.7, bsz=1024, num_updates=17860, lr=2.88948e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=118758
-2024-11-04 14:30:40 - progress_bar.py[line:274] - INFO: epoch 046:    294 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.824, ntokens=17602.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.08, wps=2975.2, ups=0.17, wpb=17602.4, bsz=1024, num_updates=17870, lr=2.88812e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=118817
-2024-11-04 14:31:36 - progress_bar.py[line:274] - INFO: epoch 046:    304 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.917, ntokens=18323.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.55, wps=3265.3, ups=0.18, wpb=18323.7, bsz=1024, num_updates=17880, lr=2.88676e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=118873
-2024-11-04 14:32:32 - progress_bar.py[line:274] - INFO: epoch 046:    314 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=3.024, ntokens=18660.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=8.14, wps=3310.3, ups=0.18, wpb=18660.5, bsz=1024, num_updates=17890, lr=2.8854e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=118929
-2024-11-04 14:33:29 - progress_bar.py[line:274] - INFO: epoch 046:    324 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.894, ntokens=18103.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=3209.2, ups=0.18, wpb=18103.5, bsz=1024, num_updates=17900, lr=2.88404e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=118986
-2024-11-04 14:34:28 - progress_bar.py[line:274] - INFO: epoch 046:    334 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.949, ntokens=18274.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.72, wps=3101.7, ups=0.17, wpb=18274.8, bsz=1024, num_updates=17910, lr=2.88268e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=32, gb_free=13.4, wall=119045
-2024-11-04 14:35:30 - progress_bar.py[line:274] - INFO: epoch 046:    344 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.852, ntokens=17981, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=2861.5, ups=0.16, wpb=17981, bsz=1024, num_updates=17920, lr=2.88132e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=32, gb_free=13.5, wall=119108
-2024-11-04 14:36:28 - progress_bar.py[line:274] - INFO: epoch 046:    354 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.953, ntokens=18433.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.74, wps=3205.2, ups=0.17, wpb=18433.2, bsz=1024, num_updates=17930, lr=2.87996e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13, wall=119165
-2024-11-04 14:37:25 - progress_bar.py[line:274] - INFO: epoch 046:    364 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.905, ntokens=18200.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.49, wps=3203.3, ups=0.18, wpb=18200.1, bsz=1024, num_updates=17940, lr=2.8786e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=119222
-2024-11-04 14:38:22 - progress_bar.py[line:274] - INFO: epoch 046:    374 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.9, ntokens=18213.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.46, wps=3173.8, ups=0.17, wpb=18213.3, bsz=1024, num_updates=17950, lr=2.87724e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=119279
-2024-11-04 14:39:19 - progress_bar.py[line:274] - INFO: epoch 046:    384 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.857, ntokens=17810.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.24, wps=3117.8, ups=0.18, wpb=17810.5, bsz=1024, num_updates=17960, lr=2.87588e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=119336
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_46.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_46.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_46.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_46.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_46.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_46.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_46.pt
-2024-11-04 14:39:56 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 46 @ 17967 updates
-2024-11-04 14:39:56 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 14:41:41 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 14:43:36 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 46 @ 17967 updates, score 0) (writing took 220.19040169898653 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_46.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 6 seek offset 300000
-slice_id 2 seek offset 100000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-2024-11-04 14:44:31 - train.py[line:336] - INFO: end of epoch 46 (average epoch stats below)
-2024-11-04 14:44:31 - progress_bar.py[line:282] - INFO: epoch 046 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.835 | ntokens 17838.6 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.14 | wps 2490.9 | ups 0.14 | wpb 17838.6 | bsz 1023 | num_updates 17967 | lr 2.87493e-05 | gnorm 0.008 | clip 0 | loss_scale 65536 | train_wall 1236 | gb_free 13.4 | wall 119648
-2024-11-04 14:44:31 - trainer.py[line:639] - INFO: loading train data for epoch 47
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 14:47:53 - trainer.py[line:703] - INFO: begin training epoch 47
-2024-11-04 14:47:53 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 14:48:14 - progress_bar.py[line:274] - INFO: epoch 047:      3 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.887, ntokens=17285.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.4, wps=323.4, ups=0.02, wpb=17285.5, bsz=985.6, num_updates=17970, lr=2.87452e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=119871
-2024-11-04 14:49:10 - progress_bar.py[line:274] - INFO: epoch 047:     13 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=17488.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3133.8, ups=0.18, wpb=17488.4, bsz=1024, num_updates=17980, lr=2.87316e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=119927
-2024-11-04 14:50:08 - progress_bar.py[line:274] - INFO: epoch 047:     23 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.805, ntokens=17637, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=3020.9, ups=0.17, wpb=17637, bsz=1024, num_updates=17990, lr=2.8718e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=119985
-2024-11-04 14:51:06 - progress_bar.py[line:274] - INFO: epoch 047:     33 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.783, ntokens=17776, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.88, wps=3094.1, ups=0.17, wpb=17776, bsz=1024, num_updates=18000, lr=2.87044e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=120043
-2024-11-04 14:52:04 - progress_bar.py[line:274] - INFO: epoch 047:     43 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.833, ntokens=17905.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.13, wps=3054.5, ups=0.17, wpb=17905.6, bsz=1024, num_updates=18010, lr=2.86908e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=120101
-2024-11-04 14:53:02 - progress_bar.py[line:274] - INFO: epoch 047:     53 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.707, ntokens=17320.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=2968.8, ups=0.17, wpb=17320.9, bsz=1024, num_updates=18020, lr=2.86772e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=120160
-2024-11-04 14:54:00 - progress_bar.py[line:274] - INFO: epoch 047:     63 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.88, ntokens=18040.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.36, wps=3111.5, ups=0.17, wpb=18040.2, bsz=1024, num_updates=18030, lr=2.86635e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=120218
-2024-11-04 14:55:02 - progress_bar.py[line:274] - INFO: epoch 047:     73 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.778, ntokens=17672.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.86, wps=2858.1, ups=0.16, wpb=17672.2, bsz=1024, num_updates=18040, lr=2.86499e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=120279
-2024-11-04 14:56:05 - progress_bar.py[line:274] - INFO: epoch 047:     83 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.778, ntokens=17574.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.86, wps=2797.3, ups=0.16, wpb=17574.7, bsz=1024, num_updates=18050, lr=2.86363e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=120342
-2024-11-04 14:57:05 - progress_bar.py[line:274] - INFO: epoch 047:     93 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.765, ntokens=17645, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=2923.1, ups=0.17, wpb=17645, bsz=1024, num_updates=18060, lr=2.86227e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=120403
-2024-11-04 14:58:03 - progress_bar.py[line:274] - INFO: epoch 047:    103 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.812, ntokens=17685.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=3090.2, ups=0.17, wpb=17685.1, bsz=1024, num_updates=18070, lr=2.86091e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=120460
-2024-11-04 14:59:02 - progress_bar.py[line:274] - INFO: epoch 047:    113 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.813, ntokens=17542, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=2965.6, ups=0.17, wpb=17542, bsz=1024, num_updates=18080, lr=2.85955e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=120519
-2024-11-04 14:59:57 - progress_bar.py[line:274] - INFO: epoch 047:    123 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.783, ntokens=17771.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.88, wps=3211.1, ups=0.18, wpb=17771.9, bsz=1024, num_updates=18090, lr=2.85819e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=120574
-2024-11-04 15:00:57 - progress_bar.py[line:274] - INFO: epoch 047:    133 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.773, ntokens=17571.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=2925.4, ups=0.17, wpb=17571.2, bsz=1024, num_updates=18100, lr=2.85683e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=120634
-2024-11-04 15:02:02 - progress_bar.py[line:274] - INFO: epoch 047:    143 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.807, ntokens=17667.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=2739.5, ups=0.16, wpb=17667.3, bsz=1024, num_updates=18110, lr=2.85547e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=32, gb_free=13.4, wall=120699
-2024-11-04 15:03:02 - progress_bar.py[line:274] - INFO: epoch 047:    153 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=17535, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=2934.1, ups=0.17, wpb=17535, bsz=1024, num_updates=18120, lr=2.85411e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=32, gb_free=13.6, wall=120759
-2024-11-04 15:04:02 - progress_bar.py[line:274] - INFO: epoch 047:    163 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.766, ntokens=17745.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=2911, ups=0.16, wpb=17745.3, bsz=1024, num_updates=18130, lr=2.85275e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=120820
-2024-11-04 15:05:04 - progress_bar.py[line:274] - INFO: epoch 047:    173 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.81, ntokens=17805.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=2892, ups=0.16, wpb=17805.9, bsz=1024, num_updates=18140, lr=2.85139e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=120881
-2024-11-04 15:06:02 - progress_bar.py[line:274] - INFO: epoch 047:    183 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.808, ntokens=17677.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3059.5, ups=0.17, wpb=17677.7, bsz=1024, num_updates=18150, lr=2.85003e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=120939
-2024-11-04 15:06:59 - progress_bar.py[line:274] - INFO: epoch 047:    193 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.781, ntokens=17599.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.87, wps=3053.2, ups=0.17, wpb=17599.1, bsz=1024, num_updates=18160, lr=2.84867e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=120997
-2024-11-04 15:07:55 - progress_bar.py[line:274] - INFO: epoch 047:    203 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.778, ntokens=17770.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.86, wps=3217.5, ups=0.18, wpb=17770.2, bsz=1024, num_updates=18170, lr=2.84731e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=121052
-2024-11-04 15:08:52 - progress_bar.py[line:274] - INFO: epoch 047:    213 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.836, ntokens=18256, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.14, wps=3171.9, ups=0.17, wpb=18256, bsz=1024, num_updates=18180, lr=2.84595e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=121109
-2024-11-04 15:09:52 - progress_bar.py[line:274] - INFO: epoch 047:    223 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.757, ntokens=17571.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.76, wps=2956.6, ups=0.17, wpb=17571.5, bsz=1024, num_updates=18190, lr=2.84459e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=121169
-2024-11-04 15:10:52 - progress_bar.py[line:274] - INFO: epoch 047:    233 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.756, ntokens=17630, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=2932.9, ups=0.17, wpb=17630, bsz=1024, num_updates=18200, lr=2.84323e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=121229
-2024-11-04 15:11:49 - progress_bar.py[line:274] - INFO: epoch 047:    243 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.815, ntokens=17589.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.04, wps=3076.5, ups=0.17, wpb=17589.1, bsz=1024, num_updates=18210, lr=2.84187e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=121286
-2024-11-04 15:12:48 - progress_bar.py[line:274] - INFO: epoch 047:    253 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.805, ntokens=17858.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=3016.8, ups=0.17, wpb=17858.8, bsz=1024, num_updates=18220, lr=2.84051e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=121345
-2024-11-04 15:13:46 - progress_bar.py[line:274] - INFO: epoch 047:    263 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.839, ntokens=17975.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=3135.2, ups=0.17, wpb=17975.6, bsz=1024, num_updates=18230, lr=2.83915e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=121403
-2024-11-04 15:14:44 - progress_bar.py[line:274] - INFO: epoch 047:    273 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.841, ntokens=18026.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.16, wps=3104.9, ups=0.17, wpb=18026.5, bsz=1024, num_updates=18240, lr=2.83779e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=121461
-2024-11-04 15:15:42 - progress_bar.py[line:274] - INFO: epoch 047:    283 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.785, ntokens=17700.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3047.5, ups=0.17, wpb=17700.6, bsz=1024, num_updates=18250, lr=2.83643e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=121519
-2024-11-04 15:16:40 - progress_bar.py[line:274] - INFO: epoch 047:    293 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.779, ntokens=17509.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.87, wps=3000.1, ups=0.17, wpb=17509.2, bsz=1024, num_updates=18260, lr=2.83507e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=121577
-2024-11-04 15:17:38 - progress_bar.py[line:274] - INFO: epoch 047:    303 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.892, ntokens=18179.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.42, wps=3133.3, ups=0.17, wpb=18179.2, bsz=1024, num_updates=18270, lr=2.83371e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=32, gb_free=13.4, wall=121635
-2024-11-04 15:18:34 - progress_bar.py[line:274] - INFO: epoch 047:    313 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.954, ntokens=18217.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.75, wps=3273.3, ups=0.18, wpb=18217.2, bsz=1024, num_updates=18280, lr=2.83234e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=121691
-2024-11-04 15:19:26 - progress_bar.py[line:274] - INFO: epoch 047:    323 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.91, ntokens=18249.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.52, wps=3473.4, ups=0.19, wpb=18249.9, bsz=1024, num_updates=18290, lr=2.83098e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=121743
-2024-11-04 15:20:22 - progress_bar.py[line:274] - INFO: epoch 047:    333 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.954, ntokens=18390.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.75, wps=3319.6, ups=0.18, wpb=18390.5, bsz=1024, num_updates=18300, lr=2.82962e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=43, gb_free=13.1, wall=121799
-2024-11-04 15:21:17 - progress_bar.py[line:274] - INFO: epoch 047:    343 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.878, ntokens=18230.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.35, wps=3268.7, ups=0.18, wpb=18230.5, bsz=1024, num_updates=18310, lr=2.82826e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=121855
-2024-11-04 15:22:13 - progress_bar.py[line:274] - INFO: epoch 047:    353 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.968, ntokens=18859, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.82, wps=3367.2, ups=0.18, wpb=18859, bsz=1024, num_updates=18320, lr=2.8269e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=121911
-2024-11-04 15:23:09 - progress_bar.py[line:274] - INFO: epoch 047:    363 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.992, ntokens=18775.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.96, wps=3408, ups=0.18, wpb=18775.7, bsz=1024, num_updates=18330, lr=2.82554e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=49, gb_free=13.3, wall=121966
-2024-11-04 15:24:05 - progress_bar.py[line:274] - INFO: epoch 047:    373 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.898, ntokens=18287.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.46, wps=3258.5, ups=0.18, wpb=18287.6, bsz=1024, num_updates=18340, lr=2.82418e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=122022
-2024-11-04 15:25:00 - progress_bar.py[line:274] - INFO: epoch 047:    383 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.88, ntokens=18008.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.36, wps=3245.2, ups=0.18, wpb=18008.7, bsz=1024, num_updates=18350, lr=2.82282e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=122077
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_47.pt
-2024-11-04 15:25:41 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 47 @ 18358 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_47.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_47.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_47.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_47.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_47.pt
-2024-11-04 15:25:41 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_47.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 15:27:27 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-04 15:29:21 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 47 @ 18358 updates, score 0) (writing took 219.64827517204685 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_47.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 5 seek offset 250000
-2024-11-04 15:30:16 - train.py[line:336] - INFO: end of epoch 47 (average epoch stats below)
-2024-11-04 15:30:16 - progress_bar.py[line:282] - INFO: epoch 047 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.828 | ntokens 17853.2 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.1 | wps 2542.8 | ups 0.14 | wpb 17853.2 | bsz 1023 | num_updates 18358 | lr 2.82173e-05 | gnorm 0.007 | clip 0 | loss_scale 131072 | train_wall 1346 | gb_free 13.3 | wall 122394
-2024-11-04 15:30:16 - trainer.py[line:639] - INFO: loading train data for epoch 48
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 15:33:17 - trainer.py[line:703] - INFO: begin training epoch 48
-2024-11-04 15:33:17 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 15:33:30 - progress_bar.py[line:274] - INFO: epoch 048:      2 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.897, ntokens=17482.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.45, wps=342.9, ups=0.02, wpb=17482.7, bsz=985.6, num_updates=18360, lr=2.82146e-05, gnorm=0.004, clip=0, loss_scale=131072, train_wall=31, gb_free=13.5, wall=122587
-2024-11-04 15:33:46 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
-2024-11-04 15:34:31 - progress_bar.py[line:274] - INFO: epoch 048:     13 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.775, ntokens=17558.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=2887.6, ups=0.16, wpb=17558.9, bsz=1024, num_updates=18370, lr=2.8201e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=34, gb_free=13.5, wall=122648
-2024-11-04 15:35:29 - progress_bar.py[line:274] - INFO: epoch 048:     23 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.785, ntokens=17584.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3042.7, ups=0.17, wpb=17584.4, bsz=1024, num_updates=18380, lr=2.81874e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=122706
-2024-11-04 15:36:29 - progress_bar.py[line:274] - INFO: epoch 048:     33 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.785, ntokens=17877.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=2966.9, ups=0.17, wpb=17877.7, bsz=1024, num_updates=18390, lr=2.81738e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=32, gb_free=13.4, wall=122766
-2024-11-04 15:37:29 - progress_bar.py[line:274] - INFO: epoch 048:     43 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.809, ntokens=17905.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=2977.6, ups=0.17, wpb=17905.4, bsz=1024, num_updates=18400, lr=2.81602e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=32, gb_free=13.4, wall=122826
-2024-11-04 15:38:29 - progress_bar.py[line:274] - INFO: epoch 048:     53 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.739, ntokens=17588.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.68, wps=2954.2, ups=0.17, wpb=17588.2, bsz=1024, num_updates=18410, lr=2.81466e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=32, gb_free=13.6, wall=122886
-2024-11-04 15:39:27 - progress_bar.py[line:274] - INFO: epoch 048:     63 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.852, ntokens=18012.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=3082.2, ups=0.17, wpb=18012.4, bsz=1024, num_updates=18420, lr=2.8133e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=122944
-2024-11-04 15:40:27 - progress_bar.py[line:274] - INFO: epoch 048:     73 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.793, ntokens=17790.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.93, wps=2989.7, ups=0.17, wpb=17790.4, bsz=1024, num_updates=18430, lr=2.81194e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=32, gb_free=13.5, wall=123004
-2024-11-04 15:41:25 - progress_bar.py[line:274] - INFO: epoch 048:     83 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17627, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3032.1, ups=0.17, wpb=17627, bsz=1024, num_updates=18440, lr=2.81058e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=123062
-2024-11-04 15:42:21 - progress_bar.py[line:274] - INFO: epoch 048:     93 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.763, ntokens=17754.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3125.3, ups=0.18, wpb=17754.5, bsz=1024, num_updates=18450, lr=2.80922e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=123119
-2024-11-04 15:43:23 - progress_bar.py[line:274] - INFO: epoch 048:    103 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.738, ntokens=17325.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.67, wps=2824.3, ups=0.16, wpb=17325.8, bsz=1024, num_updates=18460, lr=2.80786e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=32, gb_free=13.6, wall=123180
-2024-11-04 15:44:19 - progress_bar.py[line:274] - INFO: epoch 048:    113 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.75, ntokens=17110.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3065.9, ups=0.18, wpb=17110.4, bsz=1024, num_updates=18470, lr=2.8065e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=123236
-2024-11-04 15:45:17 - progress_bar.py[line:274] - INFO: epoch 048:    123 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.799, ntokens=17997.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.96, wps=3102.2, ups=0.17, wpb=17997.5, bsz=1024, num_updates=18480, lr=2.80514e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=123294
-2024-11-04 15:45:33 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-04 15:46:19 - progress_bar.py[line:274] - INFO: epoch 048:    134 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.79, ntokens=17642.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=2844.2, ups=0.16, wpb=17642.9, bsz=1024, num_updates=18490, lr=2.80378e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=34, gb_free=13.4, wall=123356
-2024-11-04 15:47:17 - progress_bar.py[line:274] - INFO: epoch 048:    144 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.83, ntokens=18067, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3110.5, ups=0.17, wpb=18067, bsz=1024, num_updates=18500, lr=2.80242e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=123414
-2024-11-04 15:48:21 - progress_bar.py[line:274] - INFO: epoch 048:    154 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.775, ntokens=17656.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=2741.8, ups=0.16, wpb=17656.5, bsz=1024, num_updates=18510, lr=2.80106e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=33, gb_free=13.3, wall=123478
-2024-11-04 15:49:23 - progress_bar.py[line:274] - INFO: epoch 048:    164 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.778, ntokens=17792.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.86, wps=2885.4, ups=0.16, wpb=17792.4, bsz=1024, num_updates=18520, lr=2.7997e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=123540
-2024-11-04 15:50:25 - progress_bar.py[line:274] - INFO: epoch 048:    174 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.824, ntokens=17778.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.08, wps=2872.9, ups=0.16, wpb=17778.8, bsz=1024, num_updates=18530, lr=2.79833e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=123602
-2024-11-04 15:51:27 - progress_bar.py[line:274] - INFO: epoch 048:    184 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.785, ntokens=17693.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=2837.9, ups=0.16, wpb=17693.3, bsz=1024, num_updates=18540, lr=2.79697e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=123664
-2024-11-04 15:52:30 - progress_bar.py[line:274] - INFO: epoch 048:    194 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.814, ntokens=17769.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=2828.1, ups=0.16, wpb=17769.7, bsz=1024, num_updates=18550, lr=2.79561e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=123727
-2024-11-04 15:53:27 - progress_bar.py[line:274] - INFO: epoch 048:    204 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.791, ntokens=17914.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=3124.9, ups=0.17, wpb=17914.1, bsz=1024, num_updates=18560, lr=2.79425e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=123784
-2024-11-04 15:54:26 - progress_bar.py[line:274] - INFO: epoch 048:    214 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.754, ntokens=17828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=3040.5, ups=0.17, wpb=17828, bsz=1024, num_updates=18570, lr=2.79289e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=123843
-2024-11-04 15:55:23 - progress_bar.py[line:274] - INFO: epoch 048:    224 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.794, ntokens=17966, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.94, wps=3131.2, ups=0.17, wpb=17966, bsz=1024, num_updates=18580, lr=2.79153e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=123900
-2024-11-04 15:56:21 - progress_bar.py[line:274] - INFO: epoch 048:    234 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.774, ntokens=17730.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3059.7, ups=0.17, wpb=17730.5, bsz=1024, num_updates=18590, lr=2.79017e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=123958
-2024-11-04 15:57:18 - progress_bar.py[line:274] - INFO: epoch 048:    244 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.794, ntokens=17582.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.93, wps=3094.3, ups=0.18, wpb=17582.2, bsz=1024, num_updates=18600, lr=2.78881e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=124015
-2024-11-04 15:58:15 - progress_bar.py[line:274] - INFO: epoch 048:    254 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.812, ntokens=17911.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=3166.1, ups=0.18, wpb=17911.4, bsz=1024, num_updates=18610, lr=2.78745e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=124072
-2024-11-04 15:59:14 - progress_bar.py[line:274] - INFO: epoch 048:    264 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.836, ntokens=17956, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.14, wps=2996.5, ups=0.17, wpb=17956, bsz=1024, num_updates=18620, lr=2.78609e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=124132
-2024-11-04 16:00:14 - progress_bar.py[line:274] - INFO: epoch 048:    274 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.808, ntokens=17971.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3001.2, ups=0.17, wpb=17971.2, bsz=1024, num_updates=18630, lr=2.78473e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=124191
-2024-11-04 16:01:12 - progress_bar.py[line:274] - INFO: epoch 048:    284 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.746, ntokens=17453.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3027.7, ups=0.17, wpb=17453.8, bsz=1024, num_updates=18640, lr=2.78337e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=124249
-2024-11-04 16:02:09 - progress_bar.py[line:274] - INFO: epoch 048:    294 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.789, ntokens=17658.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.91, wps=3101.6, ups=0.18, wpb=17658.1, bsz=1024, num_updates=18650, lr=2.78201e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=124306
-2024-11-04 16:03:04 - progress_bar.py[line:274] - INFO: epoch 048:    304 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.874, ntokens=18074.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.33, wps=3276.3, ups=0.18, wpb=18074.8, bsz=1024, num_updates=18660, lr=2.78065e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=124361
-2024-11-04 16:04:00 - progress_bar.py[line:274] - INFO: epoch 048:    314 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.961, ntokens=18423.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.78, wps=3282.5, ups=0.18, wpb=18423.8, bsz=1024, num_updates=18670, lr=2.77929e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=124417
-2024-11-04 16:04:59 - progress_bar.py[line:274] - INFO: epoch 048:    324 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.892, ntokens=18160.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.42, wps=3110.3, ups=0.17, wpb=18160.8, bsz=1024, num_updates=18680, lr=2.77793e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=124476
-2024-11-04 16:05:53 - progress_bar.py[line:274] - INFO: epoch 048:    334 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.914, ntokens=18081.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.53, wps=3335.6, ups=0.18, wpb=18081.8, bsz=1024, num_updates=18690, lr=2.77657e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=124530
-2024-11-04 16:06:46 - progress_bar.py[line:274] - INFO: epoch 048:    344 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.854, ntokens=18004.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.23, wps=3369.6, ups=0.19, wpb=18004.3, bsz=1024, num_updates=18700, lr=2.77521e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=124583
-2024-11-04 16:07:41 - progress_bar.py[line:274] - INFO: epoch 048:    354 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.917, ntokens=18442.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.55, wps=3353.6, ups=0.18, wpb=18442.7, bsz=1024, num_updates=18710, lr=2.77385e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.1, wall=124638
-2024-11-04 16:08:35 - progress_bar.py[line:274] - INFO: epoch 048:    364 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.922, ntokens=18273.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.58, wps=3404.3, ups=0.19, wpb=18273.7, bsz=1024, num_updates=18720, lr=2.77249e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=41, gb_free=13.4, wall=124692
-2024-11-04 16:09:31 - progress_bar.py[line:274] - INFO: epoch 048:    374 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.843, ntokens=17959.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.17, wps=3182.8, ups=0.18, wpb=17959.8, bsz=1024, num_updates=18730, lr=2.77113e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=49, gb_free=13.4, wall=124748
-2024-11-04 16:10:28 - progress_bar.py[line:274] - INFO: epoch 048:    384 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.877, ntokens=17937.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.35, wps=3192.1, ups=0.18, wpb=17937.9, bsz=1024, num_updates=18740, lr=2.76977e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=124805
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.pt
-2024-11-04 16:11:03 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 48 @ 18747 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.pt
-2024-11-04 16:11:03 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.pt': File exists: File exists
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 16:12:50 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 7 seek offset 350000
-2024-11-04 16:14:44 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 48 @ 18747 updates, score 0) (writing took 221.35087116301293 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_48.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-2024-11-04 16:15:39 - train.py[line:336] - INFO: end of epoch 48 (average epoch stats below)
-2024-11-04 16:15:39 - progress_bar.py[line:282] - INFO: epoch 048 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.814 | ntokens 17820.5 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 7.03 | wps 2545.9 | ups 0.14 | wpb 17820.5 | bsz 1023 | num_updates 18747 | lr 2.76881e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 1289 | gb_free 13.3 | wall 125116
-2024-11-04 16:15:39 - trainer.py[line:639] - INFO: loading train data for epoch 49
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 16:18:35 - trainer.py[line:703] - INFO: begin training epoch 49
-2024-11-04 16:18:35 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 16:18:56 - progress_bar.py[line:274] - INFO: epoch 049:      3 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.804, ntokens=17140.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.98, wps=337.4, ups=0.02, wpb=17140.5, bsz=985.6, num_updates=18750, lr=2.76841e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=45, gb_free=13.5, wall=125313
-2024-11-04 16:19:53 - progress_bar.py[line:274] - INFO: epoch 049:     13 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.752, ntokens=17456.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3059.2, ups=0.18, wpb=17456.5, bsz=1024, num_updates=18760, lr=2.76705e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=125370
-2024-11-04 16:20:52 - progress_bar.py[line:274] - INFO: epoch 049:     23 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.821, ntokens=17845.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.07, wps=3018.8, ups=0.17, wpb=17845.8, bsz=1024, num_updates=18770, lr=2.76569e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=125429
-2024-11-04 16:21:49 - progress_bar.py[line:274] - INFO: epoch 049:     33 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.691, ntokens=17343.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3041.5, ups=0.18, wpb=17343.8, bsz=1024, num_updates=18780, lr=2.76432e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=125486
-2024-11-04 16:22:45 - progress_bar.py[line:274] - INFO: epoch 049:     43 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.76, ntokens=17656.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.77, wps=3160.7, ups=0.18, wpb=17656.9, bsz=1024, num_updates=18790, lr=2.76296e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=125542
-2024-11-04 16:23:38 - progress_bar.py[line:274] - INFO: epoch 049:     53 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.766, ntokens=17664.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=3311.8, ups=0.19, wpb=17664.7, bsz=1024, num_updates=18800, lr=2.7616e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=125595
-2024-11-04 16:24:35 - progress_bar.py[line:274] - INFO: epoch 049:     63 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.863, ntokens=18264, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.28, wps=3196.5, ups=0.18, wpb=18264, bsz=1024, num_updates=18810, lr=2.76024e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=49, gb_free=13.5, wall=125652
-2024-11-04 16:25:33 - progress_bar.py[line:274] - INFO: epoch 049:     73 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.826, ntokens=17993.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.09, wps=3099.3, ups=0.17, wpb=17993.2, bsz=1024, num_updates=18820, lr=2.75888e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=125710
-2024-11-04 16:26:30 - progress_bar.py[line:274] - INFO: epoch 049:     83 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.771, ntokens=17638.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=3112.2, ups=0.18, wpb=17638.4, bsz=1024, num_updates=18830, lr=2.75752e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=125767
-2024-11-04 16:27:27 - progress_bar.py[line:274] - INFO: epoch 049:     93 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.775, ntokens=17728.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3109.9, ups=0.18, wpb=17728.8, bsz=1024, num_updates=18840, lr=2.75616e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=125824
-2024-11-04 16:28:23 - progress_bar.py[line:274] - INFO: epoch 049:    103 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.762, ntokens=17529.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.78, wps=3098.3, ups=0.18, wpb=17529.2, bsz=1024, num_updates=18850, lr=2.7548e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=125881
-2024-11-04 16:29:20 - progress_bar.py[line:274] - INFO: epoch 049:    113 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.79, ntokens=17516.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=3085.2, ups=0.18, wpb=17516.5, bsz=1024, num_updates=18860, lr=2.75344e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=125937
-2024-11-04 16:30:17 - progress_bar.py[line:274] - INFO: epoch 049:    123 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17869.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3158.6, ups=0.18, wpb=17869.5, bsz=1024, num_updates=18870, lr=2.75208e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=125994
-2024-11-04 16:31:13 - progress_bar.py[line:274] - INFO: epoch 049:    133 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.754, ntokens=17473.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=3109.4, ups=0.18, wpb=17473.1, bsz=1024, num_updates=18880, lr=2.75072e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=126050
-2024-11-04 16:32:09 - progress_bar.py[line:274] - INFO: epoch 049:    143 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.804, ntokens=17910, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=3182.6, ups=0.18, wpb=17910, bsz=1024, num_updates=18890, lr=2.74936e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=56, gb_free=13.3, wall=126106
-2024-11-04 16:33:06 - progress_bar.py[line:274] - INFO: epoch 049:    153 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.758, ntokens=17586.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.77, wps=3090.1, ups=0.18, wpb=17586.3, bsz=1024, num_updates=18900, lr=2.748e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=126163
-2024-11-04 16:34:04 - progress_bar.py[line:274] - INFO: epoch 049:    163 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.716, ntokens=17591.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3053.9, ups=0.17, wpb=17591.8, bsz=1024, num_updates=18910, lr=2.74664e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=126221
-2024-11-04 16:35:01 - progress_bar.py[line:274] - INFO: epoch 049:    173 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.806, ntokens=17811.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3125.8, ups=0.18, wpb=17811.8, bsz=1024, num_updates=18920, lr=2.74528e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.4, wall=126278
-2024-11-04 16:35:57 - progress_bar.py[line:274] - INFO: epoch 049:    183 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.758, ntokens=17611.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.77, wps=3150.2, ups=0.18, wpb=17611.5, bsz=1024, num_updates=18930, lr=2.74392e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=126334
-2024-11-04 16:36:51 - progress_bar.py[line:274] - INFO: epoch 049:    193 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.795, ntokens=17813.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.94, wps=3263.3, ups=0.18, wpb=17813.6, bsz=1024, num_updates=18940, lr=2.74256e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=126388
-2024-11-04 16:37:52 - progress_bar.py[line:274] - INFO: epoch 049:    203 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.781, ntokens=17846.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.88, wps=2942.9, ups=0.16, wpb=17846.9, bsz=1024, num_updates=18950, lr=2.7412e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=126449
-2024-11-04 16:38:53 - progress_bar.py[line:274] - INFO: epoch 049:    213 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.807, ntokens=18213.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3004.4, ups=0.16, wpb=18213.4, bsz=1024, num_updates=18960, lr=2.73984e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=126510
-2024-11-04 16:39:52 - progress_bar.py[line:274] - INFO: epoch 049:    223 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.755, ntokens=17737.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=2996.3, ups=0.17, wpb=17737.2, bsz=1024, num_updates=18970, lr=2.73848e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=126569
-2024-11-04 16:40:52 - progress_bar.py[line:274] - INFO: epoch 049:    233 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.749, ntokens=17699, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.72, wps=2942.9, ups=0.17, wpb=17699, bsz=1024, num_updates=18980, lr=2.73712e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=126629
-2024-11-04 16:41:51 - progress_bar.py[line:274] - INFO: epoch 049:    243 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.772, ntokens=17459.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=2940.5, ups=0.17, wpb=17459.2, bsz=1024, num_updates=18990, lr=2.73576e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=126688
-2024-11-04 16:42:49 - progress_bar.py[line:274] - INFO: epoch 049:    253 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.775, ntokens=17668.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3050.6, ups=0.17, wpb=17668.4, bsz=1024, num_updates=19000, lr=2.7344e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=126746
-2024-11-04 16:43:45 - progress_bar.py[line:274] - INFO: epoch 049:    263 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=18008.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3220.9, ups=0.18, wpb=18008.3, bsz=1024, num_updates=19010, lr=2.73304e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=126802
-2024-11-04 16:44:40 - progress_bar.py[line:274] - INFO: epoch 049:    273 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.796, ntokens=17895, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.94, wps=3265.8, ups=0.18, wpb=17895, bsz=1024, num_updates=19020, lr=2.73168e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=126857
-2024-11-04 16:45:34 - progress_bar.py[line:274] - INFO: epoch 049:    283 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.762, ntokens=17688.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.78, wps=3264.3, ups=0.18, wpb=17688.8, bsz=1024, num_updates=19030, lr=2.73032e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=126911
-2024-11-04 16:46:29 - progress_bar.py[line:274] - INFO: epoch 049:    293 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.827, ntokens=17933.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.1, wps=3255.7, ups=0.18, wpb=17933.2, bsz=1024, num_updates=19040, lr=2.72895e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=36, gb_free=13.4, wall=126966
-2024-11-04 16:47:25 - progress_bar.py[line:274] - INFO: epoch 049:    303 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.862, ntokens=17989.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.27, wps=3206.2, ups=0.18, wpb=17989.1, bsz=1024, num_updates=19050, lr=2.72759e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=127022
-2024-11-04 16:48:21 - progress_bar.py[line:274] - INFO: epoch 049:    313 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.913, ntokens=18203.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.53, wps=3254, ups=0.18, wpb=18203.2, bsz=1024, num_updates=19060, lr=2.72623e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=127078
-2024-11-04 16:49:17 - progress_bar.py[line:274] - INFO: epoch 049:    323 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.883, ntokens=18208, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.38, wps=3275.5, ups=0.18, wpb=18208, bsz=1024, num_updates=19070, lr=2.72487e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=127134
-2024-11-04 16:50:12 - progress_bar.py[line:274] - INFO: epoch 049:    333 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.943, ntokens=18558, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.69, wps=3336.2, ups=0.18, wpb=18558, bsz=1024, num_updates=19080, lr=2.72351e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.2, wall=127190
-2024-11-04 16:51:09 - progress_bar.py[line:274] - INFO: epoch 049:    343 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.815, ntokens=17944.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.04, wps=3181.8, ups=0.18, wpb=17944.3, bsz=1024, num_updates=19090, lr=2.72215e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=127246
-2024-11-04 16:52:05 - progress_bar.py[line:274] - INFO: epoch 049:    353 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.878, ntokens=18345.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.35, wps=3254.6, ups=0.18, wpb=18345.4, bsz=1024, num_updates=19100, lr=2.72079e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=127302
-2024-11-04 16:53:01 - progress_bar.py[line:274] - INFO: epoch 049:    363 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.876, ntokens=18117.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.34, wps=3266.2, ups=0.18, wpb=18117.9, bsz=1024, num_updates=19110, lr=2.71943e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=127358
-2024-11-04 16:53:56 - progress_bar.py[line:274] - INFO: epoch 049:    373 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.886, ntokens=18410.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3323.5, ups=0.18, wpb=18410.5, bsz=1024, num_updates=19120, lr=2.71807e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=127413
-2024-11-04 16:54:52 - progress_bar.py[line:274] - INFO: epoch 049:    383 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.851, ntokens=17882.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=3175.6, ups=0.18, wpb=17882.4, bsz=1024, num_updates=19130, lr=2.71671e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=127470
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 16:55:33 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 49 @ 19138 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.pt
-
-
-
-2024-11-04 16:55:33 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.pt': File exists
-: File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 16:57:24 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-2024-11-04 16:59:18 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 49 @ 19138 updates, score 0) (writing took 225.18289937800728 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_49.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 1 seek offset 50000
-slice_id 6 seek offset 300000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-2024-11-04 17:00:13 - train.py[line:336] - INFO: end of epoch 49 (average epoch stats below)
-2024-11-04 17:00:13 - progress_bar.py[line:282] - INFO: epoch 049 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.806 | ntokens 17834.2 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.99 | wps 2608 | ups 0.15 | wpb 17834.2 | bsz 1023 | num_updates 19138 | lr 2.71562e-05 | gnorm 0.006 | clip 0 | loss_scale 65536 | train_wall 1803 | gb_free 13.4 | wall 127790
-2024-11-04 17:00:13 - trainer.py[line:639] - INFO: loading train data for epoch 50
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 17:02:57 - trainer.py[line:703] - INFO: begin training epoch 50
-2024-11-04 17:02:57 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 17:03:10 - progress_bar.py[line:274] - INFO: epoch 050:      2 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.862, ntokens=17379.9, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.27, wps=349.4, ups=0.02, wpb=17379.9, bsz=985.6, num_updates=19140, lr=2.71535e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=46, gb_free=13.5, wall=127967
-2024-11-04 17:04:08 - progress_bar.py[line:274] - INFO: epoch 050:     12 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.724, ntokens=17167.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.61, wps=2977.8, ups=0.17, wpb=17167.2, bsz=1024, num_updates=19150, lr=2.71399e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=128025
-2024-11-04 17:05:05 - progress_bar.py[line:274] - INFO: epoch 050:     22 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.81, ntokens=18040, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=3148.3, ups=0.17, wpb=18040, bsz=1024, num_updates=19160, lr=2.71263e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=128082
-2024-11-04 17:05:59 - progress_bar.py[line:274] - INFO: epoch 050:     32 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.765, ntokens=17894.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=3278, ups=0.18, wpb=17894.8, bsz=1024, num_updates=19170, lr=2.71127e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=39, gb_free=13.5, wall=128137
-2024-11-04 17:06:52 - progress_bar.py[line:274] - INFO: epoch 050:     42 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.791, ntokens=17885.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=3384.1, ups=0.19, wpb=17885.3, bsz=1024, num_updates=19180, lr=2.70991e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=41, gb_free=13.6, wall=128189
-2024-11-04 17:07:47 - progress_bar.py[line:274] - INFO: epoch 050:     52 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.723, ntokens=17517.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.6, wps=3211.2, ups=0.18, wpb=17517.2, bsz=1024, num_updates=19190, lr=2.70855e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=128244
-2024-11-04 17:08:40 - progress_bar.py[line:274] - INFO: epoch 050:     62 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.833, ntokens=18003.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.13, wps=3390.7, ups=0.19, wpb=18003.4, bsz=1024, num_updates=19200, lr=2.70719e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=128297
-2024-11-04 17:09:34 - progress_bar.py[line:274] - INFO: epoch 050:     72 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.75, ntokens=17579.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3246.9, ups=0.18, wpb=17579.3, bsz=1024, num_updates=19210, lr=2.70583e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=36, gb_free=13.5, wall=128351
-2024-11-04 17:10:28 - progress_bar.py[line:274] - INFO: epoch 050:     82 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=17643.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3299.6, ups=0.19, wpb=17643.1, bsz=1024, num_updates=19220, lr=2.70447e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=32, gb_free=13.5, wall=128405
-2024-11-04 17:11:26 - progress_bar.py[line:274] - INFO: epoch 050:     92 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.745, ntokens=17606.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3011, ups=0.17, wpb=17606.2, bsz=1024, num_updates=19230, lr=2.70311e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=128463
-2024-11-04 17:12:25 - progress_bar.py[line:274] - INFO: epoch 050:    102 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.786, ntokens=17738.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.9, wps=3023.6, ups=0.17, wpb=17738.1, bsz=1024, num_updates=19240, lr=2.70175e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=128522
-2024-11-04 17:13:23 - progress_bar.py[line:274] - INFO: epoch 050:    112 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.793, ntokens=17676.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.93, wps=3048.9, ups=0.17, wpb=17676.1, bsz=1024, num_updates=19250, lr=2.70039e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=128580
-2024-11-04 17:14:17 - progress_bar.py[line:274] - INFO: epoch 050:    122 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.691, ntokens=17423.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3206, ups=0.18, wpb=17423.1, bsz=1024, num_updates=19260, lr=2.69903e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=128634
-2024-11-04 17:15:15 - progress_bar.py[line:274] - INFO: epoch 050:    132 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.801, ntokens=17817.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.97, wps=3076.9, ups=0.17, wpb=17817.5, bsz=1024, num_updates=19270, lr=2.69767e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=49, gb_free=13.5, wall=128692
-2024-11-04 17:16:13 - progress_bar.py[line:274] - INFO: epoch 050:    142 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.772, ntokens=17650.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=3045.6, ups=0.17, wpb=17650.4, bsz=1024, num_updates=19280, lr=2.69631e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=128750
-2024-11-04 17:17:07 - progress_bar.py[line:274] - INFO: epoch 050:    152 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.801, ntokens=17873.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.97, wps=3281.6, ups=0.18, wpb=17873.8, bsz=1024, num_updates=19290, lr=2.69494e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=47, gb_free=13.5, wall=128804
-2024-11-04 17:18:04 - progress_bar.py[line:274] - INFO: epoch 050:    162 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.69, ntokens=17233.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3038.2, ups=0.18, wpb=17233.6, bsz=1024, num_updates=19300, lr=2.69358e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=128861
-2024-11-04 17:19:01 - progress_bar.py[line:274] - INFO: epoch 050:    172 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.722, ntokens=17415.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.6, wps=3079.4, ups=0.18, wpb=17415.1, bsz=1024, num_updates=19310, lr=2.69222e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=128918
-2024-11-04 17:19:58 - progress_bar.py[line:274] - INFO: epoch 050:    182 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.746, ntokens=17464.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3069.5, ups=0.18, wpb=17464.6, bsz=1024, num_updates=19320, lr=2.69086e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=128975
-2024-11-04 17:20:55 - progress_bar.py[line:274] - INFO: epoch 050:    192 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.815, ntokens=18080.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.04, wps=3144.6, ups=0.17, wpb=18080.3, bsz=1024, num_updates=19330, lr=2.6895e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=129032
-2024-11-04 17:21:51 - progress_bar.py[line:274] - INFO: epoch 050:    202 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.762, ntokens=17715.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.78, wps=3142.8, ups=0.18, wpb=17715.5, bsz=1024, num_updates=19340, lr=2.68814e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=129088
-2024-11-04 17:22:48 - progress_bar.py[line:274] - INFO: epoch 050:    212 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.777, ntokens=18042.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=3182.8, ups=0.18, wpb=18042.5, bsz=1024, num_updates=19350, lr=2.68678e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=129145
-2024-11-04 17:23:46 - progress_bar.py[line:274] - INFO: epoch 050:    222 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.775, ntokens=17898.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=3082.5, ups=0.17, wpb=17898.7, bsz=1024, num_updates=19360, lr=2.68542e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=50, gb_free=13.4, wall=129203
-2024-11-04 17:24:44 - progress_bar.py[line:274] - INFO: epoch 050:    232 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.746, ntokens=17620.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3049.7, ups=0.17, wpb=17620.8, bsz=1024, num_updates=19370, lr=2.68406e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=129261
-2024-11-04 17:25:42 - progress_bar.py[line:274] - INFO: epoch 050:    242 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.76, ntokens=17455, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.77, wps=3008.8, ups=0.17, wpb=17455, bsz=1024, num_updates=19380, lr=2.6827e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=129319
-2024-11-04 17:26:40 - progress_bar.py[line:274] - INFO: epoch 050:    252 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.806, ntokens=17877.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=3092.2, ups=0.17, wpb=17877.6, bsz=1024, num_updates=19390, lr=2.68134e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=129377
-2024-11-04 17:27:38 - progress_bar.py[line:274] - INFO: epoch 050:    262 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.81, ntokens=17967.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=3106.1, ups=0.17, wpb=17967.9, bsz=1024, num_updates=19400, lr=2.67998e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=129435
-2024-11-04 17:28:34 - progress_bar.py[line:274] - INFO: epoch 050:    272 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.774, ntokens=17770.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3171.1, ups=0.18, wpb=17770.8, bsz=1024, num_updates=19410, lr=2.67862e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=129491
-2024-11-04 17:29:28 - progress_bar.py[line:274] - INFO: epoch 050:    282 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.745, ntokens=17557.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3251.6, ups=0.19, wpb=17557.1, bsz=1024, num_updates=19420, lr=2.67726e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=129545
-2024-11-04 17:30:25 - progress_bar.py[line:274] - INFO: epoch 050:    292 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.846, ntokens=18117.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.19, wps=3138.4, ups=0.17, wpb=18117.2, bsz=1024, num_updates=19430, lr=2.6759e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=129602
-2024-11-04 17:31:27 - progress_bar.py[line:274] - INFO: epoch 050:    302 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.825, ntokens=17845.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.09, wps=2899.8, ups=0.16, wpb=17845.2, bsz=1024, num_updates=19440, lr=2.67454e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=61, gb_free=13.4, wall=129664
-2024-11-04 17:32:23 - progress_bar.py[line:274] - INFO: epoch 050:    312 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.934, ntokens=18383.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.64, wps=3249.5, ups=0.18, wpb=18383.3, bsz=1024, num_updates=19450, lr=2.67318e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=129721
-2024-11-04 17:33:21 - progress_bar.py[line:274] - INFO: epoch 050:    322 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.852, ntokens=18028.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=3156.7, ups=0.18, wpb=18028.7, bsz=1024, num_updates=19460, lr=2.67182e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=129778
-2024-11-04 17:34:18 - progress_bar.py[line:274] - INFO: epoch 050:    332 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.908, ntokens=18424.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3189.5, ups=0.17, wpb=18424.2, bsz=1024, num_updates=19470, lr=2.67046e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=129835
-2024-11-04 17:35:16 - progress_bar.py[line:274] - INFO: epoch 050:    342 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.827, ntokens=17963.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.1, wps=3114.7, ups=0.17, wpb=17963.9, bsz=1024, num_updates=19480, lr=2.6691e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.4, wall=129893
-2024-11-04 17:36:11 - progress_bar.py[line:274] - INFO: epoch 050:    352 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.917, ntokens=18490.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.55, wps=3382.5, ups=0.18, wpb=18490.7, bsz=1024, num_updates=19490, lr=2.66774e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=129948
-2024-11-04 17:37:04 - progress_bar.py[line:274] - INFO: epoch 050:    362 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.886, ntokens=18181.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3400.3, ups=0.19, wpb=18181.5, bsz=1024, num_updates=19500, lr=2.66638e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=130001
-2024-11-04 17:37:59 - progress_bar.py[line:274] - INFO: epoch 050:    372 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=17962, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3292.8, ups=0.18, wpb=17962, bsz=1024, num_updates=19510, lr=2.66502e-05, gnorm=0.007, clip=0, loss_scale=131072, train_wall=54, gb_free=13.4, wall=130056
-2024-11-04 17:38:43 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
-2024-11-04 17:38:59 - progress_bar.py[line:274] - INFO: epoch 050:    383 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.817, ntokens=17752.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.05, wps=2939.5, ups=0.17, wpb=17752.9, bsz=1024, num_updates=19520, lr=2.66366e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=60, gb_free=13.5, wall=130116
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_50.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_50.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_50.pt
-2024-11-04 17:39:40 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 50 @ 19528 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_50.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_50.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_50.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_50.pt
-
-2024-11-04 17:39:40 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_50.pt': File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 17:41:26 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-2024-11-04 17:43:20 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 50 @ 19528 updates, score 0) (writing took 220.0079433969804 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_50.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-slice_id 2 seek offset 100000
-slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-2024-11-04 17:44:15 - train.py[line:336] - INFO: end of epoch 50 (average epoch stats below)
-2024-11-04 17:44:15 - progress_bar.py[line:282] - INFO: epoch 050 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.793 | ntokens 17784.5 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.93 | wps 2625.5 | ups 0.15 | wpb 17784.5 | bsz 1023 | num_updates 19528 | lr 2.66257e-05 | gnorm 0.007 | clip 0 | loss_scale 65536 | train_wall 1729 | gb_free 13.3 | wall 130432
-2024-11-04 17:44:15 - trainer.py[line:639] - INFO: loading train data for epoch 51
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 17:47:10 - trainer.py[line:703] - INFO: begin training epoch 51
-2024-11-04 17:47:10 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 17:47:24 - progress_bar.py[line:274] - INFO: epoch 051:      2 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.78, ntokens=16847.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.87, wps=333.8, ups=0.02, wpb=16847.3, bsz=985.6, num_updates=19530, lr=2.6623e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=35, gb_free=13.5, wall=130621
-2024-11-04 17:48:18 - progress_bar.py[line:274] - INFO: epoch 051:     12 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.74, ntokens=17460.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.68, wps=3200, ups=0.18, wpb=17460.6, bsz=1024, num_updates=19540, lr=2.66093e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=130675
-2024-11-04 17:49:15 - progress_bar.py[line:274] - INFO: epoch 051:     22 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.79, ntokens=17804.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=3155.7, ups=0.18, wpb=17804.2, bsz=1024, num_updates=19550, lr=2.65957e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=130732
-2024-11-04 17:50:12 - progress_bar.py[line:274] - INFO: epoch 051:     32 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.767, ntokens=17929.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.81, wps=3153.4, ups=0.18, wpb=17929.5, bsz=1024, num_updates=19560, lr=2.65821e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=41, gb_free=13.4, wall=130789
-2024-11-04 17:51:10 - progress_bar.py[line:274] - INFO: epoch 051:     42 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.775, ntokens=17823, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=3074.5, ups=0.17, wpb=17823, bsz=1024, num_updates=19570, lr=2.65685e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=130847
-2024-11-04 17:52:05 - progress_bar.py[line:274] - INFO: epoch 051:     52 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.695, ntokens=17468.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.48, wps=3160.6, ups=0.18, wpb=17468.8, bsz=1024, num_updates=19580, lr=2.65549e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=130902
-2024-11-04 17:53:03 - progress_bar.py[line:274] - INFO: epoch 051:     62 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.83, ntokens=18099.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3113.1, ups=0.17, wpb=18099.3, bsz=1024, num_updates=19590, lr=2.65413e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=130960
-2024-11-04 17:53:57 - progress_bar.py[line:274] - INFO: epoch 051:     72 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.739, ntokens=17612.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.68, wps=3251.3, ups=0.18, wpb=17612.2, bsz=1024, num_updates=19600, lr=2.65277e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=131014
-2024-11-04 17:54:53 - progress_bar.py[line:274] - INFO: epoch 051:     82 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.744, ntokens=17494.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3137.2, ups=0.18, wpb=17494.5, bsz=1024, num_updates=19610, lr=2.65141e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=131070
-2024-11-04 17:55:50 - progress_bar.py[line:274] - INFO: epoch 051:     92 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.774, ntokens=17991.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3162.8, ups=0.18, wpb=17991.7, bsz=1024, num_updates=19620, lr=2.65005e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=131127
-2024-11-04 17:56:46 - progress_bar.py[line:274] - INFO: epoch 051:    102 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.728, ntokens=17390.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3114.5, ups=0.18, wpb=17390.3, bsz=1024, num_updates=19630, lr=2.64869e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=131183
-2024-11-04 17:57:43 - progress_bar.py[line:274] - INFO: epoch 051:    112 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.755, ntokens=17631.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=3091.1, ups=0.18, wpb=17631.6, bsz=1024, num_updates=19640, lr=2.64733e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=131240
-2024-11-04 17:58:39 - progress_bar.py[line:274] - INFO: epoch 051:    122 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.747, ntokens=17654.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3129.3, ups=0.18, wpb=17654.6, bsz=1024, num_updates=19650, lr=2.64597e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=131296
-2024-11-04 17:59:33 - progress_bar.py[line:274] - INFO: epoch 051:    132 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.745, ntokens=17543, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3264.8, ups=0.19, wpb=17543, bsz=1024, num_updates=19660, lr=2.64461e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=131350
-2024-11-04 18:00:30 - progress_bar.py[line:274] - INFO: epoch 051:    142 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.765, ntokens=17595.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=3107.1, ups=0.18, wpb=17595.7, bsz=1024, num_updates=19670, lr=2.64325e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=131407
-2024-11-04 18:01:26 - progress_bar.py[line:274] - INFO: epoch 051:    152 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.742, ntokens=17646.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3112.1, ups=0.18, wpb=17646.4, bsz=1024, num_updates=19680, lr=2.64189e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=131463
-2024-11-04 18:02:23 - progress_bar.py[line:274] - INFO: epoch 051:    162 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.739, ntokens=17714.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.68, wps=3110.4, ups=0.18, wpb=17714.2, bsz=1024, num_updates=19690, lr=2.64053e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=131520
-2024-11-04 18:03:19 - progress_bar.py[line:274] - INFO: epoch 051:    172 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.752, ntokens=17722.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.74, wps=3164.8, ups=0.18, wpb=17722.3, bsz=1024, num_updates=19700, lr=2.63917e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=131576
-2024-11-04 18:04:17 - progress_bar.py[line:274] - INFO: epoch 051:    182 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.731, ntokens=17518, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.64, wps=3052.8, ups=0.17, wpb=17518, bsz=1024, num_updates=19710, lr=2.63781e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=131634
-2024-11-04 18:05:13 - progress_bar.py[line:274] - INFO: epoch 051:    192 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.732, ntokens=17651.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.64, wps=3120.8, ups=0.18, wpb=17651.8, bsz=1024, num_updates=19720, lr=2.63645e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=131690
-2024-11-04 18:06:10 - progress_bar.py[line:274] - INFO: epoch 051:    202 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=17829.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3131.4, ups=0.18, wpb=17829.6, bsz=1024, num_updates=19730, lr=2.63509e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=131747
-2024-11-04 18:07:06 - progress_bar.py[line:274] - INFO: epoch 051:    212 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.752, ntokens=17922.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.74, wps=3205.2, ups=0.18, wpb=17922.4, bsz=1024, num_updates=19740, lr=2.63373e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=131803
-2024-11-04 18:08:04 - progress_bar.py[line:274] - INFO: epoch 051:    222 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.774, ntokens=17877.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3079.8, ups=0.17, wpb=17877.5, bsz=1024, num_updates=19750, lr=2.63237e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=131861
-2024-11-04 18:09:01 - progress_bar.py[line:274] - INFO: epoch 051:    232 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.734, ntokens=17815.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3137.3, ups=0.18, wpb=17815.5, bsz=1024, num_updates=19760, lr=2.63101e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=131918
-2024-11-04 18:09:57 - progress_bar.py[line:274] - INFO: epoch 051:    242 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.734, ntokens=17387.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.66, wps=3075.5, ups=0.18, wpb=17387.9, bsz=1024, num_updates=19770, lr=2.62965e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=131974
-2024-11-04 18:10:53 - progress_bar.py[line:274] - INFO: epoch 051:    252 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.773, ntokens=17782.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=3190.6, ups=0.18, wpb=17782.8, bsz=1024, num_updates=19780, lr=2.62829e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=132030
-2024-11-04 18:11:51 - progress_bar.py[line:274] - INFO: epoch 051:    262 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.757, ntokens=17662.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.76, wps=3069.4, ups=0.17, wpb=17662.3, bsz=1024, num_updates=19790, lr=2.62692e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=132088
-2024-11-04 18:12:47 - progress_bar.py[line:274] - INFO: epoch 051:    272 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.773, ntokens=17816, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3164.1, ups=0.18, wpb=17816, bsz=1024, num_updates=19800, lr=2.62556e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=132144
-2024-11-04 18:13:43 - progress_bar.py[line:274] - INFO: epoch 051:    282 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.776, ntokens=17831.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=3170, ups=0.18, wpb=17831.9, bsz=1024, num_updates=19810, lr=2.6242e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=132200
-2024-11-04 18:14:39 - progress_bar.py[line:274] - INFO: epoch 051:    292 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17898.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3178.3, ups=0.18, wpb=17898.8, bsz=1024, num_updates=19820, lr=2.62284e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=132257
-2024-11-04 18:15:35 - progress_bar.py[line:274] - INFO: epoch 051:    302 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.883, ntokens=18340.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.38, wps=3285.7, ups=0.18, wpb=18340.8, bsz=1024, num_updates=19830, lr=2.62148e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=132312
-2024-11-04 18:16:29 - progress_bar.py[line:274] - INFO: epoch 051:    312 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.889, ntokens=18141.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.41, wps=3378.7, ups=0.19, wpb=18141.5, bsz=1024, num_updates=19840, lr=2.62012e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=132366
-2024-11-04 18:17:27 - progress_bar.py[line:274] - INFO: epoch 051:    322 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.862, ntokens=18214.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.27, wps=3142.4, ups=0.17, wpb=18214.1, bsz=1024, num_updates=19850, lr=2.61876e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.3, wall=132424
-2024-11-04 18:18:21 - progress_bar.py[line:274] - INFO: epoch 051:    332 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.866, ntokens=18214.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.29, wps=3380, ups=0.19, wpb=18214.7, bsz=1024, num_updates=19860, lr=2.6174e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=54, gb_free=13.3, wall=132478
-2024-11-04 18:19:17 - progress_bar.py[line:274] - INFO: epoch 051:    342 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.81, ntokens=17942.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=3201.9, ups=0.18, wpb=17942.2, bsz=1024, num_updates=19870, lr=2.61604e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=132534
-2024-11-04 18:20:13 - progress_bar.py[line:274] - INFO: epoch 051:    352 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.955, ntokens=18846.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.76, wps=3343.4, ups=0.18, wpb=18846.7, bsz=1024, num_updates=19880, lr=2.61468e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=132590
-2024-11-04 18:21:08 - progress_bar.py[line:274] - INFO: epoch 051:    362 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.829, ntokens=17978.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3263.9, ups=0.18, wpb=17978.2, bsz=1024, num_updates=19890, lr=2.61332e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=132645
-2024-11-04 18:22:04 - progress_bar.py[line:274] - INFO: epoch 051:    372 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.879, ntokens=18328, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.36, wps=3287.7, ups=0.18, wpb=18328, bsz=1024, num_updates=19900, lr=2.61196e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=132701
-2024-11-04 18:22:59 - progress_bar.py[line:274] - INFO: epoch 051:    382 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.818, ntokens=18061.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.05, wps=3261.4, ups=0.18, wpb=18061.4, bsz=1024, num_updates=19910, lr=2.6106e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=132757
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 18:23:47 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 51 @ 19919 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_51.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_51.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_51.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_51.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_51.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_51.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_51.pt
-2024-11-04 18:23:47 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_51.pt': File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 18:25:40 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-2024-11-04 18:27:34 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 51 @ 19919 updates, score 0) (writing took 227.40945749700768 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_51.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-slice_id 5 seek offset 250000
-slice_id 1 seek offset 50000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-04 18:28:30 - train.py[line:336] - INFO: end of epoch 51 (average epoch stats below)
-2024-11-04 18:28:30 - progress_bar.py[line:282] - INFO: epoch 051 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.785 | ntokens 17818.5 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.89 | wps 2623.8 | ups 0.15 | wpb 17818.5 | bsz 1023 | num_updates 19919 | lr 2.60938e-05 | gnorm 0.005 | clip 0 | loss_scale 65536 | train_wall 2115 | gb_free 13.4 | wall 133087
-2024-11-04 18:28:30 - trainer.py[line:639] - INFO: loading train data for epoch 52
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 18:31:15 - trainer.py[line:703] - INFO: begin training epoch 52
-2024-11-04 18:31:15 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 18:31:23 - progress_bar.py[line:274] - INFO: epoch 052:      1 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.892, ntokens=17508.5, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.43, wps=347.7, ups=0.02, wpb=17508.5, bsz=985.6, num_updates=19920, lr=2.60924e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=47, gb_free=13.4, wall=133260
-2024-11-04 18:32:19 - progress_bar.py[line:274] - INFO: epoch 052:     11 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.712, ntokens=17512.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.55, wps=3118.7, ups=0.18, wpb=17512.6, bsz=1024, num_updates=19930, lr=2.60788e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=133316
-2024-11-04 18:33:16 - progress_bar.py[line:274] - INFO: epoch 052:     21 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17859.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3154.3, ups=0.18, wpb=17859.5, bsz=1024, num_updates=19940, lr=2.60652e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=40, gb_free=13.4, wall=133373
-2024-11-04 18:34:12 - progress_bar.py[line:274] - INFO: epoch 052:     31 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.745, ntokens=17816.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3156.1, ups=0.18, wpb=17816.7, bsz=1024, num_updates=19950, lr=2.60516e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=133429
-2024-11-04 18:35:04 - progress_bar.py[line:274] - INFO: epoch 052:     41 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.774, ntokens=18044.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3453.9, ups=0.19, wpb=18044.1, bsz=1024, num_updates=19960, lr=2.6038e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=133482
-2024-11-04 18:36:00 - progress_bar.py[line:274] - INFO: epoch 052:     51 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.721, ntokens=17531.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3130.2, ups=0.18, wpb=17531.1, bsz=1024, num_updates=19970, lr=2.60244e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=133538
-2024-11-04 18:36:56 - progress_bar.py[line:274] - INFO: epoch 052:     61 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17765, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3192.2, ups=0.18, wpb=17765, bsz=1024, num_updates=19980, lr=2.60108e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=133593
-2024-11-04 18:37:51 - progress_bar.py[line:274] - INFO: epoch 052:     71 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.752, ntokens=17706.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3228.6, ups=0.18, wpb=17706.3, bsz=1024, num_updates=19990, lr=2.59972e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=133648
-2024-11-04 18:38:45 - progress_bar.py[line:274] - INFO: epoch 052:     81 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.765, ntokens=17752.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=3286.5, ups=0.19, wpb=17752.2, bsz=1024, num_updates=20000, lr=2.59836e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=133702
-loss_reg: 0.02589082820274328 loss_cls: 0.003806978464126587
-loss_reg: 0.013264386283241359 loss_cls: 0.003931819461286068
-loss_reg: 0.006375276986752406 loss_cls: 0.003863236401230097
-loss_reg: 0.017289880974958274 loss_cls: 0.003207497764378786
-loss_reg: 0.018673973593044214 loss_cls: 0.0042500668205320835
-loss_reg: 0.030485327053080387 loss_cls: 0.0037799689453095198
-loss_reg: 0.022149633884334517 loss_cls: 0.003560841316357255
-loss_reg: 0.002871642443228536 loss_cls: 0.0026299015153199434
-loss_reg: 0.01275364845335098 loss_cls: 0.0033174771815538406
-loss_reg: 0.011049604625877455 loss_cls: 0.0028586098924279213
-loss_reg: 0.016779386698078335 loss_cls: 0.003337868256494403
-loss_reg: 0.017617281004166215 loss_cls: 0.0033522534649819136
-loss_reg: 0.014924542857434784 loss_cls: 0.004876105114817619
-loss_reg: 0.011951928665072598 loss_cls: 0.0027779540978372097
-loss_reg: 0.02399453140209212 loss_cls: 0.0051208073273301125
-loss_reg: 0.01769124317002907 loss_cls: 0.0036857372615486383
-loss_reg: 0.024266745234664718 loss_cls: 0.0032831132411956787
-loss_reg: 0.0028835205217089942 loss_cls: 0.0028555209282785654
-loss_reg: 0.0157841813071833 loss_cls: 0.003913988824933767
-loss_reg: 0.021282098936987313 loss_cls: 0.004260902293026447
-loss_reg: 0.021909701382577487 loss_cls: 0.00372500903904438
-loss_reg: 0.01711264012015073 loss_cls: 0.003243389306589961
-loss_reg: 0.022908733282191986 loss_cls: 0.0028339263517409563
-loss_reg: 0.011727921993906635 loss_cls: 0.003421967849135399
-loss_reg: 0.0023441025470796366 loss_cls: 0.0023534870706498623
-loss_reg: 0.018458919704665247 loss_cls: 0.004718804731965065
-loss_reg: 0.01894165857158399 loss_cls: 0.0031567993573844433
-loss_reg: 0.015036974113969613 loss_cls: 0.004009539727121592
-loss_reg: 0.02296377703227137 loss_cls: 0.0034365521278232336
-loss_reg: 0.025272300568425422 loss_cls: 0.0039106798358261585
-loss_reg: 0.0174776810027744 loss_cls: 0.0036935084499418736
-loss_reg: 0.0069171625896983875 loss_cls: 0.0032495660707354546
-loss_reg: 0.018114560678420755 loss_cls: 0.004211212508380413
-loss_reg: 0.02543785831656011 loss_cls: 0.004771314095705748
-loss_reg: 0.009799092799763058 loss_cls: 0.0030468476470559835
-loss_reg: 0.011555359059532453 loss_cls: 0.0037228597793728113
-loss_reg: 0.014253863389613314 loss_cls: 0.0031556840986013412
-loss_reg: 0.013119700043657943 loss_cls: 0.0028121143113821745
-loss_reg: 0.0052935466351548294 loss_cls: 0.0029596504755318165
-loss_reg: 0.01562058704265873 loss_cls: 0.0035277847200632095
-loss_reg: 0.012641443127111669 loss_cls: 0.003888974664732814
-loss_reg: 0.01607614770168827 loss_cls: 0.003149743191897869
-loss_reg: 0.015300780212876838 loss_cls: 0.0037064708303660154
-loss_reg: 0.021755564828996744 loss_cls: 0.004035123158246279
-loss_reg: 0.020866160467161537 loss_cls: 0.0030146799981594086
-loss_reg: 0.003974192196621298 loss_cls: 0.0034675037022680044
-loss_reg: 0.014989532113375455 loss_cls: 0.003206588327884674
-loss_reg: 0.01777415681969765 loss_cls: 0.002774177584797144
-loss_reg: 0.019032544887999832 loss_cls: 0.006243404466658831
-loss_reg: 0.015868302129227553 loss_cls: 0.0054076132364571095
-loss_reg: 0.011647260542743943 loss_cls: 0.003677224274724722
-loss_reg: 0.024856091460416955 loss_cls: 0.00627283426001668
-loss_reg: 0.020523972694919394 loss_cls: 0.00479759369045496
-loss_reg: 0.001834975482356787 loss_cls: 0.0023824076633900404
-loss_reg: 0.015400550797889762 loss_cls: 0.0038019572384655476
-loss_reg: 0.01703084850109638 loss_cls: 0.003185973735526204
-loss_reg: 0.024638913626659348 loss_cls: 0.003270962508395314
-loss_reg: 0.015347907878952896 loss_cls: 0.00375892105512321
-loss_reg: 0.017530623020901206 loss_cls: 0.004170349333435297
-loss_reg: 0.01484149690379017 loss_cls: 0.005623234901577234
-loss_reg: 0.01184943434761001 loss_cls: 0.0037853417452424765
-loss_reg: 0.016212486785159998 loss_cls: 0.003692316822707653
-loss_reg: 0.016948448919589083 loss_cls: 0.004476174246519804
-loss_reg: 0.010843582961941316 loss_cls: 0.005096313543617725
-2024-11-04 18:39:39 - progress_bar.py[line:274] - INFO: epoch 052:     91 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.743, ntokens=17703.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3271.4, ups=0.18, wpb=17703.2, bsz=1024, num_updates=20010, lr=2.597e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=133756
-2024-11-04 18:40:06 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-04 18:40:39 - progress_bar.py[line:274] - INFO: epoch 052:    102 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.771, ntokens=17739.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=2950.4, ups=0.17, wpb=17739.3, bsz=1024, num_updates=20020, lr=2.59564e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=60, gb_free=13.4, wall=133816
-2024-11-04 18:41:32 - progress_bar.py[line:274] - INFO: epoch 052:    112 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.683, ntokens=17109, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3222, ups=0.19, wpb=17109, bsz=1024, num_updates=20030, lr=2.59428e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=53, gb_free=13.6, wall=133869
-2024-11-04 18:42:26 - progress_bar.py[line:274] - INFO: epoch 052:    122 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.733, ntokens=17674.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3292.1, ups=0.19, wpb=17674.8, bsz=1024, num_updates=20040, lr=2.59292e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=133923
-2024-11-04 18:43:19 - progress_bar.py[line:274] - INFO: epoch 052:    132 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.719, ntokens=17331.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3261, ups=0.19, wpb=17331.8, bsz=1024, num_updates=20050, lr=2.59155e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=133976
-2024-11-04 18:44:12 - progress_bar.py[line:274] - INFO: epoch 052:    142 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.78, ntokens=17864.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.87, wps=3354, ups=0.19, wpb=17864.1, bsz=1024, num_updates=20060, lr=2.59019e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=44, gb_free=13.5, wall=134030
-2024-11-04 18:45:05 - progress_bar.py[line:274] - INFO: epoch 052:    152 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.715, ntokens=17614.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3345, ups=0.19, wpb=17614.4, bsz=1024, num_updates=20070, lr=2.58883e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=134082
-2024-11-04 18:45:58 - progress_bar.py[line:274] - INFO: epoch 052:    162 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.705, ntokens=17582.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3305, ups=0.19, wpb=17582.4, bsz=1024, num_updates=20080, lr=2.58747e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=134135
-2024-11-04 18:46:52 - progress_bar.py[line:274] - INFO: epoch 052:    172 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.746, ntokens=17666.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3301.2, ups=0.19, wpb=17666.1, bsz=1024, num_updates=20090, lr=2.58611e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=134189
-2024-11-04 18:47:45 - progress_bar.py[line:274] - INFO: epoch 052:    182 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.723, ntokens=17455.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.6, wps=3279.8, ups=0.19, wpb=17455.9, bsz=1024, num_updates=20100, lr=2.58475e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=134242
-2024-11-04 18:48:39 - progress_bar.py[line:274] - INFO: epoch 052:    192 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.739, ntokens=17707.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.68, wps=3289, ups=0.19, wpb=17707.5, bsz=1024, num_updates=20110, lr=2.58339e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=36, gb_free=13.6, wall=134296
-2024-11-04 18:49:32 - progress_bar.py[line:274] - INFO: epoch 052:    202 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.747, ntokens=17826.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3353.7, ups=0.19, wpb=17826.2, bsz=1024, num_updates=20120, lr=2.58203e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=134349
-2024-11-04 18:50:26 - progress_bar.py[line:274] - INFO: epoch 052:    212 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.758, ntokens=18013.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.76, wps=3353.7, ups=0.19, wpb=18013.3, bsz=1024, num_updates=20130, lr=2.58067e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=134403
-2024-11-04 18:51:19 - progress_bar.py[line:274] - INFO: epoch 052:    222 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.726, ntokens=17803.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3333.8, ups=0.19, wpb=17803.4, bsz=1024, num_updates=20140, lr=2.57931e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=134456
-2024-11-04 18:52:12 - progress_bar.py[line:274] - INFO: epoch 052:    232 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.757, ntokens=17874.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.76, wps=3384.6, ups=0.19, wpb=17874.9, bsz=1024, num_updates=20150, lr=2.57795e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=134509
-2024-11-04 18:53:05 - progress_bar.py[line:274] - INFO: epoch 052:    242 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.727, ntokens=17410.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3264.4, ups=0.19, wpb=17410.8, bsz=1024, num_updates=20160, lr=2.57659e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=134562
-2024-11-04 18:53:59 - progress_bar.py[line:274] - INFO: epoch 052:    252 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.773, ntokens=17828.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3325.5, ups=0.19, wpb=17828.9, bsz=1024, num_updates=20170, lr=2.57523e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=134616
-2024-11-04 18:54:53 - progress_bar.py[line:274] - INFO: epoch 052:    262 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.797, ntokens=18022.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.95, wps=3318.6, ups=0.18, wpb=18022.3, bsz=1024, num_updates=20180, lr=2.57387e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=134670
-2024-11-04 18:55:47 - progress_bar.py[line:274] - INFO: epoch 052:    272 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.784, ntokens=17996.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.89, wps=3372.8, ups=0.19, wpb=17996.2, bsz=1024, num_updates=20190, lr=2.57251e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=134724
-2024-11-04 18:56:40 - progress_bar.py[line:274] - INFO: epoch 052:    282 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.669, ntokens=17246.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3203.6, ups=0.19, wpb=17246.2, bsz=1024, num_updates=20200, lr=2.57115e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=134777
-2024-11-04 18:57:34 - progress_bar.py[line:274] - INFO: epoch 052:    292 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.776, ntokens=17736.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=3323.5, ups=0.19, wpb=17736.1, bsz=1024, num_updates=20210, lr=2.56979e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=134831
-2024-11-04 18:58:26 - progress_bar.py[line:274] - INFO: epoch 052:    302 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.841, ntokens=18135.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.17, wps=3480, ups=0.19, wpb=18135.6, bsz=1024, num_updates=20220, lr=2.56843e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=134883
-2024-11-04 18:59:19 - progress_bar.py[line:274] - INFO: epoch 052:    312 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.92, ntokens=18431.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.57, wps=3461.6, ups=0.19, wpb=18431.1, bsz=1024, num_updates=20230, lr=2.56707e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=37, gb_free=13.3, wall=134936
-2024-11-04 19:00:12 - progress_bar.py[line:274] - INFO: epoch 052:    322 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.885, ntokens=18462.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3468.9, ups=0.19, wpb=18462.5, bsz=1024, num_updates=20240, lr=2.56571e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=44, gb_free=13.4, wall=134989
-2024-11-04 19:01:04 - progress_bar.py[line:274] - INFO: epoch 052:    332 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.914, ntokens=18517.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.54, wps=3573.2, ups=0.19, wpb=18517.3, bsz=1024, num_updates=20250, lr=2.56435e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=43, gb_free=13.4, wall=135041
-2024-11-04 19:01:56 - progress_bar.py[line:274] - INFO: epoch 052:    342 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.809, ntokens=18139.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=3464.9, ups=0.19, wpb=18139.7, bsz=1024, num_updates=20260, lr=2.56299e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=37, gb_free=13.4, wall=135094
-2024-11-04 19:02:49 - progress_bar.py[line:274] - INFO: epoch 052:    352 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.913, ntokens=18545.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.53, wps=3545.7, ups=0.19, wpb=18545.8, bsz=1024, num_updates=20270, lr=2.56163e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=37, gb_free=13.5, wall=135146
-2024-11-04 19:03:42 - progress_bar.py[line:274] - INFO: epoch 052:    362 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.829, ntokens=18058.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3410.6, ups=0.19, wpb=18058.5, bsz=1024, num_updates=20280, lr=2.56027e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=40, gb_free=13.3, wall=135199
-2024-11-04 19:04:34 - progress_bar.py[line:274] - INFO: epoch 052:    372 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.839, ntokens=18195, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=3513.5, ups=0.19, wpb=18195, bsz=1024, num_updates=20290, lr=2.55891e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=41, gb_free=13.4, wall=135251
-2024-11-04 19:05:27 - progress_bar.py[line:274] - INFO: epoch 052:    382 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.869, ntokens=18311.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.31, wps=3407.7, ups=0.19, wpb=18311.5, bsz=1024, num_updates=20300, lr=2.55754e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=135304
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt
-2024-11-04 19:06:11 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 52 @ 20309 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt
-2024-11-04 19:06:11 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt': File exists: File exists: File exists
-
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 19:08:02 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-slice_id 5 seek offset 250000
-slice_id 1 seek offset 50000
-2024-11-04 19:09:58 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 52 @ 20309 updates, score 0) (writing took 226.19532028998947 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_52.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-slice_id 2 seek offset 100000
-2024-11-04 19:10:53 - train.py[line:336] - INFO: end of epoch 52 (average epoch stats below)
-2024-11-04 19:10:53 - progress_bar.py[line:282] - INFO: epoch 052 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.777 | ntokens 17830.3 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.85 | wps 2734.4 | ups 0.15 | wpb 17830.3 | bsz 1023 | num_updates 20309 | lr 2.55632e-05 | gnorm 0.006 | clip 0 | loss_scale 32768 | train_wall 1553 | gb_free 13.4 | wall 135630
-2024-11-04 19:10:53 - trainer.py[line:639] - INFO: loading train data for epoch 53
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 19:13:38 - trainer.py[line:703] - INFO: begin training epoch 53
-2024-11-04 19:13:38 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 19:13:46 - progress_bar.py[line:274] - INFO: epoch 053:      1 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.805, ntokens=17227.2, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=345.6, ups=0.02, wpb=17227.2, bsz=985.6, num_updates=20310, lr=2.55618e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=29, gb_free=13.4, wall=135803
-2024-11-04 19:14:41 - progress_bar.py[line:274] - INFO: epoch 053:     11 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=17235.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3107.1, ups=0.18, wpb=17235.9, bsz=1024, num_updates=20320, lr=2.55482e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=135858
-2024-11-04 19:15:35 - progress_bar.py[line:274] - INFO: epoch 053:     21 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.732, ntokens=17763.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.64, wps=3286.9, ups=0.19, wpb=17763.2, bsz=1024, num_updates=20330, lr=2.55346e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=135912
-2024-11-04 19:16:29 - progress_bar.py[line:274] - INFO: epoch 053:     31 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.741, ntokens=17670.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3312.1, ups=0.19, wpb=17670.6, bsz=1024, num_updates=20340, lr=2.5521e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=135966
-2024-11-04 19:17:23 - progress_bar.py[line:274] - INFO: epoch 053:     41 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.743, ntokens=17892.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3309.6, ups=0.18, wpb=17892.1, bsz=1024, num_updates=20350, lr=2.55074e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=136020
-2024-11-04 19:18:16 - progress_bar.py[line:274] - INFO: epoch 053:     51 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.706, ntokens=17542.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3263.3, ups=0.19, wpb=17542.3, bsz=1024, num_updates=20360, lr=2.54938e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=136074
-2024-11-04 19:19:10 - progress_bar.py[line:274] - INFO: epoch 053:     61 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.819, ntokens=18117.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.06, wps=3378.3, ups=0.19, wpb=18117.4, bsz=1024, num_updates=20370, lr=2.54802e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=136127
-2024-11-04 19:20:04 - progress_bar.py[line:274] - INFO: epoch 053:     71 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.675, ntokens=17353.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=3246.2, ups=0.19, wpb=17353.5, bsz=1024, num_updates=20380, lr=2.54666e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.7, wall=136181
-2024-11-04 19:21:01 - progress_bar.py[line:274] - INFO: epoch 053:     81 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.73, ntokens=17652.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3051, ups=0.17, wpb=17652.3, bsz=1024, num_updates=20390, lr=2.5453e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=136238
-2024-11-04 19:21:58 - progress_bar.py[line:274] - INFO: epoch 053:     91 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.704, ntokens=17476.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3067.5, ups=0.18, wpb=17476.9, bsz=1024, num_updates=20400, lr=2.54394e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.6, wall=136295
-2024-11-04 19:22:51 - progress_bar.py[line:274] - INFO: epoch 053:    101 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.747, ntokens=17725.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3383.1, ups=0.19, wpb=17725.9, bsz=1024, num_updates=20410, lr=2.54258e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=136348
-2024-11-04 19:23:45 - progress_bar.py[line:274] - INFO: epoch 053:    111 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.755, ntokens=17598.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=3223.2, ups=0.18, wpb=17598.5, bsz=1024, num_updates=20420, lr=2.54122e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=136402
-2024-11-04 19:24:44 - progress_bar.py[line:274] - INFO: epoch 053:    121 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.743, ntokens=17747.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3050.8, ups=0.17, wpb=17747.1, bsz=1024, num_updates=20430, lr=2.53986e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=136461
-2024-11-04 19:25:41 - progress_bar.py[line:274] - INFO: epoch 053:    131 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.742, ntokens=17583, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3076.8, ups=0.17, wpb=17583, bsz=1024, num_updates=20440, lr=2.5385e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=136518
-2024-11-04 19:26:33 - progress_bar.py[line:274] - INFO: epoch 053:    141 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.78, ntokens=17871.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.87, wps=3387.9, ups=0.19, wpb=17871.1, bsz=1024, num_updates=20450, lr=2.53714e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=136571
-2024-11-04 19:27:26 - progress_bar.py[line:274] - INFO: epoch 053:    151 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.717, ntokens=17687.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3340.1, ups=0.19, wpb=17687.2, bsz=1024, num_updates=20460, lr=2.53578e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=136623
-2024-11-04 19:28:19 - progress_bar.py[line:274] - INFO: epoch 053:    161 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=17485.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3292, ups=0.19, wpb=17485.2, bsz=1024, num_updates=20470, lr=2.53442e-05, gnorm=0.003, clip=0, loss_scale=32768, train_wall=53, gb_free=13.6, wall=136677
-2024-11-04 19:29:15 - progress_bar.py[line:274] - INFO: epoch 053:    171 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.747, ntokens=17894.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3228.3, ups=0.18, wpb=17894.7, bsz=1024, num_updates=20480, lr=2.53306e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=136732
-2024-11-04 19:30:08 - progress_bar.py[line:274] - INFO: epoch 053:    181 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.71, ntokens=17440.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3283.9, ups=0.19, wpb=17440.1, bsz=1024, num_updates=20490, lr=2.5317e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=136785
-2024-11-04 19:31:03 - progress_bar.py[line:274] - INFO: epoch 053:    191 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=17901.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3248.2, ups=0.18, wpb=17901.8, bsz=1024, num_updates=20500, lr=2.53034e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=136840
-2024-11-04 19:31:57 - progress_bar.py[line:274] - INFO: epoch 053:    201 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=17599.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3260.3, ups=0.19, wpb=17599.6, bsz=1024, num_updates=20510, lr=2.52898e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=136894
-2024-11-04 19:32:51 - progress_bar.py[line:274] - INFO: epoch 053:    211 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.746, ntokens=17984.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3361.8, ups=0.19, wpb=17984.3, bsz=1024, num_updates=20520, lr=2.52762e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.4, wall=136948
-2024-11-04 19:33:44 - progress_bar.py[line:274] - INFO: epoch 053:    221 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.724, ntokens=17709.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.61, wps=3305.6, ups=0.19, wpb=17709.6, bsz=1024, num_updates=20530, lr=2.52626e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=137001
-2024-11-04 19:34:39 - progress_bar.py[line:274] - INFO: epoch 053:    231 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=18079.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3319.1, ups=0.18, wpb=18079.2, bsz=1024, num_updates=20540, lr=2.5249e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=137056
-2024-11-04 19:35:34 - progress_bar.py[line:274] - INFO: epoch 053:    241 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.733, ntokens=17479.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3139.2, ups=0.18, wpb=17479.5, bsz=1024, num_updates=20550, lr=2.52353e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=137111
-2024-11-04 19:36:31 - progress_bar.py[line:274] - INFO: epoch 053:    251 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.726, ntokens=17536.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3107.6, ups=0.18, wpb=17536.2, bsz=1024, num_updates=20560, lr=2.52217e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=137168
-2024-11-04 19:37:28 - progress_bar.py[line:274] - INFO: epoch 053:    261 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.737, ntokens=17610, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.67, wps=3091.1, ups=0.18, wpb=17610, bsz=1024, num_updates=20570, lr=2.52081e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=137225
-2024-11-04 19:38:24 - progress_bar.py[line:274] - INFO: epoch 053:    271 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.731, ntokens=17620.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.64, wps=3114.6, ups=0.18, wpb=17620.8, bsz=1024, num_updates=20580, lr=2.51945e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=137281
-2024-11-04 19:39:20 - progress_bar.py[line:274] - INFO: epoch 053:    281 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.733, ntokens=17758.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3173, ups=0.18, wpb=17758.8, bsz=1024, num_updates=20590, lr=2.51809e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=137337
-2024-11-04 19:40:14 - progress_bar.py[line:274] - INFO: epoch 053:    291 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.719, ntokens=17604.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3285, ups=0.19, wpb=17604.4, bsz=1024, num_updates=20600, lr=2.51673e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=137391
-2024-11-04 19:41:09 - progress_bar.py[line:274] - INFO: epoch 053:    301 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.814, ntokens=17986.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=3283.5, ups=0.18, wpb=17986.5, bsz=1024, num_updates=20610, lr=2.51537e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=137446
-2024-11-04 19:42:05 - progress_bar.py[line:274] - INFO: epoch 053:    311 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.906, ntokens=18454.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.5, wps=3271.6, ups=0.18, wpb=18454.5, bsz=1024, num_updates=20620, lr=2.51401e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=137502
-2024-11-04 19:43:01 - progress_bar.py[line:274] - INFO: epoch 053:    321 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.875, ntokens=18246.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.34, wps=3285.6, ups=0.18, wpb=18246.6, bsz=1024, num_updates=20630, lr=2.51265e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=137558
-2024-11-04 19:43:55 - progress_bar.py[line:274] - INFO: epoch 053:    331 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.871, ntokens=18393.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.32, wps=3392.8, ups=0.18, wpb=18393.7, bsz=1024, num_updates=20640, lr=2.51129e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.3, wall=137612
-2024-11-04 19:44:49 - progress_bar.py[line:274] - INFO: epoch 053:    341 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.814, ntokens=18162.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=3379.6, ups=0.19, wpb=18162.6, bsz=1024, num_updates=20650, lr=2.50993e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=54, gb_free=13.2, wall=137666
-2024-11-04 19:45:44 - progress_bar.py[line:274] - INFO: epoch 053:    351 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.886, ntokens=18548.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.39, wps=3332.6, ups=0.18, wpb=18548.7, bsz=1024, num_updates=20660, lr=2.50857e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=137721
-2024-11-04 19:46:41 - progress_bar.py[line:274] - INFO: epoch 053:    361 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.861, ntokens=18346.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.27, wps=3242.6, ups=0.18, wpb=18346.8, bsz=1024, num_updates=20670, lr=2.50721e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=137778
-2024-11-04 19:47:38 - progress_bar.py[line:274] - INFO: epoch 053:    371 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.852, ntokens=18328.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=3215.5, ups=0.18, wpb=18328.7, bsz=1024, num_updates=20680, lr=2.50585e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=137835
-2024-11-04 19:48:35 - progress_bar.py[line:274] - INFO: epoch 053:    381 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.765, ntokens=17691.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=3108.4, ups=0.18, wpb=17691.3, bsz=1024, num_updates=20690, lr=2.50449e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=137892
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 19:49:26 - progress_bar.py[line:274] - INFO: epoch 053:    391 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.86, ntokens=17460.9, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.26, wps=3404.2, ups=0.19, wpb=17460.9, bsz=985.6, num_updates=20700, lr=2.50313e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=51, gb_free=13.3, wall=137943
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt
-2024-11-04 19:49:26 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 53 @ 20700 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt
-2024-11-04 19:49:26 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt': File exists: File exists
-
-: File exists: File exists
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 19:51:18 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-
-slice_id 3 seek offset 150000
-slice_id 4 seek offset 200000
-slice_id 5 seek offset 250000
-slice_id 1 seek offset 50000
-2024-11-04 19:53:12 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 53 @ 20700 updates, score 0) (writing took 226.14665107999463 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_53.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-04 19:54:08 - train.py[line:336] - INFO: end of epoch 53 (average epoch stats below)
-2024-11-04 19:54:08 - progress_bar.py[line:282] - INFO: epoch 053 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.764 | ntokens 17800 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.79 | wps 2682.2 | ups 0.15 | wpb 17800 | bsz 1023 | num_updates 20700 | lr 2.50313e-05 | gnorm 0.005 | clip 0 | loss_scale 65536 | train_wall 1973 | gb_free 13.3 | wall 138225
-2024-11-04 19:54:08 - trainer.py[line:639] - INFO: loading train data for epoch 54
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 19:56:52 - trainer.py[line:703] - INFO: begin training epoch 54
-2024-11-04 19:56:52 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 19:57:50 - progress_bar.py[line:274] - INFO: epoch 054:     10 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=17828, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=354, ups=0.02, wpb=17828, bsz=1024, num_updates=20710, lr=2.50177e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=138447
-2024-11-04 19:58:43 - progress_bar.py[line:274] - INFO: epoch 054:     20 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.75, ntokens=17902.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3380.4, ups=0.19, wpb=17902.5, bsz=1024, num_updates=20720, lr=2.50041e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=138500
-2024-11-04 19:59:35 - progress_bar.py[line:274] - INFO: epoch 054:     30 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.706, ntokens=17639.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=3349.4, ups=0.19, wpb=17639.1, bsz=1024, num_updates=20730, lr=2.49905e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=138552
-2024-11-04 20:00:30 - progress_bar.py[line:274] - INFO: epoch 054:     40 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.713, ntokens=17795.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.56, wps=3276.1, ups=0.18, wpb=17795.6, bsz=1024, num_updates=20740, lr=2.49769e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=32, gb_free=13.5, wall=138607
-2024-11-04 20:01:24 - progress_bar.py[line:274] - INFO: epoch 054:     50 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.693, ntokens=17450.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3212.8, ups=0.18, wpb=17450.5, bsz=1024, num_updates=20750, lr=2.49633e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=50, gb_free=13.5, wall=138661
-2024-11-04 20:02:18 - progress_bar.py[line:274] - INFO: epoch 054:     60 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.744, ntokens=17783.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3290.6, ups=0.19, wpb=17783.6, bsz=1024, num_updates=20760, lr=2.49497e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=138715
-2024-11-04 20:03:10 - progress_bar.py[line:274] - INFO: epoch 054:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.776, ntokens=17992.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=3435.9, ups=0.19, wpb=17992.2, bsz=1024, num_updates=20770, lr=2.49361e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=138767
-2024-11-04 20:04:03 - progress_bar.py[line:274] - INFO: epoch 054:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.725, ntokens=17729.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.61, wps=3383, ups=0.19, wpb=17729.5, bsz=1024, num_updates=20780, lr=2.49225e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=138820
-2024-11-04 20:04:56 - progress_bar.py[line:274] - INFO: epoch 054:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.733, ntokens=17752.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3352.1, ups=0.19, wpb=17752.1, bsz=1024, num_updates=20790, lr=2.49089e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=51, gb_free=13.5, wall=138873
-2024-11-04 20:05:49 - progress_bar.py[line:274] - INFO: epoch 054:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.77, ntokens=17937.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=3370.1, ups=0.19, wpb=17937.4, bsz=1024, num_updates=20800, lr=2.48952e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=52, gb_free=13.6, wall=138926
-2024-11-04 20:06:44 - progress_bar.py[line:274] - INFO: epoch 054:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.727, ntokens=17610, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3185.6, ups=0.18, wpb=17610, bsz=1024, num_updates=20810, lr=2.48816e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=53, gb_free=13.6, wall=138981
-2024-11-04 20:07:42 - progress_bar.py[line:274] - INFO: epoch 054:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.753, ntokens=17804.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.74, wps=3103.2, ups=0.17, wpb=17804.5, bsz=1024, num_updates=20820, lr=2.4868e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=139039
-2024-11-04 20:08:38 - progress_bar.py[line:274] - INFO: epoch 054:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.683, ntokens=17330.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3077.5, ups=0.18, wpb=17330.1, bsz=1024, num_updates=20830, lr=2.48544e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=139095
-2024-11-04 20:09:34 - progress_bar.py[line:274] - INFO: epoch 054:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=17869.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3170.8, ups=0.18, wpb=17869.2, bsz=1024, num_updates=20840, lr=2.48408e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=139151
-2024-11-04 20:10:29 - progress_bar.py[line:274] - INFO: epoch 054:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.7, ntokens=17606.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3211.2, ups=0.18, wpb=17606.8, bsz=1024, num_updates=20850, lr=2.48272e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=139206
-2024-11-04 20:11:26 - progress_bar.py[line:274] - INFO: epoch 054:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.742, ntokens=17747.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3093.7, ups=0.17, wpb=17747.6, bsz=1024, num_updates=20860, lr=2.48136e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.5, wall=139264
-2024-11-04 20:12:23 - progress_bar.py[line:274] - INFO: epoch 054:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.769, ntokens=18122, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=3225, ups=0.18, wpb=18122, bsz=1024, num_updates=20870, lr=2.48e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=139320
-2024-11-04 20:13:18 - progress_bar.py[line:274] - INFO: epoch 054:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.72, ntokens=17468.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3147.9, ups=0.18, wpb=17468.4, bsz=1024, num_updates=20880, lr=2.47864e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=139375
-2024-11-04 20:14:14 - progress_bar.py[line:274] - INFO: epoch 054:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.727, ntokens=17915.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3187.4, ups=0.18, wpb=17915.6, bsz=1024, num_updates=20890, lr=2.47728e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=139431
-2024-11-04 20:15:11 - progress_bar.py[line:274] - INFO: epoch 054:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.722, ntokens=17682.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.6, wps=3134.2, ups=0.18, wpb=17682.9, bsz=1024, num_updates=20900, lr=2.47592e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=139488
-2024-11-04 20:16:07 - progress_bar.py[line:274] - INFO: epoch 054:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.731, ntokens=17842.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.64, wps=3196.2, ups=0.18, wpb=17842.9, bsz=1024, num_updates=20910, lr=2.47456e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=139544
-2024-11-04 20:17:02 - progress_bar.py[line:274] - INFO: epoch 054:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.736, ntokens=17943.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.66, wps=3237.5, ups=0.18, wpb=17943.7, bsz=1024, num_updates=20920, lr=2.4732e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=139599
-2024-11-04 20:17:57 - progress_bar.py[line:274] - INFO: epoch 054:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.7, ntokens=17709.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3223, ups=0.18, wpb=17709.1, bsz=1024, num_updates=20930, lr=2.47184e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=139654
-2024-11-04 20:18:52 - progress_bar.py[line:274] - INFO: epoch 054:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.752, ntokens=17769.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.74, wps=3201.4, ups=0.18, wpb=17769.7, bsz=1024, num_updates=20940, lr=2.47048e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=139710
-2024-11-04 20:19:49 - progress_bar.py[line:274] - INFO: epoch 054:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17335.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3072.7, ups=0.18, wpb=17335.1, bsz=1024, num_updates=20950, lr=2.46912e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=139766
-2024-11-04 20:20:46 - progress_bar.py[line:274] - INFO: epoch 054:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.715, ntokens=17621.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3096.9, ups=0.18, wpb=17621.2, bsz=1024, num_updates=20960, lr=2.46776e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=139823
-2024-11-04 20:21:42 - progress_bar.py[line:274] - INFO: epoch 054:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.753, ntokens=17801.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.74, wps=3165.5, ups=0.18, wpb=17801.1, bsz=1024, num_updates=20970, lr=2.4664e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=139879
-2024-11-04 20:22:35 - progress_bar.py[line:274] - INFO: epoch 054:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=17549, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3283.6, ups=0.19, wpb=17549, bsz=1024, num_updates=20980, lr=2.46504e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.6, wall=139933
-2024-11-04 20:23:29 - progress_bar.py[line:274] - INFO: epoch 054:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.71, ntokens=17514.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3291.3, ups=0.19, wpb=17514.7, bsz=1024, num_updates=20990, lr=2.46368e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=53, gb_free=13.6, wall=139986
-2024-11-04 20:24:22 - progress_bar.py[line:274] - INFO: epoch 054:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.8, ntokens=17912.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.96, wps=3325.8, ups=0.19, wpb=17912.2, bsz=1024, num_updates=21000, lr=2.46232e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=140040
-2024-11-04 20:25:15 - progress_bar.py[line:274] - INFO: epoch 054:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.873, ntokens=18222, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.32, wps=3440, ups=0.19, wpb=18222, bsz=1024, num_updates=21010, lr=2.46096e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=140093
-2024-11-04 20:26:09 - progress_bar.py[line:274] - INFO: epoch 054:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.895, ntokens=18483, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.44, wps=3471.1, ups=0.19, wpb=18483, bsz=1024, num_updates=21020, lr=2.4596e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=140146
-2024-11-04 20:27:02 - progress_bar.py[line:274] - INFO: epoch 054:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.875, ntokens=18439.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.34, wps=3429.2, ups=0.19, wpb=18439.1, bsz=1024, num_updates=21030, lr=2.45824e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=140200
-2024-11-04 20:27:56 - progress_bar.py[line:274] - INFO: epoch 054:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.866, ntokens=18528.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.29, wps=3451.1, ups=0.19, wpb=18528.9, bsz=1024, num_updates=21040, lr=2.45688e-05, gnorm=0.006, clip=0, loss_scale=131072, train_wall=54, gb_free=13.3, wall=140253
-2024-11-04 20:28:50 - progress_bar.py[line:274] - INFO: epoch 054:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.883, ntokens=18683.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.38, wps=3457.2, ups=0.19, wpb=18683.1, bsz=1024, num_updates=21050, lr=2.45552e-05, gnorm=0.005, clip=0, loss_scale=131072, train_wall=54, gb_free=13.3, wall=140307
-2024-11-04 20:29:46 - progress_bar.py[line:274] - INFO: epoch 054:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.837, ntokens=18181.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=3263.2, ups=0.18, wpb=18181.7, bsz=1024, num_updates=21060, lr=2.45415e-05, gnorm=0.005, clip=0, loss_scale=131072, train_wall=56, gb_free=13.2, wall=140363
-2024-11-04 20:30:41 - progress_bar.py[line:274] - INFO: epoch 054:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.805, ntokens=17932.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=3276.2, ups=0.18, wpb=17932.2, bsz=1024, num_updates=21070, lr=2.45279e-05, gnorm=0.005, clip=0, loss_scale=131072, train_wall=54, gb_free=13.3, wall=140418
-2024-11-04 20:31:35 - progress_bar.py[line:274] - INFO: epoch 054:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.793, ntokens=17896.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.93, wps=3297.4, ups=0.18, wpb=17896.2, bsz=1024, num_updates=21080, lr=2.45143e-05, gnorm=0.003, clip=0, loss_scale=131072, train_wall=54, gb_free=13.4, wall=140472
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 20:32:28 - progress_bar.py[line:274] - INFO: epoch 054:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=18091.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3417.9, ups=0.19, wpb=18091.3, bsz=1024, num_updates=21090, lr=2.45007e-05, gnorm=0.004, clip=0, loss_scale=131072, train_wall=53, gb_free=13.4, wall=140525
-2024-11-04 20:32:30 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 54 @ 21091 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.pt
-2024-11-04 20:32:30 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.pt': File exists: File exists
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 20:34:18 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-
-slice_id 5 seek offset 250000
-slice_id 1 seek offset 50000
-2024-11-04 20:36:10 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 54 @ 21091 updates, score 0) (writing took 219.97667886299314 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_54.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-
-slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-04 20:37:04 - train.py[line:336] - INFO: end of epoch 54 (average epoch stats below)
-2024-11-04 20:37:04 - progress_bar.py[line:282] - INFO: epoch 054 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.76 | ntokens 17839.4 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.78 | wps 2707.4 | ups 0.15 | wpb 17839.4 | bsz 1023 | num_updates 21091 | lr 2.44994e-05 | gnorm 0.006 | clip 0 | loss_scale 131072 | train_wall 2031 | gb_free 13.4 | wall 140802
-2024-11-04 20:37:04 - trainer.py[line:639] - INFO: loading train data for epoch 55
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 20:39:48 - trainer.py[line:703] - INFO: begin training epoch 55
-2024-11-04 20:39:48 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 20:40:38 - progress_bar.py[line:274] - INFO: epoch 055:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.659, ntokens=16568.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.31, wps=338.2, ups=0.02, wpb=16568.6, bsz=985.6, num_updates=21100, lr=2.44871e-05, gnorm=0.006, clip=0, loss_scale=131072, train_wall=29, gb_free=13.5, wall=141015
-2024-11-04 20:41:31 - progress_bar.py[line:274] - INFO: epoch 055:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.708, ntokens=17675.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3335.8, ups=0.19, wpb=17675.7, bsz=1024, num_updates=21110, lr=2.44735e-05, gnorm=0.006, clip=0, loss_scale=131072, train_wall=30, gb_free=13.5, wall=141068
-2024-11-04 20:42:24 - progress_bar.py[line:274] - INFO: epoch 055:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.714, ntokens=17720.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.56, wps=3349.8, ups=0.19, wpb=17720.7, bsz=1024, num_updates=21120, lr=2.44599e-05, gnorm=0.006, clip=0, loss_scale=131072, train_wall=31, gb_free=13.5, wall=141121
-2024-11-04 20:43:17 - progress_bar.py[line:274] - INFO: epoch 055:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.733, ntokens=17901.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3348.6, ups=0.19, wpb=17901.1, bsz=1024, num_updates=21130, lr=2.44463e-05, gnorm=0.005, clip=0, loss_scale=131072, train_wall=30, gb_free=13.4, wall=141174
-2024-11-04 20:44:10 - progress_bar.py[line:274] - INFO: epoch 055:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.708, ntokens=17654.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=3323.6, ups=0.19, wpb=17654.9, bsz=1024, num_updates=21140, lr=2.44327e-05, gnorm=0.006, clip=0, loss_scale=131072, train_wall=30, gb_free=13.6, wall=141227
-2024-11-04 20:45:04 - progress_bar.py[line:274] - INFO: epoch 055:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.72, ntokens=17734.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3327.7, ups=0.19, wpb=17734.8, bsz=1024, num_updates=21150, lr=2.44191e-05, gnorm=0.008, clip=0, loss_scale=131072, train_wall=30, gb_free=13.6, wall=141281
-2024-11-04 20:45:25 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
-2024-11-04 20:46:02 - progress_bar.py[line:274] - INFO: epoch 055:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.751, ntokens=17794.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3063.8, ups=0.17, wpb=17794.2, bsz=1024, num_updates=21160, lr=2.44055e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=33, gb_free=13.4, wall=141339
-2024-11-04 20:46:55 - progress_bar.py[line:274] - INFO: epoch 055:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.724, ntokens=17741.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.61, wps=3339.8, ups=0.19, wpb=17741.2, bsz=1024, num_updates=21170, lr=2.43919e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=141392
-2024-11-04 20:47:49 - progress_bar.py[line:274] - INFO: epoch 055:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.716, ntokens=17667.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3249.6, ups=0.18, wpb=17667.8, bsz=1024, num_updates=21180, lr=2.43783e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=38, gb_free=13.5, wall=141446
-2024-11-04 20:48:41 - progress_bar.py[line:274] - INFO: epoch 055:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.764, ntokens=17993.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3466.3, ups=0.19, wpb=17993.8, bsz=1024, num_updates=21190, lr=2.43647e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=38, gb_free=13.5, wall=141498
-2024-11-04 20:49:33 - progress_bar.py[line:274] - INFO: epoch 055:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.691, ntokens=17434.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3333.1, ups=0.19, wpb=17434.5, bsz=1024, num_updates=21200, lr=2.43511e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=141550
-2024-11-04 20:50:26 - progress_bar.py[line:274] - INFO: epoch 055:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.743, ntokens=17723.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3349.5, ups=0.19, wpb=17723.2, bsz=1024, num_updates=21210, lr=2.43375e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=141603
-2024-11-04 20:51:19 - progress_bar.py[line:274] - INFO: epoch 055:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.709, ntokens=17504.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3323.7, ups=0.19, wpb=17504.2, bsz=1024, num_updates=21220, lr=2.43239e-05, gnorm=0.003, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=141656
-2024-11-04 20:52:12 - progress_bar.py[line:274] - INFO: epoch 055:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.772, ntokens=17887.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=3387.7, ups=0.19, wpb=17887.1, bsz=1024, num_updates=21230, lr=2.43103e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=141709
-2024-11-04 20:53:04 - progress_bar.py[line:274] - INFO: epoch 055:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.69, ntokens=17541.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3335.9, ups=0.19, wpb=17541.7, bsz=1024, num_updates=21240, lr=2.42967e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=141761
-2024-11-04 20:53:57 - progress_bar.py[line:274] - INFO: epoch 055:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.699, ntokens=17611.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3314.4, ups=0.19, wpb=17611.8, bsz=1024, num_updates=21250, lr=2.42831e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=141815
-2024-11-04 20:54:50 - progress_bar.py[line:274] - INFO: epoch 055:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.746, ntokens=17931.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3413.8, ups=0.19, wpb=17931.3, bsz=1024, num_updates=21260, lr=2.42695e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=141867
-2024-11-04 20:55:43 - progress_bar.py[line:274] - INFO: epoch 055:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.742, ntokens=17766.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3369.4, ups=0.19, wpb=17766.7, bsz=1024, num_updates=21270, lr=2.42559e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=141920
-2024-11-04 20:55:53 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-04 20:56:41 - progress_bar.py[line:274] - INFO: epoch 055:    191 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.734, ntokens=17885.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.65, wps=3065.2, ups=0.17, wpb=17885.9, bsz=1024, num_updates=21280, lr=2.42423e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=34, gb_free=13.3, wall=141978
-2024-11-04 20:57:34 - progress_bar.py[line:274] - INFO: epoch 055:    201 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.773, ntokens=18002.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=3374.8, ups=0.19, wpb=18002.3, bsz=1024, num_updates=21290, lr=2.42287e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=142032
-2024-11-04 20:58:28 - progress_bar.py[line:274] - INFO: epoch 055:    211 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.691, ntokens=17744.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3327.3, ups=0.19, wpb=17744.5, bsz=1024, num_updates=21300, lr=2.42151e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=142085
-2024-11-04 20:59:21 - progress_bar.py[line:274] - INFO: epoch 055:    221 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.692, ntokens=17570.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3324.8, ups=0.19, wpb=17570.2, bsz=1024, num_updates=21310, lr=2.42014e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=142138
-2024-11-04 21:00:14 - progress_bar.py[line:274] - INFO: epoch 055:    231 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.765, ntokens=18204.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=3438.1, ups=0.19, wpb=18204.6, bsz=1024, num_updates=21320, lr=2.41878e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=142191
-2024-11-04 21:01:08 - progress_bar.py[line:274] - INFO: epoch 055:    241 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.706, ntokens=17497.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3201.1, ups=0.18, wpb=17497.4, bsz=1024, num_updates=21330, lr=2.41742e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=142245
-2024-11-04 21:02:01 - progress_bar.py[line:274] - INFO: epoch 055:    251 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.711, ntokens=17513.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.55, wps=3291.5, ups=0.19, wpb=17513.6, bsz=1024, num_updates=21340, lr=2.41606e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=142298
-2024-11-04 21:02:55 - progress_bar.py[line:274] - INFO: epoch 055:    261 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.718, ntokens=17581.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3305.4, ups=0.19, wpb=17581.5, bsz=1024, num_updates=21350, lr=2.4147e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=142352
-2024-11-04 21:03:47 - progress_bar.py[line:274] - INFO: epoch 055:    271 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.763, ntokens=17908.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.79, wps=3395.4, ups=0.19, wpb=17908.4, bsz=1024, num_updates=21360, lr=2.41334e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=142404
-2024-11-04 21:04:40 - progress_bar.py[line:274] - INFO: epoch 055:    281 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.742, ntokens=17804.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3357.9, ups=0.19, wpb=17804.8, bsz=1024, num_updates=21370, lr=2.41198e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=142457
-2024-11-04 21:05:33 - progress_bar.py[line:274] - INFO: epoch 055:    291 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.705, ntokens=17519.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3320, ups=0.19, wpb=17519.2, bsz=1024, num_updates=21380, lr=2.41062e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=142510
-2024-11-04 21:06:26 - progress_bar.py[line:274] - INFO: epoch 055:    301 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.779, ntokens=17793.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.87, wps=3395.8, ups=0.19, wpb=17793.8, bsz=1024, num_updates=21390, lr=2.40926e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=142563
-2024-11-04 21:07:18 - progress_bar.py[line:274] - INFO: epoch 055:    311 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.858, ntokens=18248.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.25, wps=3497.1, ups=0.19, wpb=18248.5, bsz=1024, num_updates=21400, lr=2.4079e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=142615
-2024-11-04 21:08:10 - progress_bar.py[line:274] - INFO: epoch 055:    321 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.86, ntokens=18243.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.26, wps=3475.2, ups=0.19, wpb=18243.8, bsz=1024, num_updates=21410, lr=2.40654e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=142667
-2024-11-04 21:09:02 - progress_bar.py[line:274] - INFO: epoch 055:    331 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.882, ntokens=18528.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.37, wps=3567.6, ups=0.19, wpb=18528.8, bsz=1024, num_updates=21420, lr=2.40518e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=142719
-2024-11-04 21:09:54 - progress_bar.py[line:274] - INFO: epoch 055:    341 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.782, ntokens=18047.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.88, wps=3446.7, ups=0.19, wpb=18047.3, bsz=1024, num_updates=21430, lr=2.40382e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.3, wall=142772
-2024-11-04 21:10:48 - progress_bar.py[line:274] - INFO: epoch 055:    351 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.892, ntokens=18628.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.42, wps=3514.2, ups=0.19, wpb=18628.3, bsz=1024, num_updates=21440, lr=2.40246e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=35, gb_free=13.4, wall=142825
-2024-11-04 21:11:39 - progress_bar.py[line:274] - INFO: epoch 055:    361 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.823, ntokens=18260.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.07, wps=3538.1, ups=0.19, wpb=18260.9, bsz=1024, num_updates=21450, lr=2.4011e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=142876
-2024-11-04 21:12:33 - progress_bar.py[line:274] - INFO: epoch 055:    371 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.827, ntokens=18179.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.1, wps=3354, ups=0.18, wpb=18179.3, bsz=1024, num_updates=21460, lr=2.39974e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=142930
-2024-11-04 21:13:29 - progress_bar.py[line:274] - INFO: epoch 055:    381 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.868, ntokens=18367, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.3, wps=3292.1, ups=0.18, wpb=18367, bsz=1024, num_updates=21470, lr=2.39838e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=142986
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 21:14:18 - progress_bar.py[line:274] - INFO: epoch 055:    391 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.837, ntokens=17524.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=7.15, wps=3581.9, ups=0.2, wpb=17524.7, bsz=985.6, num_updates=21480, lr=2.39702e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=29, gb_free=13.5, wall=143035
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt2024-11-04 21:14:18 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 55 @ 21480 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt
-2024-11-04 21:14:18 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt': File exists: File exists
-: File exists
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 21:16:07 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-slice_id 5 seek offset 250000
-slice_id 4 seek offset 200000
-2024-11-04 21:18:00 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 55 @ 21480 updates, score 0) (writing took 221.76535761897685 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_55.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 3 seek offset 150000
-slice_id 6 seek offset 300000
-2024-11-04 21:18:54 - train.py[line:336] - INFO: end of epoch 55 (average epoch stats below)
-2024-11-04 21:18:54 - progress_bar.py[line:282] - INFO: epoch 055 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.755 | ntokens 17835.8 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.75 | wps 2764.2 | ups 0.15 | wpb 17835.8 | bsz 1023 | num_updates 21480 | lr 2.39702e-05 | gnorm 0.006 | clip 0 | loss_scale 32768 | train_wall 1210 | gb_free 13.5 | wall 143312
-2024-11-04 21:18:54 - trainer.py[line:639] - INFO: loading train data for epoch 56
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 21:21:38 - trainer.py[line:703] - INFO: begin training epoch 56
-2024-11-04 21:21:38 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 21:22:33 - progress_bar.py[line:274] - INFO: epoch 056:     10 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=17378.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=351.3, ups=0.02, wpb=17378.2, bsz=1024, num_updates=21490, lr=2.39566e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=143530
-2024-11-04 21:23:26 - progress_bar.py[line:274] - INFO: epoch 056:     20 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17654.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3329.1, ups=0.19, wpb=17654.8, bsz=1024, num_updates=21500, lr=2.3943e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=143583
-2024-11-04 21:24:19 - progress_bar.py[line:274] - INFO: epoch 056:     30 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.699, ntokens=17604, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3320.7, ups=0.19, wpb=17604, bsz=1024, num_updates=21510, lr=2.39294e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=143636
-2024-11-04 21:25:12 - progress_bar.py[line:274] - INFO: epoch 056:     40 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.716, ntokens=17905.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3363.4, ups=0.19, wpb=17905.6, bsz=1024, num_updates=21520, lr=2.39158e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=143689
-2024-11-04 21:26:05 - progress_bar.py[line:274] - INFO: epoch 056:     50 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.714, ntokens=17634.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.56, wps=3348, ups=0.19, wpb=17634.1, bsz=1024, num_updates=21530, lr=2.39022e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=143742
-2024-11-04 21:26:57 - progress_bar.py[line:274] - INFO: epoch 056:     60 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.706, ntokens=17619.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3357.8, ups=0.19, wpb=17619.6, bsz=1024, num_updates=21540, lr=2.38886e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=143794
-2024-11-04 21:27:50 - progress_bar.py[line:274] - INFO: epoch 056:     70 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.728, ntokens=17670, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3355.7, ups=0.19, wpb=17670, bsz=1024, num_updates=21550, lr=2.3875e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=143847
-2024-11-04 21:28:44 - progress_bar.py[line:274] - INFO: epoch 056:     80 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.714, ntokens=17756.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.56, wps=3301.4, ups=0.19, wpb=17756.9, bsz=1024, num_updates=21560, lr=2.38613e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=143901
-2024-11-04 21:29:37 - progress_bar.py[line:274] - INFO: epoch 056:     90 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.716, ntokens=17843.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3338, ups=0.19, wpb=17843.2, bsz=1024, num_updates=21570, lr=2.38477e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=143954
-2024-11-04 21:30:30 - progress_bar.py[line:274] - INFO: epoch 056:    100 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.744, ntokens=17864, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3360.7, ups=0.19, wpb=17864, bsz=1024, num_updates=21580, lr=2.38341e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=144007
-2024-11-04 21:31:23 - progress_bar.py[line:274] - INFO: epoch 056:    110 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.667, ntokens=17284.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3259.9, ups=0.19, wpb=17284.3, bsz=1024, num_updates=21590, lr=2.38205e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=144060
-2024-11-04 21:32:16 - progress_bar.py[line:274] - INFO: epoch 056:    120 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.717, ntokens=17609.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3355.9, ups=0.19, wpb=17609.2, bsz=1024, num_updates=21600, lr=2.38069e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=144113
-2024-11-04 21:33:08 - progress_bar.py[line:274] - INFO: epoch 056:    130 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.641, ntokens=17126.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3246.1, ups=0.19, wpb=17126.8, bsz=1024, num_updates=21610, lr=2.37933e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=144166
-2024-11-04 21:34:01 - progress_bar.py[line:274] - INFO: epoch 056:    140 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.745, ntokens=17758.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3347.1, ups=0.19, wpb=17758.3, bsz=1024, num_updates=21620, lr=2.37797e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=144219
-2024-11-04 21:34:55 - progress_bar.py[line:274] - INFO: epoch 056:    150 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.67, ntokens=17485.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.37, wps=3287.9, ups=0.19, wpb=17485.5, bsz=1024, num_updates=21630, lr=2.37661e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=144272
-2024-11-04 21:35:47 - progress_bar.py[line:274] - INFO: epoch 056:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.69, ntokens=17572.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3332.5, ups=0.19, wpb=17572.5, bsz=1024, num_updates=21640, lr=2.37525e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=144324
-2024-11-04 21:36:41 - progress_bar.py[line:274] - INFO: epoch 056:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.71, ntokens=17782.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3323.7, ups=0.19, wpb=17782.8, bsz=1024, num_updates=21650, lr=2.37389e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=144378
-2024-11-04 21:37:34 - progress_bar.py[line:274] - INFO: epoch 056:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17368.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3283.4, ups=0.19, wpb=17368.2, bsz=1024, num_updates=21660, lr=2.37253e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=144431
-2024-11-04 21:38:27 - progress_bar.py[line:274] - INFO: epoch 056:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.712, ntokens=17843, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.55, wps=3383.9, ups=0.19, wpb=17843, bsz=1024, num_updates=21670, lr=2.37117e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=144484
-2024-11-04 21:39:20 - progress_bar.py[line:274] - INFO: epoch 056:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.755, ntokens=18001.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=3393, ups=0.19, wpb=18001.1, bsz=1024, num_updates=21680, lr=2.36981e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=144537
-2024-11-04 21:40:13 - progress_bar.py[line:274] - INFO: epoch 056:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.736, ntokens=18004.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.66, wps=3361.2, ups=0.19, wpb=18004.9, bsz=1024, num_updates=21690, lr=2.36845e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=144590
-2024-11-04 21:41:06 - progress_bar.py[line:274] - INFO: epoch 056:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.717, ntokens=17845.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3395.2, ups=0.19, wpb=17845.5, bsz=1024, num_updates=21700, lr=2.36709e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=144643
-2024-11-04 21:41:59 - progress_bar.py[line:274] - INFO: epoch 056:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.717, ntokens=17717.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3346.8, ups=0.19, wpb=17717.1, bsz=1024, num_updates=21710, lr=2.36573e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=144696
-2024-11-04 21:42:51 - progress_bar.py[line:274] - INFO: epoch 056:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.711, ntokens=17549.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.55, wps=3339.3, ups=0.19, wpb=17549.1, bsz=1024, num_updates=21720, lr=2.36437e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=144748
-2024-11-04 21:43:45 - progress_bar.py[line:274] - INFO: epoch 056:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.671, ntokens=17249.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.37, wps=3200.8, ups=0.19, wpb=17249.1, bsz=1024, num_updates=21730, lr=2.36301e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=144802
-2024-11-04 21:44:44 - progress_bar.py[line:274] - INFO: epoch 056:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.729, ntokens=17708.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3010.6, ups=0.17, wpb=17708.6, bsz=1024, num_updates=21740, lr=2.36165e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=144861
-2024-11-04 21:45:39 - progress_bar.py[line:274] - INFO: epoch 056:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.776, ntokens=18085.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=3265.4, ups=0.18, wpb=18085.7, bsz=1024, num_updates=21750, lr=2.36029e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=144916
-2024-11-04 21:46:32 - progress_bar.py[line:274] - INFO: epoch 056:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17605.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3322.7, ups=0.19, wpb=17605.2, bsz=1024, num_updates=21760, lr=2.35893e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=144969
-2024-11-04 21:47:25 - progress_bar.py[line:274] - INFO: epoch 056:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.707, ntokens=17558.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=3303.6, ups=0.19, wpb=17558.3, bsz=1024, num_updates=21770, lr=2.35757e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=145023
-2024-11-04 21:48:18 - progress_bar.py[line:274] - INFO: epoch 056:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.772, ntokens=17707.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=3380.8, ups=0.19, wpb=17707.7, bsz=1024, num_updates=21780, lr=2.35621e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=31, gb_free=13.1, wall=145075
-2024-11-04 21:49:09 - progress_bar.py[line:274] - INFO: epoch 056:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.892, ntokens=18489.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.42, wps=3590.8, ups=0.19, wpb=18489.1, bsz=1024, num_updates=21790, lr=2.35485e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=145126
-2024-11-04 21:50:02 - progress_bar.py[line:274] - INFO: epoch 056:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.875, ntokens=18384.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.34, wps=3477, ups=0.19, wpb=18384.8, bsz=1024, num_updates=21800, lr=2.35349e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=145179
-2024-11-04 21:50:55 - progress_bar.py[line:274] - INFO: epoch 056:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.83, ntokens=18219.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3432.8, ups=0.19, wpb=18219.2, bsz=1024, num_updates=21810, lr=2.35212e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=36, gb_free=13.4, wall=145232
-2024-11-04 21:51:47 - progress_bar.py[line:274] - INFO: epoch 056:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.783, ntokens=17978.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.88, wps=3445.1, ups=0.19, wpb=17978.3, bsz=1024, num_updates=21820, lr=2.35076e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=44, gb_free=13.3, wall=145285
-2024-11-04 21:52:40 - progress_bar.py[line:274] - INFO: epoch 056:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.867, ntokens=18500.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.3, wps=3530.9, ups=0.19, wpb=18500.2, bsz=1024, num_updates=21830, lr=2.3494e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=47, gb_free=13.4, wall=145337
-2024-11-04 21:53:32 - progress_bar.py[line:274] - INFO: epoch 056:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.811, ntokens=18093.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.02, wps=3487.7, ups=0.19, wpb=18093.9, bsz=1024, num_updates=21840, lr=2.34804e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=45, gb_free=13.2, wall=145389
-2024-11-04 21:54:25 - progress_bar.py[line:274] - INFO: epoch 056:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.844, ntokens=18346.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.18, wps=3448.6, ups=0.19, wpb=18346.6, bsz=1024, num_updates=21850, lr=2.34668e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=47, gb_free=13.3, wall=145442
-2024-11-04 21:55:18 - progress_bar.py[line:274] - INFO: epoch 056:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.822, ntokens=18254.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.07, wps=3440.7, ups=0.19, wpb=18254.5, bsz=1024, num_updates=21860, lr=2.34532e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=53, gb_free=13.4, wall=145495
-reach the end of datafile, start a new reader
-2024-11-04 21:56:10 - progress_bar.py[line:274] - INFO: epoch 056:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.853, ntokens=18276.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.23, wps=3515, ups=0.19, wpb=18276.3, bsz=1024, num_updates=21870, lr=2.34396e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=145547
-reach the end of datafile, start a new reader
-2024-11-04 21:56:12 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 56 @ 21871 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.pt
-2024-11-04 21:56:12 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.pt': File exists
-: File exists: File exists
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 21:58:01 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-
-slice_id 7 seek offset 350000
-slice_id 1 seek offset 50000
-slice_id 4 seek offset 200000
-2024-11-04 21:59:53 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 56 @ 21871 updates, score 0) (writing took 221.13734105601907 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_56.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-04 22:00:48 - train.py[line:336] - INFO: end of epoch 56 (average epoch stats below)
-2024-11-04 22:00:48 - progress_bar.py[line:282] - INFO: epoch 056 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.743 | ntokens 17785.7 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.69 | wps 2767.1 | ups 0.16 | wpb 17785.7 | bsz 1023 | num_updates 21871 | lr 2.34383e-05 | gnorm 0.006 | clip 0 | loss_scale 65536 | train_wall 1301 | gb_free 13.4 | wall 145825
-2024-11-04 22:00:48 - trainer.py[line:639] - INFO: loading train data for epoch 57
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 22:03:31 - trainer.py[line:703] - INFO: begin training epoch 57
-2024-11-04 22:03:31 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 22:04:21 - progress_bar.py[line:274] - INFO: epoch 057:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.715, ntokens=16905.9, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.56, wps=344.5, ups=0.02, wpb=16905.9, bsz=985.6, num_updates=21880, lr=2.3426e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=146038
-2024-11-04 22:05:13 - progress_bar.py[line:274] - INFO: epoch 057:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.721, ntokens=17924.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3405, ups=0.19, wpb=17924.1, bsz=1024, num_updates=21890, lr=2.34124e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=146090
-2024-11-04 22:06:07 - progress_bar.py[line:274] - INFO: epoch 057:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.698, ntokens=17615.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3309.9, ups=0.19, wpb=17615.5, bsz=1024, num_updates=21900, lr=2.33988e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=146144
-2024-11-04 22:07:01 - progress_bar.py[line:274] - INFO: epoch 057:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.748, ntokens=18080.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.72, wps=3336.4, ups=0.18, wpb=18080.1, bsz=1024, num_updates=21910, lr=2.33852e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=32, gb_free=13.4, wall=146198
-2024-11-04 22:07:58 - progress_bar.py[line:274] - INFO: epoch 057:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.717, ntokens=17719.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3094.3, ups=0.17, wpb=17719.9, bsz=1024, num_updates=21920, lr=2.33716e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=54, gb_free=13.6, wall=146255
-2024-11-04 22:08:54 - progress_bar.py[line:274] - INFO: epoch 057:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.707, ntokens=17722.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=3143.6, ups=0.18, wpb=17722.7, bsz=1024, num_updates=21930, lr=2.3358e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.6, wall=146311
-2024-11-04 22:09:50 - progress_bar.py[line:274] - INFO: epoch 057:     69 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.753, ntokens=17915.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.74, wps=3196.1, ups=0.18, wpb=17915.5, bsz=1024, num_updates=21940, lr=2.33444e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=146368
-2024-11-04 22:10:47 - progress_bar.py[line:274] - INFO: epoch 057:     79 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.661, ntokens=17394.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3067.9, ups=0.18, wpb=17394.9, bsz=1024, num_updates=21950, lr=2.33308e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=146424
-2024-11-04 22:11:40 - progress_bar.py[line:274] - INFO: epoch 057:     89 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.738, ntokens=17882.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.67, wps=3395.1, ups=0.19, wpb=17882.2, bsz=1024, num_updates=21960, lr=2.33172e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=146477
-2024-11-04 22:12:32 - progress_bar.py[line:274] - INFO: epoch 057:     99 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.717, ntokens=17779.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3398.6, ups=0.19, wpb=17779.7, bsz=1024, num_updates=21970, lr=2.33036e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.5, wall=146529
-2024-11-04 22:13:24 - progress_bar.py[line:274] - INFO: epoch 057:    109 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.699, ntokens=17540.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3376.9, ups=0.19, wpb=17540.5, bsz=1024, num_updates=21980, lr=2.329e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=50, gb_free=13.4, wall=146581
-2024-11-04 22:14:17 - progress_bar.py[line:274] - INFO: epoch 057:    119 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.731, ntokens=17769.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.64, wps=3362.1, ups=0.19, wpb=17769.2, bsz=1024, num_updates=21990, lr=2.32764e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=44, gb_free=13.5, wall=146634
-2024-11-04 22:15:10 - progress_bar.py[line:274] - INFO: epoch 057:    129 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.699, ntokens=17623.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3307.4, ups=0.19, wpb=17623.1, bsz=1024, num_updates=22000, lr=2.32628e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=48, gb_free=13.6, wall=146687
-2024-11-04 22:16:03 - progress_bar.py[line:274] - INFO: epoch 057:    139 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.706, ntokens=17626, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3359.1, ups=0.19, wpb=17626, bsz=1024, num_updates=22010, lr=2.32492e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=44, gb_free=13.5, wall=146740
-2024-11-04 22:16:55 - progress_bar.py[line:274] - INFO: epoch 057:    149 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.728, ntokens=17732.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3412.3, ups=0.19, wpb=17732.8, bsz=1024, num_updates=22020, lr=2.32356e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=39, gb_free=13.5, wall=146792
-2024-11-04 22:17:10 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-04 22:17:53 - progress_bar.py[line:274] - INFO: epoch 057:    160 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.698, ntokens=17820.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3057.1, ups=0.17, wpb=17820.5, bsz=1024, num_updates=22030, lr=2.3222e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=34, gb_free=13.5, wall=146850
-2024-11-04 22:18:46 - progress_bar.py[line:274] - INFO: epoch 057:    170 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.754, ntokens=18102.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.74, wps=3410.8, ups=0.19, wpb=18102.9, bsz=1024, num_updates=22040, lr=2.32084e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=146903
-2024-11-04 22:19:39 - progress_bar.py[line:274] - INFO: epoch 057:    180 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.653, ntokens=17261.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.29, wps=3249.4, ups=0.19, wpb=17261.1, bsz=1024, num_updates=22050, lr=2.31948e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=146956
-2024-11-04 22:20:32 - progress_bar.py[line:274] - INFO: epoch 057:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.721, ntokens=17884.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3394.2, ups=0.19, wpb=17884.8, bsz=1024, num_updates=22060, lr=2.31812e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=147009
-2024-11-04 22:21:24 - progress_bar.py[line:274] - INFO: epoch 057:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.706, ntokens=17736.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3368.7, ups=0.19, wpb=17736.9, bsz=1024, num_updates=22070, lr=2.31675e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=147062
-2024-11-04 22:22:18 - progress_bar.py[line:274] - INFO: epoch 057:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.713, ntokens=17858.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.56, wps=3343.1, ups=0.19, wpb=17858.7, bsz=1024, num_updates=22080, lr=2.31539e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=147115
-2024-11-04 22:23:11 - progress_bar.py[line:274] - INFO: epoch 057:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.651, ntokens=17460.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3276.4, ups=0.19, wpb=17460.2, bsz=1024, num_updates=22090, lr=2.31403e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=147168
-2024-11-04 22:24:04 - progress_bar.py[line:274] - INFO: epoch 057:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.697, ntokens=17778.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.48, wps=3338.3, ups=0.19, wpb=17778.9, bsz=1024, num_updates=22100, lr=2.31267e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=147222
-2024-11-04 22:24:58 - progress_bar.py[line:274] - INFO: epoch 057:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=17457.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3275.2, ups=0.19, wpb=17457.6, bsz=1024, num_updates=22110, lr=2.31131e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=147275
-2024-11-04 22:25:51 - progress_bar.py[line:274] - INFO: epoch 057:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.71, ntokens=17654, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.55, wps=3326.1, ups=0.19, wpb=17654, bsz=1024, num_updates=22120, lr=2.30995e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=147328
-2024-11-04 22:26:44 - progress_bar.py[line:274] - INFO: epoch 057:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.691, ntokens=17585, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3323.9, ups=0.19, wpb=17585, bsz=1024, num_updates=22130, lr=2.30859e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=31, gb_free=13.6, wall=147381
-2024-11-04 22:27:37 - progress_bar.py[line:274] - INFO: epoch 057:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.774, ntokens=18122.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3397.6, ups=0.19, wpb=18122.8, bsz=1024, num_updates=22140, lr=2.30723e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=147434
-2024-11-04 22:28:30 - progress_bar.py[line:274] - INFO: epoch 057:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.701, ntokens=17691.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3371, ups=0.19, wpb=17691.9, bsz=1024, num_updates=22150, lr=2.30587e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=147487
-2024-11-04 22:29:22 - progress_bar.py[line:274] - INFO: epoch 057:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.732, ntokens=17797.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.64, wps=3379.3, ups=0.19, wpb=17797.5, bsz=1024, num_updates=22160, lr=2.30451e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=147539
-2024-11-04 22:30:15 - progress_bar.py[line:274] - INFO: epoch 057:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.798, ntokens=17968.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.96, wps=3426.4, ups=0.19, wpb=17968.7, bsz=1024, num_updates=22170, lr=2.30315e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=147592
-2024-11-04 22:31:07 - progress_bar.py[line:274] - INFO: epoch 057:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.896, ntokens=18475.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.44, wps=3558.5, ups=0.19, wpb=18475.8, bsz=1024, num_updates=22180, lr=2.30179e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=147644
-2024-11-04 22:31:59 - progress_bar.py[line:274] - INFO: epoch 057:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.81, ntokens=18035.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.01, wps=3458.2, ups=0.19, wpb=18035.2, bsz=1024, num_updates=22190, lr=2.30043e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=147696
-2024-11-04 22:32:51 - progress_bar.py[line:274] - INFO: epoch 057:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.851, ntokens=18390.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.21, wps=3548.8, ups=0.19, wpb=18390.6, bsz=1024, num_updates=22200, lr=2.29907e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=147748
-2024-11-04 22:33:43 - progress_bar.py[line:274] - INFO: epoch 057:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.749, ntokens=17853.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.72, wps=3423.8, ups=0.19, wpb=17853.2, bsz=1024, num_updates=22210, lr=2.29771e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=147800
-2024-11-04 22:34:40 - progress_bar.py[line:274] - INFO: epoch 057:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.869, ntokens=18540.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.31, wps=3249.4, ups=0.18, wpb=18540.5, bsz=1024, num_updates=22220, lr=2.29635e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=147857
-2024-11-04 22:35:33 - progress_bar.py[line:274] - INFO: epoch 057:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.843, ntokens=18464.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.18, wps=3484.1, ups=0.19, wpb=18464.5, bsz=1024, num_updates=22230, lr=2.29499e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=147910
-2024-11-04 22:36:26 - progress_bar.py[line:274] - INFO: epoch 057:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.807, ntokens=18077.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3414.1, ups=0.19, wpb=18077.5, bsz=1024, num_updates=22240, lr=2.29363e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.2, wall=147963
-2024-11-04 22:37:18 - progress_bar.py[line:274] - INFO: epoch 057:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.791, ntokens=18212.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=3472.7, ups=0.19, wpb=18212.2, bsz=1024, num_updates=22250, lr=2.29227e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=148015
-reach the end of datafile, start a new reader
-2024-11-04 22:38:11 - progress_bar.py[line:274] - INFO: epoch 057:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.843, ntokens=18195.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.17, wps=3417.4, ups=0.19, wpb=18195.6, bsz=1024, num_updates=22260, lr=2.29091e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=148069
-reach the end of datafile, start a new reader
-2024-11-04 22:38:13 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 57 @ 22261 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.pt
-2024-11-04 22:38:13 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.pt': File exists: File exists
-
-: File exists
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 22:40:03 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-
-slice_id 1 seek offset 50000
-slice_id 5 seek offset 250000
-slice_id 7 seek offset 350000
-2024-11-04 22:41:56 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 57 @ 22261 updates, score 0) (writing took 222.31180876796134 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_57.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-
+2024-11-07 11:05:38 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
 
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
+
 slice_id 6 seek offset 300000
+slice_id 2 seek offset 100000
 slice_id 3 seek offset 150000
-2024-11-04 22:42:50 - train.py[line:336] - INFO: end of epoch 57 (average epoch stats below)
-2024-11-04 22:42:50 - progress_bar.py[line:282] - INFO: epoch 057 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.742 | ntokens 17838.1 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.69 | wps 2757.5 | ups 0.15 | wpb 17838.1 | bsz 1023 | num_updates 22261 | lr 2.29077e-05 | gnorm 0.008 | clip 0 | loss_scale 32768 | train_wall 1409 | gb_free 13.4 | wall 148348
-2024-11-04 22:42:50 - trainer.py[line:639] - INFO: loading train data for epoch 58
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 22:45:34 - trainer.py[line:703] - INFO: begin training epoch 58
-2024-11-04 22:45:34 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 22:46:25 - progress_bar.py[line:274] - INFO: epoch 058:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.73, ntokens=17135.1, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=346.9, ups=0.02, wpb=17135.1, bsz=985.6, num_updates=22270, lr=2.28955e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=29, gb_free=13.6, wall=148562
-2024-11-04 22:47:20 - progress_bar.py[line:274] - INFO: epoch 058:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.717, ntokens=17839.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3288.6, ups=0.18, wpb=17839.1, bsz=1024, num_updates=22280, lr=2.28819e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=148617
-2024-11-04 22:48:12 - progress_bar.py[line:274] - INFO: epoch 058:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.667, ntokens=17515.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3338.3, ups=0.19, wpb=17515.4, bsz=1024, num_updates=22290, lr=2.28683e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=148669
-2024-11-04 22:49:07 - progress_bar.py[line:274] - INFO: epoch 058:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.758, ntokens=17914.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.76, wps=3246.6, ups=0.18, wpb=17914.8, bsz=1024, num_updates=22300, lr=2.28547e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=45, gb_free=13.4, wall=148724
-2024-11-04 22:50:02 - progress_bar.py[line:274] - INFO: epoch 058:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.701, ntokens=17692.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3208.4, ups=0.18, wpb=17692.9, bsz=1024, num_updates=22310, lr=2.28411e-05, gnorm=0.011, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=148779
-2024-11-04 22:50:58 - progress_bar.py[line:274] - INFO: epoch 058:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.736, ntokens=17866.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.66, wps=3217.7, ups=0.18, wpb=17866.9, bsz=1024, num_updates=22320, lr=2.28274e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=148835
-2024-11-04 22:51:53 - progress_bar.py[line:274] - INFO: epoch 058:     69 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.737, ntokens=17833.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.67, wps=3242.7, ups=0.18, wpb=17833.8, bsz=1024, num_updates=22330, lr=2.28138e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=148890
-2024-11-04 22:52:47 - progress_bar.py[line:274] - INFO: epoch 058:     79 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.689, ntokens=17558.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3232.1, ups=0.18, wpb=17558.1, bsz=1024, num_updates=22340, lr=2.28002e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=148944
-2024-11-04 22:53:42 - progress_bar.py[line:274] - INFO: epoch 058:     89 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.708, ntokens=17792.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3265.3, ups=0.18, wpb=17792.7, bsz=1024, num_updates=22350, lr=2.27866e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=148999
-2024-11-04 22:54:37 - progress_bar.py[line:274] - INFO: epoch 058:     99 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.727, ntokens=17875.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3254.1, ups=0.18, wpb=17875.3, bsz=1024, num_updates=22360, lr=2.2773e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=149054
-2024-11-04 22:55:31 - progress_bar.py[line:274] - INFO: epoch 058:    109 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.662, ntokens=17320.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3162.8, ups=0.18, wpb=17320.6, bsz=1024, num_updates=22370, lr=2.27594e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=149109
-2024-11-04 22:56:28 - progress_bar.py[line:274] - INFO: epoch 058:    119 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.705, ntokens=17654.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3144.1, ups=0.18, wpb=17654.1, bsz=1024, num_updates=22380, lr=2.27458e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.3, wall=149165
-2024-11-04 22:57:24 - progress_bar.py[line:274] - INFO: epoch 058:    129 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.67, ntokens=17433.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3113, ups=0.18, wpb=17433.8, bsz=1024, num_updates=22390, lr=2.27322e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=149221
-2024-11-04 22:58:20 - progress_bar.py[line:274] - INFO: epoch 058:    139 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.725, ntokens=17810.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.61, wps=3140.1, ups=0.18, wpb=17810.1, bsz=1024, num_updates=22400, lr=2.27186e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=149277
-2024-11-04 22:59:17 - progress_bar.py[line:274] - INFO: epoch 058:    149 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.699, ntokens=17577.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3077, ups=0.18, wpb=17577.3, bsz=1024, num_updates=22410, lr=2.2705e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=149335
-2024-11-04 23:00:14 - progress_bar.py[line:274] - INFO: epoch 058:    159 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.682, ntokens=17674, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3121.3, ups=0.18, wpb=17674, bsz=1024, num_updates=22420, lr=2.26914e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=149391
-2024-11-04 23:01:09 - progress_bar.py[line:274] - INFO: epoch 058:    169 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.71, ntokens=17831.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3250.5, ups=0.18, wpb=17831.2, bsz=1024, num_updates=22430, lr=2.26778e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=149446
-2024-11-04 23:02:03 - progress_bar.py[line:274] - INFO: epoch 058:    179 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=17419.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3212.9, ups=0.18, wpb=17419.5, bsz=1024, num_updates=22440, lr=2.26642e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.6, wall=149500
-2024-11-04 23:02:57 - progress_bar.py[line:274] - INFO: epoch 058:    189 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.691, ntokens=17731.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3261.3, ups=0.18, wpb=17731.6, bsz=1024, num_updates=22450, lr=2.26506e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=149555
-2024-11-04 23:03:53 - progress_bar.py[line:274] - INFO: epoch 058:    199 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.753, ntokens=18060.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.74, wps=3279, ups=0.18, wpb=18060.8, bsz=1024, num_updates=22460, lr=2.2637e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=149610
-2024-11-04 23:04:47 - progress_bar.py[line:274] - INFO: epoch 058:    209 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.75, ntokens=18159.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3338.4, ups=0.18, wpb=18159.2, bsz=1024, num_updates=22470, lr=2.26234e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=149664
-2024-11-04 23:05:43 - progress_bar.py[line:274] - INFO: epoch 058:    219 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.662, ntokens=17698.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3144.8, ups=0.18, wpb=17698.6, bsz=1024, num_updates=22480, lr=2.26098e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=149720
-2024-11-04 23:06:40 - progress_bar.py[line:274] - INFO: epoch 058:    229 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.706, ntokens=17831.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=3165.4, ups=0.18, wpb=17831.7, bsz=1024, num_updates=22490, lr=2.25962e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=149777
-2024-11-04 23:07:36 - progress_bar.py[line:274] - INFO: epoch 058:    239 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.718, ntokens=17813.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3168.1, ups=0.18, wpb=17813.4, bsz=1024, num_updates=22500, lr=2.25826e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=149833
-2024-11-04 23:08:26 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 16384.0
-2024-11-04 23:08:37 - progress_bar.py[line:274] - INFO: epoch 058:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.714, ntokens=17554.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.56, wps=2849.6, ups=0.16, wpb=17554.5, bsz=1024, num_updates=22510, lr=2.2569e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=61, gb_free=13.6, wall=149895
-2024-11-04 23:09:33 - progress_bar.py[line:274] - INFO: epoch 058:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.72, ntokens=17811, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3222, ups=0.18, wpb=17811, bsz=1024, num_updates=22520, lr=2.25554e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=55, gb_free=13.6, wall=149950
-2024-11-04 23:10:28 - progress_bar.py[line:274] - INFO: epoch 058:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.692, ntokens=17661.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3211.2, ups=0.18, wpb=17661.1, bsz=1024, num_updates=22530, lr=2.25418e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=55, gb_free=13.6, wall=150005
-2024-11-04 23:11:23 - progress_bar.py[line:274] - INFO: epoch 058:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.681, ntokens=17594.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3184.2, ups=0.18, wpb=17594.8, bsz=1024, num_updates=22540, lr=2.25282e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=55, gb_free=13.6, wall=150060
-2024-11-04 23:12:16 - progress_bar.py[line:274] - INFO: epoch 058:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=17873.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3361.4, ups=0.19, wpb=17873.7, bsz=1024, num_updates=22550, lr=2.25146e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=53, gb_free=13.6, wall=150113
-2024-11-04 23:13:08 - progress_bar.py[line:274] - INFO: epoch 058:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.729, ntokens=17568, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3368.9, ups=0.19, wpb=17568, bsz=1024, num_updates=22560, lr=2.2501e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.2, wall=150165
-2024-11-04 23:14:04 - progress_bar.py[line:274] - INFO: epoch 058:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.841, ntokens=18228.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.17, wps=3268.3, ups=0.18, wpb=18228.4, bsz=1024, num_updates=22570, lr=2.24873e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=56, gb_free=13.4, wall=150221
-2024-11-04 23:14:56 - progress_bar.py[line:274] - INFO: epoch 058:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.827, ntokens=18347.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.1, wps=3504.8, ups=0.19, wpb=18347.1, bsz=1024, num_updates=22580, lr=2.24737e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=52, gb_free=13.5, wall=150273
-2024-11-04 23:15:49 - progress_bar.py[line:274] - INFO: epoch 058:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.819, ntokens=18209.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.06, wps=3478.6, ups=0.19, wpb=18209.4, bsz=1024, num_updates=22590, lr=2.24601e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=150326
-2024-11-04 23:16:41 - progress_bar.py[line:274] - INFO: epoch 058:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.783, ntokens=18158.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.88, wps=3476.9, ups=0.19, wpb=18158.8, bsz=1024, num_updates=22600, lr=2.24465e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=150378
-2024-11-04 23:17:34 - progress_bar.py[line:274] - INFO: epoch 058:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.856, ntokens=18625.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.24, wps=3507.5, ups=0.19, wpb=18625.4, bsz=1024, num_updates=22610, lr=2.24329e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=53, gb_free=13.3, wall=150431
-2024-11-04 23:18:27 - progress_bar.py[line:274] - INFO: epoch 058:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.825, ntokens=18394.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.09, wps=3505.9, ups=0.19, wpb=18394.5, bsz=1024, num_updates=22620, lr=2.24193e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=52, gb_free=13.3, wall=150484
-2024-11-04 23:19:20 - progress_bar.py[line:274] - INFO: epoch 058:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.822, ntokens=18220.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.07, wps=3432.7, ups=0.19, wpb=18220.4, bsz=1024, num_updates=22630, lr=2.24057e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=53, gb_free=13.3, wall=150537
-2024-11-04 23:20:13 - progress_bar.py[line:274] - INFO: epoch 058:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.738, ntokens=17835.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.67, wps=3347.9, ups=0.19, wpb=17835.3, bsz=1024, num_updates=22640, lr=2.23921e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=53, gb_free=13.4, wall=150590
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-04 23:21:05 - progress_bar.py[line:274] - INFO: epoch 058:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.835, ntokens=18114.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.14, wps=3500.7, ups=0.19, wpb=18114.1, bsz=1024, num_updates=22650, lr=2.23785e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=150642
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.pt2024-11-04 23:21:07 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 58 @ 22651 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.pt
-2024-11-04 23:21:07 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.pt': File exists: File exists
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-04 23:22:57 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
+2024-11-07 11:07:30 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 31 @ 12111 updates, score 0) (writing took 220.8156002679898 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_31.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-2024-11-04 23:24:49 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 58 @ 22651 updates, score 0) (writing took 222.59894327196525 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_58.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 1 seek offset 50000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-slice_id 6 seek offset 300000
-2024-11-04 23:25:44 - train.py[line:336] - INFO: end of epoch 58 (average epoch stats below)
-2024-11-04 23:25:44 - progress_bar.py[line:282] - INFO: epoch 058 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.733 | ntokens 17825.7 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.65 | wps 2701.8 | ups 0.15 | wpb 17825.7 | bsz 1023 | num_updates 22651 | lr 2.23772e-05 | gnorm 0.007 | clip 0 | loss_scale 16384 | train_wall 2049 | gb_free 13.3 | wall 150921
-2024-11-04 23:25:44 - trainer.py[line:639] - INFO: loading train data for epoch 59
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-04 23:28:27 - trainer.py[line:703] - INFO: begin training epoch 59
-2024-11-04 23:28:27 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-04 23:29:17 - progress_bar.py[line:274] - INFO: epoch 059:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.641, ntokens=16635.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=337.7, ups=0.02, wpb=16635.4, bsz=985.6, num_updates=22660, lr=2.23649e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=29, gb_free=13.5, wall=151134
-2024-11-04 23:30:11 - progress_bar.py[line:274] - INFO: epoch 059:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.719, ntokens=17842, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.58, wps=3319.8, ups=0.19, wpb=17842, bsz=1024, num_updates=22670, lr=2.23513e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=151188
-2024-11-04 23:31:04 - progress_bar.py[line:274] - INFO: epoch 059:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.669, ntokens=17612.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3295.6, ups=0.19, wpb=17612.1, bsz=1024, num_updates=22680, lr=2.23377e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=151241
-2024-11-04 23:31:58 - progress_bar.py[line:274] - INFO: epoch 059:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.684, ntokens=17640.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.43, wps=3290.8, ups=0.19, wpb=17640.7, bsz=1024, num_updates=22690, lr=2.23241e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.4, wall=151295
-2024-11-04 23:32:51 - progress_bar.py[line:274] - INFO: epoch 059:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.668, ntokens=17597.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3310.4, ups=0.19, wpb=17597.8, bsz=1024, num_updates=22700, lr=2.23105e-05, gnorm=0.01, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=151348
-2024-11-04 23:33:44 - progress_bar.py[line:274] - INFO: epoch 059:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.691, ntokens=17631.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3350.4, ups=0.19, wpb=17631.5, bsz=1024, num_updates=22710, lr=2.22969e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=151401
-2024-11-04 23:34:37 - progress_bar.py[line:274] - INFO: epoch 059:     69 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.693, ntokens=17697, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3351.6, ups=0.19, wpb=17697, bsz=1024, num_updates=22720, lr=2.22833e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=151454
-2024-11-04 23:35:30 - progress_bar.py[line:274] - INFO: epoch 059:     79 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.702, ntokens=17690, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3308.2, ups=0.19, wpb=17690, bsz=1024, num_updates=22730, lr=2.22697e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=151507
-2024-11-04 23:36:23 - progress_bar.py[line:274] - INFO: epoch 059:     89 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.669, ntokens=17519, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3299.7, ups=0.19, wpb=17519, bsz=1024, num_updates=22740, lr=2.22561e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=151560
-2024-11-04 23:37:16 - progress_bar.py[line:274] - INFO: epoch 059:     99 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.723, ntokens=17886, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.6, wps=3391.3, ups=0.19, wpb=17886, bsz=1024, num_updates=22750, lr=2.22425e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=151613
-2024-11-04 23:38:09 - progress_bar.py[line:274] - INFO: epoch 059:    109 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.7, ntokens=17631.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3335.9, ups=0.19, wpb=17631.7, bsz=1024, num_updates=22760, lr=2.22289e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=31, gb_free=13.3, wall=151666
-2024-11-04 23:39:02 - progress_bar.py[line:274] - INFO: epoch 059:    119 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.698, ntokens=17580, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3292, ups=0.19, wpb=17580, bsz=1024, num_updates=22770, lr=2.22153e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=151719
-2024-11-04 23:39:55 - progress_bar.py[line:274] - INFO: epoch 059:    129 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.654, ntokens=17452.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.29, wps=3301.6, ups=0.19, wpb=17452.5, bsz=1024, num_updates=22780, lr=2.22017e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=151772
-2024-11-04 23:40:48 - progress_bar.py[line:274] - INFO: epoch 059:    139 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.677, ntokens=17576.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.4, wps=3316.5, ups=0.19, wpb=17576.8, bsz=1024, num_updates=22790, lr=2.21881e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=151825
-2024-11-04 23:41:41 - progress_bar.py[line:274] - INFO: epoch 059:    149 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.712, ntokens=17837, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.55, wps=3383.5, ups=0.19, wpb=17837, bsz=1024, num_updates=22800, lr=2.21745e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=151878
-2024-11-04 23:42:34 - progress_bar.py[line:274] - INFO: epoch 059:    159 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17697.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3322.7, ups=0.19, wpb=17697.3, bsz=1024, num_updates=22810, lr=2.21609e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=151931
-2024-11-04 23:43:27 - progress_bar.py[line:274] - INFO: epoch 059:    169 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.716, ntokens=17996.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3424.2, ups=0.19, wpb=17996.5, bsz=1024, num_updates=22820, lr=2.21472e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.6, wall=151984
-2024-11-04 23:44:19 - progress_bar.py[line:274] - INFO: epoch 059:    179 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.669, ntokens=17397.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3287, ups=0.19, wpb=17397.9, bsz=1024, num_updates=22830, lr=2.21336e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=152037
-2024-11-04 23:45:12 - progress_bar.py[line:274] - INFO: epoch 059:    189 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.683, ntokens=17727.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3356.2, ups=0.19, wpb=17727.4, bsz=1024, num_updates=22840, lr=2.212e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=152089
-2024-11-04 23:46:05 - progress_bar.py[line:274] - INFO: epoch 059:    199 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.77, ntokens=18184.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=3461.1, ups=0.19, wpb=18184.2, bsz=1024, num_updates=22850, lr=2.21064e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=152142
-2024-11-04 23:46:58 - progress_bar.py[line:274] - INFO: epoch 059:    209 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.689, ntokens=17846.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3371, ups=0.19, wpb=17846.9, bsz=1024, num_updates=22860, lr=2.20928e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=152195
-2024-11-04 23:47:51 - progress_bar.py[line:274] - INFO: epoch 059:    219 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.685, ntokens=17689.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.43, wps=3312.6, ups=0.19, wpb=17689.3, bsz=1024, num_updates=22870, lr=2.20792e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=152248
-2024-11-04 23:48:44 - progress_bar.py[line:274] - INFO: epoch 059:    229 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=17663.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3338.6, ups=0.19, wpb=17663.6, bsz=1024, num_updates=22880, lr=2.20656e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=152301
-2024-11-04 23:49:38 - progress_bar.py[line:274] - INFO: epoch 059:    239 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.68, ntokens=17549.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3262.7, ups=0.19, wpb=17549.1, bsz=1024, num_updates=22890, lr=2.2052e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=152355
-2024-11-04 23:50:31 - progress_bar.py[line:274] - INFO: epoch 059:    249 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=17494.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3321, ups=0.19, wpb=17494.6, bsz=1024, num_updates=22900, lr=2.20384e-05, gnorm=0.009, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=152408
-2024-11-04 23:51:25 - progress_bar.py[line:274] - INFO: epoch 059:    259 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=17721.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3230, ups=0.18, wpb=17721.2, bsz=1024, num_updates=22910, lr=2.20248e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=31, gb_free=13.6, wall=152463
-2024-11-04 23:52:18 - progress_bar.py[line:274] - INFO: epoch 059:    269 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.743, ntokens=18013.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3426.8, ups=0.19, wpb=18013.3, bsz=1024, num_updates=22920, lr=2.20112e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.3, wall=152515
-2024-11-04 23:53:11 - progress_bar.py[line:274] - INFO: epoch 059:    279 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.7, ntokens=17765.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3359.7, ups=0.19, wpb=17765.2, bsz=1024, num_updates=22930, lr=2.19976e-05, gnorm=0.005, clip=0, loss_scale=16384, train_wall=31, gb_free=13.5, wall=152568
-2024-11-04 23:54:04 - progress_bar.py[line:274] - INFO: epoch 059:    289 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.708, ntokens=17843.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3344.4, ups=0.19, wpb=17843.2, bsz=1024, num_updates=22940, lr=2.1984e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=30, gb_free=13.5, wall=152621
-2024-11-04 23:55:00 - progress_bar.py[line:274] - INFO: epoch 059:    299 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.747, ntokens=17741.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3188.1, ups=0.18, wpb=17741.5, bsz=1024, num_updates=22950, lr=2.19704e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=40, gb_free=13.5, wall=152677
-2024-11-04 23:55:57 - progress_bar.py[line:274] - INFO: epoch 059:    309 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.895, ntokens=18723.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.44, wps=3267.3, ups=0.17, wpb=18723.5, bsz=1024, num_updates=22960, lr=2.19568e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=57, gb_free=13.1, wall=152734
-2024-11-04 23:56:55 - progress_bar.py[line:274] - INFO: epoch 059:    319 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.794, ntokens=18097.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.94, wps=3150.4, ups=0.17, wpb=18097.6, bsz=1024, num_updates=22970, lr=2.19432e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=57, gb_free=13.5, wall=152792
-2024-11-04 23:57:47 - progress_bar.py[line:274] - INFO: epoch 059:    329 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.8, ntokens=18242.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.96, wps=3486.9, ups=0.19, wpb=18242.3, bsz=1024, num_updates=22980, lr=2.19296e-05, gnorm=0.008, clip=0, loss_scale=16384, train_wall=52, gb_free=13.2, wall=152844
-2024-11-04 23:58:39 - progress_bar.py[line:274] - INFO: epoch 059:    339 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.791, ntokens=18259.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.92, wps=3484.4, ups=0.19, wpb=18259.5, bsz=1024, num_updates=22990, lr=2.1916e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=52, gb_free=13.4, wall=152896
-2024-11-04 23:59:32 - progress_bar.py[line:274] - INFO: epoch 059:    349 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.831, ntokens=18549.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3502.6, ups=0.19, wpb=18549.4, bsz=1024, num_updates=23000, lr=2.19024e-05, gnorm=0.007, clip=0, loss_scale=16384, train_wall=53, gb_free=13.3, wall=152949
-2024-11-05 00:00:26 - progress_bar.py[line:274] - INFO: epoch 059:    359 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.823, ntokens=18392.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.08, wps=3402, ups=0.18, wpb=18392.8, bsz=1024, num_updates=23010, lr=2.18888e-05, gnorm=0.006, clip=0, loss_scale=16384, train_wall=54, gb_free=13.3, wall=153003
-2024-11-05 00:01:19 - progress_bar.py[line:274] - INFO: epoch 059:    369 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.737, ntokens=17810.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.67, wps=3406.7, ups=0.19, wpb=17810.5, bsz=1024, num_updates=23020, lr=2.18752e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=50, gb_free=13.4, wall=153056
-2024-11-05 00:02:11 - progress_bar.py[line:274] - INFO: epoch 059:    379 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.794, ntokens=18373.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.94, wps=3514, ups=0.19, wpb=18373.4, bsz=1024, num_updates=23030, lr=2.18616e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=38, gb_free=13.4, wall=153108
-2024-11-05 00:03:03 - progress_bar.py[line:274] - INFO: epoch 059:    389 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.783, ntokens=17847.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.88, wps=3408.3, ups=0.19, wpb=17847.9, bsz=1024, num_updates=23040, lr=2.1848e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=39, gb_free=13.5, wall=153160
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-05 00:03:10 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 59 @ 23042 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.pt
 
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.pt
-2024-11-05 00:03:10 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.pt': File exists
-: File exists: File exists
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-05 00:05:00 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
 slice_id 1 seek offset 50000
-slice_id 7 seek offset 350000
 slice_id 5 seek offset 250000
-2024-11-05 00:06:53 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 59 @ 23042 updates, score 0) (writing took 222.50441667204723 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_59.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
 slice_id 4 seek offset 200000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-05 00:07:47 - train.py[line:336] - INFO: end of epoch 59 (average epoch stats below)
-2024-11-05 00:07:47 - progress_bar.py[line:282] - INFO: epoch 059 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.722 | ntokens 17809.8 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.6 | wps 2759.2 | ups 0.15 | wpb 17809.8 | bsz 1023 | num_updates 23042 | lr 2.18452e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 1379 | gb_free 13.3 | wall 153444
-2024-11-05 00:07:47 - trainer.py[line:639] - INFO: loading train data for epoch 60
+2024-11-07 11:08:25 - train.py[line:336] - INFO: end of epoch 31 (average epoch stats below)
+2024-11-07 11:08:25 - progress_bar.py[line:282] - INFO: epoch 031 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.448 | ntokens 17837.8 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.46 | wps 2799.2 | ups 0.16 | wpb 17837.8 | bsz 1023 | num_updates 12111 | lr 3.67157e-05 | gnorm 0.008 | clip 0 | loss_scale 65536 | train_wall 1383 | gb_free 13.3 | wall 77223
+2024-11-07 11:08:25 - trainer.py[line:639] - INFO: loading train data for epoch 32
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-05 00:10:31 - trainer.py[line:703] - INFO: begin training epoch 60
-2024-11-05 00:10:31 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-05 00:11:17 - progress_bar.py[line:274] - INFO: epoch 060:      8 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=17047.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=345.5, ups=0.02, wpb=17047.7, bsz=985.6, num_updates=23050, lr=2.18344e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=153654
-2024-11-05 00:12:10 - progress_bar.py[line:274] - INFO: epoch 060:     18 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.67, ntokens=17677.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3327.3, ups=0.19, wpb=17677.6, bsz=1024, num_updates=23060, lr=2.18208e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=153707
-2024-11-05 00:13:03 - progress_bar.py[line:274] - INFO: epoch 060:     28 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.67, ntokens=17546.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3317.3, ups=0.19, wpb=17546.3, bsz=1024, num_updates=23070, lr=2.18072e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=153760
-2024-11-05 00:13:56 - progress_bar.py[line:274] - INFO: epoch 060:     38 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.674, ntokens=17698.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.38, wps=3312.6, ups=0.19, wpb=17698.3, bsz=1024, num_updates=23080, lr=2.17935e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=153813
-2024-11-05 00:14:49 - progress_bar.py[line:274] - INFO: epoch 060:     48 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.679, ntokens=17622.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3346.7, ups=0.19, wpb=17622.8, bsz=1024, num_updates=23090, lr=2.17799e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=153866
-2024-11-05 00:15:41 - progress_bar.py[line:274] - INFO: epoch 060:     58 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.681, ntokens=17768.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3386.5, ups=0.19, wpb=17768.7, bsz=1024, num_updates=23100, lr=2.17663e-05, gnorm=0.01, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=153918
-2024-11-05 00:16:34 - progress_bar.py[line:274] - INFO: epoch 060:     68 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.728, ntokens=17889.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3377.3, ups=0.19, wpb=17889.6, bsz=1024, num_updates=23110, lr=2.17527e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=153971
-2024-11-05 00:17:27 - progress_bar.py[line:274] - INFO: epoch 060:     78 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.663, ntokens=17539.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3303.7, ups=0.19, wpb=17539.3, bsz=1024, num_updates=23120, lr=2.17391e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=154024
-2024-11-05 00:18:20 - progress_bar.py[line:274] - INFO: epoch 060:     88 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=17450.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3324.1, ups=0.19, wpb=17450.2, bsz=1024, num_updates=23130, lr=2.17255e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=154077
-2024-11-05 00:19:13 - progress_bar.py[line:274] - INFO: epoch 060:     98 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.687, ntokens=17789.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3367.7, ups=0.19, wpb=17789.7, bsz=1024, num_updates=23140, lr=2.17119e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=154130
-2024-11-05 00:20:06 - progress_bar.py[line:274] - INFO: epoch 060:    108 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.681, ntokens=17676.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3337.6, ups=0.19, wpb=17676.3, bsz=1024, num_updates=23150, lr=2.16983e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=154183
-2024-11-05 00:20:58 - progress_bar.py[line:274] - INFO: epoch 060:    118 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.769, ntokens=18104.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=3445.1, ups=0.19, wpb=18104.8, bsz=1024, num_updates=23160, lr=2.16847e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=154235
-2024-11-05 00:21:51 - progress_bar.py[line:274] - INFO: epoch 060:    128 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.616, ntokens=17234.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3265.8, ups=0.19, wpb=17234.2, bsz=1024, num_updates=23170, lr=2.16711e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=154288
-2024-11-05 00:22:44 - progress_bar.py[line:274] - INFO: epoch 060:    138 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.668, ntokens=17536, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3324.5, ups=0.19, wpb=17536, bsz=1024, num_updates=23180, lr=2.16575e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=154341
-2024-11-05 00:23:36 - progress_bar.py[line:274] - INFO: epoch 060:    148 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.674, ntokens=17617.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.38, wps=3340, ups=0.19, wpb=17617.7, bsz=1024, num_updates=23190, lr=2.16439e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=154394
-2024-11-05 00:24:31 - progress_bar.py[line:274] - INFO: epoch 060:    158 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17770.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3262.8, ups=0.18, wpb=17770.7, bsz=1024, num_updates=23200, lr=2.16303e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=37, gb_free=13.5, wall=154448
-2024-11-05 00:25:24 - progress_bar.py[line:274] - INFO: epoch 060:    168 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.65, ntokens=17635, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3333.4, ups=0.19, wpb=17635, bsz=1024, num_updates=23210, lr=2.16167e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=50, gb_free=13.5, wall=154501
-2024-11-05 00:26:16 - progress_bar.py[line:274] - INFO: epoch 060:    178 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.639, ntokens=17384.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.23, wps=3310.8, ups=0.19, wpb=17384.4, bsz=1024, num_updates=23220, lr=2.16031e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=49, gb_free=13.6, wall=154553
-2024-11-05 00:27:09 - progress_bar.py[line:274] - INFO: epoch 060:    188 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.669, ntokens=17629.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3316.7, ups=0.19, wpb=17629.8, bsz=1024, num_updates=23230, lr=2.15895e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=48, gb_free=13.5, wall=154607
-2024-11-05 00:28:01 - progress_bar.py[line:274] - INFO: epoch 060:    198 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.743, ntokens=18018.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3487.7, ups=0.19, wpb=18018.5, bsz=1024, num_updates=23240, lr=2.15759e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=45, gb_free=13.5, wall=154658
-2024-11-05 00:28:53 - progress_bar.py[line:274] - INFO: epoch 060:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.664, ntokens=17829.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3424.3, ups=0.19, wpb=17829.4, bsz=1024, num_updates=23250, lr=2.15623e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=154710
-2024-11-05 00:29:46 - progress_bar.py[line:274] - INFO: epoch 060:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.715, ntokens=18010.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3390.8, ups=0.19, wpb=18010.9, bsz=1024, num_updates=23260, lr=2.15487e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=154763
-2024-11-05 00:30:39 - progress_bar.py[line:274] - INFO: epoch 060:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17823.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3413, ups=0.19, wpb=17823.7, bsz=1024, num_updates=23270, lr=2.15351e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=154816
-2024-11-05 00:31:31 - progress_bar.py[line:274] - INFO: epoch 060:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.684, ntokens=17676.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3374, ups=0.19, wpb=17676.7, bsz=1024, num_updates=23280, lr=2.15215e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=154868
-2024-11-05 00:32:27 - progress_bar.py[line:274] - INFO: epoch 060:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.758, ntokens=17992.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.76, wps=3192.5, ups=0.18, wpb=17992.4, bsz=1024, num_updates=23290, lr=2.15079e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=154924
-2024-11-05 00:33:20 - progress_bar.py[line:274] - INFO: epoch 060:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.668, ntokens=17561.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3331.1, ups=0.19, wpb=17561.7, bsz=1024, num_updates=23300, lr=2.14943e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=154977
-2024-11-05 00:34:13 - progress_bar.py[line:274] - INFO: epoch 060:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.769, ntokens=18302.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=3455.3, ups=0.19, wpb=18302.8, bsz=1024, num_updates=23310, lr=2.14807e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=155030
-2024-11-05 00:35:06 - progress_bar.py[line:274] - INFO: epoch 060:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.698, ntokens=17899.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3360.3, ups=0.19, wpb=17899.2, bsz=1024, num_updates=23320, lr=2.14671e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=155083
-2024-11-05 00:36:00 - progress_bar.py[line:274] - INFO: epoch 060:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=17885.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3304.2, ups=0.18, wpb=17885.4, bsz=1024, num_updates=23330, lr=2.14534e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=155137
-2024-11-05 00:36:53 - progress_bar.py[line:274] - INFO: epoch 060:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.752, ntokens=17841.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3365.6, ups=0.19, wpb=17841.7, bsz=1024, num_updates=23340, lr=2.14398e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=155191
-2024-11-05 00:37:46 - progress_bar.py[line:274] - INFO: epoch 060:    308 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.853, ntokens=18483.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.22, wps=3544.5, ups=0.19, wpb=18483.8, bsz=1024, num_updates=23350, lr=2.14262e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=155243
-2024-11-05 00:38:38 - progress_bar.py[line:274] - INFO: epoch 060:    318 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=18393.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3510.9, ups=0.19, wpb=18393.4, bsz=1024, num_updates=23360, lr=2.14126e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.3, wall=155295
-2024-11-05 00:39:32 - progress_bar.py[line:274] - INFO: epoch 060:    328 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.795, ntokens=18197.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.94, wps=3340.4, ups=0.18, wpb=18197.4, bsz=1024, num_updates=23370, lr=2.1399e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=155350
-2024-11-05 00:40:28 - progress_bar.py[line:274] - INFO: epoch 060:    338 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=18423.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3291.8, ups=0.18, wpb=18423.8, bsz=1024, num_updates=23380, lr=2.13854e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=155405
-2024-11-05 00:41:26 - progress_bar.py[line:274] - INFO: epoch 060:    348 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.827, ntokens=18605.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.1, wps=3244.5, ups=0.17, wpb=18605.7, bsz=1024, num_updates=23390, lr=2.13718e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=31, gb_free=13.2, wall=155463
-2024-11-05 00:42:21 - progress_bar.py[line:274] - INFO: epoch 060:    358 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.772, ntokens=17919, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=3221.6, ups=0.18, wpb=17919, bsz=1024, num_updates=23400, lr=2.13582e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=31, gb_free=13.1, wall=155518
-2024-11-05 00:43:17 - progress_bar.py[line:274] - INFO: epoch 060:    368 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.798, ntokens=18173.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.95, wps=3266.3, ups=0.18, wpb=18173.9, bsz=1024, num_updates=23410, lr=2.13446e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=40, gb_free=13.4, wall=155574
-2024-11-05 00:44:12 - progress_bar.py[line:274] - INFO: epoch 060:    378 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.744, ntokens=17935.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.7, wps=3283.3, ups=0.18, wpb=17935.1, bsz=1024, num_updates=23420, lr=2.1331e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=39, gb_free=13.4, wall=155629
-2024-11-05 00:45:06 - progress_bar.py[line:274] - INFO: epoch 060:    388 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.807, ntokens=18122.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3306.4, ups=0.18, wpb=18122.1, bsz=1024, num_updates=23430, lr=2.13174e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=155684
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt2024-11-05 00:45:19 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 60 @ 23433 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt
+2024-11-07 11:11:04 - trainer.py[line:703] - INFO: begin training epoch 32
+2024-11-07 11:11:04 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 11:11:54 - progress_bar.py[line:274] - INFO: epoch 032:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.403, ntokens=16796.3, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.29, wps=345.5, ups=0.02, wpb=16796.3, bsz=985.6, num_updates=12120, lr=3.67035e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=29, gb_free=13.5, wall=77432
+2024-11-07 11:12:47 - progress_bar.py[line:274] - INFO: epoch 032:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.388, ntokens=17547.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.23, wps=3288.6, ups=0.19, wpb=17547.6, bsz=1024, num_updates=12130, lr=3.66899e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=77486
+2024-11-07 11:13:39 - progress_bar.py[line:274] - INFO: epoch 032:     29 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.406, ntokens=17660.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.3, wps=3397, ups=0.19, wpb=17660.5, bsz=1024, num_updates=12140, lr=3.66763e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=77538
+2024-11-07 11:14:32 - progress_bar.py[line:274] - INFO: epoch 032:     39 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.43, ntokens=17861.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.39, wps=3383.9, ups=0.19, wpb=17861.8, bsz=1024, num_updates=12150, lr=3.66627e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=77591
+2024-11-07 11:15:24 - progress_bar.py[line:274] - INFO: epoch 032:     49 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.385, ntokens=17492.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.22, wps=3336.5, ups=0.19, wpb=17492.3, bsz=1024, num_updates=12160, lr=3.66491e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=77643
+2024-11-07 11:16:17 - progress_bar.py[line:274] - INFO: epoch 032:     59 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.383, ntokens=17610.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.22, wps=3335.4, ups=0.19, wpb=17610.8, bsz=1024, num_updates=12170, lr=3.66355e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=77696
+2024-11-07 11:17:10 - progress_bar.py[line:274] - INFO: epoch 032:     69 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.394, ntokens=17588.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.25, wps=3309.1, ups=0.19, wpb=17588.2, bsz=1024, num_updates=12180, lr=3.66219e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=77749
+2024-11-07 11:18:03 - progress_bar.py[line:274] - INFO: epoch 032:     79 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.413, ntokens=17700.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3371.8, ups=0.19, wpb=17700.3, bsz=1024, num_updates=12190, lr=3.66083e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=77801
+2024-11-07 11:18:55 - progress_bar.py[line:274] - INFO: epoch 032:     89 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.432, ntokens=17824.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.4, wps=3389.6, ups=0.19, wpb=17824.1, bsz=1024, num_updates=12200, lr=3.65947e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=77854
+2024-11-07 11:19:49 - progress_bar.py[line:274] - INFO: epoch 032:     99 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.377, ntokens=17383.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.2, wps=3251.6, ups=0.19, wpb=17383.3, bsz=1024, num_updates=12210, lr=3.65811e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=77908
+2024-11-07 11:20:41 - progress_bar.py[line:274] - INFO: epoch 032:    109 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.399, ntokens=17444.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.27, wps=3327.5, ups=0.19, wpb=17444.9, bsz=1024, num_updates=12220, lr=3.65674e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=77960
+2024-11-07 11:21:34 - progress_bar.py[line:274] - INFO: epoch 032:    119 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.441, ntokens=17682.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3338.2, ups=0.19, wpb=17682.8, bsz=1024, num_updates=12230, lr=3.65538e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=78013
+2024-11-07 11:22:27 - progress_bar.py[line:274] - INFO: epoch 032:    129 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.379, ntokens=17337.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.2, wps=3291.8, ups=0.19, wpb=17337.7, bsz=1024, num_updates=12240, lr=3.65402e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=78066
+2024-11-07 11:23:19 - progress_bar.py[line:274] - INFO: epoch 032:    139 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.437, ntokens=17761.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.42, wps=3412.2, ups=0.19, wpb=17761.9, bsz=1024, num_updates=12250, lr=3.65266e-05, gnorm=0.011, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=78118
+2024-11-07 11:24:12 - progress_bar.py[line:274] - INFO: epoch 032:    149 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.439, ntokens=17839.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.42, wps=3378.2, ups=0.19, wpb=17839.5, bsz=1024, num_updates=12260, lr=3.6513e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=78170
+2024-11-07 11:25:04 - progress_bar.py[line:274] - INFO: epoch 032:    159 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.391, ntokens=17540.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.24, wps=3315.5, ups=0.19, wpb=17540.7, bsz=1024, num_updates=12270, lr=3.64994e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=78223
+2024-11-07 11:25:57 - progress_bar.py[line:274] - INFO: epoch 032:    169 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.431, ntokens=17937, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.39, wps=3400.7, ups=0.19, wpb=17937, bsz=1024, num_updates=12280, lr=3.64858e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=78276
+2024-11-07 11:26:50 - progress_bar.py[line:274] - INFO: epoch 032:    179 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.393, ntokens=17454.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.25, wps=3297.8, ups=0.19, wpb=17454.4, bsz=1024, num_updates=12290, lr=3.64722e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=78329
+2024-11-07 11:27:38 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
+2024-11-07 11:27:48 - progress_bar.py[line:274] - INFO: epoch 032:    190 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.461, ntokens=17970, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.51, wps=3110.5, ups=0.17, wpb=17970, bsz=1024, num_updates=12300, lr=3.64586e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=34, gb_free=13.5, wall=78387
+2024-11-07 11:28:41 - progress_bar.py[line:274] - INFO: epoch 032:    200 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.446, ntokens=17964.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.45, wps=3413.2, ups=0.19, wpb=17964.3, bsz=1024, num_updates=12310, lr=3.6445e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=78439
+2024-11-07 11:29:33 - progress_bar.py[line:274] - INFO: epoch 032:    210 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.42, ntokens=17876.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.35, wps=3407, ups=0.19, wpb=17876.3, bsz=1024, num_updates=12320, lr=3.64314e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=78492
+2024-11-07 11:30:25 - progress_bar.py[line:274] - INFO: epoch 032:    220 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.442, ntokens=17902.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3445.6, ups=0.19, wpb=17902.1, bsz=1024, num_updates=12330, lr=3.64178e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=78544
+2024-11-07 11:31:18 - progress_bar.py[line:274] - INFO: epoch 032:    230 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.419, ntokens=17641.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.35, wps=3326.2, ups=0.19, wpb=17641.2, bsz=1024, num_updates=12340, lr=3.64042e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=37, gb_free=13.5, wall=78597
+2024-11-07 11:32:10 - progress_bar.py[line:274] - INFO: epoch 032:    240 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.413, ntokens=17689.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.33, wps=3381, ups=0.19, wpb=17689.8, bsz=1024, num_updates=12350, lr=3.63906e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=39, gb_free=13.5, wall=78649
+2024-11-07 11:33:03 - progress_bar.py[line:274] - INFO: epoch 032:    250 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.377, ntokens=17330.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.19, wps=3296.5, ups=0.19, wpb=17330.9, bsz=1024, num_updates=12360, lr=3.6377e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=40, gb_free=13.6, wall=78702
+2024-11-07 11:33:55 - progress_bar.py[line:274] - INFO: epoch 032:    260 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.433, ntokens=17867.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.4, wps=3422.8, ups=0.19, wpb=17867.4, bsz=1024, num_updates=12370, lr=3.63634e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=41, gb_free=13.6, wall=78754
+2024-11-07 11:34:47 - progress_bar.py[line:274] - INFO: epoch 032:    270 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.467, ntokens=18150.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3471.9, ups=0.19, wpb=18150.8, bsz=1024, num_updates=12380, lr=3.63498e-05, gnorm=0.009, clip=0, loss_scale=65536, train_wall=37, gb_free=13.6, wall=78806
+2024-11-07 11:35:40 - progress_bar.py[line:274] - INFO: epoch 032:    280 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.441, ntokens=17897.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.43, wps=3420.6, ups=0.19, wpb=17897.8, bsz=1024, num_updates=12390, lr=3.63362e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=35, gb_free=13.6, wall=78859
+2024-11-07 11:36:32 - progress_bar.py[line:274] - INFO: epoch 032:    290 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.39, ntokens=17552.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.24, wps=3342.1, ups=0.19, wpb=17552.6, bsz=1024, num_updates=12400, lr=3.63226e-05, gnorm=0.012, clip=0, loss_scale=65536, train_wall=36, gb_free=13.5, wall=78911
+2024-11-07 11:37:25 - progress_bar.py[line:274] - INFO: epoch 032:    300 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.47, ntokens=17944.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.54, wps=3424.7, ups=0.19, wpb=17944.4, bsz=1024, num_updates=12410, lr=3.6309e-05, gnorm=0.01, clip=0, loss_scale=65536, train_wall=38, gb_free=13.1, wall=78964
+2024-11-07 11:38:17 - progress_bar.py[line:274] - INFO: epoch 032:    310 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.529, ntokens=18320.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3504.3, ups=0.19, wpb=18320.9, bsz=1024, num_updates=12420, lr=3.62954e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=46, gb_free=13.3, wall=79016
+2024-11-07 11:39:09 - progress_bar.py[line:274] - INFO: epoch 032:    320 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.527, ntokens=18238.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.77, wps=3476.2, ups=0.19, wpb=18238.7, bsz=1024, num_updates=12430, lr=3.62818e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=79068
+2024-11-07 11:40:02 - progress_bar.py[line:274] - INFO: epoch 032:    330 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.502, ntokens=18102, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.67, wps=3447.6, ups=0.19, wpb=18102, bsz=1024, num_updates=12440, lr=3.62682e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=52, gb_free=13.4, wall=79121
+2024-11-07 11:40:55 - progress_bar.py[line:274] - INFO: epoch 032:    340 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.482, ntokens=18057.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3424.7, ups=0.19, wpb=18057.9, bsz=1024, num_updates=12450, lr=3.62546e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=53, gb_free=13.3, wall=79174
+2024-11-07 11:41:47 - progress_bar.py[line:274] - INFO: epoch 032:    350 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.562, ntokens=18681.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.9, wps=3547, ups=0.19, wpb=18681.4, bsz=1024, num_updates=12460, lr=3.6241e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.3, wall=79226
+2024-11-07 11:42:42 - progress_bar.py[line:274] - INFO: epoch 032:    360 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.496, ntokens=18267.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3335.5, ups=0.18, wpb=18267.2, bsz=1024, num_updates=12470, lr=3.62273e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=79281
+2024-11-07 11:43:37 - progress_bar.py[line:274] - INFO: epoch 032:    370 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.533, ntokens=18177.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.79, wps=3339, ups=0.18, wpb=18177.6, bsz=1024, num_updates=12480, lr=3.62137e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=54, gb_free=13.1, wall=79335
+2024-11-07 11:44:30 - progress_bar.py[line:274] - INFO: epoch 032:    380 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.496, ntokens=18084.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.64, wps=3352.2, ups=0.19, wpb=18084.9, bsz=1024, num_updates=12490, lr=3.62001e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=79389
+reach the end of datafile, start a new reader
+reach the end of datafile, start a new reader
+2024-11-07 11:45:25 - progress_bar.py[line:274] - INFO: epoch 032:    390 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.482, ntokens=17953.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.59, wps=3308.9, ups=0.18, wpb=17953.9, bsz=1024, num_updates=12500, lr=3.61865e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=54, gb_free=13.4, wall=79444
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
 
-2024-11-05 00:45:19 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt': File exists: File exists: File exists
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
+2024-11-07 11:45:27 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 32 @ 12501 updates
 
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
+2024-11-07 11:45:27 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt': File exists: File exists
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
+: File exists
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-05 00:47:09 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 1 seek offset 50000
-slice_id 4 seek offset 200000
-2024-11-05 00:49:02 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 60 @ 23433 updates, score 0) (writing took 223.00416592799593 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_60.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-05 00:49:56 - train.py[line:336] - INFO: end of epoch 60 (average epoch stats below)
-2024-11-05 00:49:56 - progress_bar.py[line:282] - INFO: epoch 060 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.718 | ntokens 17836.7 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.58 | wps 2757.5 | ups 0.15 | wpb 17836.7 | bsz 1023 | num_updates 23433 | lr 2.13133e-05 | gnorm 0.007 | clip 0 | loss_scale 32768 | train_wall 1285 | gb_free 13.3 | wall 155974
-2024-11-05 00:49:56 - trainer.py[line:639] - INFO: loading train data for epoch 61
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-05 00:52:40 - trainer.py[line:703] - INFO: begin training epoch 61
-2024-11-05 00:52:40 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-05 00:53:20 - progress_bar.py[line:274] - INFO: epoch 061:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=16971.6, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=344, ups=0.02, wpb=16971.6, bsz=985.6, num_updates=23440, lr=2.13038e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=29, gb_free=13.6, wall=156177
-2024-11-05 00:54:15 - progress_bar.py[line:274] - INFO: epoch 061:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=17750.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3217.9, ups=0.18, wpb=17750.3, bsz=1024, num_updates=23450, lr=2.12902e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=37, gb_free=13.6, wall=156232
-2024-11-05 00:55:10 - progress_bar.py[line:274] - INFO: epoch 061:     27 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.626, ntokens=17385.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.17, wps=3168.4, ups=0.18, wpb=17385.1, bsz=1024, num_updates=23460, lr=2.12766e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.6, wall=156287
-2024-11-05 00:56:06 - progress_bar.py[line:274] - INFO: epoch 061:     37 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.643, ntokens=17553.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3102.7, ups=0.18, wpb=17553.3, bsz=1024, num_updates=23470, lr=2.1263e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=156344
-2024-11-05 00:57:05 - progress_bar.py[line:274] - INFO: epoch 061:     47 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.656, ntokens=17555.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3020.1, ups=0.17, wpb=17555.3, bsz=1024, num_updates=23480, lr=2.12494e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.5, wall=156402
-2024-11-05 00:58:00 - progress_bar.py[line:274] - INFO: epoch 061:     57 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.661, ntokens=17609.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3160.7, ups=0.18, wpb=17609.8, bsz=1024, num_updates=23490, lr=2.12358e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=56, gb_free=13.4, wall=156457
-2024-11-05 00:58:55 - progress_bar.py[line:274] - INFO: epoch 061:     67 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.789, ntokens=18323.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.91, wps=3341.8, ups=0.18, wpb=18323.3, bsz=1024, num_updates=23500, lr=2.12222e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=156512
-2024-11-05 00:59:48 - progress_bar.py[line:274] - INFO: epoch 061:     77 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.656, ntokens=17515.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3290.4, ups=0.19, wpb=17515.6, bsz=1024, num_updates=23510, lr=2.12086e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=42, gb_free=13.5, wall=156565
-2024-11-05 01:00:43 - progress_bar.py[line:274] - INFO: epoch 061:     87 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.688, ntokens=17736.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.44, wps=3264.5, ups=0.18, wpb=17736.9, bsz=1024, num_updates=23520, lr=2.1195e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=156620
-2024-11-05 01:01:35 - progress_bar.py[line:274] - INFO: epoch 061:     97 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.664, ntokens=17651, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3364, ups=0.19, wpb=17651, bsz=1024, num_updates=23530, lr=2.11814e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=156672
-2024-11-05 01:02:28 - progress_bar.py[line:274] - INFO: epoch 061:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.653, ntokens=17415.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.29, wps=3283, ups=0.19, wpb=17415.5, bsz=1024, num_updates=23540, lr=2.11678e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=156725
-2024-11-05 01:03:21 - progress_bar.py[line:274] - INFO: epoch 061:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=17689.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3331.4, ups=0.19, wpb=17689.2, bsz=1024, num_updates=23550, lr=2.11542e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=156778
-2024-11-05 01:04:14 - progress_bar.py[line:274] - INFO: epoch 061:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.623, ntokens=17396.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3291.7, ups=0.19, wpb=17396.1, bsz=1024, num_updates=23560, lr=2.11406e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=156831
-2024-11-05 01:05:07 - progress_bar.py[line:274] - INFO: epoch 061:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.683, ntokens=17661.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3317, ups=0.19, wpb=17661.6, bsz=1024, num_updates=23570, lr=2.1127e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=156885
-2024-11-05 01:06:00 - progress_bar.py[line:274] - INFO: epoch 061:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.701, ntokens=17754.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3355.4, ups=0.19, wpb=17754.2, bsz=1024, num_updates=23580, lr=2.11133e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=156937
-2024-11-05 01:06:54 - progress_bar.py[line:274] - INFO: epoch 061:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.697, ntokens=17891.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.48, wps=3354, ups=0.19, wpb=17891.2, bsz=1024, num_updates=23590, lr=2.10997e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=156991
-2024-11-05 01:07:46 - progress_bar.py[line:274] - INFO: epoch 061:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.678, ntokens=17778.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.4, wps=3372.3, ups=0.19, wpb=17778.2, bsz=1024, num_updates=23600, lr=2.10861e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=157043
-2024-11-05 01:08:39 - progress_bar.py[line:274] - INFO: epoch 061:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.643, ntokens=17417.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3334.5, ups=0.19, wpb=17417.9, bsz=1024, num_updates=23610, lr=2.10725e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=157096
-2024-11-05 01:09:32 - progress_bar.py[line:274] - INFO: epoch 061:    187 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.692, ntokens=17823, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3341.6, ups=0.19, wpb=17823, bsz=1024, num_updates=23620, lr=2.10589e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=157149
-2024-11-05 01:10:25 - progress_bar.py[line:274] - INFO: epoch 061:    197 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=17704.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3321.7, ups=0.19, wpb=17704.1, bsz=1024, num_updates=23630, lr=2.10453e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=157202
-2024-11-05 01:11:08 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
-2024-11-05 01:11:25 - progress_bar.py[line:274] - INFO: epoch 061:    208 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.72, ntokens=18199.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.59, wps=3065.5, ups=0.17, wpb=18199.5, bsz=1024, num_updates=23640, lr=2.10317e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=34, gb_free=13.6, wall=157262
-2024-11-05 01:12:19 - progress_bar.py[line:274] - INFO: epoch 061:    218 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.66, ntokens=17780, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3280.7, ups=0.18, wpb=17780, bsz=1024, num_updates=23650, lr=2.10181e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=33, gb_free=13.6, wall=157316
-2024-11-05 01:13:11 - progress_bar.py[line:274] - INFO: epoch 061:    228 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.707, ntokens=17950.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=3428.7, ups=0.19, wpb=17950.7, bsz=1024, num_updates=23660, lr=2.10045e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=33, gb_free=13.5, wall=157368
-2024-11-05 01:14:05 - progress_bar.py[line:274] - INFO: epoch 061:    238 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.656, ntokens=17533.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3286.8, ups=0.19, wpb=17533.8, bsz=1024, num_updates=23670, lr=2.09909e-05, gnorm=0.009, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=157422
-2024-11-05 01:14:58 - progress_bar.py[line:274] - INFO: epoch 061:    248 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.709, ntokens=17709.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3306.2, ups=0.19, wpb=17709.4, bsz=1024, num_updates=23680, lr=2.09773e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=157475
-2024-11-05 01:15:52 - progress_bar.py[line:274] - INFO: epoch 061:    258 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.654, ntokens=17455.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.29, wps=3259.9, ups=0.19, wpb=17455.3, bsz=1024, num_updates=23690, lr=2.09637e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=157529
-2024-11-05 01:16:45 - progress_bar.py[line:274] - INFO: epoch 061:    268 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.746, ntokens=18216.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.71, wps=3421.7, ups=0.19, wpb=18216.9, bsz=1024, num_updates=23700, lr=2.09501e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=157582
-2024-11-05 01:17:38 - progress_bar.py[line:274] - INFO: epoch 061:    278 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.66, ntokens=17608.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3305.1, ups=0.19, wpb=17608.9, bsz=1024, num_updates=23710, lr=2.09365e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=157635
-2024-11-05 01:18:34 - progress_bar.py[line:274] - INFO: epoch 061:    288 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.647, ntokens=17483, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3120.3, ups=0.18, wpb=17483, bsz=1024, num_updates=23720, lr=2.09229e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=39, gb_free=13.6, wall=157691
-2024-11-05 01:19:32 - progress_bar.py[line:274] - INFO: epoch 061:    298 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.751, ntokens=18025.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.73, wps=3130.6, ups=0.17, wpb=18025.8, bsz=1024, num_updates=23730, lr=2.09093e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=58, gb_free=13.3, wall=157749
-2024-11-05 01:20:26 - progress_bar.py[line:274] - INFO: epoch 061:    308 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.77, ntokens=18027.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.82, wps=3330.4, ups=0.18, wpb=18027.1, bsz=1024, num_updates=23740, lr=2.08957e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.3, wall=157803
-2024-11-05 01:21:20 - progress_bar.py[line:274] - INFO: epoch 061:    318 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.814, ntokens=18255.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=3365.3, ups=0.18, wpb=18255.5, bsz=1024, num_updates=23750, lr=2.08821e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=32, gb_free=13.4, wall=157857
-2024-11-05 01:22:13 - progress_bar.py[line:274] - INFO: epoch 061:    328 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.773, ntokens=18000.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=3432, ups=0.19, wpb=18000.6, bsz=1024, num_updates=23760, lr=2.08685e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.4, wall=157910
-2024-11-05 01:23:09 - progress_bar.py[line:274] - INFO: epoch 061:    338 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.766, ntokens=17948, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.8, wps=3192.5, ups=0.18, wpb=17948, bsz=1024, num_updates=23770, lr=2.08549e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=45, gb_free=13.4, wall=157966
-2024-11-05 01:24:04 - progress_bar.py[line:274] - INFO: epoch 061:    348 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.829, ntokens=18642.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.11, wps=3354.3, ups=0.18, wpb=18642.1, bsz=1024, num_updates=23780, lr=2.08413e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.3, wall=158021
-2024-11-05 01:24:57 - progress_bar.py[line:274] - INFO: epoch 061:    358 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.806, ntokens=18283.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.99, wps=3454.1, ups=0.19, wpb=18283.1, bsz=1024, num_updates=23790, lr=2.08277e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.2, wall=158074
-2024-11-05 01:25:49 - progress_bar.py[line:274] - INFO: epoch 061:    368 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.8, ntokens=18297.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.96, wps=3517.3, ups=0.19, wpb=18297.9, bsz=1024, num_updates=23800, lr=2.08141e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=158126
-2024-11-05 01:26:41 - progress_bar.py[line:274] - INFO: epoch 061:    378 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.737, ntokens=18036.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.67, wps=3476, ups=0.19, wpb=18036.6, bsz=1024, num_updates=23810, lr=2.08005e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.4, wall=158178
-2024-11-05 01:27:35 - progress_bar.py[line:274] - INFO: epoch 061:    388 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.732, ntokens=17703, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.64, wps=3293.4, ups=0.19, wpb=17703, bsz=1024, num_updates=23820, lr=2.07869e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=158232
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt
-2024-11-05 01:27:48 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 61 @ 23823 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
 
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt
-2024-11-05 01:27:48 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt': File exists: File exists
-
-: File exists
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-05 01:29:37 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+2024-11-07 11:47:16 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
 
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 1 seek offset 50000
-slice_id 5 seek offset 250000
-slice_id 4 seek offset 200000
-2024-11-05 01:31:29 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 61 @ 23823 updates, score 0) (writing took 221.3286895910278 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_61.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-slice_id 2 seek offset 100000
-slice_id 3 seek offset 150000
-2024-11-05 01:32:24 - train.py[line:336] - INFO: end of epoch 61 (average epoch stats below)
-2024-11-05 01:32:24 - progress_bar.py[line:282] - INFO: epoch 061 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.706 | ntokens 17790.1 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.52 | wps 2723.6 | ups 0.15 | wpb 17790.1 | bsz 1023 | num_updates 23823 | lr 2.07828e-05 | gnorm 0.006 | clip 0 | loss_scale 32768 | train_wall 1534 | gb_free 13.4 | wall 158521
-2024-11-05 01:32:24 - trainer.py[line:639] - INFO: loading train data for epoch 62
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-05 01:35:07 - trainer.py[line:703] - INFO: begin training epoch 62
-2024-11-05 01:35:07 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-05 01:35:47 - progress_bar.py[line:274] - INFO: epoch 062:      7 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.662, ntokens=16761.4, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=340.8, ups=0.02, wpb=16761.4, bsz=985.6, num_updates=23830, lr=2.07732e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=35, gb_free=13.6, wall=158724
-2024-11-05 01:36:40 - progress_bar.py[line:274] - INFO: epoch 062:     17 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.684, ntokens=17923.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.43, wps=3395.1, ups=0.19, wpb=17923.9, bsz=1024, num_updates=23840, lr=2.07596e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=158777
-2024-11-05 01:37:32 - progress_bar.py[line:274] - INFO: epoch 062:     27 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.685, ntokens=17608.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.43, wps=3345.4, ups=0.19, wpb=17608.1, bsz=1024, num_updates=23850, lr=2.0746e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.6, wall=158829
-2024-11-05 01:38:25 - progress_bar.py[line:274] - INFO: epoch 062:     37 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.658, ntokens=17670.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.31, wps=3350.4, ups=0.19, wpb=17670.2, bsz=1024, num_updates=23860, lr=2.07324e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=158882
-2024-11-05 01:39:17 - progress_bar.py[line:274] - INFO: epoch 062:     47 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=17535.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3349.2, ups=0.19, wpb=17535.4, bsz=1024, num_updates=23870, lr=2.07188e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=158934
-2024-11-05 01:40:10 - progress_bar.py[line:274] - INFO: epoch 062:     57 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.644, ntokens=17630.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3328.7, ups=0.19, wpb=17630.5, bsz=1024, num_updates=23880, lr=2.07052e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=158987
-2024-11-05 01:41:05 - progress_bar.py[line:274] - INFO: epoch 062:     67 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.708, ntokens=17812.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3248.7, ups=0.18, wpb=17812.8, bsz=1024, num_updates=23890, lr=2.06916e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=38, gb_free=13.5, wall=159042
-2024-11-05 01:42:02 - progress_bar.py[line:274] - INFO: epoch 062:     77 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.65, ntokens=17441.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3050.9, ups=0.17, wpb=17441.1, bsz=1024, num_updates=23900, lr=2.0678e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=57, gb_free=13.5, wall=159099
-2024-11-05 01:42:58 - progress_bar.py[line:274] - INFO: epoch 062:     87 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.651, ntokens=17605.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3160.4, ups=0.18, wpb=17605.7, bsz=1024, num_updates=23910, lr=2.06644e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=56, gb_free=13.6, wall=159155
-2024-11-05 01:43:54 - progress_bar.py[line:274] - INFO: epoch 062:     97 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.636, ntokens=17443.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.22, wps=3092.1, ups=0.18, wpb=17443.8, bsz=1024, num_updates=23920, lr=2.06508e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=159212
-2024-11-05 01:44:51 - progress_bar.py[line:274] - INFO: epoch 062:    107 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.648, ntokens=17537.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3103.7, ups=0.18, wpb=17537.1, bsz=1024, num_updates=23930, lr=2.06372e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=56, gb_free=13.5, wall=159268
-2024-11-05 01:45:44 - progress_bar.py[line:274] - INFO: epoch 062:    117 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.677, ntokens=17521, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=3270.9, ups=0.19, wpb=17521, bsz=1024, num_updates=23940, lr=2.06236e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=159322
-2024-11-05 01:46:40 - progress_bar.py[line:274] - INFO: epoch 062:    127 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.647, ntokens=17599.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3191.4, ups=0.18, wpb=17599.5, bsz=1024, num_updates=23950, lr=2.061e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.5, wall=159377
-2024-11-05 01:47:33 - progress_bar.py[line:274] - INFO: epoch 062:    137 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.681, ntokens=17638.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3283.4, ups=0.19, wpb=17638.4, bsz=1024, num_updates=23960, lr=2.05964e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=54, gb_free=13.4, wall=159430
-2024-11-05 01:48:26 - progress_bar.py[line:274] - INFO: epoch 062:    147 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.667, ntokens=17788.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3378.1, ups=0.19, wpb=17788.8, bsz=1024, num_updates=23970, lr=2.05828e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=159483
-2024-11-05 01:49:21 - progress_bar.py[line:274] - INFO: epoch 062:    157 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.707, ntokens=17935.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=3278.8, ups=0.18, wpb=17935.8, bsz=1024, num_updates=23980, lr=2.05692e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=159538
-2024-11-05 01:50:15 - progress_bar.py[line:274] - INFO: epoch 062:    167 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.676, ntokens=17942.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=3284.9, ups=0.18, wpb=17942.7, bsz=1024, num_updates=23990, lr=2.05556e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=55, gb_free=13.4, wall=159592
-2024-11-05 01:51:10 - progress_bar.py[line:274] - INFO: epoch 062:    177 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.627, ntokens=17405.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.18, wps=3195.7, ups=0.18, wpb=17405.5, bsz=1024, num_updates=24000, lr=2.0542e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=159647
-2024-11-05 01:52:04 - progress_bar.py[line:274] - INFO: epoch 062:    187 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.675, ntokens=17786.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=3286.5, ups=0.18, wpb=17786.9, bsz=1024, num_updates=24010, lr=2.05284e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=54, gb_free=13.5, wall=159701
-2024-11-05 01:52:56 - progress_bar.py[line:274] - INFO: epoch 062:    197 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.698, ntokens=17909.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.49, wps=3412.8, ups=0.19, wpb=17909.9, bsz=1024, num_updates=24020, lr=2.05148e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=159753
-2024-11-05 01:53:49 - progress_bar.py[line:274] - INFO: epoch 062:    207 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.691, ntokens=18031, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.46, wps=3415.9, ups=0.19, wpb=18031, bsz=1024, num_updates=24030, lr=2.05012e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=53, gb_free=13.5, wall=159806
-2024-11-05 01:54:42 - progress_bar.py[line:274] - INFO: epoch 062:    217 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.643, ntokens=17597.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3336.9, ups=0.19, wpb=17597.6, bsz=1024, num_updates=24040, lr=2.04876e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=159859
-2024-11-05 01:55:34 - progress_bar.py[line:274] - INFO: epoch 062:    227 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.671, ntokens=17819, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.37, wps=3407, ups=0.19, wpb=17819, bsz=1024, num_updates=24050, lr=2.0474e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=47, gb_free=13.5, wall=159911
-2024-11-05 01:56:27 - progress_bar.py[line:274] - INFO: epoch 062:    237 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.655, ntokens=17568.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3334.7, ups=0.19, wpb=17568.2, bsz=1024, num_updates=24060, lr=2.04604e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=37, gb_free=13.6, wall=159964
-2024-11-05 01:57:19 - progress_bar.py[line:274] - INFO: epoch 062:    247 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=17726.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3392.1, ups=0.19, wpb=17726.5, bsz=1024, num_updates=24070, lr=2.04468e-05, gnorm=0.008, clip=0, loss_scale=32768, train_wall=32, gb_free=13.5, wall=160016
-2024-11-05 01:58:12 - progress_bar.py[line:274] - INFO: epoch 062:    257 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.69, ntokens=17749, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3353.3, ups=0.19, wpb=17749, bsz=1024, num_updates=24080, lr=2.04332e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=30, gb_free=13.4, wall=160069
-2024-11-05 01:59:05 - progress_bar.py[line:274] - INFO: epoch 062:    267 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.69, ntokens=17973.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3368.6, ups=0.19, wpb=17973.3, bsz=1024, num_updates=24090, lr=2.04195e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=30, gb_free=13.5, wall=160123
-2024-11-05 01:59:58 - progress_bar.py[line:274] - INFO: epoch 062:    277 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.677, ntokens=17664.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.4, wps=3348.4, ups=0.19, wpb=17664.3, bsz=1024, num_updates=24100, lr=2.04059e-05, gnorm=0.005, clip=0, loss_scale=32768, train_wall=31, gb_free=13.5, wall=160175
-2024-11-05 02:00:51 - progress_bar.py[line:274] - INFO: epoch 062:    287 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.647, ntokens=17722.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3377.2, ups=0.19, wpb=17722.6, bsz=1024, num_updates=24110, lr=2.03923e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=41, gb_free=13.6, wall=160228
-2024-11-05 02:01:43 - progress_bar.py[line:274] - INFO: epoch 062:    297 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.681, ntokens=17487.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3340.1, ups=0.19, wpb=17487.2, bsz=1024, num_updates=24120, lr=2.03787e-05, gnorm=0.007, clip=0, loss_scale=32768, train_wall=35, gb_free=13.5, wall=160280
-2024-11-05 02:02:35 - progress_bar.py[line:274] - INFO: epoch 062:    307 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.775, ntokens=18139.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3522.8, ups=0.19, wpb=18139.9, bsz=1024, num_updates=24130, lr=2.03651e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=37, gb_free=13.1, wall=160332
-2024-11-05 02:03:28 - progress_bar.py[line:274] - INFO: epoch 062:    317 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.814, ntokens=18392.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.03, wps=3445.7, ups=0.19, wpb=18392.7, bsz=1024, num_updates=24140, lr=2.03515e-05, gnorm=0.004, clip=0, loss_scale=32768, train_wall=32, gb_free=13.3, wall=160385
-2024-11-05 02:04:20 - progress_bar.py[line:274] - INFO: epoch 062:    327 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.795, ntokens=18307.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.94, wps=3508.6, ups=0.19, wpb=18307.5, bsz=1024, num_updates=24150, lr=2.03379e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=160437
-2024-11-05 02:05:13 - progress_bar.py[line:274] - INFO: epoch 062:    337 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.754, ntokens=17960.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=3411.1, ups=0.19, wpb=17960.3, bsz=1024, num_updates=24160, lr=2.03243e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=160490
-2024-11-05 02:06:05 - progress_bar.py[line:274] - INFO: epoch 062:    347 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.762, ntokens=18317.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.78, wps=3504.2, ups=0.19, wpb=18317.3, bsz=1024, num_updates=24170, lr=2.03107e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=34, gb_free=13.4, wall=160542
-2024-11-05 02:06:57 - progress_bar.py[line:274] - INFO: epoch 062:    357 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.832, ntokens=18537.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.12, wps=3548, ups=0.19, wpb=18537.8, bsz=1024, num_updates=24180, lr=2.02971e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=37, gb_free=13.4, wall=160594
-2024-11-05 02:07:50 - progress_bar.py[line:274] - INFO: epoch 062:    367 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.775, ntokens=18166.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.84, wps=3445.9, ups=0.19, wpb=18166.5, bsz=1024, num_updates=24190, lr=2.02835e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=42, gb_free=13.2, wall=160647
-2024-11-05 02:08:42 - progress_bar.py[line:274] - INFO: epoch 062:    377 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.772, ntokens=18423.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=3511.8, ups=0.19, wpb=18423.6, bsz=1024, num_updates=24200, lr=2.02699e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=44, gb_free=13.4, wall=160700
-2024-11-05 02:09:35 - progress_bar.py[line:274] - INFO: epoch 062:    387 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.754, ntokens=17987.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.75, wps=3418.7, ups=0.19, wpb=17987.6, bsz=1024, num_updates=24210, lr=2.02563e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=48, gb_free=13.4, wall=160752
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-2024-11-05 02:09:52 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 62 @ 24214 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt
-2024-11-05 02:09:52 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt': File exists: File exists
-: File exists
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-05 02:11:20 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
 slice_id 7 seek offset 350000
-slice_id 1 seek offset 50000
-slice_id 4 seek offset 200000
-2024-11-05 02:13:30 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 62 @ 24214 updates, score 0) (writing took 217.4196638240246 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_62.pt
-2024-11-05 02:14:25 - train.py[line:336] - INFO: end of epoch 62 (average epoch stats below)
-2024-11-05 02:14:25 - progress_bar.py[line:282] - INFO: epoch 062 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.695 | ntokens 17795.8 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.48 | wps 2760.2 | ups 0.16 | wpb 17795.8 | bsz 1023 | num_updates 24214 | lr 2.02509e-05 | gnorm 0.006 | clip 0 | loss_scale 65536 | train_wall 1662 | gb_free 13.3 | wall 161042
-2024-11-05 02:14:25 - trainer.py[line:639] - INFO: loading train data for epoch 63
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
 slice_id 3 seek offset 150000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
 slice_id 6 seek offset 300000
+2024-11-07 11:49:08 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 32 @ 12501 updates, score 0) (writing took 221.09385545700206 seconds)
+cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_32.pt
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
 slice_id 2 seek offset 100000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-05 02:17:09 - trainer.py[line:703] - INFO: begin training epoch 63
-2024-11-05 02:17:09 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-05 02:17:43 - progress_bar.py[line:274] - INFO: epoch 063:      6 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.674, ntokens=16875.8, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.38, wps=345.9, ups=0.02, wpb=16875.8, bsz=985.6, num_updates=24220, lr=2.02427e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=36, gb_free=13.5, wall=161240
-2024-11-05 02:18:36 - progress_bar.py[line:274] - INFO: epoch 063:     16 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.702, ntokens=18013.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3389.4, ups=0.19, wpb=18013.1, bsz=1024, num_updates=24230, lr=2.02291e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=161293
-2024-11-05 02:19:29 - progress_bar.py[line:274] - INFO: epoch 063:     26 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.632, ntokens=17355, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3261.9, ups=0.19, wpb=17355, bsz=1024, num_updates=24240, lr=2.02155e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=161346
-2024-11-05 02:20:22 - progress_bar.py[line:274] - INFO: epoch 063:     36 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.701, ntokens=17859, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3387.3, ups=0.19, wpb=17859, bsz=1024, num_updates=24250, lr=2.02019e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=161399
-2024-11-05 02:21:15 - progress_bar.py[line:274] - INFO: epoch 063:     46 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.615, ntokens=17447.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.13, wps=3321.4, ups=0.19, wpb=17447.1, bsz=1024, num_updates=24260, lr=2.01883e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=161452
-2024-11-05 02:22:07 - progress_bar.py[line:274] - INFO: epoch 063:     56 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.647, ntokens=17725.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3357, ups=0.19, wpb=17725.7, bsz=1024, num_updates=24270, lr=2.01747e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=161504
-2024-11-05 02:23:00 - progress_bar.py[line:274] - INFO: epoch 063:     66 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.729, ntokens=18066, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3445.7, ups=0.19, wpb=18066, bsz=1024, num_updates=24280, lr=2.01611e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=161557
-2024-11-05 02:23:52 - progress_bar.py[line:274] - INFO: epoch 063:     76 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.657, ntokens=17544.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.31, wps=3341.8, ups=0.19, wpb=17544.4, bsz=1024, num_updates=24290, lr=2.01475e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=161609
-2024-11-05 02:24:45 - progress_bar.py[line:274] - INFO: epoch 063:     86 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.68, ntokens=17801.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3407, ups=0.19, wpb=17801.4, bsz=1024, num_updates=24300, lr=2.01339e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=161662
-2024-11-05 02:25:37 - progress_bar.py[line:274] - INFO: epoch 063:     96 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.664, ntokens=17896.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3407.3, ups=0.19, wpb=17896.7, bsz=1024, num_updates=24310, lr=2.01203e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=161714
-2024-11-05 02:26:30 - progress_bar.py[line:274] - INFO: epoch 063:    106 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.66, ntokens=17518.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3312.4, ups=0.19, wpb=17518.6, bsz=1024, num_updates=24320, lr=2.01067e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=161767
-2024-11-05 02:27:23 - progress_bar.py[line:274] - INFO: epoch 063:    116 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.694, ntokens=17814, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3348.6, ups=0.19, wpb=17814, bsz=1024, num_updates=24330, lr=2.00931e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=161820
-2024-11-05 02:28:16 - progress_bar.py[line:274] - INFO: epoch 063:    126 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.66, ntokens=17648.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.32, wps=3344.2, ups=0.19, wpb=17648.3, bsz=1024, num_updates=24340, lr=2.00794e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=161873
-2024-11-05 02:29:09 - progress_bar.py[line:274] - INFO: epoch 063:    136 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.664, ntokens=17654.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3324.4, ups=0.19, wpb=17654.5, bsz=1024, num_updates=24350, lr=2.00658e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=161926
-2024-11-05 02:30:02 - progress_bar.py[line:274] - INFO: epoch 063:    146 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.672, ntokens=17677.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.37, wps=3366.1, ups=0.19, wpb=17677.1, bsz=1024, num_updates=24360, lr=2.00522e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=161979
-2024-11-05 02:30:54 - progress_bar.py[line:274] - INFO: epoch 063:    156 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.704, ntokens=17921.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.52, wps=3410.7, ups=0.19, wpb=17921.3, bsz=1024, num_updates=24370, lr=2.00386e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=162031
-2024-11-05 02:31:46 - progress_bar.py[line:274] - INFO: epoch 063:    166 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.682, ntokens=18084.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3457.4, ups=0.19, wpb=18084.9, bsz=1024, num_updates=24380, lr=2.0025e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.3, wall=162084
-2024-11-05 02:32:39 - progress_bar.py[line:274] - INFO: epoch 063:    176 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.652, ntokens=17629.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3335.2, ups=0.19, wpb=17629.8, bsz=1024, num_updates=24390, lr=2.00114e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=162136
-2024-11-05 02:33:33 - progress_bar.py[line:274] - INFO: epoch 063:    186 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.693, ntokens=17833.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.47, wps=3342.6, ups=0.19, wpb=17833.3, bsz=1024, num_updates=24400, lr=1.99978e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=32, gb_free=13.4, wall=162190
-2024-11-05 02:34:30 - progress_bar.py[line:274] - INFO: epoch 063:    196 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=17749.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3107.9, ups=0.18, wpb=17749.7, bsz=1024, num_updates=24410, lr=1.99842e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=54, gb_free=13.5, wall=162247
-2024-11-05 02:35:23 - progress_bar.py[line:274] - INFO: epoch 063:    206 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.701, ntokens=18126.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.5, wps=3385, ups=0.19, wpb=18126.3, bsz=1024, num_updates=24420, lr=1.99706e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=162300
-2024-11-05 02:36:15 - progress_bar.py[line:274] - INFO: epoch 063:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.635, ntokens=17667.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.21, wps=3398.4, ups=0.19, wpb=17667.7, bsz=1024, num_updates=24430, lr=1.9957e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=52, gb_free=13.6, wall=162352
-2024-11-05 02:37:09 - progress_bar.py[line:274] - INFO: epoch 063:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.669, ntokens=17870.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=3344.4, ups=0.19, wpb=17870.4, bsz=1024, num_updates=24440, lr=1.99434e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.6, wall=162406
-2024-11-05 02:38:02 - progress_bar.py[line:274] - INFO: epoch 063:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.637, ntokens=17373.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.22, wps=3256, ups=0.19, wpb=17373.3, bsz=1024, num_updates=24450, lr=1.99298e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=53, gb_free=13.5, wall=162459
-2024-11-05 02:39:00 - progress_bar.py[line:274] - INFO: epoch 063:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.682, ntokens=17754.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.42, wps=3084.1, ups=0.17, wpb=17754.2, bsz=1024, num_updates=24460, lr=1.99162e-05, gnorm=0.008, clip=0, loss_scale=65536, train_wall=57, gb_free=13.6, wall=162517
-2024-11-05 02:39:56 - progress_bar.py[line:274] - INFO: epoch 063:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.673, ntokens=17620.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.38, wps=3144.9, ups=0.18, wpb=17620.9, bsz=1024, num_updates=24470, lr=1.99026e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=56, gb_free=13.3, wall=162573
-2024-11-05 02:40:51 - progress_bar.py[line:274] - INFO: epoch 063:    266 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.726, ntokens=18100.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.62, wps=3270.7, ups=0.18, wpb=18100.7, bsz=1024, num_updates=24480, lr=1.9889e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=162628
-2024-11-05 02:41:50 - progress_bar.py[line:274] - INFO: epoch 063:    276 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.627, ntokens=17576.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.18, wps=2975, ups=0.17, wpb=17576.6, bsz=1024, num_updates=24490, lr=1.98754e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=59, gb_free=13.5, wall=162687
-2024-11-05 02:42:48 - progress_bar.py[line:274] - INFO: epoch 063:    286 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.662, ntokens=17810.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3083.1, ups=0.17, wpb=17810.6, bsz=1024, num_updates=24500, lr=1.98618e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.6, wall=162745
-2024-11-05 02:43:46 - progress_bar.py[line:274] - INFO: epoch 063:    296 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.708, ntokens=17779.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.53, wps=3059.4, ups=0.17, wpb=17779.3, bsz=1024, num_updates=24510, lr=1.98482e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=58, gb_free=13.5, wall=162803
-2024-11-05 02:44:45 - progress_bar.py[line:274] - INFO: epoch 063:    306 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.776, ntokens=18277.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.85, wps=3084.5, ups=0.17, wpb=18277.8, bsz=1024, num_updates=24520, lr=1.98346e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=59, gb_free=13.4, wall=162862
-2024-11-05 02:45:43 - progress_bar.py[line:274] - INFO: epoch 063:    316 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.84, ntokens=18566.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7.16, wps=3229, ups=0.17, wpb=18566.5, bsz=1024, num_updates=24530, lr=1.9821e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=57, gb_free=13.4, wall=162920
-2024-11-05 02:46:38 - progress_bar.py[line:274] - INFO: epoch 063:    326 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.781, ntokens=18323.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.87, wps=3296, ups=0.18, wpb=18323.9, bsz=1024, num_updates=24540, lr=1.98074e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=162975
-2024-11-05 02:47:33 - progress_bar.py[line:274] - INFO: epoch 063:    336 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.807, ntokens=18384.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3338.5, ups=0.18, wpb=18384.9, bsz=1024, num_updates=24550, lr=1.97938e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=163031
-2024-11-05 02:48:33 - progress_bar.py[line:274] - INFO: epoch 063:    346 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.762, ntokens=18358.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.78, wps=3076.7, ups=0.17, wpb=18358.4, bsz=1024, num_updates=24560, lr=1.97802e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=60, gb_free=13.3, wall=163090
-2024-11-05 02:49:38 - progress_bar.py[line:274] - INFO: epoch 063:    356 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.779, ntokens=18355.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.86, wps=2828, ups=0.15, wpb=18355.2, bsz=1024, num_updates=24570, lr=1.97666e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=65, gb_free=13.3, wall=163155
-2024-11-05 02:50:44 - progress_bar.py[line:274] - INFO: epoch 063:    366 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.76, ntokens=18046.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.77, wps=2728.5, ups=0.15, wpb=18046.5, bsz=1024, num_updates=24580, lr=1.9753e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=66, gb_free=13.4, wall=163221
-2024-11-05 02:51:50 - progress_bar.py[line:274] - INFO: epoch 063:    376 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.679, ntokens=17802.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.4, wps=2690.3, ups=0.15, wpb=17802.6, bsz=1024, num_updates=24590, lr=1.97393e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=66, gb_free=13.4, wall=163287
-2024-11-05 02:52:57 - progress_bar.py[line:274] - INFO: epoch 063:    386 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.771, ntokens=17987, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.83, wps=2712.7, ups=0.15, wpb=17987, bsz=1024, num_updates=24600, lr=1.97257e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=66, gb_free=13.3, wall=163354
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_63.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_63.pt
-2024-11-05 02:53:27 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 63 @ 24605 updates
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_63.ptcp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_63.pt
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_63.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_63.pt
-2024-11-05 02:53:27 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_63.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-05 02:55:20 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-2024-11-05 02:57:12 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 63 @ 24605 updates, score 0) (writing took 225.09166033100337 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_63.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
+local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
 
 
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-slice_id 5 seek offset 250000
-slice_id 7 seek offset 350000
-slice_id 4 seek offset 200000
-slice_id 1 seek offset 50000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mapping
-
-
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-slice_id 3 seek offset 150000
-slice_id 2 seek offset 100000
-slice_id 6 seek offset 300000
-2024-11-05 02:58:07 - train.py[line:336] - INFO: end of epoch 63 (average epoch stats below)
-2024-11-05 02:58:07 - progress_bar.py[line:282] - INFO: epoch 063 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.696 | ntokens 17842.9 | nsentences 1023.02 | sample_size 1023.02 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.48 | wps 2660.7 | ups 0.15 | wpb 17842.9 | bsz 1023 | num_updates 24605 | lr 1.97189e-05 | gnorm 0.005 | clip 0 | loss_scale 65536 | train_wall 1757 | gb_free 13.4 | wall 163664
-2024-11-05 02:58:07 - trainer.py[line:639] - INFO: loading train data for epoch 64
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
-slice_id 0 seek offset 0
-2024-11-05 03:00:51 - trainer.py[line:703] - INFO: begin training epoch 64
-2024-11-05 03:00:51 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-05 03:01:20 - progress_bar.py[line:274] - INFO: epoch 064:      5 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.675, ntokens=17045, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=338.5, ups=0.02, wpb=17045, bsz=985.6, num_updates=24610, lr=1.97121e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=46, gb_free=13.6, wall=163857
-2024-11-05 03:02:13 - progress_bar.py[line:274] - INFO: epoch 064:     15 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.685, ntokens=17883.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.43, wps=3375, ups=0.19, wpb=17883.1, bsz=1024, num_updates=24620, lr=1.96985e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=163910
-2024-11-05 03:03:06 - progress_bar.py[line:274] - INFO: epoch 064:     25 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.644, ntokens=17672.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3342.2, ups=0.19, wpb=17672.9, bsz=1024, num_updates=24630, lr=1.96849e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=163963
-2024-11-05 03:03:59 - progress_bar.py[line:274] - INFO: epoch 064:     35 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.65, ntokens=17781.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.28, wps=3389.6, ups=0.19, wpb=17781.7, bsz=1024, num_updates=24640, lr=1.96713e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=164016
-2024-11-05 03:04:52 - progress_bar.py[line:274] - INFO: epoch 064:     45 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.647, ntokens=17590.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3303.2, ups=0.19, wpb=17590.3, bsz=1024, num_updates=24650, lr=1.96577e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=164069
-2024-11-05 03:05:44 - progress_bar.py[line:274] - INFO: epoch 064:     55 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.658, ntokens=17790.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.31, wps=3418.9, ups=0.19, wpb=17790.2, bsz=1024, num_updates=24660, lr=1.96441e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=164121
-2024-11-05 03:06:37 - progress_bar.py[line:274] - INFO: epoch 064:     65 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.703, ntokens=18000.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.51, wps=3404.8, ups=0.19, wpb=18000.4, bsz=1024, num_updates=24670, lr=1.96305e-05, gnorm=0.006, clip=0, loss_scale=131072, train_wall=31, gb_free=13.6, wall=164174
-2024-11-05 03:07:30 - progress_bar.py[line:274] - INFO: epoch 064:     75 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.646, ntokens=17512.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.26, wps=3292.8, ups=0.19, wpb=17512.9, bsz=1024, num_updates=24680, lr=1.96169e-05, gnorm=0.005, clip=0, loss_scale=131072, train_wall=30, gb_free=13.5, wall=164227
-2024-11-05 03:08:23 - progress_bar.py[line:274] - INFO: epoch 064:     85 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.666, ntokens=17790.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3367.3, ups=0.19, wpb=17790.6, bsz=1024, num_updates=24690, lr=1.96033e-05, gnorm=0.006, clip=0, loss_scale=131072, train_wall=30, gb_free=13.5, wall=164280
-2024-11-05 03:08:49 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 65536.0
-2024-11-05 03:09:20 - progress_bar.py[line:274] - INFO: epoch 064:     96 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.648, ntokens=17644.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3071.9, ups=0.17, wpb=17644.2, bsz=1024, num_updates=24700, lr=1.95897e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=33, gb_free=13.6, wall=164337
-2024-11-05 03:10:13 - progress_bar.py[line:274] - INFO: epoch 064:    106 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.654, ntokens=17542.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.3, wps=3324.3, ups=0.19, wpb=17542.7, bsz=1024, num_updates=24710, lr=1.95761e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=164390
-2024-11-05 03:11:06 - progress_bar.py[line:274] - INFO: epoch 064:    116 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.689, ntokens=17779.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3378.8, ups=0.19, wpb=17779.2, bsz=1024, num_updates=24720, lr=1.95625e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=164443
-2024-11-05 03:11:59 - progress_bar.py[line:274] - INFO: epoch 064:    126 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.61, ntokens=17464.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.11, wps=3296.7, ups=0.19, wpb=17464.9, bsz=1024, num_updates=24730, lr=1.95489e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=164496
-2024-11-05 03:12:52 - progress_bar.py[line:274] - INFO: epoch 064:    136 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.666, ntokens=17693.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.35, wps=3320.8, ups=0.19, wpb=17693.3, bsz=1024, num_updates=24740, lr=1.95353e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=164549
-2024-11-05 03:13:44 - progress_bar.py[line:274] - INFO: epoch 064:    146 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.639, ntokens=17565.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.23, wps=3357.1, ups=0.19, wpb=17565.9, bsz=1024, num_updates=24750, lr=1.95217e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=164601
-2024-11-05 03:14:37 - progress_bar.py[line:274] - INFO: epoch 064:    156 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.665, ntokens=17805.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.34, wps=3370.8, ups=0.19, wpb=17805.8, bsz=1024, num_updates=24760, lr=1.95081e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=164654
-2024-11-05 03:15:30 - progress_bar.py[line:274] - INFO: epoch 064:    166 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.68, ntokens=17983.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.41, wps=3398.2, ups=0.19, wpb=17983.4, bsz=1024, num_updates=24770, lr=1.94945e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=164707
-2024-11-05 03:16:22 - progress_bar.py[line:274] - INFO: epoch 064:    176 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.631, ntokens=17532.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.2, wps=3343.8, ups=0.19, wpb=17532.6, bsz=1024, num_updates=24780, lr=1.94809e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=164759
-2024-11-05 03:17:16 - progress_bar.py[line:274] - INFO: epoch 064:    186 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.659, ntokens=17773.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.31, wps=3327.3, ups=0.19, wpb=17773.3, bsz=1024, num_updates=24790, lr=1.94673e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=164813
-2024-11-05 03:18:08 - progress_bar.py[line:274] - INFO: epoch 064:    196 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=17619, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3360.3, ups=0.19, wpb=17619, bsz=1024, num_updates=24800, lr=1.94537e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.4, wall=164865
-2024-11-05 03:19:01 - progress_bar.py[line:274] - INFO: epoch 064:    206 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.709, ntokens=18260.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.54, wps=3483.6, ups=0.19, wpb=18260.1, bsz=1024, num_updates=24810, lr=1.94401e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=164918
-2024-11-05 03:19:53 - progress_bar.py[line:274] - INFO: epoch 064:    216 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.644, ntokens=17833.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.25, wps=3398.6, ups=0.19, wpb=17833.4, bsz=1024, num_updates=24820, lr=1.94265e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=164970
-2024-11-05 03:20:46 - progress_bar.py[line:274] - INFO: epoch 064:    226 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.739, ntokens=18373, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.68, wps=3460.3, ups=0.19, wpb=18373, bsz=1024, num_updates=24830, lr=1.94129e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=165023
-2024-11-05 03:21:39 - progress_bar.py[line:274] - INFO: epoch 064:    236 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.642, ntokens=17593.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.24, wps=3350.5, ups=0.19, wpb=17593.3, bsz=1024, num_updates=24840, lr=1.93992e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=30, gb_free=13.5, wall=165076
-2024-11-05 03:22:32 - progress_bar.py[line:274] - INFO: epoch 064:    246 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.622, ntokens=17454.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.16, wps=3246.2, ups=0.19, wpb=17454.2, bsz=1024, num_updates=24850, lr=1.93856e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.6, wall=165130
-2024-11-05 03:23:28 - progress_bar.py[line:274] - INFO: epoch 064:    256 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.689, ntokens=17833.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.45, wps=3210.2, ups=0.18, wpb=17833.1, bsz=1024, num_updates=24860, lr=1.9372e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=45, gb_free=13.5, wall=165185
-2024-11-05 03:24:24 - progress_bar.py[line:274] - INFO: epoch 064:    266 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.676, ntokens=17852.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.39, wps=3213.9, ups=0.18, wpb=17852.3, bsz=1024, num_updates=24870, lr=1.93584e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=55, gb_free=13.6, wall=165241
-2024-11-05 03:25:19 - progress_bar.py[line:274] - INFO: epoch 064:    276 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.648, ntokens=17749.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.27, wps=3182.4, ups=0.18, wpb=17749.7, bsz=1024, num_updates=24880, lr=1.93448e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=165296
-2024-11-05 03:26:15 - progress_bar.py[line:274] - INFO: epoch 064:    286 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.617, ntokens=17582.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3142.5, ups=0.18, wpb=17582.3, bsz=1024, num_updates=24890, lr=1.93312e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.5, wall=165352
-2024-11-05 03:27:11 - progress_bar.py[line:274] - INFO: epoch 064:    296 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.697, ntokens=17756.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.48, wps=3184.8, ups=0.18, wpb=17756.5, bsz=1024, num_updates=24900, lr=1.93176e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=165408
-2024-11-05 03:28:07 - progress_bar.py[line:274] - INFO: epoch 064:    306 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.749, ntokens=18196.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.72, wps=3240.6, ups=0.18, wpb=18196.5, bsz=1024, num_updates=24910, lr=1.9304e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=165464
-2024-11-05 03:29:03 - progress_bar.py[line:274] - INFO: epoch 064:    316 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.808, ntokens=18283.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=7, wps=3271, ups=0.18, wpb=18283.6, bsz=1024, num_updates=24920, lr=1.92904e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=165520
-2024-11-05 03:29:58 - progress_bar.py[line:274] - INFO: epoch 064:    326 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.761, ntokens=18095.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.78, wps=3278.2, ups=0.18, wpb=18095.1, bsz=1024, num_updates=24930, lr=1.92768e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=165575
-2024-11-05 03:30:53 - progress_bar.py[line:274] - INFO: epoch 064:    336 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.768, ntokens=18175.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.81, wps=3326.2, ups=0.18, wpb=18175.4, bsz=1024, num_updates=24940, lr=1.92632e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.5, wall=165630
-2024-11-05 03:31:48 - progress_bar.py[line:274] - INFO: epoch 064:    346 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.716, ntokens=18088.6, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.57, wps=3261.3, ups=0.18, wpb=18088.6, bsz=1024, num_updates=24950, lr=1.92496e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=165686
-2024-11-05 03:32:44 - progress_bar.py[line:274] - INFO: epoch 064:    356 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.782, ntokens=18270.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.88, wps=3308.7, ups=0.18, wpb=18270.1, bsz=1024, num_updates=24960, lr=1.9236e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=165741
-2024-11-05 03:33:39 - progress_bar.py[line:274] - INFO: epoch 064:    366 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.743, ntokens=18047.1, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.69, wps=3262.1, ups=0.18, wpb=18047.1, bsz=1024, num_updates=24970, lr=1.92224e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=55, gb_free=13.4, wall=165796
-2024-11-05 03:34:35 - progress_bar.py[line:274] - INFO: epoch 064:    376 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.722, ntokens=18091, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.6, wps=3216.9, ups=0.18, wpb=18091, bsz=1024, num_updates=24980, lr=1.92088e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=56, gb_free=13.4, wall=165852
-2024-11-05 03:35:30 - progress_bar.py[line:274] - INFO: epoch 064:    386 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.73, ntokens=17788.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.63, wps=3234.4, ups=0.18, wpb=17788.5, bsz=1024, num_updates=24990, lr=1.91952e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=55, gb_free=13.3, wall=165907
-reach the end of datafile, start a new reader
-reach the end of datafile, start a new reader
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt2024-11-05 03:35:54 - checkpoint_utils.py[line:64] - INFO: Preparing to save checkpoint for epoch 64 @ 24995 updates
-
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt
-2024-11-05 03:35:54 - trainer.py[line:431] - INFO: Saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-cp: cp: cp: cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt'cannot create regular file './polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt': File exists: File exists
-: File exists
+file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 begin to initialize row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 begin to initialize row_count and line_idx-to-offset mapping
-2024-11-05 03:37:44 - trainer.py[line:441] - INFO: Finished saving checkpoint to ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 finished initializing row_count and line_idx-to-offset mapping
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 7 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 4 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 1 row count 50000 total row count 400000
-slice_id 7 seek offset 350000
 slice_id 1 seek offset 50000
-slice_id 4 seek offset 200000
-2024-11-05 03:39:36 - checkpoint_utils.py[line:135] - INFO: Saved checkpoint ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_best.pt (epoch 64 @ 24995 updates, score 0) (writing took 222.23293120099697 seconds)
-cp ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_last.pt ./polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_64.pt
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 5 row count 50000 total row count 400000
 slice_id 5 seek offset 250000
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 finished initializing row_count and line_idx-to-offset mappinglocal datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 finished initializing row_count and line_idx-to-offset mapping
-
-local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 finished initializing row_count and line_idx-to-offset mapping
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 6 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 2 row count 50000 total row count 400000
-file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 3 row count 50000 total row count 400000
-slice_id 6 seek offset 300000
-slice_id 2 seek offset 100000
-slice_id 3 seek offset 150000
-2024-11-05 03:40:31 - train.py[line:336] - INFO: end of epoch 64 (average epoch stats below)
-2024-11-05 03:40:31 - progress_bar.py[line:282] - INFO: epoch 064 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.683 | ntokens 17816.3 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 6.42 | wps 2731 | ups 0.15 | wpb 17816.3 | bsz 1023 | num_updates 24995 | lr 1.91884e-05 | gnorm 0.005 | clip 0 | loss_scale 65536 | train_wall 1539 | gb_free 13.4 | wall 166208
-2024-11-05 03:40:31 - trainer.py[line:639] - INFO: loading train data for epoch 65
+slice_id 4 seek offset 200000
+2024-11-07 11:50:03 - train.py[line:336] - INFO: end of epoch 32 (average epoch stats below)
+2024-11-07 11:50:03 - progress_bar.py[line:282] - INFO: epoch 032 | loss 0.001 | loss_v1 0 | loss_v2 0 | nll_loss 2.439 | ntokens 17795.4 | nsentences 1023.01 | sample_size 1023.01 | sample_size_v1 0 | sample_size_v2 0 | ppl 5.42 | wps 2777.3 | ups 0.16 | wpb 17795.4 | bsz 1023 | num_updates 12501 | lr 3.61852e-05 | gnorm 0.007 | clip 0 | loss_scale 65536 | train_wall 1445 | gb_free 13.4 | wall 79722
+2024-11-07 11:50:03 - trainer.py[line:639] - INFO: loading train data for epoch 33
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 begin to initialize row_count and line_idx-to-offset mapping
 local datafile ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 finished initializing row_count and line_idx-to-offset mapping
 file ../../datasets/finetune/aihub_indoor/aihub_indoor_train.tsv slice_id 0 row count 50000 total row count 400000
 slice_id 0 seek offset 0
-2024-11-05 03:43:15 - trainer.py[line:703] - INFO: begin training epoch 65
-2024-11-05 03:43:15 - train.py[line:297] - INFO: Start iterating over samples
-2024-11-05 03:43:44 - progress_bar.py[line:274] - INFO: epoch 065:      5 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.669, ntokens=17034, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=6.36, wps=344.8, ups=0.02, wpb=17034, bsz=985.6, num_updates=25000, lr=1.91816e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=39, gb_free=13.6, wall=166401
-loss_reg: 0.013329504396735142 loss_cls: 0.0045333015732467175
-loss_reg: 0.012361892372722495 loss_cls: 0.0031015384010970592
-loss_reg: 0.012783864499016532 loss_cls: 0.003374127671122551
-loss_reg: 0.02625057309380812 loss_cls: 0.004728822968900204
-loss_reg: 0.01513249122137144 loss_cls: 0.0033096473198384047
-loss_reg: 0.02514515873378456 loss_cls: 0.003210575319826603
-loss_reg: 0.01232097617232249 loss_cls: 0.003976666368544102
-loss_reg: 0.017675849038232782 loss_cls: 0.003953460603952408
-loss_reg: 0.016971113071218347 loss_cls: 0.0033534145914018154
-loss_reg: 0.01209466104589169 loss_cls: 0.004583246074616909
-loss_reg: 0.008869981992948316 loss_cls: 0.003182005137205124
-loss_reg: 0.017733307998623795 loss_cls: 0.004592255223542452
-loss_reg: 0.010740833941382559 loss_cls: 0.002810372970998287
-loss_reg: 0.014962226740033596 loss_cls: 0.003757154569029808
-loss_reg: 0.022797350692989486 loss_cls: 0.004391951486468315
-loss_reg: 0.01216563424112238 loss_cls: 0.0032655764371156693
-loss_reg: 0.015305159087105433 loss_cls: 0.004349082242697477
-loss_reg: 0.02027516433187991 loss_cls: 0.003687010845169425
-loss_reg: 0.01860234815342079 loss_cls: 0.0035726912319660187
-loss_reg: 0.012255199995956246 loss_cls: 0.003514482406899333
-loss_reg: 0.02012446833690771 loss_cls: 0.004121749196201563
-loss_reg: 0.018455096879392903 loss_cls: 0.003771640593186021
-loss_reg: 0.009371824192670316 loss_cls: 0.003834799397736788
-loss_reg: 0.01949405918835442 loss_cls: 0.0042433408088982105
-loss_reg: 0.009942574022776145 loss_cls: 0.004148575011640787
-loss_reg: 0.016729304094757078 loss_cls: 0.004407593514770269
-loss_reg: 0.019314389341148916 loss_cls: 0.0043874033726751804
-loss_reg: 0.017447328471459547 loss_cls: 0.004181940108537674
-loss_reg: 0.01674921290565229 loss_cls: 0.003546183928847313
-loss_reg: 0.023408695816654952 loss_cls: 0.0030163757037371397
-loss_reg: 0.012653693989792238 loss_cls: 0.002970479428768158
-loss_reg: 0.025362294666604456 loss_cls: 0.003403815208002925
-loss_reg: 0.01994550040859812 loss_cls: 0.003241978120058775
-loss_reg: 0.020525655646287583 loss_cls: 0.0039258357137441635
-loss_reg: 0.021925655434687715 loss_cls: 0.003687993623316288
-loss_reg: 0.011734062533580124 loss_cls: 0.0034535296726971865
-loss_reg: 0.011198286599673875 loss_cls: 0.0034897266887128353
-loss_reg: 0.012071430282424155 loss_cls: 0.002584045520052314
-loss_reg: 0.01554061086166858 loss_cls: 0.004401541315019131
-loss_reg: 0.012185033125143869 loss_cls: 0.005030733533203602
-loss_reg: 0.018047597990205663 loss_cls: 0.0026005979161709547
-loss_reg: 0.019760386240979644 loss_cls: 0.0035239867866039276
-loss_reg: 0.01607839663678574 loss_cls: 0.0034919115714728832
-loss_reg: 0.017057343094982257 loss_cls: 0.00337665481492877
-loss_reg: 0.017039479984587366 loss_cls: 0.0035414895974099636
-loss_reg: 0.005141284569329289 loss_cls: 0.0033114254474639893
-loss_reg: 0.017903068939128904 loss_cls: 0.0031430518720299006
-loss_reg: 0.025580213160850622 loss_cls: 0.002209761179983616
-loss_reg: 0.012569511226986203 loss_cls: 0.00340187712572515
-loss_reg: 0.02140038608370016 loss_cls: 0.0026701437309384346
-loss_reg: 0.014793724316879343 loss_cls: 0.003722080960869789
-loss_reg: 0.01323026455817657 loss_cls: 0.003508408321067691
-loss_reg: 0.015400756940803648 loss_cls: 0.003998110070824623
-loss_reg: 0.004569911353627191 loss_cls: 0.002439278643578291
-loss_reg: 0.012153289055749705 loss_cls: 0.0034568493720144033
-loss_reg: 0.01089315100163158 loss_cls: 0.0036095925606787205
-loss_reg: 0.01710871316960828 loss_cls: 0.0034883308690041304
-loss_reg: 0.01607410850611196 loss_cls: 0.0035717342980206013
-loss_reg: 0.00600889062884914 loss_cls: 0.00272833532653749
-loss_reg: 0.003407044787687367 loss_cls: 0.002852275501936674
-loss_reg: 0.004346975182282305 loss_cls: 0.002969041932374239
-loss_reg: 0.0050441319712397886 loss_cls: 0.002784611890092492
-loss_reg: 0.006233676418859205 loss_cls: 0.0024678201880306005
-loss_reg: 0.003146423162625616 loss_cls: 0.0023440334480255842
-2024-11-05 03:44:38 - progress_bar.py[line:274] - INFO: epoch 065:     15 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.663, ntokens=17742.8, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.33, wps=3315.6, ups=0.19, wpb=17742.8, bsz=1024, num_updates=25010, lr=1.9168e-05, gnorm=0.004, clip=0, loss_scale=65536, train_wall=31, gb_free=13.5, wall=166455
-2024-11-05 03:45:31 - progress_bar.py[line:274] - INFO: epoch 065:     25 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.619, ntokens=17519.4, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3304, ups=0.19, wpb=17519.4, bsz=1024, num_updates=25020, lr=1.91544e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=30, gb_free=13.6, wall=166508
-2024-11-05 03:46:24 - progress_bar.py[line:274] - INFO: epoch 065:     35 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.64, ntokens=17747.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.23, wps=3342.9, ups=0.19, wpb=17747.3, bsz=1024, num_updates=25030, lr=1.91408e-05, gnorm=0.005, clip=0, loss_scale=65536, train_wall=33, gb_free=13.5, wall=166561
-2024-11-05 03:47:17 - progress_bar.py[line:274] - INFO: epoch 065:     45 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.617, ntokens=17476.7, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.14, wps=3311.6, ups=0.19, wpb=17476.7, bsz=1024, num_updates=25040, lr=1.91272e-05, gnorm=0.006, clip=0, loss_scale=65536, train_wall=31, gb_free=13.4, wall=166614
-2024-11-05 03:48:10 - progress_bar.py[line:274] - INFO: epoch 065:     55 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.636, ntokens=17676.3, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=6.22, wps=3336.2, ups=0.19, wpb=17676.3, bsz=1024, num_updates=25050, lr=1.91136e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=30, gb_free=13.3, wall=166667
+2024-11-07 11:52:44 - trainer.py[line:703] - INFO: begin training epoch 33
+2024-11-07 11:52:44 - train.py[line:297] - INFO: Start iterating over samples
+2024-11-07 11:53:33 - progress_bar.py[line:274] - INFO: epoch 033:      9 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.403, ntokens=16817.7, nsentences=985.6, sample_size=985.6, sample_size_v1=0, sample_size_v2=0, ppl=5.29, wps=344.1, ups=0.02, wpb=16817.7, bsz=985.6, num_updates=12510, lr=3.61729e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=29, gb_free=13.6, wall=79932
+2024-11-07 11:54:27 - progress_bar.py[line:274] - INFO: epoch 033:     19 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.43, ntokens=17924.5, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.39, wps=3331.2, ups=0.19, wpb=17924.5, bsz=1024, num_updates=12520, lr=3.61593e-05, gnorm=0.007, clip=0, loss_scale=65536, train_wall=36, gb_free=13.5, wall=79986
+2024-11-07 11:54:50 - trainer.py[line:922] - INFO: NOTE: gradient overflow detected, ignoring gradient, setting loss scale to: 32768.0
+2024-11-07 11:55:29 - progress_bar.py[line:274] - INFO: epoch 033:     30 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.405, ntokens=17758.2, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.3, wps=2871.3, ups=0.16, wpb=17758.2, bsz=1024, num_updates=12530, lr=3.61457e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=60, gb_free=13.5, wall=80048
+2024-11-07 11:56:22 - progress_bar.py[line:274] - INFO: epoch 033:     40 / 391 loss=0.001, loss_v1=0, loss_v2=0, nll_loss=2.466, ntokens=18046.9, nsentences=1024, sample_size=1024, sample_size_v1=0, sample_size_v2=0, ppl=5.53, wps=3433.4, ups=0.19, wpb=18046.9, bsz=1024, num_updates=12540, lr=3.61321e-05, gnorm=0.006, clip=0, loss_scale=32768, train_wall=52, gb_free=13.5, wall=80101
diff --git a/run_scripts/finetune/train_polyformer_b_aihub.sh b/run_scripts/finetune/train_polyformer_b_aihub.sh
index be9c7db..7bea840 100644
--- a/run_scripts/finetune/train_polyformer_b_aihub.sh
+++ b/run_scripts/finetune/train_polyformer_b_aihub.sh
@@ -22,8 +22,8 @@ data=${data_dir}/aihub_indoor/aihub_indoor_train.tsv,${data_dir}/aihub_indoor/ai
 selected_cols=0,5,6,2,4,3,7
 # restore_file=../../weights/polyformer_b_pretrain.pt
 # restore_file=../pretrain/polyformer_b_pretrain_aihub_indoor_checkpoints/20_5e-5_512/checkpoint_20_1000.pt
-restore_file=../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume_2/20_5e-5_512/checkpoint.best_score_0.5220.pt
-
+# restore_file=../pretrain/polyformer_b_pretrain_aihub_indoor_80_checkpoints_resume_2/20_5e-5_512/checkpoint.best_score_0.5220.pt
+restore_file=../finetune/polyformer_b_aihub_indoor_80_checkpoints/100_5e-5_512/checkpoint_epoch_73.pt
 
 
 task=refcoco
diff --git a/tasks/__pycache__/__init__.cpython-37.pyc b/tasks/__pycache__/__init__.cpython-37.pyc
index f5b0485c20dcfa178edcd56eed30da63df2f1609..a614214a0487dae3a4fa1450d3453865342536d1 100644
GIT binary patch
delta 24
ecmZo<YGmSe;^pOH0D`ZaS`)cfGnP->?gaokMFsHy

delta 24
ecmZo<YGmSe;^pOH0D_xWRVH$;W(=RW-3tIfQwDng

diff --git a/tasks/__pycache__/base_task.cpython-37.pyc b/tasks/__pycache__/base_task.cpython-37.pyc
index 013b96761234f41ce26bb9e4d2d0b78fd04d8808..30a9e75c6e7fd88a453fe59a9262c7ecf640e9f5 100644
GIT binary patch
delta 27
hcmX@+e$1WQiI<m)0SLZwYHj4^W@9YhEY7xG764#X29*E+

delta 27
hcmX@+e$1WQiI<m)0SInhRoTeR&BhqMS)6UXEC6UB2Oj_c

diff --git a/tasks/__pycache__/refcoco.cpython-37.pyc b/tasks/__pycache__/refcoco.cpython-37.pyc
index b6c20e9eb6a22666313d8032dae794e380664466..ac133005ae72678607fc3f40ef5ea2eab2a4748a 100644
GIT binary patch
delta 27
hcmbQ>Il+_LiI<m)0SLZwYHj4+!@^j;`2@=ec>rGA2c`f3

delta 27
hcmbQ>Il+_LiI<m)0SInhRoTeBhlMeG^9hy{@&IP22rvKu

diff --git a/tasks/__pycache__/refcoco_pretrain.cpython-37.pyc b/tasks/__pycache__/refcoco_pretrain.cpython-37.pyc
index 0e722f2effad9ca35e85d8f27dd3fc3eb704913a..5acec7bfcab3046748f44988954d846fe426a5b3 100644
GIT binary patch
delta 27
hcmeBo?RVvN;^pOH0D`ZaS{u2KvM`o!zR0pq764v92ciG~

delta 27
hcmeBo?RVvN;^pOH0D_xWRW@=TWnm28e350JEC6N;2rK{q

diff --git a/utils/__pycache__/__init__.cpython-37.pyc b/utils/__pycache__/__init__.cpython-37.pyc
index 1b74fa74dd90b8dc270fb84a29ff45ddbbd50e05..90f239c5f029abf49950312b126771b4b2fa42a4 100644
GIT binary patch
delta 24
ecmbQmIE#_niI<m)0SLZwYE9&JV=SK-<N*LXZ3R^T

delta 24
ecmbQmIE#_niI<m)0SInhRhh``#uz>^$O8aDdj;PB

diff --git a/utils/__pycache__/checkpoint_utils.cpython-37.pyc b/utils/__pycache__/checkpoint_utils.cpython-37.pyc
index 952d2556d652554bed660ca1df905d6be5f54c24..5233602fc6576a029633575ff796ae5abea66f78 100644
GIT binary patch
delta 29
jcmdl#mvR4GMs6owUM>b8_{yoZk$WKrWBKN_971jYdZ7pL

delta 29
jcmdl#mvR4GMs6owUM>b8xOr7&BlkiM#_-K+IfUE*f^7(U

diff --git a/utils/__pycache__/eval_utils.cpython-37.pyc b/utils/__pycache__/eval_utils.cpython-37.pyc
index 6a1e74040b834f16c6180ab3fa4a546895fc8d7b..dc6374647327a4090ef0b0e98198b6f16ae4b7fa 100644
GIT binary patch
delta 27
hcmext|Jk0~iI<m)0SLZwYHj4MXJjnj+|8IF2LNpe2X_Df

delta 27
hcmext|Jk0~iI<m)0SInhRoTd0&&U|QxtlRV4ghr32mt^9

diff --git a/utils/__pycache__/transforms.cpython-37.pyc b/utils/__pycache__/transforms.cpython-37.pyc
index 996c7ea5941f0ac7c9f6e7f34efeed5bae5153b1..6d3cc8c20b315102e1d16fefa56e9049c0e73b37 100644
GIT binary patch
delta 27
hcmX?<e<+{ZiI<m)0SLZwYHj3Rz`|I*c@2x0F#v6}2Y>(o

delta 27
hcmX?<e<+{ZiI<m)0SInhRoTeBfQ2!9^BNX0V*qu42nqlI

diff --git a/utils/__pycache__/vis_utils.cpython-37.pyc b/utils/__pycache__/vis_utils.cpython-37.pyc
index ab04880e7eb14c02a75f32eb218e45216c253cf4..7410708a1e23425fb42ff5db852b755154e6acdc 100644
GIT binary patch
delta 27
hcmbQqH<ORsiI<m)0SLZwYHj3pXJjnj9L#u(6#!291~vcy

delta 27
hcmbQqH<ORsiI<m)0SInhRoTex&d3<PIhgSnD*#r;2EYIS