From ce31be0a5f904f3aeae92089e592be7b8faffaa4 Mon Sep 17 00:00:00 2001 From: Rodrigo Fuentes Date: Fri, 16 Aug 2024 09:35:53 -0300 Subject: [PATCH] Filter ingestion messages by date range --- .../excel_to_bq/data/pan_ingestion.xlsx | Bin 320732 -> 320792 bytes .../excel_to_bq/test_pan_ingestion.py | 24 ++++++++++++------ .../ingestion/excel_to_bq/feeds/pan_ingest.py | 13 ++-------- .../ingestion/excel_to_bq/pipeline.py | 13 ++++++++-- 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/packages/pipe-vms-ingestion/tests/vms_ingestion/ingestion/excel_to_bq/data/pan_ingestion.xlsx b/packages/pipe-vms-ingestion/tests/vms_ingestion/ingestion/excel_to_bq/data/pan_ingestion.xlsx index c0d8ce81fe453c8a0534c8cb8a81fbf7cd1cc038..f6adb0853182e1a11402b50bcead433c2ed35c23 100644 GIT binary patch delta 13571 zcmc(`2{={T|37Z6y$zShP*jpAl#ES68E&_cqT#0b7%G*76dE{6(Y;Ny%T~&8RqkyN zomfd3LMkOuA+t(l$aKxa`K`Tis5?C0&+~uo^Y3}iyM2behWGG#zuxb)l{}RTc_t@m zZ9$Nugv7+eg!H#a$4O2G1pc2l6U>I}g)`AZuot^T71*yRe=!~Iq64B8-9F;x?M^%T zUXlq48;slJpCp{vhJ+7fE#9kjRQBhy3GaLVoOfmNeD%cT)*1VEYkg___`to)uxE*a zPgq~mr`pV?-JLtV;~(tz7#DUaJnXrPPB3j_U}|FA+g-l7H!?izrU5TByU2e346a9) zTe3;-`xfs0F1y689lai0lk!WJ5+ygoFDIAo+MQx(yn*xJGUw%6t6V- z-CXWc->8#S)KO=`-JHoUUm9tNy83n=T-jt7S9^lvP8aarST*eC>)rA;URD!uBlAu5 z`>+01;ijdx_yupChTh2BmfV%yyvOK+0Wb4_$wLq0ZnwVPLoMyarM00c+I*u;US6Gb zo%y_O15@sS{j)A-c{pm^=nUmLMsvl>A4q*lq}QN*Nq6}7FLKwKcJ*!P_2{g8b2r=b zL{XRBtJ2JZ8#l5WVs=;Ozj;wp+MCJW=;IZ)v1k9qCx6D8_*8PM^FL3}xf$bg$hf{G zsWbFNw3-1=IXPUnS|POau<+@DlhK*=sW^)dh4 zwHci)hm4!PWPYhJy+(O;bo#VqC#i4^xTYM-H%Tw%CeOa*X&CmkY1evZ zAGo=-LsPWq>xKje?f1Q#!b-iZzxr%_F`M_v$|ifH#*Iz&o8BkAd>y@cr1IN`+9hv% zD)W_mx*ZYm}#UYD7)=<5Wq?%!ZB)Io+}Ct`D;f>v}Uy zJCgS_==Q1odHx``>5Sj)lI{~J+9}LOT^-LJri4Uu5$!kWc7N##Nn4*_G6{$^#O@*?PB(YIDLOP+crir#EGbW=bQN5Ww29k&!vy= zUWfkWwZ6PBU{%53=Ze+qeG}zvV}wJTz+W!KVitrr$yBdDdRA&QHBo9T{@vCPr?J)R z{S$-jhAycfsm;_wspwh@yni?A*)2)mWjkKJIYVC zj*5_-Nkz%kQK@n>D2LG&l<#Pr?_NrjpS%SXA+Ph@YdkMsejK10<%Owcg%?wsY9FNW zfshcNpHXZa?`$|i$b6~I+bnsmy^FF4^<0y@yl)MEm7b3k_e~v?0MZv#H?!SNipMR_SudDm(^eA#HkO#;l4NL^6+nCX= z!y_OCkn1UEMaFZWDgYhPFI6GX>kdzDQ|Q;w5@#V+Ll?5!8a0BiN>5#>@|nZ02`_GR z%27Pz+%fBuWK|Z`q^pRK8ypRIxi{qitveompdvJNdA4@ykY`}&MhXX-;-Awp8-N@y z)%q78+*7L91F5eUlt^Vf6jiBq0riZ93dF!|Uc#Ug;kC?#lbZ}&8YTX2F`3c}G|Apt zPOpVAvU>!i0XmTb<`U=%3C04-)>H&(B`~{Oas}_0cHaWKslZ9{DD`(x0@Zu46A1T@ z8s{dYFD8^0tR)S*T%MQv&Et*U=uy-+pK;2jaqft7kMY42UuLZClVCo9{0+cV$UCt> z3!q{nJUDMvkpWQSS4;W~c)H%WuwLbx~>hUr?+Qumx1b zgvZG8A~SZnFu);Gp(28oUcs{n)=GSmtVU&*;8H-v>b0UU5`RgZ7@wxjXpi`I^(W1I zhp#?TGDU<3Y&if;or>?`<%M9|-?|5}mK0rq4B+kOVSBeQI^Ikm)AJCU?b$nEq0WCV z*hLz)J)@W-l4H7Dv~1>&+M!uk-d8=ajYmvaszX#I!Yp1-RRp!j;?2;UP;im5p5)0c z-fb&1h0MZDA?5JllX{P-z?c@!W~`3(;m8YLtL%S0Z(?C_S@lRpN%~6ayt3aiA=Rk7 z7u7j{Sz;Pp9KoG~J~_77y-7n}`{?2zadUo`4R=@iZF5S0YA7#-o(hLHD+fRz(sctj zMKnI}5BhP0`W0umuYdlaeT7w4TH!zb`oKK4)T1RJq5BpE^t@Dy0`udH_gridVZ+G;Qe!#VtTE4?$B~h%D1x-rAw~)9`xp(WJSoO5GRRL0 zNZ>0iG2R4i89?MH*3qY2;T3NDFtr*V-oh$mTCKwg)3{Jt$?wP9b&jnZMV}Y@{r(#{ z<^MV~$_WD-Nyd~I1pW!ZB&sE=8_w+ayL~IUjW(_TJx-d*tQE z9gIkfnvA6iJh;_XyENuV9W%Rdr^|(jYMJd{dA^+|M9^t%4D>(?rOkrMK-ADhnma77 zXATl zzd$7>eC+IG0ctdd@<3oQp5a*;fFAinU4R9 zwn_CO!PlV{FIFFWj`uE{fOd&vPeQjWfgu6A+)?BRs0q-BcqR$WsFDV>_6$6HbGc#< zZ9Njmi_aq&9YQ98DwV=fVFRE=U+WT!Ze|>Ac9I8@5y8e z9Pb^r&G!w4XHJ`dde_iu4KbBq`Q#FeHPuGs>5Zps?>y~d*b)!;eT8hO^OjQf z1TVY`+i`*iUN{fllK^zYdr+g^iUim^1oPiy6E^u)Dox|vD7_3(>_PBcY;s?y;I2VL zA0f*k`VqN)L^pLtLE_*UL@sS$6$`=#gw^?91)w%G8Iw~4xsb`lC}%ks4bYw-z!lf% z9hyl%4$?$0IVkw?%HmaNeQgH(5`T^#Qkerz15A)`?#Fx+o3-pWoPNG_U~{NZm@<@( z2gieBb`2*REIW*}YQQ-C5mLU3T@hv}cq~dCE6&~y2P7M$Ormaw>fJ+JagbgttLsP~ zMlZtOVYZp3y^;=`1V~=?vuS{L`8u{l>3zl$3h6k`;O&jrOG9dq^fEXJ(%GuedQzLG zz$2Gn8#75Nh|Ry*Du3)GTS7Vf`E*bKc*n3{M{+RRVqp9puVMv>pGvhZQ|-$y7|t0P z^eay2es5LhpDxUzpCh%xSRU$`Qf>5H-&_U2i zZ-ElYlBvjXSN`62r@_I*N%_rh<>l7s0JY%#ex03mWfe|0ZhJeo#xuB$>#gL3u6G4% zINk4RoH!f2>*>Q4pn}eYU@A}9T@0vq45jcvZq{Ef2q=0SlWr?lULUj|){N8ya7`m5V`l8GZoDne`7-w`JTQoWO3RxwZu38LgX#)$C|1u23+g4cD%n!uU- zHNu(Pjigd&Gxgpxg5do2{%;5(vh#(;B2?@Kao+FNW;jmWS<-$}2GC`m4A%Y)|GSlr z0ZcFo%m2)$>NDHljUr@V_7K-#GZ$XhpUk>8GuU$h!rt3-?i8pX=}__HjhK4qP5$fbQx0j3pKz%Z@(fzt&{3#BfF#<;=>Y3&ax-kWVWiN4TfN>P>@ zp%y)WDYu@wFz;+0lW8zPdb1iv;Hqk<9vNhE>x!B2Ck4(Hj8B7N~8N7cTN>i3z|=wFe`?K#@C0E<_KL47r#q%8-j{4i)!$66HHTR*;c#)z&<$ zJgULn$0+c92=#L=N&K(Vy7{s-|uXTTCZ`0e_+mc*Ks`97nTxM|RHYZMK z@4JF+DQ)pZ7@@v<+tJ^Y_)jD|{eL+D+fVuDrO#s8E&u%W-+tTfgeng~Ol@=3B>vKx z!A>)OuMBS0DN6C=v~VpFt0jh3`@tUxo$|f`MmWf8r^R>SGF?tRNUX*tkWJtP7KfPfh&Yb9PJ&;B&)}}P>@?BEl zr?d@p;s!e=)3V>DhZipMkY+k2qjyrTbmd{qCuDaB-Y0mvKlg`m2FaAsc${38$(jSz zD$IBXTy)UW7eUt3=Lxek`YsKw!_}C_U1d6pcSx4!;R19afjXjLIZ&&@u`)2iVE921 z2ONIo(Jz-pEHCYK@9zh#M>r;JQC|MakCBzE zY+@xi%6uZjN)=Y>v(lWED_CjAN;g&>W@Qj7BUu@TC0BWQ8vEuID=S&q#7c0Coq&}p ztkh?vIV)GN(vFpGtUS!hAXY}QGLDsLtbE1FN>(+-&mRa^pSaf z;I2COMiqSRY`S7yKspGYOS37!Ls!aHtVMb-wCg;Pl+2pVq}PT zMn*N)ZT}ovzY9J|I8Wk|Q-w=|Eu;;yZ6&%O9aczc(7x61D(rHgM=T@9Nc?i@C+n0h zwd2|8OX$i}XpYKe5RWmpzfHK~o|`CXMIh(b@JEPBb;ucz&fQ7QQ|H@`8!TK0>#pFt zx?Fn`GG@ zss<{=n_)KrUr}W=<5_x-;Ry85o5a;=jUtJg$WhN=y2$aAW$C&?bZG-rgj^-s?mB6S zH;FYf{}t?_Q?sBts#Rp}iMmP>c&0B++;3Em`wb)0bMqFTg zgwOa$CH4p8`hxfZm>9*q+O}^MH4|3~js(X2h(27QA={nAGN{()AQ*ay-9)Awvkcy) z)bop)oK(`W2yX8U{&+foe)W?4gmg!dlc4b13;yK~kjVjZ3LsZW>i8g;r;LT1w&5Ak zj<1Yfr1hCktg^A45zn2(9IC>JjG4lJGPv*3IGIiQVhhrOPqGIE4K}up#`Ar6)U7|m z-`{biWzwNNTCR*80a?x=y2RwG%db`H4yeo=2MN>LIc7!|&mvQTGoUG|8_i66FpxY1 z=?FFAEehO&eVefl{y2Rvvl`dOc;|Gw85u)&SrSqbq7#3<$abl*!!&({&iyJwkG2rF zC)z%r+)56pX+CyuOfDlB#g+Hq0qHY|q`XKW_X2hodgLB}!jUB&B=1dEV3QvkK@QXn+Y?Gh653E`(sODG1aM`1B(8DKU?r z8&jben)HfsQygj{{iZ_LRE7k&iOlohcpyq$t2EEoGo^jY;T7RNJ;@XyI(rt0qxB@c zLX|j-B6tLDX<#V9+R=N)D0 z1K<_7kEo;d3W-B>;3w?AtcsCuFFq`Qg{E|x3!#+okh};)j+a}en+nloXNmFTjeB!1 z(LpKD7u|jfKY`#!*at-3K@Et+Pm;Syk()ClJ6ql(9c@Ab(B@}}5IQgg9#U^gxmGz) zfF`=x>W}yd6J$SzAa@s>n-+cJv2w%y-tM_b<}R575=00^?6;qw6`F2Mjv7EPxnKs& z{smRm9U~WW`P8R7uDLT0QO9s*4AmMxQ?h(ZWyFr-%G<{!yRVz`UrVR|dV-VJ z39Ou`Y-8z(HMBeUW0;X3Wl3e6OQkrmY#E>|8UMKN4H zfn(8(7YxF(Xf_m@IHmy;kE0aj#&XNlmv?^dSG;Wdq z)`$D$DESD14_>35FffD@$zwu|yGwx3g0OC74%)INe(68e!QhXAHWQJ;OorSIB2mQy z@;8!>*hf59>q^-%Sh)V}T2dIIfRcBTYPk73l(HO(7{wbJV7!C4G{E6M?VlhC>(PTE zxL^|BMX*o-?`IFOIM=M)bCf#KtlpcA zbNRrzB8GalKgc$i3rO={S;TI@7C?u^2y=*Pl`+g>RKbaN6M<&eF(B#~mN;VmC19AOB5Ufx~uA3 zIhQc3@-VD=+@lXmkR7zN4k62s4I2^7YD`*`2a-{uv#mJ;BN>$=&h*F?r~Dum__JAe z{k+?K9y$^lC{CK3f?Lav$(exLV}?mE=E$EgDNdYL;`vp){rk87C${--WRu&xQC{kz zzsq;%(?l#w=X&Z$h%QV2|4mN+U#FLnKz1pNpWx(z>C|^@!cHwE#GgAz3GitbLkUEG zz{fpFE8?y)yhRz?`U-lricAA&_ZK({0KOnW&lI6^tKk^bYC-_0@mYCjC}TIXftIv4 zK1!c(nQ%7wV}3t;xROX#jQoM68VX4K7U3ie(|hP} zEz(+-U*VcEV9LQQ^`CyHnCF!;r{Dbd%}hSUIJbRVwJ0q!g2+bUs^kIT0g5-{33Jq_ zfhs2u;s7&32~=;-BdM-tBBbjNe3M4n`cPVBhoKbBKSk#j5)^Iw2rfgadkE}ar%|O71Lyq-zZWNb{A?Xg zU)awA#}jwexLXD3Z?FaDXu-7LB#*veA6jz39>&;bJ#yHIBNHq}P4$=))46$=^cek2 ztR?7NK5QnBT^pz+Xk6pJMlrR_gFjwxNhAEkN!rZ8r8PqbfR_i?mj&R zUF;?i)qk}ZTO$Ex**KGzprAbrS;u)7%l_JksDwAvd!ht%L12S{lj!R+S#()6z~Stf zb`XZAyE)_!;=+B7c=PnLLQ+q5ybS+Eo92x7aa*SbG~75lgM+&lMPvZ2Q$*^ZdqtRN zj5WO!AbiUrGh3#{Xeax&=Pfp0Mc07AhI#2A%3I)Am z@f!NK7T7!5M;}4gGZfD$D9)ESB6cpBo;e!$wqEh?eu=^_J3%?xh# zj;x+JJKl6Yd~5glg3wx?`B;l3wc8YTUKC%t%_(eKZ_T0k*Y5uDV8?ak6X$$ZJ8v{z z-gLld`@3BwjdOP7?Z}&E|Hj-#Yv$=s#>EO4t3UQ%{YHPHOS9u(G#Bw zaP4-@sw3fUQ_rey*itj@{MofjHtIcTI(tvcyy9#l=WO%wme#RWDu=i3lbxRRc1iOB zcvLG>T*H^Y$Rljx(}L^`s=C^TRi6gO|MUTRhxv$`1&7nH-TbL|o%|Z^-ShbA_m&aX z7NTNDZ|a%+f&0B`xc~gEF<+LDlew>!`{|AHZlAWeJW}t;^dB64x!74HOWDm#ezPa* z`rVn%V?$FuYIHcrkD0S?q`LRL_DlKh4bCoi3S07u4(HvJbsyzrvd1h})A6^EsDNEx zass0yWelfI1MYi;PRV~+@Z(m?X~ON-&qjeo(+bkO{?-Ra~>{kMB1E^U9Qc~s4S zd%h~8aP{c5vRwr zyz$I@RpymA<%=F4My6J)$t-Z@m%7K;m_4HThF|L(Kgs&0No{93 zkL;-CJhmGX@apLyKXaRPX{X~)ge=_jO5Zu@ZEN;4>-;k_)o0Isel9nCeutxBzAj^# z1>2uljrPQ*`HTtmx2EwgBPVgoogV2)XO*|xoQt&yU0r;anmK*U!FR5)n#t$25_c{P zJ*qC(+n{j%kC(<`CAuc%NfH~z`)Fa9vU+)%&Iz;D&$>;CtC@&J{p2F~ z`m@*s?GB=Q%q>6t?Tt$&kKsgZ65D+`jy4{1)XJeKW2s}7`Oz__H|$DUZs8TjZFUrr zYdk0YYo&DR6l*z8;ol_Ol1K87X->0h>D=eOHz;cz;a9srD5vp-^Db%cDA5YTQ=Lv3 zVV7&;N2!DzP5E+LCfHGODQKu{p`Hi@DL;Q)w63wft z+a@nrlRtrs`e^Jl56PiV)H45QQF6!Tytfx z&hF#aw>o=WPQGFObO+t6Ag{51yF>q6Mb2hEqEAqaM`m} z$IE$_wFQ8qgam(kv~KuLq`$fSF0KE8RAyIF`Buo6#$Pt!m)ti$A}VOt52T7Rvy!Zs c(8g_8A+PZn=(oQrrei;l>LN#~NPP4E1DRl#HUIzs delta 12723 zcmc(G30O_*+qdUgt3hc_p^+v-G>FR3ZkwZ295O^QlOzsNw8@ZJSw~AknVl0Fwt5il zMh=aHRHqR|8g(kE_Wqu=cDr!=-~a!6zwf=?u4~<`cI{`lhu`maKWlgPnc81Z)f8Oj z5TtrPSy|bB`D+#86bu1@zw@oYc*x#(6I}#*vq?si{S|dcb`0-HxJ*T7w|Y>6*9Ku8 znRx8=$x^BKr0E3>@X;#62^v$5jC3gR<#g-LJ3Vy%Emid$7bD(GsCoZzTbXUwQ_Wo$ zg~FPw%;%jQ8~4OL^xvU<@w~2HPE2lCjiwvNZlbWn!=w9-p5AS1esoLG%3b4lJGy+6 z?S$PeqMcto?|QL2n6th8!>QJc8gtxcfWqVy?%$5|p+7Pxa<{pZq^ ziOEZU^=odet?EiT7TxjdcD}z~K(kqvl3Ca>j+0vW^4Fc#Lc7Qretw?Yo8Ojfh%C=3 z-()xXet36N_ihVLNp@Z%uYp(Qtudc!> zotND4;rUm+q*dV!dnP2*r?rcCfqqsoI~sF-tw7Y->rFh(ng~>~XYzA7$Rv!h{X25$h&)-0En#QD9$~B-%VP>Flx? z6zrGup(v?!?XF!((GhQV1^5Qq2`0SRQP{Gzt>x~Mk9o(UOCvw!JSlB<+|Ns1Y@BH_ zdee=THA^3tgf(x;dXc<4>(>%Xo%MqSX$I9LfgSb&7s0b%vb?jl-dw{w{g(Dw?6dUs z)Y0i~wtwy@iPzaU*eCMyyXs?GTSj>$hEL`_SQ2Qnq<*8%kn-I(>{=gm#y5CZUT+Ka zY256gq`TR_+MCbWB$$_kc7JX1fAEzj?D$wfwU^YFF6?Udoc`ruRc89pFS|0cf(L$V zIqIXg@Lk@h<&$p>k92z&xX6{W;Jjl&w^h}bH-`%-;k96GKP~z8@vqwsb;Vw8ROs-` ze53U_{b;OTSh?=-qKVmVGPB#}wzTd${p-&BkJh@se$C`^Zpj9i6(1vv0(^?I2&3&h zpO^TL+CJCXKP~%RVCQ0~cxr;^@s(VGO>OW%-^M+=eWO-myNj`1ZT3%>KYTdBCEl+t zMlbBIi6`=AEDL<=N_%LxrXRi6{w&$XB_7*JF7bsQO7Ijhb20@vyq%Dppt$XJy89>d!XIzFQ{1)83#GFY zCWCFg_u^aA!ml4kWF`D={m(18TcE$pC(LU;d!6+Hl!BrbHC9p0=)ipq&pXr4diAef z5_CUo#T}!c+10&Xz*70=#gqUBgqTf z7mEC}0u?A<0}fH6(8d?7)gvi;oq2ruk+uYyKM&{w#2X6Cp-i~-O#!D`fm&jyz429H zp!b(?)e8SlwqEqLxOZ_X?iR{;amzRGs+_YM2if?J3wmn&NmN8N8EW#p#`-jQrN>zO zFWIsGjfs)cA3y^j-62p9pokpoM!vy4Wk5gUzz}K+=MWOvLu&%1S_3senGYKvQ}dA9 z1_|{*f#3RJv5oG(Nh7FhnzxrgC2N8J-w-l}E z)Wfl)Qip`{&2rQLx^o#~JJkpd0My>e>S%Wq-a=v!vxDCkts&8-b$H40vpy)O`tR?e z@YvKIa~)Dp(%{9nxCa4cZz;_W@xnv!IKUY44afVENHiLb21tG|9+^f};EExyO(fE^I63J3B4+4}=i$X_sK7$7^oiw45ckOSyIC6ESG><|V1m5xJjnDCuKAP~=B zv2Qx|EqBgLNOox@)l_O;S&>vDs?U8A^O`g9Hlo{;33G3=X=zxRSFkV+NW{EJD@HWR3bmu4wNDW38- z5mohWaE`Yb;f*;An!cHi`r>)6f6u^Wsh_#qd~05VI!$x5}%=%)5sXrKXs z*yH>wn{f8*or-;V_W_s$r2iHK;I+`pLx2vhnW(sGlotY30Ka3)Fu><2V@r3mxZxV0JtIIo<>Imr);@_} zHwqvtSAa{Mh{L7NPxr%q(H<2EjYOqdU9hL1JaN^gq?`$|kZ$uZ<#3<BQ5CVyknw`=`prGvAkTZE{q%hpRNAC8x-Y2 zI1ZxNUm1QCNdrd`O4@%YQAdBC_UrmRA*2O}kZSzN;SC7RG@{ZewsV?V{s zuIAfqz+Ujs2ORi=FVLpl;4_dq%xjkx(4m$>S*y?0wV)Eq^#`#c1Ry;%gKuv{SAM_9-;y8$U?;1i`SQs!Bj1i%)g5QpVwglN)^Dvci zJ*-9R3BZ`sGs@Ad2@rD>^_whCe+zkrXZ-YA330k|&q%QoMD)WGbhf;nHsZt5ayhJf zDviQ>eSl%&h}&Wk2$@}u!pM{|k1+a^=hM_Ph%H_v1Gpxi;xZ%O%>7kej=S!{nT<&? z1Np8G6ac^O1NNf92jEIb{S(FW;jr$2{gQx6*n(!)6Ej(gVT&1e>7ld1R-DCnC4P_n zzo`UsM5=;%7qEMp}QNhSm)nP<|uNDQa$MH>>KN z|5c5ahU2jEZ@eHOKWYgVz!fae+Cl!&VjOPE1Pp+D?X+QlnlduONVpTDqS$3j&WY*e z38ohr{q54JJ4$kRiIs(D-$JkqpvddYz?z4~$x=-^muWnYlkr#QzFO{)E6W{l&Q?$8 zH@8$QR}=)&RH&B#Q-U=?;C$yVrEW=6A(ql~Lq4cA;`#q`1#3Q4I-3UTuMw}{Ze<~z zx(Td5t(S{t`GbXWl#8D1w=?(399354>@yH!bU7n)JV%8yZ^D7X>L>@VTvygmSkMb= z5{wL8qf_l{syF4F7$bT0{8VR5i?Ie$hzrb6vWgy%kxY5q{ySP=@DM#O z1N(u@>s|Z5{yRl6gFvei85xo?A6fyF>j-8;gT6DlnbI zu|+Cga3EmzjHBJF=PQ6oH>2Iep_Xu~Q{8vmDk^1ABGtg*=ZhctlWfEZ!GD1jt+K%Y z{KKtOrAk3=i(GF!b92rMB##Sh0g_a0OS@k5;nG)K& zy%Ceh&LD+BSav=bqAHf1983mSc%J{Sl{;SJzmjmWSy;k#nfo-+8zC4%+)i$zT?sHL zi~uL_Wsf53fHE*@##1oC)6j0p`prfo4*l1n9gk;G97RP7@h%I3>dH`I@C&N7mRHE;z9wu;r70MVFoYvLGB5dp&?gl`lz$?b^IFEVU%ziec*K`SH3R`J4^U_r1$c)N&b+>axb0X2J!P zyeU`QW3o$O)dAXY6&PUhd4$Dh{|^fPNwEY#W8tgb!tGr%t4_a@hv>NutYd+B9zlwJ z6uz2-;Y(Hv_Ml#mND_ryyxYn=Yg5t&-Rt3R(JKh~ZcG z3%LhPOiP*KbmcwjD->fOMf%f(gau^1`?S<@Ej&X)9t zskAQyYI3;55?&Q1Cw?uapI3mmu&rPbk=R+x=Wfsjz=j|Cn;a_$P{AF>f5+vRFg>t? zQGn@>nEfRg|1Id_y}0j`t-!vxSK0DI%L)LsV)SJO55IheM?Xn;jAksDQ?bL|Y>mo#ogVon=hATYtO`R=jY9JHu5&HY z9$JApm{X}sX509i>jwh^Fd6c7ai`{U}9H9Xv>10~{3W}h;CU(5t~R=g!t7E(jC=3zC7 zE2$^m{n0}NRk6xqN*A~z5Q~fXw7wcrSdG~Z^vUVz^a)EKKbS#oWs*Dl;+z*vCe^ZL zGV9cCdWh!Tt)zTUUclPgRE$8F2)2;C&(h5;)`Mr}wPij}*yH=Rw}cd~XbYY($Fgtcc`8^zk2tc_*uJ=UhM_D|O4v-S;ZD_Hx1wN0%3!rFf6 zcmviqWSMU_lvz87wfd|b&RTQU+OXE1wX@PSgOxPt)MxN-S~&~q(h(Fe({4dBP(d*(V$yZ0v~l>@K{IRH8@+fa*7r7=vzIB@H2J77&9$SQYG#-0KLO zDVs}ii1lTh(i;Vn{0??j%eChWY={{3m$lnHpF3O1-7Aytr|qHW)Pv*fSgP>y@Idl-KnGrTG0V*fz(aW*+Y8tP%SWU0FkD0Br32qOLQzNogP{QQ~K6| zSJWaPG=EsjRubXe)c`$7jXH#io0n0aVsZ0gb@;?AvNhV{I-QyY?a-ayhzSHrT0p82 zbpA(zgRX^;_7H7cN?JnNF@c<6Bw9Tru%~Q(XPfktu$2MjLanUk*pT1t9XAOF)4>#a zE`Z^H&QF0{lov^0q)>WJ8bdfV5all>@Ug+l{Ea`O*PC(hQBAlsM$=oO-AF~A>qF3l{B}2)I=$@_@D;qO(U^f&Ps(Yz(|y9 zB{`Rnz=pU-Q#g>rBAS`+E*pAZ6U?UvsFJnx-V8Vb-6_5Eq1W8$r2oVeBdhq9v5YY-DN-zHefMA{HaZNFY= z-H6=J6DB}P_{;udddPP$fg6A=kKq8OP%t#E-#~N_BSh;Z`BWOGvdltudTQ8lWu|v( ztT}nUZa_!Q+{72W<5_7Av}Z9~j2y=k`tnTNWTnpXA`S^?UoN2yMMvpVT;e0zFc8-z zDf}01M~=G*+!o++z@@S>n#9b&mm!n^%FiJ25xv?iLK`4OZPEbHB6T7f^1NszCqi?; z6a9}Yr+P5SW03(@(h-XY4Z4X-b`qC1{E0w0fi?8$AvlJ}VSiWx zscwY9g~@MiQk8L4<=?PQdi{`EnRAEe=GBB2%AZTjCz0b9Fkk1g(MwqSD@g z_7+}D`zJx1_5h``av2%bKJK(dCiFrva|wL*+3dn3)^A3*!cYO9z`2w!fbwYSFU-pY zilhO_X8(6vxk=-#=o*TA?@7{aqObU|PSM~z`6+Q4NlT%O z`y_sVadi}l&&sPM`=RlELWT?ddPm}hijT!3?co>eMCXi!2|@RBZo9hcD&mqxaq;9~ zTnt%+0Yp(U1nvNqTq1W$k=aV$_(vP3Qjt{`ppT^zIv@`@49WX8Rw~SRb>0N3i zg8s503+y}VC&vU%3_dsqsmvo}z&9aBL1XZ~z)ZBS8Oi`AJc{37A02d%bf9Gpk+*2G zL*yiye~5g{#MFfjPl3mcyQmG4wol?}`Tooqm&Dbg$zzmEXq9ZZ7e%Ziu>#BUAQzD+ zsvKf~$2Fk>3{(qxPq-v!K5Hi9;}5q3+@tIfv8FS#Xk5FCEXk7uZJr5}nT*l_M3lG= z-V#zGuOWeeHgh6%z%lyNJ7OTMoDFxNSQ!GhOoSGxh&}%Vs-ih_m`t=5F~wL|K;Xwh zlbp%PgyB2A0*NGc@WmZ5K4)u~Gq1%YacWwqY^|Gs#8rs6_JjfklXNJHl=PVq^z)%F z2|Q8WVuAyExzh?VwKUWCtDa-Jk~k*48QqG{?iT3=WnW#wX7ph(pN68x`DE&GO1%dt1U##~{*xv5ut~(cb=D=G35u30d z0fkiZpc0!B(&$qXiIwEI0us9{`4vNu4Tj_d!dU2y9WUNwFUKzp{)Xd)UU*#EsT0op z5n<-f!McHtT}52)rP=QN|Dai{s+DgsL0dV4BrrRsKxKyAd4ODdy;6o=B138uwa4br zp5uwD$oVhmOt8fhm}|A0UOt-8kU6i@aEzb6hi*PkP;`?&xdUxmLt=>h-kU4XolS%k zH0cW7@TG@n@B6N>U?}Mp;zsmiy~bVL?b{a97CBHGIR=pzh~DbZ?X3=H zR4X9P0~!k{8m*bh1VFg~_XIb_BD@96>S}aZ8eD56Tr))SIE~BnefR<1!2|t-Ct4lv zURlBGjNq|R1;3E@aqnr3V+TU|5+WC;Vm4gVS66b-h=+AjPc-^EG^Us&Y{Geqb;6|= z1U}#64#(?~k#U;(UoqD>nD~}?WSD8egn+-7QB^ogeSuUb%Pn^T2rc>qm$>^Mu;3|K zTEU5s*XmUh;)gf(e^~YAAH6RFjzSlm@K(w1SZbRTQ~h`$Vk&{TxS>UR-5A6T$%OV& zqBTtYqeaAP=}Q~V{s*!OVOeDa`}k8H>hdIw*-uG>=4L`G^5FGLJfu>~sL7wzqzAfe zMBqAEQOZ1VZdgH1#D%<=a5K0kvQ&_aFCo&Z1eQpZd&%3DNh75x`d_Tu8dWUj%lQ{3ebvKZ&$p`6 zvr-V;o232Uung_tn%UHj#Tt^D#jFf-B~FPcYNQEdL>&H%My}P%^2o6Y^d^5&3;B*F zbL5ju#{s~h$7^;3& z^I}DhyC0VJY5yC<4B8&E5>N1S+hUvbLz17SaSNK9G%&gQ#N+*DC|N0#-#5kk{hzgF z|5p>(n$oU=h-{+_kz=~#%u}N2ThC6vzs%MfLfEDD)Q@SyQKQJ*fCWKw)& zeLa?BiUZh8@?iiM6vBL_Kf(0JvJ||;c0qPxnqrXYg0ZL%yCsKrrtA}hM%_u`Pbkb` zhO$V34!z_i0%+@5%nr!@7XqJTeOrh<()1=HO)>F+%UIsy7F8ZuRuByTzJ}6_YOp4m zjeplz&e3GlX=pWkMSFC@aX%8p+e+r>jQ0v_J*InmggvhW0+mY+TpEJmDSoO5f$iW}zN4y=x)vNq#1Ud(~<;GMG61v{ik|jyr-8KXK1s8S-w&u)y@PEOoPFgC8RPXRR8R76lcMZ`2jn zdsHBS(3J%9{9C|%QOZ33)?-=)&qBC_2|TXQ(p<8?9K9k9%F^ENi2iir8`4zOy;4xL ze9hWPiKg$H_DtVeE&qg0DI$l_K1Jj<#%Q9dBGL#9zM{qP8u^0k_euXnzwbU9OzW4B zCcMx8?@@ zv19(^b4Jw{+pnfi9}^>J)v>L8uC;E|GqtE`C65yxH;j6alypCPL^HQ&;r86xqk)AN zmlyiXZX}M}Y#liHfVukdj_Rm@_iEPbwaO>mi*p;wvpX439qbzYTJNsno+o6=xz=Zk zW)HD0a&f(Xd3AiX?~BH2B5M0br)73#Bf6r_-J3YK*?-c;B|iA$svBQyd@*9>D_7d| zrBT2zA-D8Hb=2DRTgt4eYm|-+D^F?`r7Rujcy@{V>!QgMx_oOJbw+k9_Pb+oJ1wU) zxwLf8z0y5X4)XSm{j;iFeebcUsYYYMFF1d!o-)12ZRMHORaL%QE01mp{WZ|;_KU&6 zhVRD;r4F|I>Xd8pm+|Lu6+sbeuE#^0txX+dk5{zuBJv?Za*^H;g;c; zLGN}txNGQdaoBn0>ZGdO-9a}(QsO7{4`21kuwxjQqOtOzT8ik)xJdn{g;`67)r6%O zKMhNm_O8kdn3n0Znz_u8k$pJlvK)2KZH)clr-dJ%}qB~e%LZFwDCyG@SD>d zMuL%=BZnr)X5`qEyi}eTR&O<0kZIDi%Em!W?(e_BoT-14DQ^X&eiwiYBhbVpm~d8*}&E7xus1u zeQJO01J7-mVAt(&QTOb2U0QYWxAWnT4p1-lX}_A7GsCmMYPs6>c_qBPks5wkn}4B? zwEaFj%GB0Mb-q-FeZ!*t_sp!#Pd#tV*?2NOU&-VSCt=+{$CaGe8e3kfpaC7)vk9_TUhecBT`b$|i`GsCsMcAYKpn z9Q~-(W}iZRGtqXcBWV45kHmd=FqkbNIJA8hdx**nGgBWcj|VQp-pKo|6}}P(IYyJ#fy4S56Dtqfehu zpY1$a4ZcE4ZfV-hj*2`bf+1YPK zbB?!D#nk8-D|R`05MMXdN0Ca+!|I&&*e<$gwB=0o2#0m28r@Af)}`0SBw0E)On7iF z-TLs}&IfxRs$LD7x7Imm_4M6~bF=&xhHMCHA2=c_xkA0%>%xJ;(RAdB#i6Iy*BlaP zXQC^755{Mmo}+z~vCHn`>wA|eSCFrme@Z|fD#+{XeN!Rr_4Sx>zg@iYcYm}CgEFbduI$u1Qj2&}PSW}nq`J}%^U(SDw;+> ReadMatches() | "Convert Excel Files to Dict" >> beam.FlatMap(lambda x: read_excel_to_dict(x.read())) | "Ingest data" >> FeedIngestionFactory.get_ingestion(feed="pan") + | "Filter messages inside date range" + >> beam.Filter( + lambda x: x["timestamp"] >= datetime.datetime(2020, 7, 1, 0, 0) + and x["timestamp"] < datetime.datetime(2020, 9, 1, 0, 0) + ) + | "Map ingested message" + >> map_ingested_message.MapIngestedMessage(feed="pan", fleet="trawler") ) # Assert that the output PCollection matches the EXPECTED data. diff --git a/packages/pipe-vms-ingestion/vms_ingestion/ingestion/excel_to_bq/feeds/pan_ingest.py b/packages/pipe-vms-ingestion/vms_ingestion/ingestion/excel_to_bq/feeds/pan_ingest.py index 3d50ff0..afdb976 100644 --- a/packages/pipe-vms-ingestion/vms_ingestion/ingestion/excel_to_bq/feeds/pan_ingest.py +++ b/packages/pipe-vms-ingestion/vms_ingestion/ingestion/excel_to_bq/feeds/pan_ingest.py @@ -3,9 +3,6 @@ import apache_beam as beam from shipdataprocess.standardize import standardize_str -from vms_ingestion.ingestion.excel_to_bq.transforms.map_ingested_message import ( - MapIngestedMessage, -) # Function to convert DMS to decimal @@ -60,15 +57,9 @@ def get_ingested_at(): class PANIngest(beam.PTransform): - def __init__(self, feed, fleet=None) -> None: + def __init__(self, feed) -> None: self.feed = feed - self.fleet = fleet def expand(self, pcoll): - return ( - pcoll - | "map position fields" >> beam.Map(lambda x: map_pan_fields(x)) - | "Map ingested message" - >> MapIngestedMessage(feed=self.feed, fleet=self.fleet) - ) + return pcoll | "Map position fields" >> beam.Map(lambda x: map_pan_fields(x)) diff --git a/packages/pipe-vms-ingestion/vms_ingestion/ingestion/excel_to_bq/pipeline.py b/packages/pipe-vms-ingestion/vms_ingestion/ingestion/excel_to_bq/pipeline.py index d2bc719..f64b2b0 100644 --- a/packages/pipe-vms-ingestion/vms_ingestion/ingestion/excel_to_bq/pipeline.py +++ b/packages/pipe-vms-ingestion/vms_ingestion/ingestion/excel_to_bq/pipeline.py @@ -8,6 +8,9 @@ FeedIngestionFactory, ) from vms_ingestion.ingestion.excel_to_bq.options import IngestionExcelToBQOptions +from vms_ingestion.ingestion.excel_to_bq.transforms.map_ingested_message import ( + MapIngestedMessage, +) from vms_ingestion.ingestion.excel_to_bq.transforms.read_excel_to_dict import ( read_excel_to_dict, ) @@ -77,8 +80,14 @@ def __init__(self, options): self.pipeline | "Read source" >> ReadSource(source=self.source) | "Read Excel Files" >> beam.FlatMap(lambda x: read_excel_to_dict(x.read())) - | "Ingest data" - >> FeedIngestionFactory.get_ingestion(feed=self.feed, fleet=self.fleet) + | "Ingest data" >> FeedIngestionFactory.get_ingestion(feed=self.feed) + | "Filter messages inside date range" + >> beam.Filter( + lambda x: x["timestamp"] >= self.start_date + and x["timestamp"] < self.end_date + ) + | "Map ingested message" + >> MapIngestedMessage(feed=self.feed, fleet=self.fleet) | PickOutputFields(fields=[f"{field}" for field in self.output_fields]) # | "Print" >> beam.Map(print) | "Write Sink"