From f23b3af1d4f3aef779a2c7b72f3a56264c1b2016 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Wed, 28 Aug 2024 09:30:04 +0100 Subject: [PATCH] feat(data-warehouse): Added MS SQL source support (#24608) --- frontend/public/services/sql-azure.png | Bin 0 -> 13280 bytes frontend/src/lib/constants.tsx | 1 + .../data-warehouse/new/NewSourceWizard.tsx | 6 +- .../data-warehouse/new/sourceWizardLogic.tsx | 148 +++++++++++++++++- .../DataWarehouseManagedSourcesTable.tsx | 5 +- frontend/src/types.ts | 4 +- latest_migrations.manifest | 2 +- ...61_alter_externaldatasource_source_type.py | 29 ++++ .../data_imports/pipelines/pipeline.py | 2 + .../data_imports/pipelines/schemas.py | 3 + .../pipelines/sql_database/__init__.py | 4 + .../workflow_activities/import_data.py | 6 +- posthog/warehouse/api/external_data_schema.py | 11 +- posthog/warehouse/api/external_data_source.py | 35 ++++- .../warehouse/models/external_data_schema.py | 56 +++++++ .../warehouse/models/external_data_source.py | 1 + production.Dockerfile | 6 + requirements.in | 2 + requirements.txt | 6 +- 19 files changed, 309 insertions(+), 18 deletions(-) create mode 100644 frontend/public/services/sql-azure.png create mode 100644 posthog/migrations/0461_alter_externaldatasource_source_type.py diff --git a/frontend/public/services/sql-azure.png b/frontend/public/services/sql-azure.png new file mode 100644 index 0000000000000000000000000000000000000000..87648c4078e30e854e5afd7e2b29baa3a57dbc6e GIT binary patch literal 13280 zcmeIZ^;cWb(mx#B3lu5Z7AR04SaB=vP#l5>D8-9Alu#U6C>E@^dx|^3i#rr3?i$?X zo8Ei>i1%IV`QfZ&?VK}v=CjAoBtrdzJOM5hE&u=^P*jl71OQNg|1NAyIO6cPKFLBfNg_*9_N-O zoMO3a)pw6X9P8STRWd0Oq4kFTS$KFLy26;ubA{Iy0v|v4DEKt^-(_em&lfgl@48)y z)rT5xFD=#17}qOiw%W}GWIk%>6qZB-Ug$!|z!;}8D6!1WF3H5|5=#2dvQQ-~M+F{? zpqAFOfO((y<>&0d)}s?xrtekAz+i&hZe^B2R0=$>(Wn(ZWbY;5LMprsM~dMVQhepf zKTRJ_3`Y$|5yXJ3%-k)vw%egN;lpE>Vu*4&K7kKJgmJlGIq`Vq3V8|DC_{TUDwtet zgvr?IwFYD|%MHbZ-OsM>@1H{!&xxg4C?1%cf>TJrU`x%nQte5aA5bO0dk;ieTjvt2 zQn7OUq+e(80N5s#e!)ao4Ak6MmsA)OAmSFfyo4yg&qXu_)aU-#n!I7%JuRrb=#0sS zdni%<$y@) zfOkE$wdgGzuBo9qE~n%z1hE8cU7lx+Xu9=$`b?IUK@n(&1?dYWifJOSx}F+=S0&rA zFcNGop{_)}R3Co_;0P%M5U{*LJC+9!;?g}MH6h6zr^ke5iz=`v`-ZwzVmUdFTIJDQ z9Wb?%3gUoCAm7C@t%8aA?`@@cb1lI(8?s7@${0(yVDC}SY);xSbvsZRKk7BwMO)De zrr<0_3M0~g2(cK9t_|7&$^gfhfD@qc@n)AH3L$(Q@DgyWY@!*vI1Gp5W+x?_`T-CJ zfuv&N>z%mCnh@v*qLvvvx<|7Co9tUNFhSg0My>i(ypP@}Nvl*V)>n^o`v7wG{s#pQ zTKS~e$eN^J=7rUp^YpI>#??XXw27jdKq#%^}4J&~MmoFoCh z^c+)NkA4rFmVbdn5mCf^5u3s6>s>y2GSi;R*!ZG6{lZ#}R;p?rU3ii?_VR=Dz#H@? zq8HCkKwrMuiY*6WDJc@84-m)bW$|+3GRo#^Xo#XDGHPiUzemd*c(HZ&Wb|>Aq^}tu z`X)Hb?9GeUzaL?onggjX6&CTqN^RH^e@;ZBYeb39qYdKk#uMM81d^Id(Z@0qRps+_ zu(M;lR#N{-@x3H&HoL+crB}~1GIb)QuB3Lg{w3TK<-VRMXHE{|X2TVPrmQ6N3f_m} z_j;go5Jw>_O~5Uv*j3!G5{EnX<)ymjgS4`inkqRR@vDe|xZR>sqwt7nM;QnGyxEh9 zlDka6bpg@Wx@QD~?|^}f`F=bQLlh&*{4!4;)jt8J0)k6T80ECSW1HzRQhCHNJvpSG zI(nZ&@EPAm#z^VR0lyVD0jlMVU({YspszXt{8X_e?OH>1D0Sg+niKa^4uGBF%rlqk zDH3bzTQRDII{NhYkvYl1L|v?mBd$MK0H(J)r|WMyQXv|N+h(?hB3w;zb-r>r_Lxp# zL|%5fDFAOGfb25{Jenp2S zyhB-u2fWYR?>|rQq*@^K#|ldg2n)8BRXM)bl6>66pD3z9pyfQ%+OO!HTNkG(V;^+R z?!O7ZIwqi#5$jhPRmhXp`F3-0J1;uVhHcvs}bxQX#b zyoyrm{>J^viQD^$7eYurKR$+m24t}rSB1%}ildD6PDQuqwb(5qPENDVpG+BW= z%lRh?yO6DdjQJ-=!Nrp|R|RFoqAfIqqOUCMbium`3$7>}2Ha2Y>Cg$t&gwS!klP9X zuHOUfc+#eFeNN$p;}f^&k=HDyOG`T%&wj%BvM(1wqBqrb9f_z;)kJGy8Z#C}e@>r+ z#Ljfi-+cV8x+);>H`-#j-AsLsG{wI?2f!Ya%=4LS>s#Ye9jhs+R9UTY1i_Ox7Ldh~ z7;*;~8|Ty=$cBFRD)>UCwAP}y6bV0YQ^$Do#&2qMjnZ>rDer*+@alQa6)@{wLFqlJ z9is!FmJyNZ(Ms4>-v{-*9K;RQgvY;^(_!zs(L!Lmx30;S1`~O*&I@x7vVj#@Dsv98 zAUE=C{S|WeHrp_qy5uo*H z__{4DzI@@A^D>sk^U;2Wt#68}c%Z;G+`Ja^`^h=N?3$vHP%c^k2Z@xZWmlt3rZ)7OZzXrfg>-)>>s@W?u)5dHrQv*Y|m10TQ6D@MB_bEWtd19%{HJGSeE~st= zXbO>CKQL6O7oVH{7UpRZ8s}y58{Bkrz&T2~6Q|>KC+cHbns~}ihdxXw-z4+MsiP#6 zTj(iEo%$V$Rc{)(M$uZ6gva*d6>b6)*hy{m2$zG^X!ukSt%Vb87x-@7z|jKHvgRc^*4fJw%igKZDqL<|=z;BjSbBvd={kFlvgh#~tOf$DbJB{H)3 zskCB0n5c$JB#vLU2m2oBv)~P>gytPG>x3i4mYmph^mlF5MQDQq*%kr5 zyMuzxk>o?JW*Dbhf!x!G-WVJj8RVM6ZaIMT7X~&PN*@C+;Fs|)DwFl<$D!tg4yF*9q3a0_!__KN-4!y7 ziHxw#OQyw>Np|MZC-_Znos7omZ`A#qr+IGI$&$uqhltC&?IZ>!hE}-|-N`w}47S%j zZ;ZT0eC*0WkBi2xU#?E7O>1-mL)U+!;*|=hy>Ia_>uO*XejM-<^0~FKw`-VdvAFz= z3({S=-xZg1xDnE|6`Gk3!E0}_n1AGCFJMSy(4!|P6ZiAd?K0e&0+AdNiY#NSzJhIK z(pIAukP)5CHLL-BqSDh(uiGXYR^@#uK>CBtK4%-19GPRsyMtwAd^nva`C(yoron4w zK2_EO%7G@$8BiOqi6F#XR?zy_%+;>5R=uZ%t61pEopR3elko@kd_vRn$G;DVjW8u3 zx%9Uswl_`U+=|P;jxv28#hB37=wBKsFFQ-}J9qm&rTK+}j@M!u6tdnzC7l0$s1(`v ze>*t3_NFi z8kIeRl6(_++)Z6Z><#3PiMT0Hr+?b*?h zc~L{OQMI*t5oqq48B%=x9Cd5!nWaP-`6jeOV_Iu9W;AtTSi=sRg!JI449h^|fci^L zX>Sddf#;STPQ18`Ub`j9koxrMm)vIUh1b7K5BnIJ4JklM{RkBDxufeT)FE2EiVY^i zT!OpLJ;gKP*>C&K-n5;Mr`mSyR$Q($x0cc>#*8M7@&_ThaxA} zM*))hol7S?w7VY(v*w#n5|1D?Rm6&l{``Bn05OW!5FK)ztda&)9a*xD`{eIZttAO% zJyrVR-Qg29{3U{2R=xvWz5y-N={wJ|JN<+&aWN(eQh=bDX<9lGrrSr`7bfZ`7o}G5f zdTh{jKTn_OjX*}1W@G;?N|e#xfOUH#v4Fft0ZVFAkJa7VTEUr9!vvxh zuk*Zb^gS1zqj=wuM~Ho=+Fkk||20pP78B7E!@LKDj3B6L{PjGL^ld~8e9oe>V+7|+ufkSj^5bku4Y2TYeJ(b=U%jWm&m#iA?y`d`? zE`#w!j`NV)g1!^Y!T6oi>z?xYPPyNbHaX;4w#$%Wz9g{iA!9O^IgUB~{p5xb-{yV{ zZEt|MJFm7oewq5ai{`g@;MmRmLcP@`J1g_vBF)5kmrt%ROP;;7SACDY@@PNuTV{Sv z50mY(shjZPPTD4!`7EY!xer?(i(#hY9k+pDI;={*X|VIQ49$p;IC-PKK{wx{aG+i8!xU zdt~&+fMr_yes3;5OE&wEG8FRgzys4J)dT1{^66#h1ifnW#AMQL_Kz%@I0)ThcG{U* zc4~28j;=kzelHqPne*|IpI7mHfeS~kE(d&PpqjhQR)ed1u-*m67&7|w$Y?Jr!Ra#6 z#s#aI#dni=N6VROuEfuvwBGQL$+S9SNuq5m(|e@7Jt%hU^Ne}HGq=$~Jazsm`3d^GY*4FC zyTxbrnHKNoBbjk`xl-DxjspdZbC9iQH0uvDSVSU=-V_eS7S^h?vWCY(p&y&(wB2{Lt~X=YdFa*0gMHQ5a7MEpDB3Hg$Ey*sB?(ZGZb zw0@#QGN@X?(!spMMgHWo3Yvn-I(B46y4h=iaCz$rPsA5KthG5XJ6|U}Xjvo8^-3)o zGEHruuYvzc(2;kk-HVj)h0bXYe=k{<*>cdR00wNp(hOW&lx_jOAaGW(B(5;##XU#; z7KMAho1>)kr?(B&kg#smQzY$V?92@r5^^IcK&;*#*kx_289a3~ubQ0jE5sdn<#dmK zXfP()^VFaY)7q2;R5UZo7P2&O-~$u!Wza$X(gE9lgch@X04lnlA8s|K=P>bNTeT!PBH!gr>?QrF?=3sN>&ywG)7oYt5q*?X@nL- z$~o%;nMR&GU<4x1!o(*S@8%Emk-?>A7zvgm2g-gQ^yo8@TWR)MD?!D1h0|(PWkUqTAM!cXYK(z z+fwXEq=v`^VYkXs+mohTwa=BAJp;C4II)JmJNLJu-cgJE+LpnpdsRS)Qo_D2w9^y^Dq(1WHC@eb61)`{q)h0G8RN=3$Od zyxPy2GPLZC3(U`9K(kYr-VxksTp#$#NQVAPKX-tO?V1Cll`ai$Y+`j)LhGHEuxA!a z$ZhWjUS2H(OR+oLINNRzi8&V%yjpl@XwteIshOXS^RZY(355kKNix^k^wUc{o!}FO zQv|k+?efVC2aTihx{Te64>g5B^%MeizJg0LAtdFyD`^$|=fZ53fp~v^1m?$qF`<#R z`U~2meP5G2waNQ5G@j$N*)RywavN*VAa0EDH+!756T2m5J)eB9JZ;u$?M5QkP$TG~ z_jgt9tP!V@b_qM1*ylX{asE#to4I_O?oWn@_QDpugR0MOs6XZd(Ad}-eK_YYgqx-0GDI|r`<8Re{P>hRA@qCvtL64V*@ZArDob`FIeQPX_m;3Oo(3zio!(?qG05s*z&%b%V z#NoO_rCr~Z<-<_5>f(^@6@TRd9?f*`?ufV3o~w@Jz^&f5-9jwe=i+4jO05AtG+F-~Y+AK>BiBh`){WIWg zZtM0uWrM9rIuDLbQNw}n=w{iYiF$d9o6(2WzY7w3HX3S-vn%G{%>(j55HT8Cu8ft5 z#1DGVW|cgjdU7Kw)K2}vQ6zAGFyMVSGE{2(Ho0WNFFW9?w(NipNX@-HFS!j(!vnLY z>3{cK5jNVb@>wz2Q3_OHJPtKoVnIbyMP8ro}iR(+K4kppB#qNGeB|f#&t(5fqH4T3|-mu|Hg08Z{;TprUOXLloBaH`ExF<#655dp8(h)C7Mg zMw&;5I1KQ@QJXHMC6oqV@13&9b;#l=?*>o_m#$KR<^ynhX^CA58&5Q@H*!?0O^HEn*JV2apqdFIYVBJly!>0?xsjqc$`J zjkl+ulHW9a`a;`QE@%44`g8BS_=;3gM2%seqj?)DpH@P8BR-IEu8e{k7D@?I88#mOIgKoLN7P3@5|+;1s4a<4{(g=< zO?HeAWINDs8Gu&-fT?<^kNMWma8yMpk$`qOii)hzq!WP0C&J_j>|=8n$< zB_e*qNyUvBOP?kquQ=b|S6zhUN%z~EH;;qWFz;sUNI=?z>1qYd+guB|R>;~T0PbPm z5XKi?!4#_IZ#)?Mk7EYR?li9D)pw_=;;^F|#(*=Vab{b$fS1LaP80T~`x%?7{i z5Yk{j#CLVJ>HG@}V+#{v9drxLaCAsdx!R|)8Ge{b*_Tm|@Lu>p^5susIMPMzq1lmJ z*p{bXoopg&3R);D06>!VZx*1bbm7Ulh#lmcSo{ZbX3kQZcX z@{_JTWDp(JsnW1r64nE+^2S1KDU*~n4I8Ao4O5oKnap#q$eR8e+nY`uy%I&(o|p5N zlAkHG4!;)>6XXsE_R+x{=J5bqo}~)a5Va=Ypv$_S}A<0O3+i$cRb~PWj?> zg-W;@@+Og9K7_V@Q3bFaI1AlObgj|R5#n?`E`~I7m?PiZVyMQ??N_9wRQG#DN@5r= zLCEqIJxQdcFp*(^CcM)m5euGEpzgRoLZCI^78G0tvx|n2>9X9-p3p$e zzT3oZ&9gDtw9j&>fNILORhNA^*7KRjqeK0+}sC`s^ z@++w^*!E@W4i z07$ZJc7pQdZ`ZpgE7R9vWsjX>B+ZFQAI3uEP1{qJK1$D~3c@*ux|BZU#Ks#wN>?n?h00>?Dq-dTqZWUoV(HBy%u77gly7jFn_()fcn$RL4Medjwvrt zo4rs(40!X&qq58jTwLO?6dh=3GCB5Awe`a{)}g68ZfA2e_WUB_YRli3bLdbG!MDfu z%zyjk>X+=dhoqJzpd1^Do@8ctZVG=)aka*Ba;;|`Pd4ciXT;nSep;n+R^tFk5R4ly zia$MCd<0w0>{es!bM`(*}E3b4l8f~_G;Uvq0T z5N|^MtnV&4k|>(8o_qBrei_PnUiM@Yd!OsAt+z3_e0dyOSe8dC8x6a@;=RD6W%d{G zG4vK2u)pOE>6p33q6>nj?_^W{8EfPC;p&N&6yqEguQ{0DH6DAF5mz>iUZtnytf(U-C*9Y_6)i4A!d<2c(`Cn&W~Bz_Q%CmrfKQgZhI1 zcqH(6r9*pqN~krj_9RORdGzI#Lsimr>A33Ep`=}57&0zz77)>3>k607-RV{=G}D{n z12^5M1U`a805fdT#jS;3>v_cM3VT5GMOLz6z{NNlyAJ`#h8V@=g|Zr9`_mNny_ue? zuJ6TfEY*?#r+#aOGaY}6i9=&WXnABsT8^a~?i7B!N^bNLcYY>H(gZUSD>Np>^l zzjw11Ydaeq=F8qSXzfc&;pu(CuzgS;<(WKsQAwQOl4vQFV}`Lqnf$ashCSgPbs2k; zSlL~@0H(noj+$(dG}XnRo-VdUop+l~`M&&JjKa)&fwZciCnJlZ`nj<&1TqW9$ja&W zAYq*JNa@(ix_#ArLz9~ZQ?2O>CM1#WSQ2y#w9FAR^Gn`Q)fB@o>^*uZ6mh-L zPFiTOUG)lRdSbAGoJJD)t zt@DWVtE-r3hT(4Jbn4?{F%p)|$Tsr_Oz4N1q4{^nWHdfhT7ZL=V9)rE*NM#^(ur(8 zx96zrGT)_~F)F0li$AN2xfmkSHe4-~&CIkJ`kYXssTZT51twKzWW3aW+ORaAJU*h>9{sPR_C3ZlS=mQ+0OugVF_kx+ccw zTS-H8KOenBW^Nw)s;=+ypSnEAQuy_B^(w zgz+JP`SkDq_Y*|6^5;|koz+?y_bq}_+n>9{HJ1J*vf^cw{4dFmsN@*G$6PWB=+MC6 z;*BsQ*;LmwE_lxC-QCsOfDDqK`xS=PY0qd0hkCvtXZFZwku`0JP(2PF5I12)VElLt zy~g^1n1Hx(hYk4}a~#?4`$xbYBHNK_Bs((pF8&LW|LFg1hJ5Hc^~SwpP)n6G6xtSv zT>e)hBk5_NkYhNQo7p(7+wh5&M;7uK8dd2*0ux(5A({OrWeGe7u39#3ZoRwwkB$F5 zHjaGksnt`pZ?n4Odrgq9f$h6@Fm1R0_Wa~tfS6{jV=3G^(bCW1X$4)&RQP4b2CY7f z|0UT3&%sO%B}a%49n~RUM-=K^P5+nAWB@r|=LQw~Y2O%bfE^dC)2>GhNLEh&(E=A# zmvwP+ceTD7I)lwZ-l~o~H=)&+$o}Stg9>UCm9(K3lgZ$*`RYBlp zi)0rU09hwa{8wNzu#j4y2LQPt&i^eS72y9rcu?xU?E4*QzU}M;*u!=6cA0}ujqaiHOqJv#~+tWiAq3^rOf>%*%jMU1hn(4GW54<_a z4`eGZbw6zKo1_FE{C$|~$?v75cA;3YPY{Ud@F%dWeseDLS2QTB{o zH+xff9&XTj@?N4vtu(~`rxwzC2qi9fS+<2`9_d%_p#6u6Gg!=<;WgraM38q7&6$%K zXWXwK*$puNh08uWjyW?ts*_W5L-e|&#-P7fNDFCjJjLJj-sCSYC6-WogAdR1oVP=1 z&}F2WYY*T} zNGsicZdfd#G{+Ly@B6;~x*-u4U+DXL*jJO8-df1@*KGc3>L}zqGBDC^ymotV;cBVw zS{$fr*6scU5S7|wJE=v~09PoTi{QiXqJ{F=lnX)OB+cT@JJ0RNDBOOE!<*dKwXgqR z(%ItIWg1iJLVQ7?7_|o~*@f&ISs9Mbut1WWcI&v$%`v%C2Ye*Z&JD8-!e3i1&*Ch0|{_ zAR4`a8y(7S{zJg>)y>(AL0NC={ym1Sr`mod%KccX@PVKmhMW#*cE!)6Axn0nXZSmB zLTkbZWxe-RnJ7<@$Tt~gv;erDW{J%G3VKfwhiq^$;2l-z)a(4GxFqP{)nePn=16~S z`_oq-l$s2xsN^dPY-d^RVA=}jV;bG_cL?eC$x>^fp`#gTS6Jrcn6R(DGPq^)=Nj04 zXhgAM5TH+e^;|eoU|lsa$13*2)%d#^V8KHJh6XM+NbCmmaBi=#=WtTu^}_S*zpkX! z%&zq3(~yA0wo#x>xSY(VoVTLji@hqJ$-dW{dUP`Rl*NCPDKNYgDC8J@XBAisMPFsI zne?U!?4@dUV7_jsiL2fqckF)@2+Fc-&O~LfBrIw5>#*cBHnPcaz_8<73 z&Zp@DHoA>j0JoYvux_{vz-jlpEOwX8&-C-<$2G0H*`LiFHHe}c>X&dI3?sERv>$e_ zoxV&x*LZv4M9Aiua_)blr^fX8)w8TU9d|F3{pp+XC=@?jxClzOi)R$s7gc;%H zY`M#ELTX7PU?Hx3p`6`kDU2CrLd!Z~Q}Bej;$wnqwzaiEmrY4KdQQJE(FO`@xXKK< z+~B0{2^E0(B<9qsL`4%yqEnX?eqwlSdS~!YcP^uk^ZmxJHWn4t9+#!n_mFDd&r3Zc>iF{oa6=%7p>!Uj_p`jx<`G2!$Fb5CUmeDvze=W#jzn2`0rr`K46 znMcve%q} zL?`yfq^B36Z)k*@H3#w%y#B_Y3J~=?IX*d@#gLyqfSVkUBm1)AXI|{W7LF+B7cCXR z66w*+QZP$!dW!6Fl6-!5*ZX@ckx8#`=Fw?{8z!{L!^&W(=|oe(2$jWz@OM<`I`tPX z1qI)a8{9a*^l_hx-IBa&9xr>;Qd@>Yt2r^D!v~e~qAwIou~+D^jnG$U(xzpRvjyl- z&WA(I$ijS1f&2-t`%TV_dS=|lx7`#JpvIxS!pvg1l6VtLWy2uskK5AWyWh zfGiJP&wsVw{4iYcPSYearZzS4cMfwfo(rf)x~5rARI$K>WJ1MOu0~Q#V5mgnK1_R` z^9^!_RZTI%q3t*`)@DyK@v|Vz^AntJ2jwb7Y?$CmU@(`JrMNH6C`HkAMHZzk^i0PvBh znr;KAz>SBBUagQ&gc`L(!N9neAJZCc<_X(mqz!l(abDr)t?K*c`GFudCUhO0qBJ8q z^m4MAC^sYdCq7HBDM(0>m5_xkfSZ1iZw71;R(O1A}Pp4P50 zfnCzkpzK}avt}d<2Yb})hS-rzNz0>Y_` zA}s7|7h}As<3PR(URCM5I(6J1pbaP|lgL5M(8D1vg>aglh2{`5_QXf7Q<&<1wYn29 z>dyPtv+nRkrOrnY;a0-)MVMQ7Za{4gcbojz;&V)>L&%&s8yuHHYvl{sourceConfig.name} + return ( + + {sourceConfig.label ?? sourceConfig.name} + + ) }, }, { diff --git a/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx b/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx index 6c16e79e369f5..d597b90b99f14 100644 --- a/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx +++ b/frontend/src/scenes/data-warehouse/new/sourceWizardLogic.tsx @@ -3,6 +3,8 @@ import { actions, connect, kea, listeners, path, props, reducers, selectors } fr import { forms } from 'kea-forms' import { router, urlToAction } from 'kea-router' import api from 'lib/api' +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import posthog from 'posthog-js' import { preflightLogic } from 'scenes/PreflightCheck/preflightLogic' import { Scene } from 'scenes/sceneTypes' @@ -331,6 +333,138 @@ export const SOURCE_DETAILS: Record = { }, ], }, + MSSQL: { + name: 'MSSQL', + label: 'Azure SQL Server', + caption: ( + <> + Enter your MS SQL Server/Azure SQL Server credentials to automatically pull your SQL data into the + PostHog Data warehouse. + + ), + fields: [ + { + name: 'host', + label: 'Host', + type: 'text', + required: true, + placeholder: 'localhost', + }, + { + name: 'port', + label: 'Port', + type: 'number', + required: true, + placeholder: '1433', + }, + { + name: 'dbname', + label: 'Database', + type: 'text', + required: true, + placeholder: 'msdb', + }, + { + name: 'user', + label: 'User', + type: 'text', + required: true, + placeholder: 'sa', + }, + { + name: 'password', + label: 'Password', + type: 'password', + required: true, + placeholder: '', + }, + { + name: 'schema', + label: 'Schema', + type: 'text', + required: true, + placeholder: 'dbo', + }, + { + name: 'ssh-tunnel', + label: 'Use SSH tunnel?', + type: 'switch-group', + default: false, + fields: [ + { + name: 'host', + label: 'Tunnel host', + type: 'text', + required: true, + placeholder: 'localhost', + }, + { + name: 'port', + label: 'Tunnel port', + type: 'number', + required: true, + placeholder: '22', + }, + { + type: 'select', + name: 'auth_type', + label: 'Authentication type', + required: true, + defaultValue: 'password', + options: [ + { + label: 'Password', + value: 'password', + fields: [ + { + name: 'username', + label: 'Tunnel username', + type: 'text', + required: true, + placeholder: 'User1', + }, + { + name: 'password', + label: 'Tunnel password', + type: 'password', + required: true, + placeholder: '', + }, + ], + }, + { + label: 'Key pair', + value: 'keypair', + fields: [ + { + name: 'username', + label: 'Tunnel username', + type: 'text', + required: false, + placeholder: 'User1', + }, + { + name: 'private_key', + label: 'Tunnel private key', + type: 'textarea', + required: true, + placeholder: '', + }, + { + name: 'passphrase', + label: 'Tunnel passphrase', + type: 'password', + required: false, + placeholder: '', + }, + ], + }, + ], + }, + ], + }, + ], + }, Snowflake: { name: 'Snowflake', caption: ( @@ -534,6 +668,8 @@ export const sourceWizardLogic = kea([ ['dataWarehouseSources'], preflightLogic, ['preflight'], + featureFlagLogic, + ['featureFlags'], ], actions: [ dataWarehouseTableLogic, @@ -714,15 +850,21 @@ export const sourceWizardLogic = kea([ (selectedConnector, isManualLinkFormVisible) => selectedConnector || isManualLinkFormVisible, ], connectors: [ - (s) => [s.dataWarehouseSources], - (sources): SourceConfig[] => { - return Object.values(SOURCE_DETAILS).map((connector) => ({ + (s) => [s.dataWarehouseSources, s.featureFlags], + (sources, featureFlags): SourceConfig[] => { + const connectors = Object.values(SOURCE_DETAILS).map((connector) => ({ ...connector, disabledReason: sources && sources.results.find((source) => source.source_type === connector.name) ? 'Already linked' : null, })) + + if (!featureFlags[FEATURE_FLAGS.MSSQL_SOURCE]) { + return connectors.filter((n) => n.name !== 'MSSQL') + } + + return connectors }, ], manualConnectors: [ diff --git a/frontend/src/scenes/data-warehouse/settings/DataWarehouseManagedSourcesTable.tsx b/frontend/src/scenes/data-warehouse/settings/DataWarehouseManagedSourcesTable.tsx index 69c12250eea14..e983ce363bd95 100644 --- a/frontend/src/scenes/data-warehouse/settings/DataWarehouseManagedSourcesTable.tsx +++ b/frontend/src/scenes/data-warehouse/settings/DataWarehouseManagedSourcesTable.tsx @@ -12,12 +12,14 @@ import IconMySQL from 'public/services/mysql.png' import IconPostgres from 'public/services/postgres.png' import IconSalesforce from 'public/services/salesforce.png' import IconSnowflake from 'public/services/snowflake.png' +import IconMSSQL from 'public/services/sql-azure.png' import IconStripe from 'public/services/stripe.png' import IconZendesk from 'public/services/zendesk.png' import { urls } from 'scenes/urls' import { manualLinkSources, PipelineNodeTab, PipelineStage } from '~/types' +import { SOURCE_DETAILS } from '../new/sourceWizardLogic' import { dataWarehouseSettingsLogic } from './dataWarehouseSettingsLogic' const StatusTagSetting = { @@ -56,7 +58,7 @@ export function DataWarehouseManagedSourcesTable(): JSX.Element { `managed-${source.id}`, PipelineNodeTab.Schemas )} - title={source.source_type} + title={SOURCE_DETAILS[source.source_type]?.label ?? source.source_type} description={source.prefix} /> ) @@ -185,6 +187,7 @@ export function RenderDataWarehouseSourceIcon({ 'cloudflare-r2': IconCloudflare, azure: Iconazure, Salesforce: IconSalesforce, + MSSQL: IconMSSQL, }[type] return ( diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 7bdb1a9b8b306..d66b0beb1e788 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -3854,6 +3854,7 @@ export const externalDataSources = [ 'Hubspot', 'Postgres', 'MySQL', + 'MSSQL', 'Zendesk', 'Snowflake', 'Salesforce', @@ -3875,7 +3876,7 @@ export interface ExternalDataStripeSource { source_id: string connection_id: string status: string - source_type: string + source_type: ExternalDataSourceType prefix: string last_run_at?: Dayjs schemas: ExternalDataSourceSchema[] @@ -4249,6 +4250,7 @@ export type SourceFieldConfig = export interface SourceConfig { name: ExternalDataSourceType + label?: string caption: string | React.ReactNode fields: SourceFieldConfig[] disabledReason?: string | null diff --git a/latest_migrations.manifest b/latest_migrations.manifest index af30ae6589b0d..5f32a0ea3f522 100644 --- a/latest_migrations.manifest +++ b/latest_migrations.manifest @@ -5,7 +5,7 @@ contenttypes: 0002_remove_content_type_name ee: 0016_rolemembership_organization_member otp_static: 0002_throttling otp_totp: 0002_auto_20190420_0723 -posthog: 0460_alertconfiguration_threshold_alertsubscription_and_more +posthog: 0461_alter_externaldatasource_source_type sessions: 0001_initial social_django: 0010_uid_db_index two_factor: 0007_auto_20201201_1019 diff --git a/posthog/migrations/0461_alter_externaldatasource_source_type.py b/posthog/migrations/0461_alter_externaldatasource_source_type.py new file mode 100644 index 0000000000000..8ede6c0fd01eb --- /dev/null +++ b/posthog/migrations/0461_alter_externaldatasource_source_type.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.14 on 2024-08-23 09:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("posthog", "0460_alertconfiguration_threshold_alertsubscription_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="externaldatasource", + name="source_type", + field=models.CharField( + choices=[ + ("Stripe", "Stripe"), + ("Hubspot", "Hubspot"), + ("Postgres", "Postgres"), + ("Zendesk", "Zendesk"), + ("Snowflake", "Snowflake"), + ("Salesforce", "Salesforce"), + ("MySQL", "MySQL"), + ("MSSQL", "MSSQL"), + ], + max_length=128, + ), + ), + ] diff --git a/posthog/temporal/data_imports/pipelines/pipeline.py b/posthog/temporal/data_imports/pipelines/pipeline.py index b0715f952f854..0dbfd8f5c3fd7 100644 --- a/posthog/temporal/data_imports/pipelines/pipeline.py +++ b/posthog/temporal/data_imports/pipelines/pipeline.py @@ -54,6 +54,8 @@ def __init__( self.should_chunk_pipeline = ( incremental and inputs.job_type != ExternalDataSource.Type.POSTGRES + and inputs.job_type != ExternalDataSource.Type.MYSQL + and inputs.job_type != ExternalDataSource.Type.MSSQL and inputs.job_type != ExternalDataSource.Type.SNOWFLAKE ) diff --git a/posthog/temporal/data_imports/pipelines/schemas.py b/posthog/temporal/data_imports/pipelines/schemas.py index c8c0d9729c2d8..0acd00e8bd6f3 100644 --- a/posthog/temporal/data_imports/pipelines/schemas.py +++ b/posthog/temporal/data_imports/pipelines/schemas.py @@ -28,6 +28,7 @@ ExternalDataSource.Type.SNOWFLAKE: (), ExternalDataSource.Type.SALESFORCE: SALESFORCE_ENDPOINTS, ExternalDataSource.Type.MYSQL: (), + ExternalDataSource.Type.MSSQL: (), } PIPELINE_TYPE_INCREMENTAL_ENDPOINTS_MAPPING = { @@ -38,6 +39,7 @@ ExternalDataSource.Type.SNOWFLAKE: (), ExternalDataSource.Type.SALESFORCE: SALESFORCE_INCREMENTAL_ENDPOINTS, ExternalDataSource.Type.MYSQL: (), + ExternalDataSource.Type.MSSQL: (), } PIPELINE_TYPE_INCREMENTAL_FIELDS_MAPPING: dict[ExternalDataSource.Type, dict[str, list[IncrementalField]]] = { @@ -48,4 +50,5 @@ ExternalDataSource.Type.SNOWFLAKE: {}, ExternalDataSource.Type.SALESFORCE: SALESFORCE_INCREMENTAL_FIELDS, ExternalDataSource.Type.MYSQL: {}, + ExternalDataSource.Type.MSSQL: {}, } diff --git a/posthog/temporal/data_imports/pipelines/sql_database/__init__.py b/posthog/temporal/data_imports/pipelines/sql_database/__init__.py index 65a4ca9527cd6..0fc7f7394b6ad 100644 --- a/posthog/temporal/data_imports/pipelines/sql_database/__init__.py +++ b/posthog/temporal/data_imports/pipelines/sql_database/__init__.py @@ -69,6 +69,10 @@ def sql_source_for_type( ) elif source_type == ExternalDataSource.Type.MYSQL: credentials = ConnectionStringCredentials(f"mysql+pymysql://{user}:{password}@{host}:{port}/{database}") + elif source_type == ExternalDataSource.Type.MSSQL: + credentials = ConnectionStringCredentials( + f"mssql+pyodbc://{user}:{password}@{host}:{port}/{database}?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes" + ) else: raise Exception("Unsupported source_type") diff --git a/posthog/temporal/data_imports/workflow_activities/import_data.py b/posthog/temporal/data_imports/workflow_activities/import_data.py index 974edc7ca3460..6ce4237f53711 100644 --- a/posthog/temporal/data_imports/workflow_activities/import_data.py +++ b/posthog/temporal/data_imports/workflow_activities/import_data.py @@ -102,7 +102,11 @@ async def import_data_activity(inputs: ImportDataActivityInputs): schema=schema, reset_pipeline=reset_pipeline, ) - elif model.pipeline.source_type in [ExternalDataSource.Type.POSTGRES, ExternalDataSource.Type.MYSQL]: + elif model.pipeline.source_type in [ + ExternalDataSource.Type.POSTGRES, + ExternalDataSource.Type.MYSQL, + ExternalDataSource.Type.MSSQL, + ]: from posthog.temporal.data_imports.pipelines.sql_database import sql_source_for_type host = model.pipeline.job_inputs.get("host") diff --git a/posthog/warehouse/api/external_data_schema.py b/posthog/warehouse/api/external_data_schema.py index 5b982e54b8434..6db5379a2fe96 100644 --- a/posthog/warehouse/api/external_data_schema.py +++ b/posthog/warehouse/api/external_data_schema.py @@ -23,6 +23,7 @@ cancel_external_data_workflow, ) from posthog.warehouse.models.external_data_schema import ( + filter_mssql_incremental_fields, filter_mysql_incremental_fields, filter_postgres_incremental_fields, filter_snowflake_incremental_fields, @@ -269,7 +270,11 @@ def incremental_fields(self, request: Request, *args: Any, **kwargs: Any): source: ExternalDataSource = instance.source incremental_columns: list[IncrementalField] = [] - if source.source_type in [ExternalDataSource.Type.POSTGRES, ExternalDataSource.Type.MYSQL]: + if source.source_type in [ + ExternalDataSource.Type.POSTGRES, + ExternalDataSource.Type.MYSQL, + ExternalDataSource.Type.MSSQL, + ]: # TODO(@Gilbert09): Move all this into a util and replace elsewhere host = source.job_inputs.get("host") port = source.job_inputs.get("port") @@ -312,8 +317,10 @@ def incremental_fields(self, request: Request, *args: Any, **kwargs: Any): columns = db_schemas.get(instance.name, []) if source.source_type == ExternalDataSource.Type.POSTGRES: incremental_fields_func = filter_postgres_incremental_fields - else: + elif source.source_type == ExternalDataSource.Type.MYSQL: incremental_fields_func = filter_mysql_incremental_fields + elif source.source_type == ExternalDataSource.Type.MSSQL: + incremental_fields_func = filter_mssql_incremental_fields incremental_columns = [ {"field": name, "field_type": field_type, "label": name, "type": field_type} diff --git a/posthog/warehouse/api/external_data_source.py b/posthog/warehouse/api/external_data_source.py index 3fa01db7c1a9b..c3756e2fecddf 100644 --- a/posthog/warehouse/api/external_data_source.py +++ b/posthog/warehouse/api/external_data_source.py @@ -33,6 +33,8 @@ get_hubspot_access_token_from_code, ) from posthog.warehouse.models.external_data_schema import ( + filter_mssql_incremental_fields, + filter_mysql_incremental_fields, filter_postgres_incremental_fields, filter_snowflake_incremental_fields, get_sql_schemas_for_source_type, @@ -264,7 +266,11 @@ def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: new_source_model = self._handle_zendesk_source(request, *args, **kwargs) elif source_type == ExternalDataSource.Type.SALESFORCE: new_source_model = self._handle_salesforce_source(request, *args, **kwargs) - elif source_type in [ExternalDataSource.Type.POSTGRES, ExternalDataSource.Type.MYSQL]: + elif source_type in [ + ExternalDataSource.Type.POSTGRES, + ExternalDataSource.Type.MYSQL, + ExternalDataSource.Type.MSSQL, + ]: try: new_source_model, sql_schemas = self._handle_sql_source(request, *args, **kwargs) except InternalPostgresError: @@ -280,7 +286,11 @@ def create(self, request: Request, *args: Any, **kwargs: Any) -> Response: payload = request.data["payload"] schemas = payload.get("schemas", None) - if source_type in [ExternalDataSource.Type.POSTGRES, ExternalDataSource.Type.MYSQL]: + if source_type in [ + ExternalDataSource.Type.POSTGRES, + ExternalDataSource.Type.MYSQL, + ExternalDataSource.Type.MSSQL, + ]: default_schemas = sql_schemas elif source_type == ExternalDataSource.Type.SNOWFLAKE: default_schemas = snowflake_schemas @@ -668,7 +678,11 @@ def database_schema(self, request: Request, *arg: Any, **kwargs: Any): ) # Get schemas and validate SQL credentials - if source_type in [ExternalDataSource.Type.POSTGRES, ExternalDataSource.Type.MYSQL]: + if source_type in [ + ExternalDataSource.Type.POSTGRES, + ExternalDataSource.Type.MYSQL, + ExternalDataSource.Type.MSSQL, + ]: host = request.data.get("host", None) port = request.data.get("port", None) database = request.data.get("dbname", None) @@ -775,9 +789,18 @@ def database_schema(self, request: Request, *arg: Any, **kwargs: Any): data={"message": get_generic_sql_error(source_type)}, ) - filtered_results = [ - (table_name, filter_postgres_incremental_fields(columns)) for table_name, columns in result.items() - ] + if source_type == ExternalDataSource.Type.POSTGRES: + filtered_results = [ + (table_name, filter_postgres_incremental_fields(columns)) for table_name, columns in result.items() + ] + elif source_type == ExternalDataSource.Type.MYSQL: + filtered_results = [ + (table_name, filter_mysql_incremental_fields(columns)) for table_name, columns in result.items() + ] + elif source_type == ExternalDataSource.Type.MSSQL: + filtered_results = [ + (table_name, filter_mssql_incremental_fields(columns)) for table_name, columns in result.items() + ] result_mapped_to_options = [ { diff --git a/posthog/warehouse/models/external_data_schema.py b/posthog/warehouse/models/external_data_schema.py index f42cf3248b8ad..a4e91d45c577f 100644 --- a/posthog/warehouse/models/external_data_schema.py +++ b/posthog/warehouse/models/external_data_schema.py @@ -9,6 +9,7 @@ import uuid import psycopg2 import pymysql +import pymssql from .external_data_source import ExternalDataSource from posthog.warehouse.data_load.service import ( external_data_workflow_exists, @@ -339,6 +340,59 @@ def get_schemas(mysql_host: str, mysql_port: int): return get_schemas(host, int(port)) +def filter_mssql_incremental_fields(columns: list[tuple[str, str]]) -> list[tuple[str, IncrementalFieldType]]: + results: list[tuple[str, IncrementalFieldType]] = [] + for column_name, type in columns: + type = type.lower() + if type == "date": + results.append((column_name, IncrementalFieldType.Date)) + elif type == "datetime" or type == "datetime2" or type == "smalldatetime": + results.append((column_name, IncrementalFieldType.DateTime)) + elif type == "tinyint" or type == "smallint" or type == "int" or type == "bigint": + results.append((column_name, IncrementalFieldType.Integer)) + + return results + + +def get_mssql_schemas( + host: str, port: str, database: str, user: str, password: str, schema: str, ssh_tunnel: SSHTunnel +) -> dict[str, list[tuple[str, str]]]: + def get_schemas(postgres_host: str, postgres_port: int): + connection = pymssql.connect( + server=postgres_host, + port=str(postgres_port), + database=database, + user=user, + password=password, + login_timeout=5, + ) + + with connection.cursor(as_dict=False) as cursor: + cursor.execute( + "SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = %(schema)s ORDER BY table_name ASC", + {"schema": schema}, + ) + + schema_list = defaultdict(list) + + for row in cursor: + if row: + schema_list[row[0]].append((row[1], row[2])) + + connection.close() + + return schema_list + + if ssh_tunnel.enabled: + with ssh_tunnel.get_tunnel(host, int(port)) as tunnel: + if tunnel is None: + raise Exception("Can't open tunnel to SSH server") + + return get_schemas(tunnel.local_bind_host, tunnel.local_bind_port) + + return get_schemas(host, int(port)) + + def get_sql_schemas_for_source_type( source_type: ExternalDataSource.Type, host: str, @@ -353,6 +407,8 @@ def get_sql_schemas_for_source_type( schemas = get_postgres_schemas(host, port, database, user, password, schema, ssh_tunnel) elif source_type == ExternalDataSource.Type.MYSQL: schemas = get_mysql_schemas(host, port, database, user, password, schema, ssh_tunnel) + elif source_type == ExternalDataSource.Type.MSSQL: + schemas = get_mssql_schemas(host, port, database, user, password, schema, ssh_tunnel) else: raise Exception("Unsupported source_type") diff --git a/posthog/warehouse/models/external_data_source.py b/posthog/warehouse/models/external_data_source.py index 49c91d7781764..6f9fe14e01dd9 100644 --- a/posthog/warehouse/models/external_data_source.py +++ b/posthog/warehouse/models/external_data_source.py @@ -22,6 +22,7 @@ class Type(models.TextChoices): SNOWFLAKE = "Snowflake", "Snowflake" SALESFORCE = "Salesforce", "Salesforce" MYSQL = "MySQL", "MySQL" + MSSQL = "MSSQL", "MSSQL" class Status(models.TextChoices): RUNNING = "Running", "Running" diff --git a/production.Dockerfile b/production.Dockerfile index 07906afd9bc4c..ff7bd3d619b55 100644 --- a/production.Dockerfile +++ b/production.Dockerfile @@ -158,6 +158,12 @@ RUN apt-get update && \ "libxml2" \ "gettext-base" +# Install MS SQL dependencies +RUN curl https://packages.microsoft.com/keys/microsoft.asc | tee /etc/apt/trusted.gpg.d/microsoft.asc +RUN curl https://packages.microsoft.com/config/debian/11/prod.list | tee /etc/apt/sources.list.d/mssql-release.list +RUN apt-get update +RUN ACCEPT_EULA=Y apt-get install -y msodbcsql18 + # Install NodeJS 18. RUN apt-get install -y --no-install-recommends \ "curl" \ diff --git a/requirements.in b/requirements.in index ac315f2c5e719..806129962e009 100644 --- a/requirements.in +++ b/requirements.in @@ -57,11 +57,13 @@ Pillow==10.2.0 pdpyras==5.2.0 posthoganalytics==3.5.0 psycopg2-binary==2.9.7 +pymssql==2.3.0 PyMySQL==1.1.1 psycopg[binary]==3.1.20 pyarrow==17.0.0 pydantic==2.5.3 pyjwt==2.4.0 +pyodbc==5.1.0 python-dateutil>=2.8.2 python3-saml==1.12.0 pytz==2023.3 diff --git a/requirements.txt b/requirements.txt index 8e64d768b0ec7..9b00b7283f7f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -258,8 +258,6 @@ googleapis-common-protos==1.60.0 # via # google-api-core # grpcio-status -greenlet==3.0.3 - # via sqlalchemy grpcio==1.57.0 # via # google-api-core @@ -456,10 +454,14 @@ pyjwt==2.4.0 # simple-salesforce # snowflake-connector-python # social-auth-core +pymssql==2.3.0 + # via -r requirements.in pymysql==1.1.1 # via -r requirements.in pynacl==1.5.0 # via paramiko +pyodbc==5.1.0 + # via -r requirements.in pyopenssl==23.0.0 # via # snowflake-connector-python