From 985ed85d7219ed69403dcda6161a2bddc977b3c2 Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Mon, 22 Apr 2024 13:53:35 +0530 Subject: [PATCH] E2E Excel as ITN --- .../features/excelplugin/ExcelToFile.feature | 49 ++++++++++++++++++ .../common/stepsdesign/TestSetupHooks.java | 10 +++- .../resources/pluginParameters.properties | 9 ++++ .../testdata/exceldata/testExcel.xlsx | Bin 0 -> 4818 bytes .../OUTPUT_FOR_EXCEL_TEST.csv | 5 ++ 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 core-plugins/src/e2e-test/features/excelplugin/ExcelToFile.feature create mode 100644 core-plugins/src/e2e-test/resources/testdata/exceldata/testExcel.xlsx create mode 100644 core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_EXCEL_TEST.csv diff --git a/core-plugins/src/e2e-test/features/excelplugin/ExcelToFile.feature b/core-plugins/src/e2e-test/features/excelplugin/ExcelToFile.feature new file mode 100644 index 000000000..c891857b5 --- /dev/null +++ b/core-plugins/src/e2e-test/features/excelplugin/ExcelToFile.feature @@ -0,0 +1,49 @@ +# Copyright © 2024 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@File_Sink +Feature:File Sink - Verification of Excel Plugin to File Successful data transfer + + @File_Sink_Required @EXCEL_TEST @FILE_SINK_TEST + Scenario: To verify data is getting transferred from Excel source to File sink + Given Open Datafusion Project to configure pipeline + When Select plugin: "Excel" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "File" from the plugins list as: "Sink" + Then Connect plugins: "Excel" and "File" to establish connection + Then Navigate to the properties page of plugin: "Excel" + Then Enter input plugin property: "referenceName" with value: "FileReferenceName" + Then Enter input plugin property: "filePath" with value: "excelTestFile" + Then Enter input plugin property: "filePattern" with value: "Pattern" + Then Select dropdown plugin property: "sheet" with option value: "Sheet Name" + Then Enter input plugin property: "sheetValue" with value: "Sheet1" + Then Enter Value for plugin property table key : "columnList" with values: "columnvalues" + Then Enter key value pairs for plugin property: "columnMapping" with values from json: "columnmappings" + Then Validate "Excel" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "File" + Then Enter input plugin property: "referenceName" with value: "FileReferenceName" + Then Enter input plugin property: "path" with value: "fileSinkTargetBucket" + Then Replace input plugin property: "pathSuffix" with value: "yyyy-MM-dd-HH-mm" + Then Select dropdown plugin property: "format" with option value: "csv" + Then Validate "File" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForExcelTest" diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 8542f8c17..76269ab75 100644 --- a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -285,7 +285,7 @@ public static void createBucketWithFileCSVDataTypeTest1() throws IOException, UR BeforeActions.scenario.write("CSV Datatype test bucket name - " + fileSourceBucket1); } - @After(order = 1, value = "@CSV_DATATYPE_TEST1") + @After(order = 1, value = "@CSV_DATATYPE_TEST1 or @EXCEL_TEST") public static void deleteSourceBucketWithFileCSVDataTypeTest1() { deleteGCSBucket(fileSourceBucket1); fileSourceBucket1 = StringUtils.EMPTY; @@ -486,4 +486,12 @@ public static String createGCSBucketWithMultipleFiles(String folderPath) throws + files.size() + " files in " + folderPath); return bucketName; } + + @Before(order = 1, value = "@EXCEL_TEST") + public static void createBucketWithExcelFile() throws IOException, URISyntaxException { + fileSourceBucket1 = createGCSBucketWithFile(PluginPropertyUtils.pluginProp("excelFile")); + PluginPropertyUtils.addPluginProp("excelTestFile", "gs://" + fileSourceBucket1 + "/" + + PluginPropertyUtils.pluginProp("excelFile")); + BeforeActions.scenario.write("excel test bucket name - " + fileSourceBucket1); + } } diff --git a/core-plugins/src/e2e-test/resources/pluginParameters.properties b/core-plugins/src/e2e-test/resources/pluginParameters.properties index 397bb7fc8..0e4fbf0df 100644 --- a/core-plugins/src/e2e-test/resources/pluginParameters.properties +++ b/core-plugins/src/e2e-test/resources/pluginParameters.properties @@ -291,3 +291,12 @@ quotedValueDelimitedTestFiles=dummy fileSinkTargetBucket=file-plugin-output recursiveTest=dummy testOnCdap=true + +## EXCEL-PLUGIN-PROPERTIES-START ## +excelTestFile=dummy +excelFile=testdata/exceldata/testExcel.xlsx +Pattern=.*\.xlsx$ +outputForExcelTest=e2e-tests/file/expected_outputs/OUTPUT_FOR_EXCEL_TEST.csv +columnvalues=A +columnmappings=[{"key":"A","value":"id"}] +## EXCEL-PLUGIN-PROPERTIES-END ## diff --git a/core-plugins/src/e2e-test/resources/testdata/exceldata/testExcel.xlsx b/core-plugins/src/e2e-test/resources/testdata/exceldata/testExcel.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..e4e849a79fd9305d4c152cca9ec0a7e542a59c39 GIT binary patch literal 4818 zcmai22RNJU8b-|8Vnu6}s!=00HQE?eYHO=%6b)j88Z~Rh-eOa2t=N>bHZ^K1s>3F= zD@9S18YlEWr~aPv_dLm!eBX8Dy>q?y`#kq^Kc9{UfRG-Kl#~>YsSas~cPf~1*B zr2`H5=U2Q<^DAi|MYS2J-;$Tbh*B)QdqFwJfkMg_lJ#GIc&op7S!Lr2X}b68nx{N7 z%PoL(!0hIs=5SDz#F?F|ndkPTIfEE2_K0?@FeQZ~WxDb@`}iGxfnpH^p_0k`HCVn; zHOPS+ne-vc)?}AnQ_lypu1We%d{*tYOSMW~U7-~`rcDV@Rrl3^Y8wD+KyO!#^!M4x z3jS`%m%IY@6{b_AedzPmW;z-qq$CzF1Q9+S9ti*sPv`$p>F(%k=VAkey1EFTeEp;{ z@eMpjgjwb3y(xT8_CCjSj%VprMX36s^!^f)Swevk)he>?@mS9H=cOZU9(EW%b%83x zD?TxtWeqnl_(~AuiznAwQNTORgxTD|+5+VAXv=lDr1_fDc)uDRc6oDB7x5^26F1+) zw=Er~fCSsM#%-u*)OezSNKOZ~dA6ckbOzC(hJYLnL!}U25+(@Gnt9k{jK#)zr%uvg zrZ8^GAyiWlthfpGWe&sKuE$&PPJz+>d>Sz`63Anj_2wpL$QtZ*oSf@Q569hiGm(-8 zPdiN+T`pa@=qU-lRn_6dBX;$f8{bG$aE0QER;rk!EP z2y)~R^A8miE#D^Xu28SBYhBjk1`TbuKI6mPaPe$mf6xSM7O`q4Z{I8$x;;_zUFmwc zm7OE=tS~24TiabXH=8GqSQTp?%xJ+{eB&d}6Pl_#EU^2AwmBS9AGQ+S7_OK+qG_be zKU9*%T$>>uD(K5wl9zGnIY%A&G^9YaOy*hNY6XR}hj`N0L=R`+C%2hVO=U^^=(o09 z#nC7(u6{FqE)r(pMELpI(x_6=CwyGjK$n|cvs{7L@t)WG32CAa6T<;MhP;!n7pg>G z_PC;MoP#doRg>pskRpPC{R@T20x3VZw$*zIv`=yCgC{Ekb^9d8`6I@+@3n`^pRYuJ z${((urFbfT%$%Y2r?W`t_&$0-<4mgv4KW7j5h~)H@G&?{d_^?~p2v1i4MlCw#=ZV_ zJR>?A(>mWb;L~O%K`n+p8!Rm&kTpjnQrYvln~JV~@RMsu5Ik!m&;X^C6* zNIiJ$b01Wp+nS1;Ca3+NM$F~@#9qsbi2@}Ta@E6t>71H=BOs2IXsI*KdN59Pm58|)Y8uEZC61klY#~Ry^){M#~;pGuqz+(v=6B z1d_1BCT4F3AB}7)E~pv7tXQa+SaV!&@hcD!bx@?ZwVjQbAq@@VoO#cT#Qv7cktcpY zTxxLYLR85oAnjM%&Chh@_aUMLiI;%_4I$x)?PoY?dBqr+Jg=tiG0w}i>1ldKL1Ne$ z98}{i?}#OSN^BW>`l>@QHd~ydUQXI7s5>GE7?Fn*LcXA(5qZoP9U4T#hgsm>NJb7R zC5HHKC_PJ}e*US^oa|MJu@6AI!A|W09h5Kob$F6}66E+(O zmS&1py8V6l)q()Q^$UEc3-YG4a&YR`sjORJ=)}f%{DNRnzz1{d$%0o{uELxcJ$34h zaAY||gHA@XtxVp{=EX>^?_Yl1Ot{ssM4*+G7VK_zv9+o{ z&q5~K!dx+*0CZEAga}h<_uVv)Gz?FlF4?VUU;Bf+cV0enlyV$I=J+|YbcmI<;~_Re zq{=){X-Mj6?Z@7EkHSn{&}O^$dlt$y;+{IOuLb>&u>$FMOm@tL*c+5+$T}ILl0U43 z?P5R)@IX;Yqa}F4Jl;U$pfEyFti^M)jN#Op#waxwYQIFQ)8ArT*H`V;hdm?z>+O(i^gRSZ~kAPPz34 zZ~3Hncq@tI3U=hx&JLlxIzjA&9nR?R2APBpK9*8TYex4Q1hz_cw8y(EcTdCyniF8o zi)&P+RDUWo`k#g7Vq@+My|3@;eBAtxdp>kfSF25g0;eucpggLZ$_2w2FXX;)pdA&W>RbF6 zhsvwk8K{z6^us^an&)8VoEF>$-T$aHlz&m+>S+%>+51a_mj9^CllJVa%SykC8CUSKUUFj8LInwDtwujuGbgL@w#0%eiXl2G?2_CXE&#E_9*+} zsCXB{LGtR8tXz3p;LF~oNGeWisVC_Pjroe22PY*_DO=g_0hbG9gnudt>Yw>=+z!qi3#l5p=aw&oF_#iwicGsGvF65^Dz?X%1FA3G|H z9>J$sGQ}|nXUU89X-L8nC%sX9t)^E~jAxNSU8Qqlce=V62u7K->)96vt}XVHeH@QV z;O~&l5f3{4FqwVNt~R4nKammNCVM z2f|}eF8fh|0jp{{jNM6?iZt#kZ4beA9#$`^f1!mCdK#EsD7mHwwda=h! zBhQV5gvjdAX2TmSCYm7!=#Hgz=iZN-4eV?#8rAOGg5HPh;dzM_iUksWK9Rdd#y7ad zsKpMtXyPp*#zXVl%N9e;zYKxEsnKVa3j(;4&l>waw5C9hN0WrV8yG-!i9VDVZH;B#*g zD2SjV^a9#1-X&TKP5E(=ChbLa9W@(6RI2&iX$RAa;n!lLZeVh%^*aGv5d(TwvrdDx zq_pzW;+&M}hNxSr4l#8qwfw!;(iT8EX7?i+Z^ms`5ji<+?4bp&c5IQmM79YU z0$*0%1EiI(8z`XKyCT_d`vHzsEE&|Izl`Ifmf%lWOOE@+PrCkchgYcrMQDfyaqiG+ zThGZnD(g9z$DWeyS(Os!m3&xn>&l8=GZ`N-cRy;FCjsZ)N!_dqy2x$K$-))3+V$3})fn8jd1o40#T@6{o^F zE@d?rBmLwI8*yfkgf!PNi=z^^5#B!v_OUsLIly(nyq=T`x6gvHs?nc$W&Cp~t9J7?XDE~_jW!uo#ff%|2a`z>DtcfAN7b<&e*5{WpSSBtNsQJlhxRFB_| zC$fl7K#%w98Nun1_xKsX-{YU280cvH9p&`k^LG>t-2dWGP7OQ1hUMQ;PLB@9L(;E^ z#EHq@QU2%1^gG~bi*%e+zakb#1rG4{`Qvwh(+1<%N&SlNBqsp><*R;2INg$tjmWR~ zLHZNnKepuerKgqjxM%$ea-10aMaRE8*zb!^i^{Rs{)$+Nlg0lay}#p}uGnJ<`V}p> n?VJ*%-xr=vy1#Q@f-|iDqr7!Ah=`BT&){C^I4L|&bA0zNHYCWy literal 0 HcmV?d00001 diff --git a/core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_EXCEL_TEST.csv b/core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_EXCEL_TEST.csv new file mode 100644 index 000000000..51654d0b7 --- /dev/null +++ b/core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_EXCEL_TEST.csv @@ -0,0 +1,5 @@ +testExcel.xlsx,Sheet1,A +testExcel.xlsx,Sheet1,1.0 +testExcel.xlsx,Sheet1,2.0 +testExcel.xlsx,Sheet1,3.0 +testExcel.xlsx,Sheet1,4.0 \ No newline at end of file