diff --git a/core-plugins/src/e2e-test/features/excelplugin/ExcelToFile.feature b/core-plugins/src/e2e-test/features/excelplugin/ExcelToFile.feature new file mode 100644 index 000000000..85dd03fae --- /dev/null +++ b/core-plugins/src/e2e-test/features/excelplugin/ExcelToFile.feature @@ -0,0 +1,49 @@ +# Copyright © 2024 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@File_Sink +Feature:File Sink - Verification of Excel Plugin to File Successful data transfer + + @EXCEL_TEST @FILE_SINK_TEST + Scenario: To verify data is getting transferred from Excel source to File sink + Given Open Datafusion Project to configure pipeline + When Select plugin: "Excel" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "File" from the plugins list as: "Sink" + Then Connect plugins: "Excel" and "File" to establish connection + Then Navigate to the properties page of plugin: "Excel" + Then Enter input plugin property: "referenceName" with value: "FileReferenceName" + Then Enter input plugin property: "filePath" with value: "excelTestFile" + Then Enter input plugin property: "filePattern" with value: "Pattern" + Then Select dropdown plugin property: "sheet" with option value: "Sheet Name" + Then Enter input plugin property: "sheetValue" with value: "Sheet1" + Then Enter Value for plugin property table key : "columnList" with values: "columnvalues" + Then Enter key value pairs for plugin property: "columnMapping" with values from json: "columnmappings" + Then Validate "Excel" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "File" + Then Enter input plugin property: "referenceName" with value: "FileReferenceName" + Then Enter input plugin property: "path" with value: "fileSinkTargetBucket" + Then Replace input plugin property: "pathSuffix" with value: "yyyy-MM-dd-HH-mm" + Then Select dropdown plugin property: "format" with option value: "csv" + Then Validate "File" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForExcelTest" diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 8542f8c17..76269ab75 100644 --- a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -285,7 +285,7 @@ public static void createBucketWithFileCSVDataTypeTest1() throws IOException, UR BeforeActions.scenario.write("CSV Datatype test bucket name - " + fileSourceBucket1); } - @After(order = 1, value = "@CSV_DATATYPE_TEST1") + @After(order = 1, value = "@CSV_DATATYPE_TEST1 or @EXCEL_TEST") public static void deleteSourceBucketWithFileCSVDataTypeTest1() { deleteGCSBucket(fileSourceBucket1); fileSourceBucket1 = StringUtils.EMPTY; @@ -486,4 +486,12 @@ public static String createGCSBucketWithMultipleFiles(String folderPath) throws + files.size() + " files in " + folderPath); return bucketName; } + + @Before(order = 1, value = "@EXCEL_TEST") + public static void createBucketWithExcelFile() throws IOException, URISyntaxException { + fileSourceBucket1 = createGCSBucketWithFile(PluginPropertyUtils.pluginProp("excelFile")); + PluginPropertyUtils.addPluginProp("excelTestFile", "gs://" + fileSourceBucket1 + "/" + + PluginPropertyUtils.pluginProp("excelFile")); + BeforeActions.scenario.write("excel test bucket name - " + fileSourceBucket1); + } } diff --git a/core-plugins/src/e2e-test/resources/pluginParameters.properties b/core-plugins/src/e2e-test/resources/pluginParameters.properties index 397bb7fc8..0e4fbf0df 100644 --- a/core-plugins/src/e2e-test/resources/pluginParameters.properties +++ b/core-plugins/src/e2e-test/resources/pluginParameters.properties @@ -291,3 +291,12 @@ quotedValueDelimitedTestFiles=dummy fileSinkTargetBucket=file-plugin-output recursiveTest=dummy testOnCdap=true + +## EXCEL-PLUGIN-PROPERTIES-START ## +excelTestFile=dummy +excelFile=testdata/exceldata/testExcel.xlsx +Pattern=.*\.xlsx$ +outputForExcelTest=e2e-tests/file/expected_outputs/OUTPUT_FOR_EXCEL_TEST.csv +columnvalues=A +columnmappings=[{"key":"A","value":"id"}] +## EXCEL-PLUGIN-PROPERTIES-END ## diff --git a/core-plugins/src/e2e-test/resources/testdata/exceldata/testExcel.xlsx b/core-plugins/src/e2e-test/resources/testdata/exceldata/testExcel.xlsx new file mode 100644 index 000000000..e4e849a79 Binary files /dev/null and b/core-plugins/src/e2e-test/resources/testdata/exceldata/testExcel.xlsx differ diff --git a/core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_EXCEL_TEST.csv b/core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_EXCEL_TEST.csv new file mode 100644 index 000000000..51654d0b7 --- /dev/null +++ b/core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_EXCEL_TEST.csv @@ -0,0 +1,5 @@ +testExcel.xlsx,Sheet1,A +testExcel.xlsx,Sheet1,1.0 +testExcel.xlsx,Sheet1,2.0 +testExcel.xlsx,Sheet1,3.0 +testExcel.xlsx,Sheet1,4.0 \ No newline at end of file