From 6866c1d2d9ebf56137bba77fac7d94e770d53e43 Mon Sep 17 00:00:00 2001 From: Ainur Date: Thu, 5 Sep 2024 14:13:19 +0200 Subject: [PATCH] CB-5762 add simple tests for data transfer api --- .../service/WebServiceBindingBase.java | 1 + .../service/WebServiceServletBase.java | 9 +- .../service/sql/WebSQLContextInfo.java | 8 +- .../service/sql/WebServiceBindingSQL.java | 2 +- .../META-INF/MANIFEST.MF | 3 +- .../data/transfer/DBWServiceDataTransfer.java | 26 ++-- .../WebServiceBindingDataTransfer.java | 7 +- .../impl/WebDataTransferImportServlet.java | 17 ++- .../impl/WebDataTransferOutputSettings.java | 15 +- .../impl/WebDataTransferParameters.java | 42 +++--- .../transfer/impl/WebDataTransferServlet.java | 5 +- .../impl/WebDataTransferSessionConfig.java | 6 +- .../impl/WebDataTransferStreamProcessor.java | 4 +- .../impl/WebDataTransferTaskConfig.java | 17 ++- .../transfer/impl/WebDataTransferUtils.java | 26 ++-- .../transfer/impl/WebServiceDataTransfer.java | 134 ++++++++++-------- .../META-INF/MANIFEST.MF | 1 + .../test/platform/CEServerTestSuite.java | 3 +- .../test/platform/DataTransferTest.java | 125 ++++++++++++++++ 19 files changed, 321 insertions(+), 130 deletions(-) create mode 100644 server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/DataTransferTest.java diff --git a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/WebServiceBindingBase.java b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/WebServiceBindingBase.java index e6609823b2..5e04b6800d 100644 --- a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/WebServiceBindingBase.java +++ b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/WebServiceBindingBase.java @@ -102,6 +102,7 @@ protected static DBWBindingContext getBindingContext(DataFetchingEnvironment env return GraphQLEndpoint.getBindingContext(env); } + @NotNull protected static WebSession getWebSession(DataFetchingEnvironment env) throws DBWebException { return CBPlatform.getInstance().getSessionManager().getWebSession( getServletRequest(env), getServletResponse(env)); diff --git a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/WebServiceServletBase.java b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/WebServiceServletBase.java index 3768ff0617..d9924b3fb7 100644 --- a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/WebServiceServletBase.java +++ b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/WebServiceServletBase.java @@ -20,16 +20,15 @@ import com.google.gson.GsonBuilder; import io.cloudbeaver.model.app.WebApplication; import io.cloudbeaver.model.session.WebSession; -import io.cloudbeaver.server.CBApplication; import io.cloudbeaver.server.CBPlatform; -import org.jkiss.dbeaver.DBException; -import org.jkiss.dbeaver.Log; -import org.jkiss.dbeaver.model.data.json.JSONUtils; - import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServlet; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import org.jkiss.dbeaver.DBException; +import org.jkiss.dbeaver.Log; +import org.jkiss.dbeaver.model.data.json.JSONUtils; + import java.io.IOException; import java.lang.reflect.Type; import java.util.Map; diff --git a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/sql/WebSQLContextInfo.java b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/sql/WebSQLContextInfo.java index 1dc81cbad9..2dfbdb21cb 100644 --- a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/sql/WebSQLContextInfo.java +++ b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/sql/WebSQLContextInfo.java @@ -52,7 +52,9 @@ public class WebSQLContextInfo implements WebSessionProvider { private static final Log log = Log.getLog(WebSQLContextInfo.class); + @NotNull private final transient WebSQLProcessor processor; + @NotNull private final String id; private final String projectId; private final Map resultInfoMap = new HashMap<>(); @@ -60,7 +62,11 @@ public class WebSQLContextInfo implements WebSessionProvider { private final AtomicInteger resultId = new AtomicInteger(); public WebSQLContextInfo( - WebSQLProcessor processor, String id, String catalogName, String schemaName, String projectId + @NotNull WebSQLProcessor processor, + @NotNull String id, + String catalogName, + String schemaName, + String projectId ) throws DBCException { this.processor = processor; this.id = id; diff --git a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/sql/WebServiceBindingSQL.java b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/sql/WebServiceBindingSQL.java index 78065051a4..9a237e4d90 100644 --- a/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/sql/WebServiceBindingSQL.java +++ b/server/bundles/io.cloudbeaver.server/src/io/cloudbeaver/service/sql/WebServiceBindingSQL.java @@ -240,7 +240,7 @@ private WebDataFormat getDataFormat(DataFetchingEnvironment env) { } @NotNull - public static WebSQLConfiguration getSQLConfiguration(WebSession webSession) { + private static WebSQLConfiguration getSQLConfiguration(WebSession webSession) { return webSession.getAttribute("sqlConfiguration", cfg -> new WebSQLConfiguration(), WebSQLConfiguration::dispose); } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/META-INF/MANIFEST.MF b/server/bundles/io.cloudbeaver.service.data.transfer/META-INF/MANIFEST.MF index d4aec562b9..23d3340606 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/META-INF/MANIFEST.MF +++ b/server/bundles/io.cloudbeaver.service.data.transfer/META-INF/MANIFEST.MF @@ -8,7 +8,8 @@ Bundle-Release-Date: 20241021 Bundle-RequiredExecutionEnvironment: JavaSE-17 Bundle-ActivationPolicy: lazy Bundle-ClassPath: . +Export-Package: io.cloudbeaver.service.data.transfer.impl Require-Bundle: io.cloudbeaver.server, - org.jkiss.dbeaver.data.transfer, + org.jkiss.dbeaver.data.transfer;visibility:=reexport, org.jkiss.dbeaver.registry Automatic-Module-Name: io.cloudbeaver.service.data.transfer diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java index 61b80c8dd5..75b4351f93 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/DBWServiceDataTransfer.java @@ -28,6 +28,7 @@ import io.cloudbeaver.service.sql.WebSQLProcessor; import io.cloudbeaver.service.sql.WebSQLResultsInfo; import org.jkiss.code.NotNull; +import org.jkiss.code.Nullable; import java.nio.file.Path; import java.util.List; @@ -37,18 +38,22 @@ */ public interface DBWServiceDataTransfer extends DBWService { + @NotNull @WebAction - List getAvailableStreamProcessors(WebSession session) throws DBWebException; + List getAvailableStreamProcessors(@NotNull WebSession session) throws DBWebException; + @NotNull @WebAction - List getAvailableImportStreamProcessors(WebSession session) throws DBWebException; + List getAvailableImportStreamProcessors(@NotNull WebSession session) throws DBWebException; + @NotNull @WebAction WebAsyncTaskInfo dataTransferExportDataFromContainer( - WebSQLProcessor sqlProcessor, - String containerNodePath, - WebDataTransferParameters parameters) throws DBWebException; + @NotNull WebSQLProcessor sqlProcessor, + @NotNull String containerNodePath, + @NotNull WebDataTransferParameters parameters) throws DBWebException; + @NotNull @WebAction WebAsyncTaskInfo asyncImportDataContainer( @NotNull String processorId, @@ -56,14 +61,17 @@ WebAsyncTaskInfo asyncImportDataContainer( @NotNull WebSQLResultsInfo webSQLResultsInfo, @NotNull WebSession webSession) throws DBWebException; + @NotNull @WebAction WebAsyncTaskInfo dataTransferExportDataFromResults( - WebSQLContextInfo sqlContextInfo, - String resultsId, - WebDataTransferParameters parameters) throws DBWebException; + @NotNull WebSQLContextInfo sqlContextInfo, + @NotNull String resultsId, + @NotNull WebDataTransferParameters parameters) throws DBWebException; + @Nullable @WebAction - Boolean dataTransferRemoveDataFile(WebSession session, String dataFileId) throws DBWebException; + Boolean dataTransferRemoveDataFile(@NotNull WebSession session, @NotNull String dataFileId) throws DBWebException; + @NotNull WebDataTransferDefaultExportSettings defaultExportSettings(); } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java index 10151d5467..941610b51c 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/WebServiceBindingDataTransfer.java @@ -21,17 +21,18 @@ import io.cloudbeaver.service.DBWServiceBindingServlet; import io.cloudbeaver.service.DBWServletContext; import io.cloudbeaver.service.WebServiceBindingBase; +import io.cloudbeaver.service.data.transfer.impl.WebDataTransferImportServlet; import io.cloudbeaver.service.data.transfer.impl.WebDataTransferParameters; import io.cloudbeaver.service.data.transfer.impl.WebDataTransferServlet; import io.cloudbeaver.service.data.transfer.impl.WebServiceDataTransfer; -import io.cloudbeaver.service.data.transfer.impl.WebDataTransferImportServlet; import io.cloudbeaver.service.sql.WebServiceBindingSQL; import org.jkiss.dbeaver.DBException; /** * Web service implementation */ -public class WebServiceBindingDataTransfer extends WebServiceBindingBase implements DBWServiceBindingServlet { +public class WebServiceBindingDataTransfer extends WebServiceBindingBase + implements DBWServiceBindingServlet> { public WebServiceBindingDataTransfer() { super(DBWServiceDataTransfer.class, new WebServiceDataTransfer(), "schema/service.data.transfer.graphqls"); @@ -65,7 +66,7 @@ public void bindWiring(DBWBindingContext model) { } @Override - public void addServlets(CBApplication application, DBWServletContext servletContext) throws DBException { + public void addServlets(CBApplication application, DBWServletContext servletContext) throws DBException { if (!application.isMultiuser()) { return; } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferImportServlet.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferImportServlet.java index f645d07404..b6bfc3ef0d 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferImportServlet.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferImportServlet.java @@ -35,6 +35,7 @@ import jakarta.servlet.annotation.MultipartConfig; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import org.jkiss.code.NotNull; import org.jkiss.dbeaver.Log; import org.jkiss.dbeaver.model.data.json.JSONUtils; @@ -50,21 +51,19 @@ public class WebDataTransferImportServlet extends WebServiceServletBase { private static final Log log = Log.getLog(WebDataTransferImportServlet.class); - public static final String ECLIPSE_JETTY_MULTIPART_CONFIG = "org.eclipse.jetty.multipartConfig"; + private static final String ECLIPSE_JETTY_MULTIPART_CONFIG = "org.eclipse.jetty.multipartConfig"; + private final DBWServiceDataTransfer dbwServiceDataTransfer; - DBWServiceDataTransfer dbwServiceDataTransfer; - - - public WebDataTransferImportServlet(CBApplication application, DBWServiceDataTransfer dbwServiceDataTransfer) { + public WebDataTransferImportServlet(@NotNull CBApplication application, @NotNull DBWServiceDataTransfer dbwServiceDataTransfer) { super(application); this.dbwServiceDataTransfer = dbwServiceDataTransfer; } @Override protected void processServiceRequest( - WebSession session, - HttpServletRequest request, - HttpServletResponse response + WebSession session, + HttpServletRequest request, + HttpServletResponse response ) throws IOException, DBWebException { if (!session.isAuthorizedInSecurityManager()) { response.sendError(HttpServletResponse.SC_FORBIDDEN, "Import for users only"); @@ -108,7 +107,7 @@ protected void processServiceRequest( } WebAsyncTaskInfo asyncImportDataContainer = - dbwServiceDataTransfer.asyncImportDataContainer(processorId, filePath, webSQLResultsInfo, session); + dbwServiceDataTransfer.asyncImportDataContainer(processorId, filePath, webSQLResultsInfo, session); response.setContentType(CBConstants.APPLICATION_JSON); Map parameters = new LinkedHashMap<>(); parameters.put("id", asyncImportDataContainer.getId()); diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferOutputSettings.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferOutputSettings.java index 7c288644b6..7cf9863d07 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferOutputSettings.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferOutputSettings.java @@ -16,15 +16,19 @@ */ package io.cloudbeaver.service.data.transfer.impl; +import org.jkiss.code.Nullable; import org.jkiss.dbeaver.model.data.json.JSONUtils; import java.util.Map; public class WebDataTransferOutputSettings { private final boolean insertBom; + @Nullable private final String encoding; + @Nullable private final String timestampPattern; private final boolean compress; + @Nullable private final String fileName; public WebDataTransferOutputSettings(Map outputSettings) { @@ -35,7 +39,13 @@ public WebDataTransferOutputSettings(Map outputSettings) { this.fileName = JSONUtils.getString(outputSettings, "fileName"); } - public WebDataTransferOutputSettings(boolean insertBom, String encoding, String timestampPattern, boolean compress, String fileName) { + public WebDataTransferOutputSettings( + boolean insertBom, + @Nullable String encoding, + @Nullable String timestampPattern, + boolean compress, + @Nullable String fileName + ) { this.insertBom = insertBom; this.encoding = encoding; this.timestampPattern = timestampPattern; @@ -47,10 +57,12 @@ public boolean isInsertBom() { return insertBom; } + @Nullable public String getEncoding() { return encoding; } + @Nullable public String getTimestampPattern() { return timestampPattern; } @@ -59,6 +71,7 @@ public boolean isCompress() { return compress; } + @Nullable public String getFileName() { return fileName; } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferParameters.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferParameters.java index ab35121bc8..d831e49d1d 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferParameters.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferParameters.java @@ -17,61 +17,53 @@ package io.cloudbeaver.service.data.transfer.impl; import io.cloudbeaver.service.sql.WebSQLDataFilter; +import org.jkiss.code.NotNull; import org.jkiss.dbeaver.model.data.json.JSONUtils; import java.util.Map; public class WebDataTransferParameters { - private String processorId; - private Map dbProducerSettings; - private Map processorProperties; - private WebSQLDataFilter filter; - private WebDataTransferOutputSettings outputSettings; - - public WebDataTransferParameters() { - } + @NotNull + private final String processorId; + @NotNull + private final Map dbProducerSettings; + @NotNull + private final Map processorProperties; + @NotNull + private final WebSQLDataFilter filter; + @NotNull + private final WebDataTransferOutputSettings outputSettings; public WebDataTransferParameters(Map params) { - this.processorId = JSONUtils.getString(params, "processorId"); + this.processorId = JSONUtils.getString(params, "processorId", ""); this.dbProducerSettings = JSONUtils.getObject(params, "settings"); this.processorProperties = JSONUtils.getObject(params, "processorProperties"); this.filter = new WebSQLDataFilter(JSONUtils.getObject(params, "filter")); this.outputSettings = new WebDataTransferOutputSettings(JSONUtils.getObject(params, "outputSettings")); } + @NotNull public String getProcessorId() { return processorId; } - public void setProcessorId(String processorId) { - this.processorId = processorId; - } - + @NotNull public Map getDbProducerSettings() { return dbProducerSettings; } - public void setDbProducerSettings(Map dbProducerSettings) { - this.dbProducerSettings = dbProducerSettings; - } - + @NotNull public Map getProcessorProperties() { return processorProperties; } - public void setProcessorProperties(Map processorProperties) { - this.processorProperties = processorProperties; - } - + @NotNull public WebSQLDataFilter getFilter() { return filter; } - public void setFilter(WebSQLDataFilter filter) { - this.filter = filter; - } - + @NotNull public WebDataTransferOutputSettings getOutputSettings() { return outputSettings; } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferServlet.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferServlet.java index 073cbc38a8..dc9c4e12a8 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferServlet.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferServlet.java @@ -23,6 +23,7 @@ import io.cloudbeaver.service.data.transfer.DBWServiceDataTransfer; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import org.jkiss.code.NotNull; import org.jkiss.dbeaver.DBException; import org.jkiss.dbeaver.Log; import org.jkiss.dbeaver.tools.transfer.registry.DataTransferProcessorDescriptor; @@ -34,15 +35,15 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Map; public class WebDataTransferServlet extends WebServiceServletBase { private static final Log log = Log.getLog(WebDataTransferServlet.class); + @NotNull private final DBWServiceDataTransfer dtManager; - public WebDataTransferServlet(CBApplication application, DBWServiceDataTransfer dtManager) { + public WebDataTransferServlet(@NotNull CBApplication application, @NotNull DBWServiceDataTransfer dtManager) { super(application); this.dtManager = dtManager; } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferSessionConfig.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferSessionConfig.java index 9575c8d517..517ec8a24b 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferSessionConfig.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferSessionConfig.java @@ -16,6 +16,9 @@ */ package io.cloudbeaver.service.data.transfer.impl; +import org.jkiss.code.NotNull; +import org.jkiss.code.Nullable; + import java.util.HashMap; import java.util.Map; @@ -26,7 +29,8 @@ public class WebDataTransferSessionConfig { public WebDataTransferSessionConfig() { } - public WebDataTransferTaskConfig getTask(String dataFileId) { + @Nullable + public WebDataTransferTaskConfig getTask(@NotNull String dataFileId) { return tasks.get(dataFileId); } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferStreamProcessor.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferStreamProcessor.java index 64a0c62cb8..f1dcfeb034 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferStreamProcessor.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferStreamProcessor.java @@ -28,10 +28,12 @@ public class WebDataTransferStreamProcessor { + @NotNull private final WebSession session; + @NotNull private final DataTransferProcessorDescriptor processor; - public WebDataTransferStreamProcessor(WebSession session, DataTransferProcessorDescriptor processor) { + public WebDataTransferStreamProcessor(@NotNull WebSession session, @NotNull DataTransferProcessorDescriptor processor) { this.session = session; this.processor = processor; } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferTaskConfig.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferTaskConfig.java index 6a550245c1..1c5e0b5b4e 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferTaskConfig.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferTaskConfig.java @@ -16,6 +16,8 @@ */ package io.cloudbeaver.service.data.transfer.impl; +import org.jkiss.code.NotNull; +import org.jkiss.code.Nullable; import org.jkiss.dbeaver.Log; import java.io.IOException; @@ -26,32 +28,39 @@ public class WebDataTransferTaskConfig { private static final Log log = Log.getLog(WebDataTransferTaskConfig.class); - private Path dataFile; - private WebDataTransferParameters parameters; + @NotNull + private final Path dataFile; + @NotNull + private final WebDataTransferParameters parameters; + @Nullable private String exportFileName; - public WebDataTransferTaskConfig(Path dataFile, WebDataTransferParameters parameters) { + public WebDataTransferTaskConfig(@NotNull Path dataFile, @NotNull WebDataTransferParameters parameters) { this.dataFile = dataFile; this.parameters = parameters; } + @NotNull public Path getDataFile() { return dataFile; } + @NotNull public String getDataFileId() { return dataFile.getFileName().toString(); } + @NotNull public WebDataTransferParameters getParameters() { return parameters; } + @Nullable public String getExportFileName() { return exportFileName; } - public void setExportFileName(String exportFileName) { + public void setExportFileName(@NotNull String exportFileName) { this.exportFileName = exportFileName; } diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferUtils.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferUtils.java index d05129736a..2189a17082 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferUtils.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebDataTransferUtils.java @@ -28,23 +28,28 @@ class WebDataTransferUtils { private static final Log log = Log.getLog(WebDataTransferUtils.class); - public static final String EXTENSION = "extension"; + private static final String EXTENSION = "extension"; - - public static String getProcessorFileExtension(DataTransferProcessorDescriptor processor) { - DBPPropertyDescriptor extProperty = processor.getProperty("extension"); + @NotNull + public static String getProcessorFileExtension(@NotNull DataTransferProcessorDescriptor processor) { + DBPPropertyDescriptor extProperty = processor.getProperty(EXTENSION); String ext = extProperty == null ? processor.getAppFileExtension() : CommonUtils.toString(extProperty.getDefaultValue(), null); return CommonUtils.isEmpty(ext) ? "data" : ext; } - public static String getProcessorFileExtension(DataTransferProcessorDescriptor processor, Map processorProperties) { - if (processorProperties != null && processorProperties.get(EXTENSION) != null) { + @NotNull + public static String getProcessorFileExtension( + @NotNull DataTransferProcessorDescriptor processor, + @NotNull Map processorProperties + ) { + if (processorProperties.get(EXTENSION) != null) { return CommonUtils.toString(processorProperties.get(EXTENSION), "data"); } return getProcessorFileExtension(processor); } + @NotNull public static String normalizeFileName( @NotNull String fileName, @NotNull WebDataTransferOutputSettings outputSettings @@ -52,7 +57,12 @@ public static String normalizeFileName( return outputSettings.isCompress() ? fileName + ".zip" : fileName; } - public static WebDataTransferSessionConfig getSessionDataTransferConfig(WebSession session) { - return session.getAttribute("dataTransfer", x -> new WebDataTransferSessionConfig(), WebDataTransferSessionConfig::deleteExportFiles); + @NotNull + public static WebDataTransferSessionConfig getSessionDataTransferConfig(@NotNull WebSession session) { + return session.getAttribute( + "dataTransfer", + x -> new WebDataTransferSessionConfig(), + WebDataTransferSessionConfig::deleteExportFiles + ); } } \ No newline at end of file diff --git a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java index 6367944b53..836fd15690 100644 --- a/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java +++ b/server/bundles/io.cloudbeaver.service.data.transfer/src/io/cloudbeaver/service/data/transfer/impl/WebServiceDataTransfer.java @@ -37,12 +37,12 @@ import org.jkiss.dbeaver.model.runtime.DBRProgressMonitor; import org.jkiss.dbeaver.model.runtime.VoidProgressMonitor; import org.jkiss.dbeaver.model.sql.DBQuotaException; -import org.jkiss.dbeaver.model.sql.DBSQLException; import org.jkiss.dbeaver.model.struct.DBSDataContainer; import org.jkiss.dbeaver.model.struct.DBSDataManipulator; import org.jkiss.dbeaver.model.struct.DBSEntity; import org.jkiss.dbeaver.model.struct.DBSObjectContainer; import org.jkiss.dbeaver.tools.transfer.IDataTransferConsumer; +import org.jkiss.dbeaver.tools.transfer.IDataTransferNode; import org.jkiss.dbeaver.tools.transfer.IDataTransferProcessor; import org.jkiss.dbeaver.tools.transfer.database.*; import org.jkiss.dbeaver.tools.transfer.registry.DataTransferProcessorDescriptor; @@ -51,15 +51,11 @@ import org.jkiss.dbeaver.utils.ContentUtils; import org.jkiss.utils.CommonUtils; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.BatchUpdateException; import java.util.*; -import java.util.stream.Collectors; /** * Web service implementation @@ -83,37 +79,46 @@ public WebServiceDataTransfer() { } } + @NotNull @Override - public List getAvailableStreamProcessors(WebSession session) { - List processors = DataTransferRegistry.getInstance() - .getAvailableProcessors(StreamTransferConsumer.class, DBSEntity.class); - if (CommonUtils.isEmpty(processors)) { - return Collections.emptyList(); - } - - return processors.stream().map(x -> new WebDataTransferStreamProcessor(session, x)).collect(Collectors.toList()); + public List getAvailableStreamProcessors(@NotNull WebSession session) { + return getStreamProcessors(session, StreamTransferConsumer.class); } + @NotNull @Override - public List getAvailableImportStreamProcessors(WebSession session) { - List processors = - DataTransferRegistry.getInstance().getAvailableProcessors(StreamTransferProducer.class, DBSEntity.class); + public List getAvailableImportStreamProcessors(@NotNull WebSession session) { + return getStreamProcessors(session, StreamTransferProducer.class); + } + + @NotNull + private List getStreamProcessors( + @NotNull WebSession session, + @NotNull Class> nodeType + ) { + List processors = DataTransferRegistry.getInstance() + .getAvailableProcessors(nodeType, DBSEntity.class); if (CommonUtils.isEmpty(processors)) { return Collections.emptyList(); } - - return processors.stream().map(x -> new WebDataTransferStreamProcessor(session, x)).collect(Collectors.toList()); + return processors.stream().map(x -> new WebDataTransferStreamProcessor(session, x)).toList(); } + @NotNull @Override public WebAsyncTaskInfo dataTransferExportDataFromContainer( - WebSQLProcessor sqlProcessor, - String containerNodePath, - WebDataTransferParameters parameters) throws DBWebException { + @NotNull WebSQLProcessor sqlProcessor, + @NotNull String containerNodePath, + @NotNull WebDataTransferParameters parameters + ) throws DBWebException { DBSDataContainer dataContainer; try { - dataContainer = sqlProcessor.getDataContainerByNodePath(sqlProcessor.getWebSession().getProgressMonitor(), containerNodePath, DBSDataContainer.class); + dataContainer = sqlProcessor.getDataContainerByNodePath( + sqlProcessor.getWebSession().getProgressMonitor(), + containerNodePath, + DBSDataContainer.class + ); } catch (DBException e) { throw new DBWebException("Invalid node path: " + containerNodePath, e); } @@ -123,30 +128,36 @@ public WebAsyncTaskInfo dataTransferExportDataFromContainer( @NotNull private String makeUniqueFileName( - WebSQLProcessor sqlProcessor, - DataTransferProcessorDescriptor processor, - Map processorProperties + @NotNull WebSQLProcessor sqlProcessor, + @NotNull DataTransferProcessorDescriptor processor, + @NotNull Map processorProperties ) { - if (processorProperties != null && processorProperties.get(StreamConsumerSettings.PROP_FILE_EXTENSION) != null) { - return sqlProcessor.getWebSession().getSessionId() + "_" + UUID.randomUUID() + - "." + processorProperties.get(StreamConsumerSettings.PROP_FILE_EXTENSION); + StringBuilder builder = new StringBuilder() + .append(sqlProcessor.getWebSession().getSessionId()) + .append("_") + .append(UUID.randomUUID()) + .append("."); + if (processorProperties.get(StreamConsumerSettings.PROP_FILE_EXTENSION) != null) { + return builder.append(processorProperties.get(StreamConsumerSettings.PROP_FILE_EXTENSION)).toString(); } - return sqlProcessor.getWebSession().getSessionId() + "_" + UUID.randomUUID() + "." + WebDataTransferUtils.getProcessorFileExtension(processor); + return builder.append(WebDataTransferUtils.getProcessorFileExtension(processor)).toString(); } + @NotNull @Override public WebAsyncTaskInfo dataTransferExportDataFromResults( - WebSQLContextInfo sqlContext, - String resultsId, - WebDataTransferParameters parameters) throws DBWebException { + @NotNull WebSQLContextInfo sqlContext, + @NotNull String resultsId, + @NotNull WebDataTransferParameters parameters) throws DBWebException { WebSQLResultsInfo results = sqlContext.getResults(resultsId); return asyncExportFromDataContainer(sqlContext.getProcessor(), parameters, results.getDataContainer(), results); } + @Nullable @Override - public Boolean dataTransferRemoveDataFile(WebSession webSession, String dataFileId) throws DBWebException { + public Boolean dataTransferRemoveDataFile(@NotNull WebSession webSession, @NotNull String dataFileId) throws DBWebException { WebDataTransferSessionConfig dtConfig = WebDataTransferUtils.getSessionDataTransferConfig(webSession); WebDataTransferTaskConfig taskInfo = dtConfig.getTask(dataFileId); if (taskInfo == null) { @@ -165,23 +176,28 @@ public Boolean dataTransferRemoveDataFile(WebSession webSession, String dataFile return true; } + @NotNull @Override public WebDataTransferDefaultExportSettings defaultExportSettings() { return new WebDataTransferDefaultExportSettings(); } - private WebAsyncTaskInfo asyncExportFromDataContainer(WebSQLProcessor sqlProcessor, WebDataTransferParameters parameters, DBSDataContainer dataContainer, - @Nullable WebSQLResultsInfo resultsInfo) { + private WebAsyncTaskInfo asyncExportFromDataContainer( + @NotNull WebSQLProcessor sqlProcessor, + @NotNull WebDataTransferParameters parameters, + @NotNull DBSDataContainer dataContainer, + @Nullable WebSQLResultsInfo resultsInfo + ) { sqlProcessor.getWebSession().addInfoMessage("Export data"); DataTransferProcessorDescriptor processor = DataTransferRegistry.getInstance().getProcessor(parameters.getProcessorId()); - WebAsyncTaskProcessor runnable = new WebAsyncTaskProcessor() { + WebAsyncTaskProcessor runnable = new WebAsyncTaskProcessor<>() { @Override public void run(DBRProgressMonitor monitor) throws InvocationTargetException { monitor.beginTask("Export data", 1); try { monitor.subTask("Export data using " + processor.getName()); Path exportFile = dataExportFolder.resolve( - makeUniqueFileName(sqlProcessor, processor, parameters.getProcessorProperties())); + makeUniqueFileName(sqlProcessor, processor, parameters.getProcessorProperties())); try { exportData(monitor, processor, dataContainer, parameters, resultsInfo, exportFile); } catch (Exception e) { @@ -200,7 +216,7 @@ public void run(DBRProgressMonitor monitor) throws InvocationTargetException { var outputSettings = parameters.getOutputSettings(); Path finallyExportFile = outputSettings.isCompress() ? exportFile.resolveSibling(WebDataTransferUtils.normalizeFileName( - exportFile.getFileName().toString(), outputSettings)) + exportFile.getFileName().toString(), outputSettings)) : exportFile; WebDataTransferTaskConfig taskConfig = new WebDataTransferTaskConfig(finallyExportFile, parameters); String exportFileName = CommonUtils.isEmpty(outputSettings.getFileName()) ? @@ -220,10 +236,13 @@ public void run(DBRProgressMonitor monitor) throws InvocationTargetException { return sqlProcessor.getWebSession().createAndRunAsyncTask("Data export", runnable); } - public WebAsyncTaskInfo asyncImportDataContainer(@NotNull String processorId, - @NotNull Path path, - @NotNull WebSQLResultsInfo sqlContext, - @NotNull WebSession webSession) throws DBWebException { + @NotNull + public WebAsyncTaskInfo asyncImportDataContainer( + @NotNull String processorId, + @NotNull Path path, + @NotNull WebSQLResultsInfo sqlContext, + @NotNull WebSession webSession + ) throws DBWebException { webSession.addInfoMessage("Import data"); DataTransferProcessorDescriptor processor = DataTransferRegistry.getInstance().getProcessor(processorId); @@ -258,18 +277,16 @@ public void run(DBRProgressMonitor monitor) throws InvocationTargetException { } private void exportData( - DBRProgressMonitor monitor, - DataTransferProcessorDescriptor processor, + @NotNull DBRProgressMonitor monitor, + @NotNull DataTransferProcessorDescriptor processor, DBSDataContainer dataContainer, WebDataTransferParameters parameters, WebSQLResultsInfo resultsInfo, - Path exportFile) throws DBException, IOException - { + Path exportFile) throws DBException { IDataTransferProcessor processorInstance = processor.getInstance(); - if (!(processorInstance instanceof IStreamDataExporter)) { + if (!(processorInstance instanceof IStreamDataExporter exporter)) { throw new DBException("Invalid processor. " + IStreamDataExporter.class.getSimpleName() + " expected"); } - IStreamDataExporter exporter = (IStreamDataExporter) processorInstance; Number fileSizeLimit = CBApplication.getInstance().getAppConfiguration().getResourceQuota(QUOTA_PROP_FILE_LIMIT); @@ -304,7 +321,6 @@ public void fetchRow(@NotNull DBCSession session, @NotNull DBCResultSet resultSe Map properties = new HashMap<>(); Map processorProperties = parameters.getProcessorProperties(); - if (processorProperties == null) processorProperties = Collections.emptyMap(); for (DBPPropertyDescriptor prop : processor.getProperties()) { Object propValue = processorProperties.get(CommonUtils.toString(prop.getId())); properties.put(prop.getId(), propValue != null ? propValue : prop.getDefaultValue()); @@ -315,7 +331,8 @@ public void fetchRow(@NotNull DBCSession session, @NotNull DBCResultSet resultSe DatabaseTransferProducer producer = new DatabaseTransferProducer( dataContainer, - parameters.getFilter() == null ? null : parameters.getFilter().makeDataFilter(resultsInfo)); + parameters.getFilter().makeDataFilter(resultsInfo) + ); DatabaseProducerSettings producerSettings = new DatabaseProducerSettings(); producerSettings.setExtractType(DatabaseProducerSettings.ExtractType.SINGLE_QUERY); producerSettings.setQueryRowCount(false); @@ -335,10 +352,11 @@ public void fetchRow(@NotNull DBCSession session, @NotNull DBCResultSet resultSe } private void importData( - DBRProgressMonitor monitor, - DataTransferProcessorDescriptor processor, - @NotNull DBSDataManipulator dataContainer, - Path path) throws DBException { + @NotNull DBRProgressMonitor monitor, + @NotNull DataTransferProcessorDescriptor processor, + @NotNull DBSDataManipulator dataContainer, + @NotNull Path path + ) throws DBException { IDataTransferProcessor processorInstance = processor.getInstance(); StreamTransferProducer producer; @@ -357,10 +375,10 @@ private void importData( } producerSettings.setProcessorProperties(properties); producerSettings.updateProducerSettingsFromStream( - monitor, - producer, - processorInstance, - properties); + monitor, + producer, + processorInstance, + properties); DatabaseMappingContainer databaseMappingContainer = new DatabaseMappingContainer(databaseConsumerSettings, producer.getDatabaseObject()); databaseMappingContainer.getAttributeMappings(monitor); diff --git a/server/test/io.cloudbeaver.test.platform/META-INF/MANIFEST.MF b/server/test/io.cloudbeaver.test.platform/META-INF/MANIFEST.MF index 29e094999e..2c61dcd84a 100644 --- a/server/test/io.cloudbeaver.test.platform/META-INF/MANIFEST.MF +++ b/server/test/io.cloudbeaver.test.platform/META-INF/MANIFEST.MF @@ -23,6 +23,7 @@ Require-Bundle: org.eclipse.core.runtime, io.cloudbeaver.resources.drivers.base, io.cloudbeaver.product.ce, io.cloudbeaver.service.auth, + io.cloudbeaver.service.data.transfer, io.cloudbeaver.service.rm, io.cloudbeaver.service.rm.nio, org.jkiss.dbeaver.ext.mysql, diff --git a/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/CEServerTestSuite.java b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/CEServerTestSuite.java index 3ca615b5ea..571e79d021 100644 --- a/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/CEServerTestSuite.java +++ b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/CEServerTestSuite.java @@ -43,7 +43,8 @@ AuthenticationTest.class, ResourceManagerTest.class, RMLockTest.class, - RMNIOTest.class + RMNIOTest.class, + DataTransferTest.class } ) public class CEServerTestSuite { diff --git a/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/DataTransferTest.java b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/DataTransferTest.java new file mode 100644 index 0000000000..8ff5e04fc9 --- /dev/null +++ b/server/test/io.cloudbeaver.test.platform/src/io/cloudbeaver/test/platform/DataTransferTest.java @@ -0,0 +1,125 @@ +/* + * DBeaver - Universal Database Manager + * Copyright (C) 2010-2024 DBeaver Corp and others + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.cloudbeaver.test.platform; + +import io.cloudbeaver.server.CBApplication; +import io.cloudbeaver.service.data.transfer.impl.WebDataTransferDefaultExportSettings; +import io.cloudbeaver.service.data.transfer.impl.WebDataTransferOutputSettings; +import io.cloudbeaver.test.WebGQLClient; +import org.jkiss.code.NotNull; +import org.jkiss.dbeaver.model.auth.SMAuthStatus; +import org.jkiss.dbeaver.model.data.json.JSONUtils; +import org.jkiss.dbeaver.model.struct.DBSEntity; +import org.jkiss.dbeaver.tools.transfer.IDataTransferNode; +import org.jkiss.dbeaver.tools.transfer.registry.DataTransferProcessorDescriptor; +import org.jkiss.dbeaver.tools.transfer.registry.DataTransferRegistry; +import org.jkiss.dbeaver.tools.transfer.stream.StreamTransferConsumer; +import org.jkiss.dbeaver.tools.transfer.stream.StreamTransferProducer; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Data transfer test. + */ +public class DataTransferTest { + private static WebGQLClient client; + private static final String GQL_TRANSFER_STREAM_PROCESSORS = """ + query dataTransferAvailableStreamProcessors { + result: dataTransferAvailableStreamProcessors { + id + name + } + }"""; + private static final String GQL_TRANSFER_IMPORT_STREAM_PROCESSORS = """ + query dataTransferAvailableImportStreamProcessors { + result: dataTransferAvailableImportStreamProcessors { + id + name + } + }"""; + private static final String GQL_TRANSFER_DEFAULT_SETTINGS = """ + query dataTransferDefaultExportSettings { + result: dataTransferDefaultExportSettings { + outputSettings { + insertBom + encoding + timestampPattern + compress + } + supportedEncodings + } + }"""; + + @BeforeClass + public static void init() throws Exception { + Assert.assertTrue(CBApplication.getInstance().getAppConfiguration().isResourceManagerEnabled()); + client = CEServerTestSuite.createClient(); + Map authInfo = CEServerTestSuite.authenticateTestUser(client); + Assert.assertEquals(SMAuthStatus.SUCCESS.name(), JSONUtils.getString(authInfo, "authStatus")); + } + + @Test + public void availableStreamProcessors() throws Exception { + checkStreamProcessors(GQL_TRANSFER_STREAM_PROCESSORS, StreamTransferConsumer.class); + checkStreamProcessors(GQL_TRANSFER_IMPORT_STREAM_PROCESSORS, StreamTransferProducer.class); + } + + @Test + public void checkDefaultSettings() throws Exception { + WebDataTransferDefaultExportSettings defaultSettings = new WebDataTransferDefaultExportSettings(); + Map result = client.sendQuery(GQL_TRANSFER_DEFAULT_SETTINGS, null); + + WebDataTransferOutputSettings responseOutputSettings = new WebDataTransferOutputSettings( + JSONUtils.getObject(result, "outputSettings") + ); + WebDataTransferOutputSettings defaultOutputSettings = defaultSettings.getOutputSettings(); + Assert.assertEquals(defaultOutputSettings.getTimestampPattern(), responseOutputSettings.getTimestampPattern()); + Assert.assertEquals(defaultOutputSettings.isCompress(), responseOutputSettings.isCompress()); + Assert.assertEquals(defaultOutputSettings.getEncoding(), responseOutputSettings.getEncoding()); + Assert.assertEquals(defaultOutputSettings.isInsertBom(), responseOutputSettings.isInsertBom()); + + Set responseSupportedEncodings = new HashSet<>(JSONUtils.getStringList(result, "supportedEncodings")); + Assert.assertEquals(defaultSettings.getSupportedEncodings(), responseSupportedEncodings); + } + + private void checkStreamProcessors( + @NotNull String gqlScript, + @NotNull Class> nodeType + ) throws Exception { + List> result = client.sendQuery(gqlScript, null); + // get set of processor ids + Set processorIds = result.stream() + .map(p -> JSONUtils.getString(p, "id")) + .collect(Collectors.toSet()); + List processors = DataTransferRegistry.getInstance() + .getAvailableProcessors(nodeType, DBSEntity.class); + assert processors != null; + Assert.assertEquals(result.size(), processors.size()); + for (DataTransferProcessorDescriptor processor : processors) { + Assert.assertTrue(processorIds.contains(processor.getFullId())); + } + } + + +}