From e7b453cd63c6bde955b543cc17da5018d3d68708 Mon Sep 17 00:00:00 2001 From: mozhenghua Date: Fri, 11 Oct 2024 14:33:43 +0800 Subject: [PATCH] refactor create DDL process --- .../plugin/datax/DataXClickhouseWriter.java | 14 +- .../dameng/writer/DataXDaMengWriter.java | 6 +- .../plugin/datax/doris/BasicDorisWriter.java | 21 +-- .../plugin/datax/doris/DataXDorisWriter.java | 2 +- .../tis/plugin/datax/DataXOdpsWriter.java | 4 +- .../tis/plugin/datax/DataXOracleWriter.java | 6 +- .../plugin/datax/DataXPostgresqlWriter.java | 130 +------------- .../PostgreSQLCreateTableSqlBuilder.java | 158 ++++++++++++++++++ .../datax/TestDataXPostgresqlWriter.java | 4 +- .../tis/plugin/datax/multi-pks-create-ddl.txt | 12 +- .../plugin/datax/single-pks-create-ddl.txt | 12 +- .../plugin/datax/DataXSqlserverWriter.java | 15 +- .../plugin/datax/BasicStarRocksWriter.java | 25 +-- .../tis/plugin/common/WriterTemplate.java | 13 +- .../tis/plugin/datax/DataxMySQLWriter.java | 6 +- .../tis/plugin/datax/DataXHiveWriter.java | 4 +- 16 files changed, 240 insertions(+), 192 deletions(-) create mode 100644 tis-datax/tis-datax-postgresql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/PostgreSQLCreateTableSqlBuilder.java diff --git a/tis-datax/tis-datax-clickhouse-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXClickhouseWriter.java b/tis-datax/tis-datax-clickhouse-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXClickhouseWriter.java index 42c757a55..9ad0cd239 100644 --- a/tis-datax/tis-datax-clickhouse-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXClickhouseWriter.java +++ b/tis-datax/tis-datax-clickhouse-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXClickhouseWriter.java @@ -32,7 +32,7 @@ import com.qlangtech.tis.plugin.datax.common.BasicDataXRdbmsWriter; import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules; import com.qlangtech.tis.plugin.ds.CMeta; - + import com.qlangtech.tis.plugin.ds.DataType; import com.qlangtech.tis.plugin.ds.IColMetaGetter; import com.qlangtech.tis.plugin.ds.clickhouse.ClickHouseDataSourceFactory; @@ -40,6 +40,7 @@ import org.apache.commons.lang.StringUtils; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -86,10 +87,10 @@ protected void appendExtraColDef(List pks) { @Override protected ColWrapper createColWrapper(IColMetaGetter c) { - return new ColWrapper(c) { + return new ColWrapper(c, this.pks) { @Override public String getMapperType() { - return convertType(this.meta); + return convertType(this.getType()); } }; } @@ -103,9 +104,10 @@ protected void appendTabMeta(List pk) { script.append(" SETTINGS index_granularity = 8192"); } - private String convertType(IColMetaGetter col) { - DataType type = col.getType(); - switch (type.getJdbcType()) { + private String convertType(DataType type) { + + switch (Objects.requireNonNull(type, "type can not be null") + .getJdbcType()) { case INTEGER: case TINYINT: case SMALLINT: diff --git a/tis-datax/tis-datax-dameng-plugin/src/main/java/com/qlangtech/tis/plugin/datax/dameng/writer/DataXDaMengWriter.java b/tis-datax/tis-datax-dameng-plugin/src/main/java/com/qlangtech/tis/plugin/datax/dameng/writer/DataXDaMengWriter.java index 0d5e312f3..630eeb066 100644 --- a/tis-datax/tis-datax-dameng-plugin/src/main/java/com/qlangtech/tis/plugin/datax/dameng/writer/DataXDaMengWriter.java +++ b/tis-datax/tis-datax-dameng-plugin/src/main/java/com/qlangtech/tis/plugin/datax/dameng/writer/DataXDaMengWriter.java @@ -169,10 +169,10 @@ protected void appendTabMeta(List pks) { @Override protected ColWrapper createColWrapper(IColMetaGetter c) { - return new ColWrapper(c) { + return new ColWrapper(c,this.pks) { @Override public String getMapperType() { - return convertType(this.meta); + return convertType(this); } }; } @@ -183,7 +183,7 @@ public String getMapperType() { * @param col * @return */ - private String convertType(IColMetaGetter col) { + private String convertType(ColWrapper col) { DataType type = col.getType(); switch (type.getJdbcType()) { case CHAR: { diff --git a/tis-datax/tis-datax-doris-plugin/src/main/java/com/qlangtech/tis/plugin/datax/doris/BasicDorisWriter.java b/tis-datax/tis-datax-doris-plugin/src/main/java/com/qlangtech/tis/plugin/datax/doris/BasicDorisWriter.java index c283b7aec..3a4f986e5 100644 --- a/tis-datax/tis-datax-doris-plugin/src/main/java/com/qlangtech/tis/plugin/datax/doris/BasicDorisWriter.java +++ b/tis-datax/tis-datax-doris-plugin/src/main/java/com/qlangtech/tis/plugin/datax/doris/BasicDorisWriter.java @@ -107,8 +107,8 @@ protected static class DorisColWrapper extends ColWrapper { protected DorisType dorisType; private final BasicCreateTableSqlBuilder sqlBuilder; - public DorisColWrapper(IColMetaGetter meta, BasicCreateTableSqlBuilder sqlBuilder) { - super(meta); + public DorisColWrapper(IColMetaGetter meta, List pks, BasicCreateTableSqlBuilder sqlBuilder) { + super(meta, pks); this.sqlBuilder = sqlBuilder; this.dorisType = convertType(meta); } @@ -120,7 +120,7 @@ public final String getMapperType() { @Override protected final void appendExtraConstraint(BlockScriptBuffer ddlScript) { - if (sqlBuilder.isPK(this.meta.getName())) { + if (sqlBuilder.isPK(this.getName())) { ddlScript.append(" NOT NULL"); } } @@ -154,10 +154,10 @@ public BasicCreateTableSqlBuilder(IDataxProcessor.TableMap tableMapper this.dorisTab = tableMapper.getSourceTab(); this.primaryKeys = Lists.newArrayList(); for (String pk : this.dorisTab.getPrimaryKeys()) { - for (IColMetaGetter c : this.getCols()) { + for (DorisColWrapper c : this.getCols()) { if (pk.equalsIgnoreCase(c.getName())) { // result.add(createColWrapper(c)); - this.primaryKeys.add(createColWrapper(c)); + this.primaryKeys.add((c)); } } } @@ -178,7 +178,7 @@ public boolean isPK(String colName) { protected abstract String getUniqueKeyToken(); @Override - protected List preProcessCols(List pks, List cols) { + protected List preProcessCols(List pks, List cols) { //return super.preProcessCols(pks, cols); // 将主键排在最前面 @@ -187,7 +187,7 @@ protected List preProcessCols(List pks, List !this.pks.contains(c.getName())).forEach((c) -> { - result.add(createColWrapper(c)); + result.add((c)); }); return result; } @@ -204,8 +204,8 @@ protected void appendTabMeta(List pks) { if (pks.size() > 0) { script.append(primaryKeys.stream().map((pk) -> wrapWithEscape(pk.getName())).collect(Collectors.joining(","))); } else { - List cols = this.getCols(); - Optional firstCol = cols.stream().findFirst(); + List cols = this.getCols(); + Optional firstCol = cols.stream().findFirst(); if (firstCol.isPresent()) { script.append(firstCol.get().getName()); } else { @@ -226,7 +226,8 @@ protected void appendTabMeta(List pks) { @Override protected DorisColWrapper createColWrapper(IColMetaGetter c) { - return new DorisColWrapper(c, this); + + return new DorisColWrapper(c, this.pks, this); } diff --git a/tis-datax/tis-datax-doris-plugin/src/main/java/com/qlangtech/tis/plugin/datax/doris/DataXDorisWriter.java b/tis-datax/tis-datax-doris-plugin/src/main/java/com/qlangtech/tis/plugin/datax/doris/DataXDorisWriter.java index 446dec1da..8e8ff57c5 100644 --- a/tis-datax/tis-datax-doris-plugin/src/main/java/com/qlangtech/tis/plugin/datax/doris/DataXDorisWriter.java +++ b/tis-datax/tis-datax-doris-plugin/src/main/java/com/qlangtech/tis/plugin/datax/doris/DataXDorisWriter.java @@ -115,7 +115,7 @@ protected String getUniqueKeyToken() { @Override protected DorisColWrapper createColWrapper(IColMetaGetter col) { - return new DorisColWrapper(col, this) { + return new DorisColWrapper(col, this.pks, this) { @Override protected DorisType convertType(IColMetaGetter col) { DorisType type = super.convertType(col); diff --git a/tis-datax/tis-datax-odps-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXOdpsWriter.java b/tis-datax/tis-datax-odps-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXOdpsWriter.java index 4b1dc1472..ca56b8a46 100644 --- a/tis-datax/tis-datax-odps-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXOdpsWriter.java +++ b/tis-datax/tis-datax-odps-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXOdpsWriter.java @@ -40,7 +40,7 @@ import com.qlangtech.tis.plugin.datax.odps.OdpsDataSourceFactory; import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules; import com.qlangtech.tis.plugin.ds.CMeta; - + import com.qlangtech.tis.plugin.ds.DataSourceMeta; import com.qlangtech.tis.plugin.ds.DataType; import com.qlangtech.tis.plugin.ds.IColMetaGetter; @@ -234,7 +234,7 @@ protected void appendTabMeta(List pks) { @Override protected ColWrapper createColWrapper(IColMetaGetter c) { - return new ColWrapper(c) { + return new ColWrapper(c, this.pks) { @Override public String getMapperType() { return c.getType().accept(typeTransfer); diff --git a/tis-datax/tis-datax-oracle-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXOracleWriter.java b/tis-datax/tis-datax-oracle-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXOracleWriter.java index 1bb821948..4675afc05 100644 --- a/tis-datax/tis-datax-oracle-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXOracleWriter.java +++ b/tis-datax/tis-datax-oracle-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXOracleWriter.java @@ -95,10 +95,10 @@ protected void appendTabMeta(List pks) { @Override protected ColWrapper createColWrapper(IColMetaGetter c) { - return new ColWrapper(c) { + return new ColWrapper(c,this.pks) { @Override public String getMapperType() { - return convertType(this.meta); + return convertType(this); } }; } @@ -109,7 +109,7 @@ public String getMapperType() { * @param col * @return */ - private String convertType(IColMetaGetter col) { + private String convertType(ColWrapper col) { DataType type = col.getType(); switch (type.getJdbcType()) { case CHAR: { diff --git a/tis-datax/tis-datax-postgresql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXPostgresqlWriter.java b/tis-datax/tis-datax-postgresql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXPostgresqlWriter.java index 8c34bdcc4..876175d40 100644 --- a/tis-datax/tis-datax-postgresql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXPostgresqlWriter.java +++ b/tis-datax/tis-datax-postgresql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXPostgresqlWriter.java @@ -23,19 +23,12 @@ import com.qlangtech.tis.datax.IDataxProcessor; import com.qlangtech.tis.extension.TISExtension; import com.qlangtech.tis.extension.impl.IOUtils; -import com.qlangtech.tis.plugin.datax.CreateTableSqlBuilder.ColWrapper; import com.qlangtech.tis.plugin.datax.common.BasicDataXRdbmsWriter; import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules; -import com.qlangtech.tis.plugin.ds.CMeta; -import com.qlangtech.tis.plugin.ds.DataType; -import com.qlangtech.tis.plugin.ds.IColMetaGetter; import com.qlangtech.tis.plugin.ds.postgresql.PGDataSourceFactory; import org.apache.commons.lang.StringUtils; -import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; /** * @author: baisui 百岁 @@ -55,128 +48,9 @@ public static void main(String[] args) { @Override public CreateTableSqlBuilder.CreateDDL generateCreateDDL(IDataxProcessor.TableMap tableMapper, Optional transformers) { -// if (!this.autoCreateTable) { -// return null; -// } - PGDataSourceFactory ds = this.getDataSourceFactory(); - // 多个主键 - boolean multiPk = Objects.requireNonNull(tableMapper.getSourceCols(), "sourceCols can not be null") - .stream().filter((col) -> col.isPk()).count() > 1; - - final CreateTableSqlBuilder createTableSqlBuilder = new CreateTableSqlBuilder(tableMapper, ds, transformers) { - @Override - protected CreateTableName getCreateTableName() { - return new CreateTableName(ds.tabSchema, tableMapper.getTo(), this); - } - - @Override - protected void appendExtraColDef(List pks) { -// if (!pks.isEmpty()) { -// script.append(" PRIMARY KEY (").append(pks.stream().map((pk) -> "`" + pk.getName() + "`") -// .collect(Collectors.joining(","))).append(")").append("\n"); -// } - if (multiPk) { - this.script.append(", CONSTRAINT ").append("uk_" + tableMapper.getTo() + "_unique_" + pks.stream().map((c) -> c).collect(Collectors.joining("_"))) - .append(" UNIQUE(") - .append(pks.stream().map((c) -> c).collect(Collectors.joining(","))).append(")"); - } - } - - - @Override - protected ColWrapper createColWrapper(IColMetaGetter c) { - return new ColWrapper(c) { - @Override - public String getMapperType() { - return convertType(this.meta); - } - }; - } - - @Override - protected void appendTabMeta(List pks) { - - } - - /** - * https://www.runoob.com/mysql/mysql-data-types.html - * @param col - * @return - */ - private String convertType(IColMetaGetter col) { - DataType type = col.getType(); - String colType = type.accept(new DataType.TypeVisitor() { - @Override - public String bigInt(DataType type) { - return "BIGINT"; - } - - @Override - public String doubleType(DataType type) { - return "FLOAT8"; - } - - @Override - public String dateType(DataType type) { - return "DATE"; - } - - @Override - public String timestampType(DataType type) { - return "TIMESTAMP"; - } - - @Override - public String bitType(DataType type) { - return "BIT"; - } - - @Override - public String blobType(DataType type) { - return "BYTEA"; - } - - @Override - public String varcharType(DataType type) { - return "VARCHAR(" + type.getColumnSize() + ")"; - } - - @Override - public String intType(DataType type) { - return "INTEGER"; - } - - @Override - public String floatType(DataType type) { - return "FLOAT4"; - } - - @Override - public String decimalType(DataType type) { - return "DECIMAL"; - } - - @Override - public String timeType(DataType type) { - return "TIME"; - } - - @Override - public String tinyIntType(DataType dataType) { - return smallIntType(dataType); - } - - @Override - public String smallIntType(DataType dataType) { - return "SMALLINT"; - } - }); - - return colType + (!multiPk && col.isPk() ? " PRIMARY KEY" : StringUtils.EMPTY); - } - - }; + + final CreateTableSqlBuilder createTableSqlBuilder = new PostgreSQLCreateTableSqlBuilder(tableMapper, ds, transformers); return createTableSqlBuilder.build(); } diff --git a/tis-datax/tis-datax-postgresql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/PostgreSQLCreateTableSqlBuilder.java b/tis-datax/tis-datax-postgresql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/PostgreSQLCreateTableSqlBuilder.java new file mode 100644 index 000000000..512f6535a --- /dev/null +++ b/tis-datax/tis-datax-postgresql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/PostgreSQLCreateTableSqlBuilder.java @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.qlangtech.tis.plugin.datax; + +import com.qlangtech.tis.datax.IDataxProcessor.TableMap; +import com.qlangtech.tis.plugin.datax.CreateTableSqlBuilder.ColWrapper; +import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules; +import com.qlangtech.tis.plugin.ds.DataType; +import com.qlangtech.tis.plugin.ds.IColMetaGetter; +import com.qlangtech.tis.plugin.ds.postgresql.PGDataSourceFactory; +import org.apache.commons.lang.StringUtils; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author: 百岁（baisui@qlangtech.com） + * @create: 2024-10-11 12:41 + **/ +public class PostgreSQLCreateTableSqlBuilder extends CreateTableSqlBuilder { + private final boolean multiPk; + private final PGDataSourceFactory ds; + private final TableMap tableMapper; + + public PostgreSQLCreateTableSqlBuilder(TableMap tableMapper, PGDataSourceFactory dsMeta, Optional transformers) { + super(tableMapper, dsMeta, transformers); + this.multiPk = this.pks.size() > 1; + this.ds = dsMeta; + this.tableMapper = tableMapper; + } + + @Override + protected CreateTableName getCreateTableName() { + return new CreateTableName(ds.tabSchema, tableMapper.getTo(), this); + } + + @Override + protected void appendExtraColDef(List pks) { + if (multiPk) { + this.script.append(", CONSTRAINT ").append("uk_" + tableMapper.getTo() + "_unique_" + pks.stream().map((c) -> c).collect(Collectors.joining("_"))) + .append(" UNIQUE(") + .append(pks.stream().map((c) -> c).collect(Collectors.joining(","))).append(")"); + } + } + + + @Override + protected ColWrapper createColWrapper(IColMetaGetter c) { + return new ColWrapper(c, this.pks) { + @Override + public String getMapperType() { + return convertType(this.getType(), this.isPk()); + } + }; + } + + @Override + protected void appendTabMeta(List pks) { + + } + + /** + * https://www.runoob.com/mysql/mysql-data-types.html + * + * @param type + * @return + */ + private String convertType(DataType type, boolean isPk) { + + String colType = Objects.requireNonNull(type, "type can not be null") + .accept(new DataType.TypeVisitor() { + @Override + public String bigInt(DataType type) { + return "BIGINT"; + } + + @Override + public String doubleType(DataType type) { + return "FLOAT8"; + } + + @Override + public String dateType(DataType type) { + return "DATE"; + } + + @Override + public String timestampType(DataType type) { + return "TIMESTAMP"; + } + + @Override + public String bitType(DataType type) { + return "BIT"; + } + + @Override + public String blobType(DataType type) { + return "BYTEA"; + } + + @Override + public String varcharType(DataType type) { + return "VARCHAR(" + type.getColumnSize() + ")"; + } + + @Override + public String intType(DataType type) { + return "INTEGER"; + } + + @Override + public String floatType(DataType type) { + return "FLOAT4"; + } + + @Override + public String decimalType(DataType type) { + return "DECIMAL"; + } + + @Override + public String timeType(DataType type) { + return "TIME"; + } + + @Override + public String tinyIntType(DataType dataType) { + return smallIntType(dataType); + } + + @Override + public String smallIntType(DataType dataType) { + return "SMALLINT"; + } + }); + + return colType + ((!multiPk && isPk) ? " PRIMARY KEY" : StringUtils.EMPTY); + } +} diff --git a/tis-datax/tis-datax-postgresql-plugin/src/test/java/com/qlangtech/tis/plugin/datax/TestDataXPostgresqlWriter.java b/tis-datax/tis-datax-postgresql-plugin/src/test/java/com/qlangtech/tis/plugin/datax/TestDataXPostgresqlWriter.java index a01237136..71ec541c4 100644 --- a/tis-datax/tis-datax-postgresql-plugin/src/test/java/com/qlangtech/tis/plugin/datax/TestDataXPostgresqlWriter.java +++ b/tis-datax/tis-datax-postgresql-plugin/src/test/java/com/qlangtech/tis/plugin/datax/TestDataXPostgresqlWriter.java @@ -116,7 +116,7 @@ public void testAutoCreateDDL() { IDataxProcessor.TableMap tableMapper = WriterTemplate.createCustomer_order_relationTableMap(); - CreateTableSqlBuilder.CreateDDL createDDL = dataXPostgresqlWriter.generateCreateDDL(tableMapper); + CreateTableSqlBuilder.CreateDDL createDDL = dataXPostgresqlWriter.generateCreateDDL(tableMapper, Optional.empty()); Assert.assertNotNull(createDDL); // 多主键 Assert.assertEquals(IOUtils.loadResourceFromClasspath(TestDataXPostgresqlWriter.class, "multi-pks-create-ddl.txt"), createDDL.toString()); @@ -125,7 +125,7 @@ public void testAutoCreateDDL() { Assert.assertTrue(firstCustomerregisterId.isPresent()); firstCustomerregisterId.get().setPk(false); - createDDL = dataXPostgresqlWriter.generateCreateDDL(tableMapper); + createDDL = dataXPostgresqlWriter.generateCreateDDL(tableMapper, Optional.empty()); Assert.assertNotNull(createDDL); // 单主键 Assert.assertEquals(IOUtils.loadResourceFromClasspath(TestDataXPostgresqlWriter.class, "single-pks-create-ddl.txt"), createDDL.toString()); diff --git a/tis-datax/tis-datax-postgresql-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/multi-pks-create-ddl.txt b/tis-datax/tis-datax-postgresql-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/multi-pks-create-ddl.txt index 1bfddad7b..ff25090c1 100644 --- a/tis-datax/tis-datax-postgresql-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/multi-pks-create-ddl.txt +++ b/tis-datax/tis-datax-postgresql-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/multi-pks-create-ddl.txt @@ -1,8 +1,8 @@ -CREATE TABLE customer_order_relation +CREATE TABLE "customer_order_relation" ( - customerregister_id VARCHAR(256), - waitingorder_id VARCHAR(256), - kind INTEGER, - create_time BIGINT, - last_ver INTEGER + "customerregister_id" VARCHAR(256), + "waitingorder_id" VARCHAR(256), + "kind" INTEGER, + "create_time" BIGINT, + "last_ver" INTEGER , CONSTRAINT uk_customer_order_relation_unique_customerregister_id_waitingorder_id UNIQUE(customerregister_id,waitingorder_id)) diff --git a/tis-datax/tis-datax-postgresql-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/single-pks-create-ddl.txt b/tis-datax/tis-datax-postgresql-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/single-pks-create-ddl.txt index 2c0cd6250..d8bcd07ab 100644 --- a/tis-datax/tis-datax-postgresql-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/single-pks-create-ddl.txt +++ b/tis-datax/tis-datax-postgresql-plugin/src/test/resources/com/qlangtech/tis/plugin/datax/single-pks-create-ddl.txt @@ -1,8 +1,8 @@ -CREATE TABLE customer_order_relation +CREATE TABLE "customer_order_relation" ( - customerregister_id VARCHAR(256), - waitingorder_id VARCHAR(256) PRIMARY KEY, - kind INTEGER, - create_time BIGINT, - last_ver INTEGER + "customerregister_id" VARCHAR(256), + "waitingorder_id" VARCHAR(256) PRIMARY KEY, + "kind" INTEGER, + "create_time" BIGINT, + "last_ver" INTEGER ) diff --git a/tis-datax/tis-datax-sqlserver-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXSqlserverWriter.java b/tis-datax/tis-datax-sqlserver-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXSqlserverWriter.java index 5995035a8..7527156d6 100644 --- a/tis-datax/tis-datax-sqlserver-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXSqlserverWriter.java +++ b/tis-datax/tis-datax-sqlserver-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXSqlserverWriter.java @@ -71,10 +71,11 @@ public CreateTableSqlBuilder.CreateDDL generateCreateDDL( final CreateTableSqlBuilder createTableSqlBuilder = new CreateTableSqlBuilder(tableMapper, this.getDataSourceFactory(), transformers) { - private String convertType(IColMetaGetter col) { + private String convertType(DataType type, boolean isPk) { //https://www.cnblogs.com/liberty777/p/10748570.html - StringBuffer createSql = new StringBuffer(getSqlServerType(col)); - if (col.isPk()) { + StringBuffer createSql = new StringBuffer(getSqlServerType(type)); + + if (isPk) { createSql.append(" primary key "); } return createSql.toString(); @@ -82,16 +83,16 @@ private String convertType(IColMetaGetter col) { @Override protected ColWrapper createColWrapper(IColMetaGetter c) { - return new ColWrapper(c) { + return new ColWrapper(c, this.pks) { @Override public String getMapperType() { - return convertType(this.meta); + return convertType(this.getType(), this.isPk()); } }; } - private String getSqlServerType(IColMetaGetter col) { - DataType type = col.getType(); + private String getSqlServerType(DataType type) { + switch (type.getJdbcType()) { case INTEGER: case TINYINT: diff --git a/tis-datax/tis-datax-starrocks-plugin/src/main/java/com/qlangtech/tis/plugin/datax/BasicStarRocksWriter.java b/tis-datax/tis-datax-starrocks-plugin/src/main/java/com/qlangtech/tis/plugin/datax/BasicStarRocksWriter.java index 30fb61218..c97b35501 100644 --- a/tis-datax/tis-datax-starrocks-plugin/src/main/java/com/qlangtech/tis/plugin/datax/BasicStarRocksWriter.java +++ b/tis-datax/tis-datax-starrocks-plugin/src/main/java/com/qlangtech/tis/plugin/datax/BasicStarRocksWriter.java @@ -42,6 +42,7 @@ import java.io.Serializable; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -122,19 +123,19 @@ protected void appendExtraColDef(List pks) { @Override - protected List preProcessCols(List pks, List cols) { + protected List preProcessCols(List pks, List cols) { // 将主键排在最前面 List result = Lists.newArrayList(); for (String pk : pks) { - for (IColMetaGetter c : cols) { + for (T c : cols) { if (pk.equalsIgnoreCase(c.getName())) { - result.add((T) createColWrapper(c)); + result.add(c); } } } cols.stream().filter((c) -> !pks.contains(c.getName())).forEach((c) -> { - result.add((T) createColWrapper(c)); + result.add(c); }); return result; } @@ -153,8 +154,8 @@ protected void appendTabMeta(List pks) { .map((pk) -> wrapWithEscape(pk)) .collect(Collectors.joining(","))); } else { - List cols = this.getCols(); - Optional firstCol = cols.stream().findFirst(); + List cols = this.getCols(); + Optional firstCol = cols.stream().findFirst(); if (firstCol.isPresent()) { script.append(firstCol.get().getName()); } else { @@ -168,24 +169,24 @@ protected void appendTabMeta(List pks) { @Override protected T createColWrapper(IColMetaGetter c) { - return (T) new ColWrapper(c) { + return (T) new ColWrapper(c, this.pks) { @Override public String getMapperType() { - return convertType(this.meta).token; + return convertType(getType()).token; } @Override protected void appendExtraConstraint(BlockScriptBuffer ddlScript) { - if (this.meta.isPk()) { + if (this.isPk()) { ddlScript.append(" NOT NULL"); } } }; } - protected DorisType convertType(IColMetaGetter col) { - DataType type = col.getType(); - return type.accept(columnTokenRecognise); + protected DorisType convertType(DataType type) { + return Objects.requireNonNull(type, "type can not be null") + .accept(columnTokenRecognise); } } diff --git a/tis-datax/tis-datax-test-common/src/main/java/com/qlangtech/tis/plugin/common/WriterTemplate.java b/tis-datax/tis-datax-test-common/src/main/java/com/qlangtech/tis/plugin/common/WriterTemplate.java index f7b8de583..79d76b1ef 100644 --- a/tis-datax/tis-datax-test-common/src/main/java/com/qlangtech/tis/plugin/common/WriterTemplate.java +++ b/tis-datax/tis-datax-test-common/src/main/java/com/qlangtech/tis/plugin/common/WriterTemplate.java @@ -48,6 +48,7 @@ import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; +import java.util.stream.Collectors; /** * @author: 百岁（baisui@qlangtech.com） @@ -338,14 +339,24 @@ public static IDataxProcessor.TableMap createCustomer_order_relationTableMap() { public static IDataxProcessor.TableMap createCustomer_order_relationTableMap(Optional tab) { IDataxProcessor.TableMap tableMap = new IDataxProcessor.TableMap(tab.isPresent() ? tab.get() : new ISelectedTab() { + List _cols; + @Override public String getName() { return TAB_customer_order_relation; } + @Override + public List getPrimaryKeys() { + return getCols().stream().filter((col) -> col.isPk()).map((col) -> col.getName()).collect(Collectors.toList()); + } + @Override public List getCols() { - return createColMetas(); + if (_cols == null) { + _cols = createColMetas(); + } + return _cols; } }); tableMap.setTo(TAB_customer_order_relation); diff --git a/tis-datax/tis-ds-mysql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataxMySQLWriter.java b/tis-datax/tis-ds-mysql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataxMySQLWriter.java index 6d33e0209..284a16555 100644 --- a/tis-datax/tis-ds-mysql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataxMySQLWriter.java +++ b/tis-datax/tis-ds-mysql-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataxMySQLWriter.java @@ -214,10 +214,10 @@ protected void appendTabMeta(List pks) { @Override protected ColWrapper createColWrapper(IColMetaGetter c) { - return new ColWrapper(c) { + return new ColWrapper(c, this.pks) { @Override public String getMapperType() { - return convertType(this.meta); + return convertType(this); } }; } @@ -227,7 +227,7 @@ public String getMapperType() { * @param col * @return */ - private String convertType(IColMetaGetter col) { + private String convertType(ColWrapper col) { DataType type = col.getType(); switch (type.getJdbcType()) { case CHAR: { diff --git a/tis-datax/tis-hive-flat-table-builder-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXHiveWriter.java b/tis-datax/tis-hive-flat-table-builder-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXHiveWriter.java index 6c9c38e24..0670ca9b4 100644 --- a/tis-datax/tis-hive-flat-table-builder-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXHiveWriter.java +++ b/tis-datax/tis-hive-flat-table-builder-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXHiveWriter.java @@ -62,7 +62,7 @@ import com.qlangtech.tis.plugin.datax.transformer.RecordTransformerRules; import com.qlangtech.tis.plugin.ds.CMeta; import com.qlangtech.tis.plugin.ds.DBConfig; - + import com.qlangtech.tis.plugin.ds.DataSourceMeta; import com.qlangtech.tis.plugin.ds.IColMetaGetter; import com.qlangtech.tis.plugin.ds.IDataSourceFactoryGetter; @@ -218,7 +218,7 @@ private HdfsFormat parseFSFormat() { @Override protected ColWrapper createColWrapper(IColMetaGetter c) { - return new ColWrapper(c) { + return new ColWrapper(c, this.pks) { @Override public String getMapperType() { return c.getType().accept(HiveColumn.hiveTypeVisitor);