support create or alter table with external cooldown config properties …

…StarRocks#51649

Signed-off-by: hoffermei <[email protected]>

fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java

@@ -88,6 +88,7 @@
 import com.starrocks.common.util.concurrent.MarkedCountDownLatch;
 import com.starrocks.common.util.concurrent.lock.LockType;
 import com.starrocks.common.util.concurrent.lock.Locker;
+import com.starrocks.externalcooldown.ExternalCoolDownConfig;
 import com.starrocks.persist.TableAddOrDropColumnsInfo;
 import com.starrocks.qe.ConnectContext;
 import com.starrocks.qe.ShowResultSet;
@@ -132,6 +133,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -2130,6 +2132,85 @@ public void updateTableMeta(Database db, String tableName, Map<String, String> p
  }
  }
 
+ public boolean updateExternalCoolDownConfigMeta(Database db, Long tableId, Map<String, String> properties) {
+ OlapTable olapTable;
+ ExternalCoolDownConfig newExternalCoolDownConfig;
+ boolean hasChanged = false;
+ boolean isModifiedSuccess = true;
+ Locker locker = new Locker();
+ locker.lockDatabase(db, LockType.READ);
+ try {
+ olapTable = (OlapTable) db.getTable(tableId);
+ if (olapTable == null) {
+ return false;
+ }
+ if (!olapTable.containsExternalCoolDownConfig()) {
+ newExternalCoolDownConfig = new ExternalCoolDownConfig();
+ hasChanged = true;
+ } else {
+ newExternalCoolDownConfig = new ExternalCoolDownConfig(olapTable.getCurExternalCoolDownConfig());
+ }
+ } finally {
+ locker.unLockDatabase(db, LockType.READ);
+ }
+
+ // judge whether the attribute has changed
+ // no exception will be thrown, for the analyzer has checked
+ if (properties.containsKey(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_TARGET)) {
+ String externalCoolDownTarget = properties.get(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_TARGET);
+ if (!Objects.equals(externalCoolDownTarget, newExternalCoolDownConfig.getTarget())) {
+ newExternalCoolDownConfig.setTarget(externalCoolDownTarget);
+ hasChanged = true;
+ }
+ }
+ if (properties.containsKey(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_SCHEDULE)) {
+ String externalCoolDownSchedule = properties.get(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_SCHEDULE);
+ if (!Objects.equals(externalCoolDownSchedule, newExternalCoolDownConfig.getSchedule())) {
+ newExternalCoolDownConfig.setSchedule(externalCoolDownSchedule);
+ hasChanged = true;
+ }
+ }
+ if (properties.containsKey(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_WAIT_SECOND)) {
+ long externalCoolDownWaitSecond = Long.parseLong(properties.get(
+ PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_WAIT_SECOND));
+ if (externalCoolDownWaitSecond != newExternalCoolDownConfig.getWaitSecond()) {
+ newExternalCoolDownConfig.setWaitSecond(externalCoolDownWaitSecond);
+ hasChanged = true;
+ }
+ }
+ if (!hasChanged) {
+ LOG.info("table {} external cool down config is same as the previous version, so nothing need to do",
+ olapTable.getName());
+ return true;
+ }
+ locker.lockDatabase(db, LockType.WRITE);
+ try {
+ ExternalCoolDownConfig oldExternalCoolDownConfig = olapTable.getCurExternalCoolDownConfig();
+ GlobalStateMgr.getCurrentState().getLocalMetastore().modifyExternalCoolDownMeta(
+ db, olapTable, newExternalCoolDownConfig);
+ if (oldExternalCoolDownConfig != null) {
+ LOG.info("update external cool down config of table {} successfully, the external cool down config after " +
+ "modified is : {}, previous is {}",
+ olapTable.getName(),
+ olapTable.getCurExternalCoolDownConfig().toString(),
+ oldExternalCoolDownConfig.toString());
+ } else {
+ LOG.info("update external cool down config of table {} successfully, the external cool down config"
+ + " after modified is : {}, ",
+ olapTable.getName(), olapTable.getCurExternalCoolDownConfig().toString());
+ }
+ } catch (Exception e) {
+ // defensive programming, it normally should not throw an exception,
+ // here is just to ensure that a correct result can be returned
+ LOG.warn("update external cool down config of table {} failed", olapTable.getName());
+ isModifiedSuccess = false;
+ } finally {
+ locker.unLockDatabase(db, LockType.WRITE);
+ }
+
+ return isModifiedSuccess;
+ }
+
  // return true means that the modification of FEMeta is successful,
  // and as long as the modification of metadata is successful,
  // the final consistency will be achieved through the report handler

fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java

@@ -87,6 +87,7 @@
 import com.starrocks.common.util.Util;
 import com.starrocks.common.util.WriteQuorum;
 import com.starrocks.common.util.concurrent.MarkedCountDownLatch;
+import com.starrocks.externalcooldown.ExternalCoolDownConfig;
 import com.starrocks.lake.DataCacheInfo;
 import com.starrocks.lake.StarOSAgent;
 import com.starrocks.lake.StorageInfo;
@@ -485,6 +486,31 @@ public void setBinlogTxnId(long binlogTxnId) {
  this.binlogTxnId = binlogTxnId;
  }
 
+ public ExternalCoolDownConfig getCurExternalCoolDownConfig() {
+ if (tableProperty != null) {
+ return tableProperty.getExternalCoolDownConfig();
+ }
+ return null;
+ }
+
+ public void setCurExternalCoolDownConfig(ExternalCoolDownConfig externalCoolDownConfig) {
+ if (tableProperty == null) {
+ tableProperty = new TableProperty(Maps.newHashMap());
+ }
+ tableProperty.modifyTableProperties(externalCoolDownConfig.toProperties());
+ tableProperty.setExternalCoolDownConfig(externalCoolDownConfig);
+ }
+
+ public boolean containsExternalCoolDownConfig() {
+ if (tableProperty == null ||
+ tableProperty.getExternalCoolDownConfig() == null) {
+ return false;
+ }
+
+ return tableProperty.getExternalCoolDownConfig().getTarget() != null &&
+ !tableProperty.getExternalCoolDownConfig().getTarget().isEmpty();
+ }
+
  public void setTableProperty(TableProperty tableProperty) {
  this.tableProperty = tableProperty;
  }
@@ -3391,6 +3417,27 @@ public boolean hasRowStorageType() {
  return TStorageType.ROW == getStorageType() || TStorageType.COLUMN_WITH_ROW == getStorageType();
  }
 
+ public String getExternalCoolDownTarget() {
+ if (tableProperty != null) {
+ return tableProperty.getExternalCoolDownTarget();
+ }
+ return null;
+ }
+
+ public String getExternalCoolDownSchedule() {
+ if (tableProperty != null) {
+ return tableProperty.getExternalCoolDownSchedule();
+ }
+ return null;
+ }
+
+ public Long getExternalCoolDownWaitSecond() {
+ if (tableProperty != null) {
+ return tableProperty.getExternalCoolDownWaitSecond();
+ }
+ return null;
+ }
+
  // ------ for lake table and lake materialized view start ------
  @Nullable
  public FilePathInfo getDefaultFilePathInfo() {

fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java

@@ -53,6 +53,7 @@
 import com.starrocks.common.util.PropertyAnalyzer;
 import com.starrocks.common.util.TimeUtils;
 import com.starrocks.common.util.WriteQuorum;
+import com.starrocks.externalcooldown.ExternalCoolDownConfig;
 import com.starrocks.lake.StorageInfo;
 import com.starrocks.persist.OperationType;
 import com.starrocks.persist.gson.GsonPostProcessable;
@@ -294,6 +295,7 @@ public static String valueList() {
  private boolean useFastSchemaEvolution;
 
  private PeriodDuration dataCachePartitionDuration;
+ private ExternalCoolDownConfig externalCoolDownConfig;
 
  private Multimap<String, String> location;
 
@@ -382,12 +384,19 @@ public TableProperty buildProperty(short opCode) {
  case OperationType.OP_MODIFY_TABLE_CONSTRAINT_PROPERTY:
  buildConstraint();
  break;
+ case OperationType.OP_MODIFY_EXTERNAL_COOLDOWN_CONFIG:
+ buildExternalCoolDownConfig();
+ break;
  default:
  break;
  }
  return this;
  }
 
+ public TableProperty buildExternalCoolDownProperties() {
+ return this;
+ }
+
  // TODO: refactor the postProcessing code into listener-based instead of procedure-oriented
  public TableProperty buildMvProperties() {
  buildPartitionTTL();
@@ -421,6 +430,22 @@ public void setBinlogConfig(BinlogConfig binlogConfig) {
  this.binlogConfig = binlogConfig;
  }
 
+ public TableProperty buildExternalCoolDownConfig() {
+ String externalCoolDownTarget = properties.get(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_TARGET);
+ String externalCoolDownSchedule = properties.get(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_SCHEDULE);
+ long externalCoolDownWaitSecond = Long.parseLong(properties.getOrDefault(
+ PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_WAIT_SECOND,
+ String.valueOf(0)));
+ externalCoolDownConfig = new ExternalCoolDownConfig(
+ externalCoolDownTarget, externalCoolDownSchedule, externalCoolDownWaitSecond);
+ return this;
+ }
+
+ // just modify externalCoolDownConfig, not properties
+ public void setExternalCoolDownConfig(ExternalCoolDownConfig externalCoolDownConfig) {
+ this.externalCoolDownConfig = externalCoolDownConfig;
+ }
+
  public TableProperty buildDynamicProperty() {
  HashMap<String, String> dynamicPartitionProperties = new HashMap<>();
  for (Map.Entry<String, String> entry : properties.entrySet()) {
@@ -973,6 +998,10 @@ public BinlogConfig getBinlogConfig() {
  return binlogConfig;
  }
 
+ public ExternalCoolDownConfig getExternalCoolDownConfig() {
+ return externalCoolDownConfig;
+ }
+
  public List<UniqueConstraint> getUniqueConstraints() {
  return uniqueConstraints;
  }
@@ -1021,6 +1050,27 @@ public boolean getUseFastSchemaEvolution() {
  return useFastSchemaEvolution;
  }
 
+ public String getExternalCoolDownTarget() {
+ if (externalCoolDownConfig == null) {
+ return null;
+ }
+ return externalCoolDownConfig.getTarget();
+ }
+
+ public String getExternalCoolDownSchedule() {
+ if (externalCoolDownConfig == null) {
+ return null;
+ }
+ return externalCoolDownConfig.getSchedule();
+ }
+
+ public long getExternalCoolDownWaitSecond() {
+ if (externalCoolDownConfig == null) {
+ return 0;
+ }
+ return externalCoolDownConfig.getWaitSecond();
+ }
+
  @Override
  public void write(DataOutput out) throws IOException {
  Text.writeString(out, GsonUtils.GSON.toJson(this));
@@ -1057,5 +1107,6 @@ public void gsonPostProcess() throws IOException {
  buildStorageType();
  buildMvProperties();
  buildLocation();
+ buildExternalCoolDownConfig();
  }
 }

fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java

@@ -53,6 +53,7 @@
 import com.starrocks.catalog.ColumnId;
 import com.starrocks.catalog.DataProperty;
 import com.starrocks.catalog.Database;
+import com.starrocks.catalog.IcebergTable;
 import com.starrocks.catalog.InternalCatalog;
 import com.starrocks.catalog.KeysType;
 import com.starrocks.catalog.MaterializedView;
@@ -70,6 +71,7 @@
 import com.starrocks.common.ErrorReport;
 import com.starrocks.common.FeConstants;
 import com.starrocks.common.Pair;
+import com.starrocks.externalcooldown.ExternalCoolDownConfig;
 import com.starrocks.lake.DataCacheInfo;
 import com.starrocks.qe.ConnectContext;
 import com.starrocks.server.GlobalStateMgr;
@@ -247,6 +249,14 @@ public class PropertyAnalyzer {
  */
  public static final String MULTI_LOCATION_LABELS_REGEX = "\\s*" + SINGLE_LOCATION_LABEL_REGEX +
  "\\s*(,\\s*" + SINGLE_LOCATION_LABEL_REGEX + "){0,9}\\s*";
+ // "external_cooldown_target"="iceberg_catalog.iceberg_db.iceberg_tbl",
+ public static final String PROPERTIES_EXTERNAL_COOLDOWN_TARGET = "external_cooldown_target";
+
+ // "external_cooldown_schedule"="START <start_time> END <end_time> EVERY INTERVAL <cooldown_interval>"
+ public static final String PROPERTIES_EXTERNAL_COOLDOWN_SCHEDULE = "external_cooldown_schedule";
+
+ // "external_cooldown_wait_second"="86400"
+ public static final String PROPERTIES_EXTERNAL_COOLDOWN_WAIT_SECOND = "external_cooldown_wait_second";
 
  public static DataProperty analyzeDataProperty(Map<String, String> properties,
  DataProperty inferredDataProperty,
@@ -1655,6 +1665,57 @@ public static DataProperty analyzeMVDataProperty(MaterializedView materializedVi
  return dataProperty;
  }
 
+ public static ExternalCoolDownConfig analyzeExternalCoolDownConfig(Map<String, String> properties) throws AnalysisException {
+ ExternalCoolDownConfig externalCoolDownConfig = new ExternalCoolDownConfig();
+
+ if (properties.containsKey(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_TARGET)) {
+ String target = properties.get(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_TARGET);
+ properties.remove(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_TARGET);
+ if (!target.isEmpty()) {
+ Pattern targetPattern = Pattern.compile("^\\w+\\.\\w+\\.\\w+$", Pattern.CASE_INSENSITIVE);
+ if (!targetPattern.matcher(target).find()) {
+ throw new AnalysisException("Property " + PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_TARGET +
+ " must be format of {catalog}.{db}.{tbl}");
+ }
+ String[] parts = target.split("\\.");
+ Table table = GlobalStateMgr.getCurrentState().getMetadataMgr().getTable(parts[0], parts[1], parts[2]);
+ if (table == null) {
+ throw new AnalysisException("Property " + PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_TARGET +
+ " table " + target + " not exist");
+ }
+ if (!(table instanceof IcebergTable)) {
+ throw new AnalysisException("Property " + PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_TARGET +
+ " only support iceberg table");
+ }
+ }
+ externalCoolDownConfig.setTarget(target);
+ }
+ if (properties.containsKey(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_SCHEDULE)) {
+ String schedule = properties.get(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_SCHEDULE);
+ properties.remove(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_SCHEDULE);
+ Pattern schedulePattern = Pattern.compile(
+ "^\\s*START\\s+\\d+:\\d+\\s+END\\s+\\d+:\\d+\\s+EVERY\\s+INTERVAL\\s+\\d+[smh]\\s*$",
+ Pattern.CASE_INSENSITIVE);
+ if (!schedulePattern.matcher(schedule).find()) {
+ throw new AnalysisException("Property " + PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_SCHEDULE +
+ " must be format like `START 01:00 END 07:59 EVERY INTERVAL 1m`");
+ }
+ externalCoolDownConfig.setSchedule(schedule);
+ }
+ if (properties.containsKey(PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_WAIT_SECOND)) {
+ long waitSecond;
+ try {
+ waitSecond = PropertyAnalyzer.analyzeLongProp(properties,
+ PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_WAIT_SECOND, 0);
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("Property " +
+ PropertyAnalyzer.PROPERTIES_EXTERNAL_COOLDOWN_WAIT_SECOND + " must be long");
+ }
+ externalCoolDownConfig.setWaitSecond(waitSecond);
+ }
+ return externalCoolDownConfig;
+ }
+
  /**
  * Generate a string representation of properties like ('a'='1', 'b'='2')
  */