Skip to content

Commit

Permalink
update name
Browse files Browse the repository at this point in the history 
Signed-off-by: Seaven <[email protected]>
  • Loading branch information
@Seaven
Seaven committed Nov 13, 2024
1 parent 45694f1 commit c9963a4
Show file tree
Hide file tree
Showing 5 changed files with 34 additions and 36 deletions.
23 changes: 16 additions & 7 deletions fe/fe-core/src/main/java/com/starrocks/statistic/StatisticExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,9 +143,8 @@ private static Table lookupTable(Long dbId, Long tableId) {
List<TStatisticData> columnStats = Lists.newArrayList();
if (CollectionUtils.isNotEmpty(columnWithFullStats)) {
List<String> columnNamesForStats = columnWithFullStats.stream().map(ColumnStatsMeta::getColumnName)
.collect(Collectors.toList());
List<Type> columnTypesForStats =
columnWithFullStats.stream()
.collect(Collectors.toList());
List<Type> columnTypesForStats = columnWithFullStats.stream()
.map(x -> StatisticUtils.getQueryStatisticsColumnType(table, x.getColumnName()))
.collect(Collectors.toList());

Expand All @@ -156,10 +155,20 @@ private static Table lookupTable(Long dbId, Long tableId) {
}
if (CollectionUtils.isNotEmpty(columnWithSampleStats)) {
List<String> columnNamesForStats = columnWithSampleStats.stream().map(ColumnStatsMeta::getColumnName)
.collect(Collectors.toList());
String statsSql = StatisticSQLBuilder.buildQuerySampleStatisticsSQL(dbId, tableId, columnNamesForStats);
List<TStatisticData> tStatisticData = executeStatisticDQL(context, statsSql);
columnStats.addAll(tStatisticData);
.collect(Collectors.toList());
if (Config.statistic_use_meta_statistics) {
List<Type> columnTypesForStats = columnWithSampleStats.stream()
.map(x -> StatisticUtils.getQueryStatisticsColumnType(table, x.getColumnName()))
.collect(Collectors.toList());
String statsSql = StatisticSQLBuilder.buildQueryFullStatisticsSQL(
dbId, tableId, columnNamesForStats, columnTypesForStats);
List<TStatisticData> tStatisticData = executeStatisticDQL(context, statsSql);
columnStats.addAll(tStatisticData);
} else {
String statsSql = StatisticSQLBuilder.buildQuerySampleStatisticsSQL(dbId, tableId, columnNamesForStats);
List<TStatisticData> tStatisticData = executeStatisticDQL(context, statsSql);
columnStats.addAll(tStatisticData);
}
}
return columnStats;
}
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/com/starrocks/statistic/hyper/FullQueryJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,13 @@ protected List<String> buildQuerySQL() {
}

for (ColumnStats columnStat : columnStats) {
VelocityContext context = StatisticSQLs.buildBaseContext(db, table, partition, columnStat);
VelocityContext context = HyperStatisticSQLs.buildBaseContext(db, table, partition, columnStat);
context.put("dataSize", columnStat.getFullDateSize());
context.put("countNullFunction", columnStat.getFullNullCount());
context.put("hllFunction", columnStat.getNDV());
context.put("maxFunction", columnStat.getMax());
context.put("minFunction", columnStat.getMin());
String sql = StatisticSQLs.build(context, StatisticSQLs.BATCH_FULL_STATISTIC_TEMPLATE);
String sql = HyperStatisticSQLs.build(context, HyperStatisticSQLs.BATCH_FULL_STATISTIC_TEMPLATE);
metaSQL.add(sql);
}
}
Expand Down
31 changes: 10 additions & 21 deletions ...rrocks/statistic/hyper/StatisticSQLs.java → ...s/statistic/hyper/HyperStatisticSQLs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
import java.util.Objects;
import java.util.stream.Collectors;

public class StatisticSQLs {
public class HyperStatisticSQLs {
private static final VelocityEngine DEFAULT_VELOCITY_ENGINE;

static {
Expand All @@ -54,8 +54,8 @@ public class StatisticSQLs {
//| max | varchar(1048576) | NO | false | <null> | |
//| min | varchar(1048576) | NO | false | <null> | |
//| update_time | datetime | NO | false | <null> | |
public static final String TABLE_NAME = "column_statistics";
public static final String BATCH_FULL_STATISTIC_TEMPLATE = "SELECT cast($version as INT)" +
static final String TABLE_NAME = "column_statistics";
static final String BATCH_FULL_STATISTIC_TEMPLATE = "SELECT cast($version as INT)" +
", cast($partitionId as BIGINT)" + // BIGINT
", '$columnNameStr'" + // VARCHAR
", cast(COUNT(1) as BIGINT)" + // BIGINT
Expand All @@ -66,7 +66,7 @@ public class StatisticSQLs {
", $minFunction " + // VARCHAR
" FROM `$dbName`.`$tableName` partition `$partitionName`";

public static final String BATCH_META_STATISTIC_TEMPLATE = "SELECT cast($version as INT)" +
static final String BATCH_META_STATISTIC_TEMPLATE = "SELECT cast($version as INT)" +
", cast($partitionId as BIGINT)" + // BIGINT, partition_id
", '$columnNameStr'" + // VARCHAR, column_name
", cast(COUNT(*) as BIGINT)" + // BIGINT, row_count
Expand All @@ -77,18 +77,7 @@ public class StatisticSQLs {
", $minFunction " + // VARCHAR, min
" FROM `$dbName`.`$tableName` partitions(`$partitionName`) [_META_]";

public static final String BATCH_DATA_STATISTIC_TEMPLATE = "SELECT cast($version as INT)" +
", cast($partitionId as BIGINT)" + // BIGINT, partition_id
", '$columnNameStr'" + // VARCHAR, column_name
", cast(0 as BIGINT)" + // BIGINT, row_count
", cast($dataSize as BIGINT)" + // BIGINT, data_size
", $hllFunction" + // VARBINARY, ndv
", cast($countNullFunction as BIGINT)" + // BIGINT, null_count
", ''" + // VARCHAR, max
", '' " + // VARCHAR, min
" FROM `$dbName`.`$tableName` partitions(`$partitionName`)";

public static final String BATCH_DATA_STATISTIC_SELECT_TEMPLATE = "SELECT cast($version as INT)" +
static final String BATCH_DATA_STATISTIC_SELECT_TEMPLATE = "SELECT cast($version as INT)" +
", cast($partitionId as BIGINT)" + // BIGINT, partition_id
", '$columnNameStr'" + // VARCHAR, column_name
", cast(0 as BIGINT)" + // BIGINT, row_count
Expand All @@ -99,7 +88,7 @@ public class StatisticSQLs {
", '' " + // VARCHAR, min
" FROM base_cte_table ";

public static final String BATCH_SAMPLE_STATISTIC_SELECT_TEMPLATE = "SELECT cast($version as INT)" +
static final String BATCH_SAMPLE_STATISTIC_SELECT_TEMPLATE = "SELECT cast($version as INT)" +
", cast($partitionId as BIGINT)" + // BIGINT
", '$columnNameStr'" + // VARCHAR
", cast($rowCount as BIGINT)" + // BIGINT
Expand All @@ -110,13 +99,13 @@ public class StatisticSQLs {
", $minFunction " + // VARCHAR
" FROM base_cte_table ";

public static String build(VelocityContext context, String template) {
static String build(VelocityContext context, String template) {
StringWriter sw = new StringWriter();
DEFAULT_VELOCITY_ENGINE.evaluate(context, sw, "", template);
return sw.toString();
}

public static VelocityContext buildBaseContext(Database db, Table table, Partition p, ColumnStats stats) {
static VelocityContext buildBaseContext(Database db, Table table, Partition p, ColumnStats stats) {
VelocityContext context = new VelocityContext();
String columnNameStr = stats.getColumnNameStr();
String quoteColumnName = stats.getQuotedColumnName();
Expand All @@ -130,7 +119,7 @@ public static VelocityContext buildBaseContext(Database db, Table table, Partiti
return context;
}

public static String buildSampleSQL(Database db, Table table, Partition p, List<ColumnStats> stats,
static String buildSampleSQL(Database db, Table table, Partition p, List<ColumnStats> stats,
PartitionSampler sampler, String template) {
String tableName = "`" + db.getOriginName() + "`.`" + table.getName() + "`";

Expand Down Expand Up @@ -162,7 +151,7 @@ public static String buildSampleSQL(Database db, Table table, Partition p, List<
context.put("countNullFunction", stat.getSampleNullCount(info));
context.put("maxFunction", stat.getMax());
context.put("minFunction", stat.getMin());
groupSQLs.add(StatisticSQLs.build(context, template));
groupSQLs.add(HyperStatisticSQLs.build(context, template));
}
sqlBuilder.append(String.join(" UNION ALL ", groupSQLs));
return sqlBuilder.toString();
Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/main/java/com/starrocks/statistic/hyper/MetaQueryJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,10 @@ private List<String> buildBatchMetaQuerySQL(List<ColumnStats> queryColumns) {
}

for (ColumnStats columnStat : queryColumns) {
VelocityContext context = StatisticSQLs.buildBaseContext(db, table, partition, columnStat);
VelocityContext context = HyperStatisticSQLs.buildBaseContext(db, table, partition, columnStat);
context.put("maxFunction", columnStat.getMax());
context.put("minFunction", columnStat.getMin());
String sql = StatisticSQLs.build(context, StatisticSQLs.BATCH_META_STATISTIC_TEMPLATE);
String sql = HyperStatisticSQLs.build(context, HyperStatisticSQLs.BATCH_META_STATISTIC_TEMPLATE);
metaSQL.add(sql);
}
}
Expand Down Expand Up @@ -140,8 +140,8 @@ private List<String> buildBatchNDVQuerySQL(List<ColumnStats> queryColumns) {
continue;
}
for (List<ColumnStats> part : partColumns) {
String sql = StatisticSQLs.buildSampleSQL(db, table, partition, part, sampler,
StatisticSQLs.BATCH_DATA_STATISTIC_SELECT_TEMPLATE);
String sql = HyperStatisticSQLs.buildSampleSQL(db, table, partition, part, sampler,
HyperStatisticSQLs.BATCH_DATA_STATISTIC_SELECT_TEMPLATE);
metaSQL.add(sql);
}
}
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/com/starrocks/statistic/hyper/SampleQueryJob.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ protected List<String> buildQuerySQL() {
continue;
}
for (List<ColumnStats> stats : partColumns) {
String sql = StatisticSQLs.buildSampleSQL(db, table, partition, stats, sampler,
StatisticSQLs.BATCH_SAMPLE_STATISTIC_SELECT_TEMPLATE);
String sql = HyperStatisticSQLs.buildSampleSQL(db, table, partition, stats, sampler,
HyperStatisticSQLs.BATCH_SAMPLE_STATISTIC_SELECT_TEMPLATE);
sampleSQLs.add(sql);
}
}
Expand Down

0 comments on commit c9963a4

Please sign in to comment.