[Fix][dryrun] 完善样例数据生成，支持groupUniqArray 单参

5 files changed, 102 insertions, 56 deletions

diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/ExampleDataHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/ExampleDataHelper.java
index c3eb9e52..f4d3c4d6 100644
--- a/src/main/java/com/mesalab/common/utils/sqlparser/ExampleDataHelper.java
+++ b/src/main/java/com/mesalab/common/utils/sqlparser/ExampleDataHelper.java
@@ -3,8 +3,10 @@ package com.mesalab.common.utils.sqlparser;
 import cn.hutool.core.date.DatePattern;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.util.BooleanUtil;
 import cn.hutool.core.util.CharsetUtil;
 import com.alibaba.fastjson2.JSONPath;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.mesalab.common.entity.DataTypeMapping;
 import com.mesalab.common.enums.HttpStatusCodeEnum;
@@ -211,31 +213,37 @@ public class ExampleDataHelper {
         return result;
     }
 
-    private static List getExampleValue(Map<String, String> meta, Object initValue, Boolean isDesc, boolean canRepeat, int size, Map<String, List<Object>> schemaDataDict) {
+    private static List getExampleValue(Map<String, String> meta, Object initValue, Boolean isDesc, boolean repeatable, int size, Map<String, List<Object>> schemaDataDict) {
+        boolean isArray = BooleanUtil.toBoolean(meta.get(MetaConst.IS_ARRAY));
+        List result = inExampleData(meta, schemaDataDict)
+                ? getExampleDataValues(meta, repeatable, size, schemaDataDict)
+                : getOtherExampleValues(meta, isDesc, size);
+        return isArray ? convertToArrayList(result) : result;
+    }
 
-        if (inExampleData(meta, schemaDataDict)) {
-            return getExampleDataValues(meta, canRepeat, size, schemaDataDict);
-        }
-        String dataType = meta.get(MetaConst.META_DATA_TYPE) == null ? meta.get(MetaConst.META_TYPE) : meta.get(MetaConst.META_DATA_TYPE);
+    private static List<List<Object>> convertToArrayList(List<Object> result) {
+        return result.stream()
+                .map(Lists::newArrayList)
+                .collect(Collectors.toList());
+    }
 
+    private static List getOtherExampleValues(Map<String, String> meta, Boolean isDesc, int size) {
+        String dataType = meta.get(MetaConst.META_DATA_TYPE) == null ? meta.get(MetaConst.META_TYPE) : meta.get(MetaConst.META_DATA_TYPE);
         if (DataTypeMapping.LONG.equals(dataType) || DataTypeMapping.INT.equals(dataType)) {
             return getExampleIntegerValues(null, isDesc, size);
-        }
-        if (DataTypeMapping.DOUBLE.equals(dataType) || DataTypeMapping.FLOAT.equals(dataType)) {
+        } else if (DataTypeMapping.DOUBLE.equals(dataType) || DataTypeMapping.FLOAT.equals(dataType)) {
             return getExampleDoubleValues(null, isDesc, size);
-        }
-        if (DataTypeMapping.BOOLEAN.equals(dataType)) {
+        } else if (DataTypeMapping.BOOLEAN.equals(dataType)) {
             return getExampleBooleanValues(size);
-        }
-        if (DataTypeMapping.DATE.equals(dataType) || DataTypeMapping.TIMESTAMP.equals(dataType)
+        } else if (DataTypeMapping.DATE.equals(dataType) || DataTypeMapping.TIMESTAMP.equals(dataType)
                 || DataTypeConst.TIMESTAMP_FORMAT.equals(dataType)
                 || DataTypeConst.TIMESTAMP_MS_FORMAT.equals(dataType) || DataTypeConst.UNIX_TIMESTAMP.equals(dataType)
                 || DataTypeConst.UNIX_TIMESTAMP_MS.equals(dataType)
-                || DataTypeConst.DATE_TIME_64.equals(dataType)
-        ) {
+                || DataTypeConst.DATE_TIME_64.equals(dataType)) {
             return getExampleDateTimeValues(dataType, size);
+        } else {
+            return getExampleStrValues(size);
         }
-        return getExampleStrValues(size);
     }
 
     private static List<Object> getExampleDateTimeValues(String dataType, int rowCount) {
@@ -254,6 +262,7 @@ public class ExampleDataHelper {
                 value = DateUtil.format(DateUtil.offsetHour(DateUtil.beginOfHour(currentDate), -rowCount), DatePattern.UTC_WITH_ZONE_OFFSET_PATTERN);
                 break;
             case DataTypeConst.TIMESTAMP_MS_FORMAT:
+            case DataTypeConst.DATE_TIME_64:
                 value = DateUtil.format(DateUtil.offsetHour(currentDate, -rowCount), DatePattern.NORM_DATETIME_MS_FORMAT);
                 break;
             case DataTypeConst.UNIX_TIMESTAMP:
@@ -265,9 +274,6 @@ public class ExampleDataHelper {
             case DataTypeConst.DATE:
                 value = DateUtil.format(DateUtil.offsetHour(currentDate, -rowCount), DatePattern.NORM_DATE_PATTERN);
                 break;
-            case DataTypeConst.DATE_TIME_64:
-                value = DateUtil.format(DateUtil.offsetHour(currentDate, -rowCount), DatePattern.NORM_DATETIME_MS_FORMAT);
-                break;
             default:
                 value = DateUtil.format(DateUtil.offsetHour(currentDate, -rowCount), DatePattern.NORM_DATETIME_FORMAT);
                 break;
@@ -303,12 +309,12 @@ public class ExampleDataHelper {
         return result;
     }
 
-    private static List<Object> getExampleDataValues(Map<String, String> meta, boolean canRepeat, int size, Map<String, List<Object>> schemaDataDict) {
+    private static List<Object> getExampleDataValues(Map<String, String> meta, boolean repeatable, int size, Map<String, List<Object>> schemaDataDict) {
         List<Object> result = new ArrayList<>();
 
         if (schemaDataDict.containsKey(meta.get(MetaConst.META_FIELD_NAME))) {
             List<Object> list = schemaDataDict.get(meta.get(MetaConst.META_FIELD_NAME));
-            buildEnumDataList(canRepeat, size, result, list);
+            buildEnumDataList(repeatable, size, result, list);
             return result;
         }
 
@@ -329,7 +335,7 @@ public class ExampleDataHelper {
             }
         } else if (ExampleDataModeConst.ENUM.equalsIgnoreCase(mode.toString())) {
             List list = (List) JSONPath.extract(exampleJsonData, path + ".values");
-            buildEnumDataList(canRepeat, size, result, list);
+            buildEnumDataList(repeatable, size, result, list);
         } else if (ExampleDataModeConst.SEQUENCE.equalsIgnoreCase(mode.toString())) {
             List list = (List) JSONPath.extract(exampleJsonData, path + ".values");
             result.addAll(list.subList(0, size));
@@ -337,15 +343,15 @@ public class ExampleDataHelper {
         return result;
     }
 
-    private static void buildEnumDataList(boolean canRepeat, int size, List<Object> result, List<Object> list) {
-        if (canRepeat) {
+    private static void buildEnumDataList(boolean repeatable, int size, List<Object> result, List<Object> list) {
+        if (repeatable) {
             for (int i = 0; i < size; i++) {
                 int index = RandomNumberGenerator.generateRangeRandomInt(0, list.size() - 1);
                 result.add(list.get(index));
             }
         } else {
             Collections.shuffle(list);
-            result.addAll(list.subList(0, Math.min(size, list.size() )));
+            result.addAll(list.subList(0, Math.min(size, list.size())));
         }
     }
 
diff --git a/src/main/java/com/mesalab/common/utils/sqlparser/SelectItemHelper.java b/src/main/java/com/mesalab/common/utils/sqlparser/SelectItemHelper.java
index 7333383d..09b236d5 100644
--- a/src/main/java/com/mesalab/common/utils/sqlparser/SelectItemHelper.java
+++ b/src/main/java/com/mesalab/common/utils/sqlparser/SelectItemHelper.java
@@ -7,6 +7,8 @@ import lombok.SneakyThrows;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.Function;
+import net.sf.jsqlparser.expression.Parenthesis;
+import net.sf.jsqlparser.expression.RowConstructor;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
 import net.sf.jsqlparser.schema.Column;
 import net.sf.jsqlparser.statement.select.*;
@@ -22,6 +24,27 @@ import java.util.*;
  */
 public class SelectItemHelper {
 
+    private static final Map<String, String> FUNCTION_DATE_TYPE_MAPPING = Maps.newHashMap();
+
+    static {
+        FUNCTION_DATE_TYPE_MAPPING.put("FROM_UNIXTIME", DataTypeConst.TIMESTAMP_FORMAT);
+        FUNCTION_DATE_TYPE_MAPPING.put("DATE_FORMAT", DataTypeConst.TIMESTAMP_FORMAT);
+        FUNCTION_DATE_TYPE_MAPPING.put("CONVERT_TZ", DataTypeConst.TIMESTAMP_FORMAT);
+        FUNCTION_DATE_TYPE_MAPPING.put("TIME_FORMAT", DataTypeConst.TIMESTAMP_FORMAT);
+        FUNCTION_DATE_TYPE_MAPPING.put("FROM_UNIXTIME_MILLIS", DataTypeConst.TIMESTAMP_MS_FORMAT);
+        FUNCTION_DATE_TYPE_MAPPING.put("UNIX_TIMESTAMP", DataTypeConst.UNIX_TIMESTAMP);
+        FUNCTION_DATE_TYPE_MAPPING.put("TIME_FLOOR_WITH_FILL", DataTypeConst.UNIX_TIMESTAMP);
+        FUNCTION_DATE_TYPE_MAPPING.put("UNIX_TIMESTAMP_MILLIS", DataTypeConst.UNIX_TIMESTAMP_MS);
+        FUNCTION_DATE_TYPE_MAPPING.put("PERCENTILES_HDR", DataTypeConst.HDR_HISTOGRAM);
+        FUNCTION_DATE_TYPE_MAPPING.put("IP_TO_GEO", DataTypeConst.IP);
+        FUNCTION_DATE_TYPE_MAPPING.put("IP_TO_CITY", DataTypeConst.IP);
+        FUNCTION_DATE_TYPE_MAPPING.put("IP_TO_COUNTRY", DataTypeConst.IP);
+        FUNCTION_DATE_TYPE_MAPPING.put("IP_TO_ISP", DataTypeConst.IP);
+        FUNCTION_DATE_TYPE_MAPPING.put("IP_TO_ASN", DataTypeConst.IP);
+        FUNCTION_DATE_TYPE_MAPPING.put("IP_TO_ASN_DETAIL", DataTypeConst.IP);
+        FUNCTION_DATE_TYPE_MAPPING.put("IP_TO_ASN_ORG", DataTypeConst.IP);
+    }
+
     private SelectItemHelper() {
     }
 
@@ -92,44 +115,56 @@ public class SelectItemHelper {
             } else if (expression instanceof Function) {
                 Function function = (Function) expression;
                 String functionName = function.getName().toUpperCase();
-                Map<String, String> functionDateTypeMap = Maps.newHashMap();
-                functionDateTypeMap.put("FROM_UNIXTIME", DataTypeConst.TIMESTAMP_FORMAT);
-                functionDateTypeMap.put("DATE_FORMAT", DataTypeConst.TIMESTAMP_FORMAT);
-                functionDateTypeMap.put("CONVERT_TZ", DataTypeConst.TIMESTAMP_FORMAT);
-                functionDateTypeMap.put("TIME_FORMAT", DataTypeConst.TIMESTAMP_FORMAT);
-                functionDateTypeMap.put("FROM_UNIXTIME_MILLIS", DataTypeConst.TIMESTAMP_MS_FORMAT);
-                functionDateTypeMap.put("UNIX_TIMESTAMP", DataTypeConst.UNIX_TIMESTAMP);
-                functionDateTypeMap.put("TIME_FLOOR_WITH_FILL", DataTypeConst.UNIX_TIMESTAMP);
-                functionDateTypeMap.put("DATETIME_FLOOR_WITH_FILL", DataTypeConst.TIMESTAMP_FORMAT);
-                functionDateTypeMap.put("UNIX_TIMESTAMP_MILLIS", DataTypeConst.UNIX_TIMESTAMP_MS);
-                functionDateTypeMap.put("PERCENTILES_HDR", DataTypeConst.HDR_HISTOGRAM);
-                functionDateTypeMap.put("IP_TO_GEO", DataTypeConst.IP);
-                functionDateTypeMap.put("IP_TO_CITY", DataTypeConst.IP);
-                functionDateTypeMap.put("IP_TO_COUNTRY", DataTypeConst.IP);
-                functionDateTypeMap.put("IP_TO_ISP", DataTypeConst.IP);
-                functionDateTypeMap.put("IP_TO_ASN", DataTypeConst.IP);
-                functionDateTypeMap.put("IP_TO_ASN_DETAIL", DataTypeConst.IP);
-                functionDateTypeMap.put("IP_TO_ASN_ORG", DataTypeConst.IP);
-
-                if (functionDateTypeMap.containsKey(functionName)) {
+                if (SQLFunctionUtil.GROUP_UNIQ_ARRAY.equalsIgnoreCase(functionName)
+                        && function.getParameters() != null
+                        && function.getParameters().getExpressions() != null) {
+                    List<Expression> paramExpressions = function.getParameters().getExpressions();
+                    Expression expr1 = paramExpressions.get(0);
+                    if (expr1 instanceof Column) {
+                        Column column = (Column) expr1;
+                        AliasColumn aliasColumn = new AliasColumn();
+                        aliasColumn.setName(name);
+                        aliasColumn.setFieldName(column.getColumnName());
+                        aliasColumn.setArray(true);
+                        this.aliasExpr = aliasColumn;
+                        return;
+                    } else if ((expr1 instanceof Parenthesis) && ((Parenthesis) expr1).getExpression() instanceof Column) {
+                        Column column = (Column) ((Parenthesis) expr1).getExpression();
+                        AliasColumn aliasColumn = new AliasColumn();
+                        aliasColumn.setName(name);
+                        aliasColumn.setFieldName(column.getColumnName());
+                        aliasColumn.setArray(true);
+                        this.aliasExpr = aliasColumn;
+                        return;
+                    } else if (expr1 instanceof RowConstructor
+                            && ((RowConstructor) expr1).getExprList() != null
+                            && ((RowConstructor) expr1).getExprList().getExpressions().get(0) instanceof Column) {
+                        AliasColumn aliasColumn = new AliasColumn();
+                        aliasColumn.setName(name);
+                        aliasColumn.setArray(true);
+                        this.aliasExpr = aliasColumn;
+                        return;
+                    }
+                }
+                if (FUNCTION_DATE_TYPE_MAPPING.containsKey(functionName)) {
                     AliasFunExpr aliasFunExpr = new AliasFunExpr();
                     aliasFunExpr.setName(name);
-                    aliasFunExpr.setDateType(functionDateTypeMap.get(functionName));
+                    aliasFunExpr.setDateType(FUNCTION_DATE_TYPE_MAPPING.get(functionName));
                     this.aliasExpr = aliasFunExpr;
                     return;
                 }
+                AliasObject aliasObject = new AliasObject();
+                aliasObject.setName(name);
+                this.aliasExpr = aliasObject;
             }
-            AliasObject aliasObject = new AliasObject();
-            aliasObject.setName(name);
-            this.aliasExpr = aliasObject;
         }
-
     }
 
     @Data
     public static class AliasObject {
         private String name;
         private int index;
+        private boolean isArray;
     }
 
     @Data
diff --git a/src/main/java/com/mesalab/qgw/constant/MetaConst.java b/src/main/java/com/mesalab/qgw/constant/MetaConst.java
index 2f3f964a..5ec2fa66 100644
--- a/src/main/java/com/mesalab/qgw/constant/MetaConst.java
+++ b/src/main/java/com/mesalab/qgw/constant/MetaConst.java
@@ -9,10 +9,12 @@ package com.mesalab.qgw.constant;
  */
 public class MetaConst {
     public static final String META_CATEGORY = "category";
+    public static final String META_CATEGORY_METRIC = "Metric";
+    public static final String META_CATEGORY_DIMENSION = "Dimension";
     public static final String META_NAME = "name";
     public static final String META_TYPE = "type";
     public static final String META_DATA_TYPE = "data_type";
     public static final String META_FIELD_NAME = "field_name";
-    public static final String META_CATEGORY_METRIC = "Metric";
-    public static final String META_CATEGORY_DIMENSION = "Dimension";
+    public static final String IS_ARRAY = "is_array";
+
 }
diff --git a/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java b/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java
index 50b7df85..99970843 100644
--- a/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java
+++ b/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java
@@ -38,7 +38,6 @@ import org.apache.http.NameValuePair;
 import org.apache.http.client.utils.URLEncodedUtils;
 import org.springframework.util.CollectionUtils;
 
-import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
 import java.util.regex.Matcher;
@@ -46,7 +45,7 @@ import java.util.regex.Pattern;
 
 public class ClickHouseDialect extends AbstractDataSourceDialect {
     private static final Log log = LogFactory.get();
-    private static final Pattern pNullable = Pattern.compile("\\bNullable\\((.*?)\\)", Pattern.CASE_INSENSITIVE);
+    private static final Pattern pNullable = Pattern.compile("^Nullable\\((.*?)\\)", Pattern.CASE_INSENSITIVE);
     private static final Pattern pDateTime = Pattern.compile("\\b(DateTime|DateTime64|Date)\\((.*?)\\)", Pattern.CASE_INSENSITIVE);
     private static final Pattern pIn = Pattern.compile("(?<!\\bnot)\\s+in\\s*\\(\\s*SELECT\\b", Pattern.CASE_INSENSITIVE);
     private static final Pattern pNotIn = Pattern.compile("\\bnot\\s+in\\s*\\(\\s*SELECT\\b", Pattern.CASE_INSENSITIVE);
@@ -820,6 +819,7 @@ public class ClickHouseDialect extends AbstractDataSourceDialect {
             SelectItemHelper.AliasColumn aliasColumn = (SelectItemHelper.AliasColumn) aliasObject;
             String fieldName = aliasColumn.getFieldName();
             meta.put(MetaConst.META_FIELD_NAME, fieldName);
+            meta.put(MetaConst.IS_ARRAY, String.valueOf(aliasColumn.isArray()));
             Map schemaInfo = databaseService.getSchemaInfo(MetadataType.FIELDS.getValue(), sqlQueryContext.getDbSelectStatement().getTableNames().get(0), false);
             Object dateType = JSONPath.extract(JSON.toJSONString(schemaInfo), "$.fields[?(@.name == \"" + fieldName + "\")].doc.constraints.type");
             if (dateType instanceof List && ((List<?>) dateType).size() > 0) {
diff --git a/src/main/java/com/mesalab/qgw/dialect/FederationDialect.java b/src/main/java/com/mesalab/qgw/dialect/FederationDialect.java
index 2278ac61..5dc72f3a 100644
--- a/src/main/java/com/mesalab/qgw/dialect/FederationDialect.java
+++ b/src/main/java/com/mesalab/qgw/dialect/FederationDialect.java
@@ -171,11 +171,14 @@ public class FederationDialect extends AbstractEngineDialect {
         if (baseResult.getMeta() != null) {
             List<Map<String, Object>> metaList = (List<Map<String, Object>>) baseResult.getMeta();
             metaList.forEach(x -> {
-                x.remove(MetaConst.META_CATEGORY);
-                x.remove(MetaConst.META_DATA_TYPE);
-                x.remove(MetaConst.META_FIELD_NAME);
+                Iterator<String> iterator = x.keySet().iterator();
+                while (iterator.hasNext()) {
+                    String key = iterator.next();
+                    if (!key.equals(MetaConst.META_NAME) && !key.equals(MetaConst.META_TYPE)) {
+                        iterator.remove();
+                    }
+                }
             });
-
         }
         if (sqlQueryContext.getFormat().equalsIgnoreCase(OutputMode.CSV.getValue())) {
             convertJsonToCSV();