diff options
36 files changed, 2378 insertions, 104 deletions
diff --git a/galaxy-admin-server/pom.xml b/galaxy-admin-server/pom.xml index bd5f13f..e800ddb 100644 --- a/galaxy-admin-server/pom.xml +++ b/galaxy-admin-server/pom.xml @@ -23,14 +23,6 @@ <artifactId>spring-boot-starter-web</artifactId> </dependency> <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-starter-netflix-hystrix</artifactId> - </dependency> - <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-security</artifactId> </dependency> diff --git a/galaxy-auth-center/pom.xml b/galaxy-auth-center/pom.xml index 8667385..c7622a3 100644 --- a/galaxy-auth-center/pom.xml +++ b/galaxy-auth-center/pom.xml @@ -14,10 +14,6 @@ <dependencies> <dependency> - <groupId>com.mesalab</groupId> - <artifactId>galaxy-common</artifactId> - </dependency> - <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> </dependency> @@ -30,14 +26,6 @@ <artifactId>spring-boot-starter-web</artifactId> </dependency> <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-starter-netflix-hystrix</artifactId> - </dependency> - <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-actuator</artifactId> </dependency> diff --git a/galaxy-auth-center/src/test/java/com/mesalab/api/test/JwtAuthorizeServiceTest.java b/galaxy-auth-center/src/test/java/com/mesalab/api/test/JwtAuthorizeServiceTest.java deleted file mode 100644 index 953ec17..0000000 --- a/galaxy-auth-center/src/test/java/com/mesalab/api/test/JwtAuthorizeServiceTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.mesalab.api.test; - -import com.mesalab.auth.GalaxyAuthCenterApp; -import com.mesalab.auth.service.AuthorizeService; -import lombok.extern.slf4j.Slf4j; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -/** - * @Date: 2020-08-17 14:33 - * @Author : liuyongqiang - * @ClassName : JwtAuthorizeServiceTest - * @Description : JWT授权服务测试 - */ -@RunWith(SpringRunner.class) -@SpringBootTest(classes = {GalaxyAuthCenterApp.class})// 指定启动类 -@Slf4j -public class JwtAuthorizeServiceTest { - - @Autowired - AuthorizeService jwtAuthorizeService; - - //ArangoDBJwt登录测试 - @Test - public void arangoJwtLoginTest() { - log.info(jwtAuthorizeService.arangoJwtLogin()); - } -} diff --git a/galaxy-common/pom.xml b/galaxy-common/pom.xml deleted file mode 100644 index 72d01ca..0000000 --- a/galaxy-common/pom.xml +++ /dev/null @@ -1,20 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <parent> - <artifactId>galaxy-data-platform</artifactId> - <groupId>com.mesalab</groupId> - <version>1.0-SNAPSHOT</version> - </parent> - - <modelVersion>4.0.0</modelVersion> - <artifactId>galaxy-common</artifactId> - <packaging>jar</packaging> - - - <build> - <finalName>galaxy-common</finalName> - </build> - -</project>
\ No newline at end of file diff --git a/galaxy-gateway/pom.xml b/galaxy-gateway/pom.xml index e5fe68a..a7d8781 100644 --- a/galaxy-gateway/pom.xml +++ b/galaxy-gateway/pom.xml @@ -14,18 +14,6 @@ <dependencies> <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-starter-gateway</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-starter-netflix-hystrix</artifactId> - </dependency> - <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-actuator</artifactId> </dependency> diff --git a/galaxy-gateway/src/main/java/com/mesalab/api/gateway/GalaxyGatewayApp.java b/galaxy-gateway/src/main/java/com/mesalab/api/gateway/GalaxyGatewayApp.java index 661d03a..0f72ed3 100644 --- a/galaxy-gateway/src/main/java/com/mesalab/api/gateway/GalaxyGatewayApp.java +++ b/galaxy-gateway/src/main/java/com/mesalab/api/gateway/GalaxyGatewayApp.java @@ -1,19 +1,11 @@ package com.mesalab.api.gateway; import org.springframework.boot.SpringApplication; -import org.springframework.cloud.client.SpringCloudApplication; -import org.springframework.cloud.client.loadbalancer.LoadBalanced; -import org.springframework.context.annotation.Bean; -import org.springframework.web.client.RestTemplate; +import org.springframework.boot.autoconfigure.SpringBootApplication; -@SpringCloudApplication +@SpringBootApplication public class GalaxyGatewayApp { - @Bean - @LoadBalanced - RestTemplate restTemplate() { - return new RestTemplate(); - } public static void main(String[] args) { SpringApplication.run(GalaxyGatewayApp.class); diff --git a/galaxy-query-engine/config/application.properties b/galaxy-query-engine/config/application.properties index d884eee..014575d 100644 --- a/galaxy-query-engine/config/application.properties +++ b/galaxy-query-engine/config/application.properties @@ -1,8 +1,29 @@ #application config server.port=8803 spring.application.name=galaxy-query-engine + #management config management.endpoints.web.exposure.include=* management.endpoint.health.show-details=always + #log file path config -logging.config=./config/logback-spring.xml
\ No newline at end of file +logging.config=classpath:logback-spring.xml + +#engine config +engine.maxCacheNum=500000 +engine.defaultResultNum=300000 + +#druid config +druid.address=192.168.40.152:8082 +druid.dbname=druid +druid.sqlTimeZone=Asia/Shanghai +druid.query.url=http://${druid.address}/druid/v2/sql + +#clickhouse config +clickhouse.address=192.168.44.12:8123 +clickhouse.query.url=http://${clickhouse.address} +clickhouse.dbname=tsg_galaxy_v3 +clickhouse.real.time.username=tsg_query +clickhouse.real.time.password=ceiec2018 +clickhouse.long.term.username=tsg_report +clickhouse.long.term.password=ceiec2019
\ No newline at end of file diff --git a/galaxy-query-engine/pom.xml b/galaxy-query-engine/pom.xml index caea719..b7d8c32 100644 --- a/galaxy-query-engine/pom.xml +++ b/galaxy-query-engine/pom.xml @@ -12,4 +12,118 @@ <artifactId>galaxy-query-engine</artifactId> <packaging>jar</packaging> + <dependencies> + <dependency> + <groupId>org.projectlombok</groupId> + <artifactId>lombok</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-actuator</artifactId> + </dependency> + <dependency> + <groupId>cn.hutool</groupId> + <artifactId>hutool-all</artifactId> + </dependency> + <dependency> + <groupId>org.apache.avro</groupId> + <artifactId>avro</artifactId> + </dependency> + <dependency> + <groupId>com.github.java-json-tools</groupId> + <artifactId>json-schema-validator</artifactId> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-test</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.zdjizhi</groupId> + <artifactId>galaxy</artifactId> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>commons-lang</groupId> + <artifactId>commons-lang</artifactId> + <version>2.6</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + </dependency> + + </dependencies> + + <build> + <finalName>galaxy-query-engine</finalName> + <plugins> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + <version>2.0.2.RELEASE</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>repackage</goal> + </goals> + </execution> + </executions> + <configuration> + <includeSystemScope>true</includeSystemScope> + <mainClass>com.mesalab.engine.GalaxyDataEngineApp</mainClass> + </configuration> + </plugin> + <plugin> + <groupId>com.spotify</groupId> + <artifactId>docker-maven-plugin</artifactId> + <version>1.0.0</version> + <configuration> + <serverId>153-docker-repo</serverId> + <registryUrl>${docker.registry}:${docker.registry.port}</registryUrl> + <pushImage>true</pushImage> + <imageName>${docker.registry}:${docker.registry.port}/${docker.image.prefix}/${project.artifactId} + </imageName> + <forceTags>true</forceTags> + <dockerHost>http://192.168.40.153:2375</dockerHost> + <dockerDirectory>docker</dockerDirectory> + <buildArgs> + <JDK_IMAGE>192.168.40.153:9080/common/jdk:1.8.0_73</JDK_IMAGE> + <JAR_FILE>${project.build.finalName}.jar</JAR_FILE> + </buildArgs> + <resources> + <resource> + <targetPath>/</targetPath> + <directory>${project.build.directory}</directory> + <include>${project.build.finalName}.jar</include> + </resource> + <resource> + <targetPath>/mmdb</targetPath> + <directory>mmdb</directory> + </resource> + <resource> + <targetPath>/config</targetPath> + <directory>config</directory> + </resource> + <resource> + <targetPath>/schema</targetPath> + <directory>schema</directory> + </resource> + </resources> + </configuration> + </plugin> + </plugins> + </build> + </project>
\ No newline at end of file diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/GalaxyQueryEngine.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/GalaxyQueryEngine.java new file mode 100644 index 0000000..ade63bd --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/GalaxyQueryEngine.java @@ -0,0 +1,12 @@ +package com.mesalab.engine; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class GalaxyQueryEngine { + + public static void main(String[] args) { + SpringApplication.run(GalaxyQueryEngine.class); + } +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/constant/SqlKeywords.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/constant/SqlKeywords.java new file mode 100644 index 0000000..f5ff188 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/constant/SqlKeywords.java @@ -0,0 +1,71 @@ +package com.mesalab.engine.common.constant; + +/** + * @Date: 2020-07-22 19:41 + * @Author : liuyongqiang + * @ClassName : SqlKeywords + * @Description : SQL关键字常量 + */ +public class SqlKeywords { + + public static final String SELECT = " select "; + + public static final String AND = " and "; + + public static final String WHERE = " where "; + + public static final String LIKE = " like "; + + public static final String OR = " or "; + + public static final String FROM = " from "; + + public static final String BWTWEEN = " between "; + + public static final String ORDER_BY = " order by "; + + public static final String LIMIT = " limit "; + + public static final String D_TIME = " __time "; + + public static final String D_TIME_FLOOR = " time_floor(__time,''{0}'') "; + + public static final String C_TO_DATE_TIME = "toDateTime(''{0}'')"; + + public static final String A_SORT ="SORT"; + + public static final String A_TIME = "_TIME"; + + public static final String A_FILTER = "FILTER"; + + public static final String A_AND = "AND"; + + public static final String A_LIKE = "LIKE"; + + public static final String A_OR = "OR"; + + public static final String A_SPACE = " "; + + public static final String A_SPOT = "."; + + public static final String A_LIMIT = "LIMIT" ; + + public static final String A_RETURN = "RETURN"; + + public static final String C_SECOND = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} SECOND)))) as granularity"; + + public static final String C_MINUTE = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} MINUTE)))) as granularity"; + + public static final String C_HOUR = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} HOUR)))) as granularity"; + + public static final String C_DAY = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} DAY)))) as granularity"; + + public static final String C_WEEK = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} WEEK)))) as granularity"; + + public static final String C_MONTH = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} MONTH)))) as granularity"; + + public static final String C_YEAR = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} YEAR)))) as granularity"; + + + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/dto/ClickHouseResult.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/dto/ClickHouseResult.java new file mode 100644 index 0000000..51786da --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/dto/ClickHouseResult.java @@ -0,0 +1,31 @@ +package com.mesalab.engine.common.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +/** + * @Date: 2020-07-23 16:21 + * @Author : liuyongqiang + * @ClassName : ClickHouseResult + * @Description : ClickHouse查询结果封装 + */ +@Data +@AllArgsConstructor +public class ClickHouseResult implements Serializable { + + private int rows; + private List<MetaBean> meta; + private List<Map<String, Object>> data; + private Map<String, Object> statistics; + + @Data + public static class MetaBean { + private String name; + private String type; + } + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/EngineTypeEnum.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/EngineTypeEnum.java new file mode 100644 index 0000000..979716a --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/EngineTypeEnum.java @@ -0,0 +1,36 @@ +package com.mesalab.engine.common.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @Date: 2020-07-20 18:45 + * @Author : liuyongqiang + * @ClassName : EngineTypeEnum + * @Description : 查询引擎映射枚举类 + */ +@Getter +@AllArgsConstructor +public enum EngineTypeEnum { + + //AnalysisEngine-AnangoDB 分析、学习引擎 + //CaculationEngine-Druid 保存、计算引擎 + //BusinessEngine-ClickHouse 业务数据引擎 + + ANALYSIS_ENGINE("AnalysisEngine", "AnangoDB"), + CACULATION_ENGINE("CaculationEngine", "Druid"), + BUSINESS_ENGINE("BusinessEngine", "ClickHouse"); + + private String engine;//引擎名称 + private String dbtype;//数据库类型 + + public static EngineTypeEnum getByEngine(String engine) { + for (EngineTypeEnum constants : values()) { + if (constants.engine.equalsIgnoreCase(engine)) { + return constants; + } + } + return null; + } + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/IntervalTypeEnum.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/IntervalTypeEnum.java new file mode 100644 index 0000000..8be621f --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/IntervalTypeEnum.java @@ -0,0 +1,28 @@ +package com.mesalab.engine.common.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @Date: 2020-07-29 15:45 + * @Author : wangwei + * @ClassName : IntervalTypeEnum + * @Description : 区间查询类型枚举 + */ +@Getter +@AllArgsConstructor +public enum IntervalTypeEnum { + EXACTLY("BETWEEN", "闭区间间隔"); + + private String type; + private String name; + + public static IntervalTypeEnum getByType(String type) { + for (IntervalTypeEnum constants : values()) { + if (constants.type.equalsIgnoreCase(type)) { + return constants; + } + } + return null; + } +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/MatchTypeEnum.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/MatchTypeEnum.java new file mode 100644 index 0000000..36b7e01 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/MatchTypeEnum.java @@ -0,0 +1,33 @@ +package com.mesalab.engine.common.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @Date: 2020-07-22 17:45 + * @Author : liuyongqiang + * @ClassName : MatchTypeEnum + * @Description : 匹配查询类型枚举 + */ +@Getter +@AllArgsConstructor +public enum MatchTypeEnum { + + EXACTLY("exactly", "完全匹配"), + PREFIX("prefix", "前缀匹配"), + SUFFIX("suffix", "后缀匹配"), + SUBSTRING("substring", "字串匹配"), + REGEX("regex", "正则匹配"); + + private String type; + private String name; + + public static MatchTypeEnum getByType(String type) { + for (MatchTypeEnum constants : values()) { + if (constants.type.equalsIgnoreCase(type)) { + return constants; + } + } + return null; + } +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/RangeTypeEnum.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/RangeTypeEnum.java new file mode 100644 index 0000000..a9c5fba --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/RangeTypeEnum.java @@ -0,0 +1,34 @@ +package com.mesalab.engine.common.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @Date: 2020-07-22 17:44 + * @Author : liuyongqiang + * @ClassName : RangeTypeEnum + * @Description : 范围查询类型枚举 + */ +@Getter +@AllArgsConstructor +public enum RangeTypeEnum { + + GT("gt", " > "), + LT("lt", " < "), + EQ("eq", " = "), + GE("ge", " >= "), + LE("le", " <= "), + NE("ne", " != "); + + private String type; + private String expr; + + public static RangeTypeEnum getByType(String type) { + for (RangeTypeEnum constants : values()) { + if (constants.type.equalsIgnoreCase(type)) { + return constants; + } + } + return null; + } +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/ResultCodeEnum.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/ResultCodeEnum.java new file mode 100644 index 0000000..ab5fe58 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/ResultCodeEnum.java @@ -0,0 +1,24 @@ +package com.mesalab.engine.common.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * BaseResult的业务状态码 + */ +@Getter +@AllArgsConstructor +public enum ResultCodeEnum { + + EXECUTE_SUCCESS("200666", "成功"), + PARAM_SYNTAX_ERROR("400001", "参数检查异常"), + SQL_SYNTAX_ERROR("400010", "SQL语句检查异常"), + SQL_EXECUTION_ERROR("500001", "SQL 执行异常"), + ENGINE_STATISTICS_ERROR("500010", "引擎计算异常"), + UNKNOW_ERROR("500999", "执行失败"); + + private String code; + private String message; + +} + diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/ResultStatusEnum.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/ResultStatusEnum.java new file mode 100644 index 0000000..64cb304 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/enums/ResultStatusEnum.java @@ -0,0 +1,29 @@ +package com.mesalab.engine.common.enums; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * BaseResult的http状态编码枚举 + */ +@Getter +@AllArgsConstructor +public enum ResultStatusEnum { + + /** + * SUCCESS: 200 成功 + * FAIL: 400 失败 + * NOT_FOUND： 404 不存在 + * SERVER_ERROR: 500 网络服务异常 + */ + SUCCESS(200, "成功"), + FAIL(400, "失败"), + NOT_FOUND(404, "不存在"), + REQ_FORBIDDEN(403, "重复请求"), + SERVER_ERROR(500, "服务异常"); + + private int code; + private String message; + +} + diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/exception/BusinessException.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/exception/BusinessException.java new file mode 100644 index 0000000..2f5c476 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/exception/BusinessException.java @@ -0,0 +1,52 @@ +package com.mesalab.engine.common.exception; + +import com.mesalab.engine.common.enums.ResultCodeEnum; +import com.mesalab.engine.common.enums.ResultStatusEnum; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * 业务异常 + * + * @author dazzlzy + * @date 2018/3/22 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode(callSuper = false) +public class BusinessException extends RuntimeException { + + /** + * 返回HTTP状态码 + */ + private int errorStatus = ResultStatusEnum.SERVER_ERROR.getCode(); + + /** + * 内部执行错误码 + */ + private String errorCode = ResultCodeEnum.UNKNOW_ERROR.getCode(); + + /** + * 异常信息 + */ + private String errorMessage; + + + public BusinessException(String errorMessage) { + this.errorMessage = errorMessage; + } + + + public BusinessException(String errorMessage, Throwable e) { + super(errorMessage, e); + } + + public BusinessException(int errorStatus, String errorCode, String errorMessage, Throwable e) { + super(errorMessage, e); + this.errorStatus = errorStatus; + this.errorCode = errorCode; + } +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/BaseResult.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/BaseResult.java new file mode 100644 index 0000000..3784e6d --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/BaseResult.java @@ -0,0 +1,51 @@ +package com.mesalab.engine.common.pojo; + +import com.mesalab.engine.common.enums.ResultStatusEnum; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.Map; + +/** + * 响应结果 + * + * @author dazzlzy + * @date 2018/3/21 + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class BaseResult<T> implements Serializable { + + private Integer status; + + private String code; + + private boolean success; + + private String message; + + private Map<String, Object> statistics; + + private String formatType; + + private T meta; + + private T data; + + + /** + * 判断是否是成功结果 + * JsonIgnore使之不在json序列化结果当中 + * + * @return 是否为成功结果 + */ + public boolean isSuccess() { + return ResultStatusEnum.SUCCESS.getCode() == this.status; + } + +}
\ No newline at end of file diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/ClickHouseParser.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/ClickHouseParser.java new file mode 100644 index 0000000..a2b4c32 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/ClickHouseParser.java @@ -0,0 +1,122 @@ +package com.mesalab.engine.common.pojo; + +import cn.hutool.core.util.StrUtil; +import com.mesalab.engine.common.constant.SqlKeywords; +import com.mesalab.engine.service.ClickHouseEngineService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.text.MessageFormat; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @Date: 2020-07-22 18:00 + * @Author : liuyongqiang + * @ClassName : ClickHouseDSLParser + * @Description : ClickHouse DSL语法解析器 + */ +@Slf4j +@Component +public class ClickHouseParser extends DSLParser { + + @Autowired + ClickHouseEngineService clickHouseEngineService; + + //表字段和数据类型查询SQL + public final static String SCHEAM_QUERY = " desc {0} "; + + //查询所有表 + public final static String TABLES_QUERY = " show tables "; + + public static Pattern periodOfPT = Pattern.compile("PT(\\d+)(\\w+)", Pattern.CASE_INSENSITIVE); + + public static Pattern periodOfP = Pattern.compile("P(\\d+)(\\w+)", Pattern.CASE_INSENSITIVE); + + + @Override + public String parseGranularity(DSLObject dslObject, String sql) { + if (StrUtil.isEmpty(dslObject.getQuery().getGranularity())) { + return sql; + } + String granularity = dslObject.getQuery().getGranularity(); + Matcher matcherPT = periodOfPT.matcher(granularity); + Matcher matcherP = periodOfP.matcher(granularity); + String format = ""; + + if (matcherPT.find()) { + String PTnum = matcherPT.group(1); + String PTunit = matcherPT.group(2); + if ("S".equalsIgnoreCase(PTunit)) { + format = MessageFormat.format(SqlKeywords.C_SECOND, PTnum); + } else if ("M".equalsIgnoreCase(PTunit)) { + format = MessageFormat.format(SqlKeywords.C_MINUTE, PTnum); + } else if ("H".equalsIgnoreCase(PTunit)) { + format = MessageFormat.format(SqlKeywords.C_HOUR, PTnum); + } + } + + if (matcherP.find()) { + String Pnum = matcherP.group(1); + String Punit = matcherP.group(2); + if ("D".equalsIgnoreCase(Punit)) { + format = MessageFormat.format(SqlKeywords.C_DAY, Pnum); + } else if ("W".equalsIgnoreCase(Punit)) { + format = MessageFormat.format(SqlKeywords.C_WEEK, Pnum); + } else if ("M".equalsIgnoreCase(Punit)) { + format = MessageFormat.format(SqlKeywords.C_MONTH, Pnum); + } else if ("Y".equalsIgnoreCase(Punit)) { + format = MessageFormat.format(SqlKeywords.C_YEAR, Pnum); + } + } + sql = sql.replace(SqlKeywords.SELECT, SqlKeywords.SELECT.concat(format).concat(",")); + return sql; + } + + @Override + public String parseSort(DSLObject dslObject, String sql) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getSort())) { + return sql; + } + String sortType = dslObject.getQuery().getSort().get(0).getType(); + sql = sql.concat(SqlKeywords.ORDER_BY + .concat(dslObject.getQuery().getSort().get(0).getFieldKey())) + .concat(" ") + .concat(sortType); + if (dslObject.getQuery().getSort().size() > 1) { + List<DSLObject.QueryBean.SortBean> sortBeans = dslObject.getQuery().getSort(); + for (int i = 1; i < sortBeans.size(); i++) { + sql = sql.concat(" , ") + .concat(sortBeans.get(i).getFieldKey()) + .concat(" ") + .concat(sortBeans.get(i).getType()); + } + } + return sql; + } + + @Override + public String parseIntervals(DSLObject dslObject, String sql) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getIntervals())) { + return sql; + } + + List<DSLObject.QueryBean.IntervalsBean> intervalsBeanList = dslObject.getQuery().getIntervals(); + for (DSLObject.QueryBean.IntervalsBean intervalsBean : intervalsBeanList) { + String fieldKey = intervalsBean.getFieldKey(); + List<String> values = intervalsBean.getFieldValues(); + if (CollectionUtils.isEmpty(values)) break; + sql = sql.concat(sql.contains(SqlKeywords.WHERE) ? SqlKeywords.AND : SqlKeywords.WHERE) + .concat(fieldKey) + .concat(SqlKeywords.BWTWEEN) + .concat(MessageFormat.format(SqlKeywords.C_TO_DATE_TIME, values.get(0))) + .concat(SqlKeywords.AND) + .concat(MessageFormat.format(SqlKeywords.C_TO_DATE_TIME, values.get(1))); + } + return sql; + } + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/DSLObject.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/DSLObject.java new file mode 100644 index 0000000..8afcc6d --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/DSLObject.java @@ -0,0 +1,61 @@ +package com.mesalab.engine.common.pojo; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * @Date: 2020-07-20 09:48 + * @Author : liuyongqiang + * @ClassName : DSLObject + * @Description : 标准服务接口模型 + */ + +@Data +public class DSLObject implements Serializable { + + private static final long serialVersionUID = 4956257906217545258L; + + private String limit;//数据条数 + private String dataEngine;//数据引擎 + private String dataSource;//数据来源 + private QueryBean query;//查询结构 + + @Data + public static class QueryBean { + private String granularity;//时间粒度 + private List<MatchBean> match;//匹配查询 + private List<RangeBean> range;//区间查询 + private List<IntervalsBean> intervals;//时间范围 + private List<SortBean> sort;//排序方式 + + @Data + public static class MatchBean { + private String type; + private String fieldKey; + private List<String> fieldValues; + } + + @Data + public static class RangeBean { + private String type; + private String fieldKey; + private List<String> fieldValues; + } + + @Data + public static class IntervalsBean { + private String type; + private String fieldKey; + private List<String> fieldValues; + } + + @Data + public static class SortBean { + private String type; + private String fieldKey; + } + } +} + diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/DSLParser.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/DSLParser.java new file mode 100644 index 0000000..cabf0fa --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/DSLParser.java @@ -0,0 +1,285 @@ +package com.mesalab.engine.common.pojo; + +import cn.hutool.core.util.StrUtil; +import com.mesalab.engine.common.constant.SqlKeywords; +import com.mesalab.engine.common.enums.EngineTypeEnum; +import com.mesalab.engine.common.enums.MatchTypeEnum; +import com.mesalab.engine.common.enums.RangeTypeEnum; +import com.mesalab.engine.service.ClickHouseEngineService; +import com.mesalab.engine.service.DruidEngineService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.text.MessageFormat; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * @Date: 2020-07-22 17:35 + * @Author : liuyongqiang + * @ClassName : DslParser + * @Description : DSLParse解析抽象类 + */ +@Slf4j +@Component +public abstract class DSLParser { + + //默认数据大小 + private final static String DEFAULT_LIMIT = "100000"; + + @Autowired + DruidEngineService druidEngine; + + @Autowired + ClickHouseEngineService clickHouseEngine; + + + /** + * 解析granularity字段 + */ + public abstract String parseGranularity(DSLObject dslObject, String sql); + + /** + * 解析 Sort对象 + */ + public abstract String parseSort(DSLObject dslObject, String sql); + + /** + * 解析 Intervals对象 + */ + public abstract String parseIntervals(DSLObject dslObject, String sql); + + /** + * @param dslObject: + * @Description: 单表全列查询 + * @Author: liuyongqiang + * @Date: 2020/7/23 18:23 + * @return: java.lang.String + **/ + public String allQuery(DSLObject dslObject) { + String selectSql = this.allColumn(dslObject); + selectSql = parseQuery(dslObject, selectSql); + return selectSql; + } + + /** + * @param dslObject: + * @Description: 查询表中的所有列 + * @Author: liuyongqiang + * @Date: 2020/7/23 17:24 + * @return: java.lang.String + **/ + public String allColumn(DSLObject dslObject) { + BaseResult baseResult = null; + + switch (EngineTypeEnum.getByEngine(dslObject.getDataEngine())) { + case CACULATION_ENGINE: + baseResult = druidEngine.schemaQuery(dslObject); + break; + case BUSINESS_ENGINE: + baseResult = clickHouseEngine.schemaQuery(dslObject); + break; + } + + List<Map<String, Object>> results = (List<Map<String, Object>>) baseResult.getData(); + String column = ""; + for (Map<String, Object> map : results) { + String name = String.valueOf(map.get("name")); + //ClickHouse数据表中的时间字段做日期转换 + if (dslObject.getDataEngine().equals(EngineTypeEnum.BUSINESS_ENGINE.getEngine())) { + if (name.lastIndexOf("_time") > 0) { + name = MessageFormat.format(" toDateTime({0}) as c_{1} ", name, name); + } + } + //Druid数据表中的时间字段做日期转换 + if (dslObject.getDataEngine().equals(EngineTypeEnum.CACULATION_ENGINE.getEngine())) { + if (name.equalsIgnoreCase("__time")) { + name = "TIME_FORMAT(__time,'yyyy-MM-dd HH:mm:ss') as c_time"; + } + } + //拼接表中的列名称 + column = column.concat(name).concat(","); + } + //去掉最后一个逗号 + column = column.substring(0, column.length() - 1); + return SqlKeywords.SELECT + .concat(column) + .concat(SqlKeywords.FROM) + .concat(dslObject.getDataSource()); + } + + + /** + * @param dslObject: + * @param sql: + * @Description: 解析query对象 + * @Author: liuyongqiang + * @Date: 2020/7/23 14:29 + * @return: java.lang.String + **/ + public String parseQuery(DSLObject dslObject, String sql) { + if (Objects.nonNull(dslObject.getQuery())) { + sql = parseGranularity(dslObject, sql);//解析Granularity + sql = parseMatch(dslObject, sql);//解析Match + sql = parseRange(dslObject, sql);//range解析 + sql = parseIntervals(dslObject, sql);//解析Intervals + sql = parseSort(dslObject, sql);//解析Sort + } + sql = parseLimit(dslObject, sql);//limit解析 + log.info("Parse Query SQL:{}", sql); + return sql; + } + + /** + * @param dslObject: + * @param sql: + * @Description: 解析 Range对象 + * @Author: liuyongqiang + * @Date: 2020/7/23 17:47 + * @return: java.lang.String + **/ + public String parseRange(DSLObject dslObject, String sql) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getRange())) { + return sql; + } + List<DSLObject.QueryBean.RangeBean> rangeBeanList = dslObject.getQuery().getRange(); + for (DSLObject.QueryBean.RangeBean rangeBean : rangeBeanList) { + String type = rangeBean.getType(); + String fieldKey = rangeBean.getFieldKey(); + List<String> values = rangeBean.getFieldValues(); + + if (CollectionUtils.isEmpty(values)) break; + + String join = sql.contains(SqlKeywords.WHERE) ? SqlKeywords.AND : SqlKeywords.WHERE; + String expr = RangeTypeEnum.getByType(type).getExpr(); + sql = sql.concat(join).concat(fieldKey).concat(expr).concat(values.get(0)); + + if (values.size() > 1) { + for (int i = 1; i < values.size(); i++) { + sql = sql.concat(SqlKeywords.OR) + .concat(fieldKey) + .concat(expr) + .concat(values.get(i)); + } + } + } + return sql; + } + + /** + * @param dslObject: + * @param sql: + * @Description: 解析 Match对象 + * @Author: liuyongqiang + * @Date: 2020/7/23 17:45 + * @return: java.lang.String + **/ + public String parseMatch(DSLObject dslObject, String sql) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getMatch())) { + return sql; + } + List<DSLObject.QueryBean.MatchBean> matchBeanList = dslObject.getQuery().getMatch(); + int index = 0; + for (DSLObject.QueryBean.MatchBean matchBean : matchBeanList) { + index++; + String type = matchBean.getType(); + String fieldKey = matchBean.getFieldKey(); + List<String> values = matchBean.getFieldValues(); + + if (CollectionUtils.isEmpty(values)) break; + + String join = index == 1 ? SqlKeywords.WHERE : SqlKeywords.AND; + sql = sql.concat(join) + .concat(fieldKey) + .concat(SqlKeywords.LIKE) + .concat(parseLike(values.get(0), type)); + + if (values.size() > 1) { + for (int i = 1; i < values.size(); i++) { + sql = sql.concat(SqlKeywords.OR) + .concat(fieldKey) + .concat(SqlKeywords.LIKE) + .concat(parseLike(values.get(i), type)); + } + } + } + return sql; + } + + + /** + * @param dslObject: + * @param sql: + * @Description: 解析Limit对象 + * @Author: liuyongqiang + * @Date: 2020/7/23 17:44 + * @return: java.lang.String + **/ + public String parseLimit(DSLObject dslObject, String sql) { + sql = sql.concat(SqlKeywords.LIMIT); + if (StrUtil.isNotEmpty(dslObject.getLimit())) { + sql = sql.concat(dslObject.getLimit()); + } else { + sql = sql.concat(DEFAULT_LIMIT); + } + return sql; + } + + /** + * @param value: + * @param type: + * @Description: match.type转换 + * @Author: liuyongqiang + * @Date: 2020/7/23 11:47 + * @return: java.lang.String + **/ + public String parseLike(String value, String type) { + switch (MatchTypeEnum.getByType(type)) { + case REGEX: + return parseRegex(value);//正则匹配 + case PREFIX: + return "'" + value + "%'";//前缀匹配 + case SUFFIX: + return "'%" + value + "'";//后缀匹配 + case EXACTLY: + return "'" + value + "'";//完全匹配 + case SUBSTRING: + return "'%" + value + "%'";//子串匹配 + } + return ""; + } + + /** + * @param value: + * @Description: parseRegex + * @Author: liuyongqiang + * @Date: 2020/7/23 11:49 + * @return: java.lang.String + **/ + public String parseRegex(String value) { + if (value.startsWith("$")) { + //以$开头，不以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示完整匹配 + value = value.replaceAll("\\$", ""); + return "'" + value + "'"; + } else if (value.startsWith("*") && value.endsWith("*")) { + //以*开始，以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示子串匹配 + value = value.replaceAll("\\*", ""); + return "'%" + value + "%'"; + } else if (value.startsWith("*")) { + //以*开始，不以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示右匹配（后缀匹配） + value = value.replaceAll("\\*", ""); + return "'%" + value + "'"; + } else if (value.endsWith("*")) { + //以*结尾，不以*或者$开始，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示左匹配(前缀匹配) + value = value.replaceAll("\\*", ""); + return "'" + value + "%'"; + } else { + //不以*或者$开头，不以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示子串匹配 + return "'%" + value + "%'"; + } + } + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/DruidParser.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/DruidParser.java new file mode 100644 index 0000000..04728dd --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/pojo/DruidParser.java @@ -0,0 +1,82 @@ +package com.mesalab.engine.common.pojo; + +import cn.hutool.core.util.StrUtil; +import com.mesalab.engine.common.constant.SqlKeywords; +import com.mesalab.engine.service.DruidEngineService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.text.MessageFormat; +import java.util.List; + +/** + * @Date: 2020-07-22 17:00 + * @Author : liuyongqiang + * @ClassName : DruidDSLParse + * @Description : Druid DSL语法解析器 + */ +@Slf4j +@Component +public class DruidParser extends DSLParser { + + @Autowired + DruidEngineService druidEngineService; + + //表字段和数据类型查询SQL + public final static String SCHEAM_QUERY = "SELECT COLUMN_NAME as name, DATA_TYPE as type FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ''{0}''"; + + //查询所有表 + public final static String TABLES_QUERY = "SELECT TABLE_NAME AS name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'TABLE'"; + + + @Override + public String parseGranularity(DSLObject dslObject, String sql) { + if (StrUtil.isEmpty(dslObject.getQuery().getGranularity())) { + return sql; + } + String granularity = dslObject.getQuery().getGranularity(); + sql = sql.replace(SqlKeywords.SELECT, SqlKeywords.SELECT + .concat(MessageFormat.format(SqlKeywords.D_TIME_FLOOR, granularity)) + .concat(",")); + return sql; + } + + + @Override + public String parseSort(DSLObject dslObject, String sql) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getSort())) { + return sql; + } + String sortType = dslObject.getQuery().getSort().get(0).getType(); + sql = sql.concat(SqlKeywords.ORDER_BY + .concat(SqlKeywords.D_TIME)) + .concat(sortType); + return sql; + } + + @Override + public String parseIntervals(DSLObject dslObject, String sql) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getIntervals())) { + return sql; + } + List<DSLObject.QueryBean.IntervalsBean> intervalsBeanList = dslObject.getQuery().getIntervals(); + for (DSLObject.QueryBean.IntervalsBean intervalsBean : intervalsBeanList) { + String fieldKey = intervalsBean.getFieldKey(); + List<String> values = intervalsBean.getFieldValues(); + + if (CollectionUtils.isEmpty(values)) break; + + String join = sql.contains(SqlKeywords.WHERE) ? SqlKeywords.AND : SqlKeywords.WHERE; + + sql = sql.concat(join).concat(fieldKey) + .concat(SqlKeywords.BWTWEEN) + .concat("'" + values.get(0) + "'") + .concat(SqlKeywords.AND) + .concat("'" + values.get(1) + "'"); + } + return sql; + } + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/BaseResultUtil.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/BaseResultUtil.java new file mode 100644 index 0000000..80f6020 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/BaseResultUtil.java @@ -0,0 +1,208 @@ +package com.mesalab.engine.common.util; + + +import com.mesalab.engine.common.enums.ResultCodeEnum; +import com.mesalab.engine.common.enums.ResultStatusEnum; +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; + +import java.util.Map; + +/** + * BaseResult生成器 + * + * @author dazzlzy + * @date 2018/4/1 + */ +public class BaseResultUtil { + + /** + * 生成返回结果 + * + * @param status 返回HTTP响应码 + * @param code 返回业务编码 + * @param message 返回消息 + * @param data 返回数据 + * @param formatType 返回数据格式 JSON/CSV + * @param <T> 返回数据类型 + * @return 返回结果 + */ + public static <T> BaseResult<T> generate(final int status, final String code, final String message, T data, T meta, final Map<String, Object> statistics, final String formatType) { + return new BaseResult<>(status, code, false, message, statistics, formatType, meta, data); + } + + /** + * 操作成功响应结果， 默认结果 + * + * @return 操作成功的默认响应结果 + */ + public static <T> BaseResult<T> success() { + return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), + true, ResultCodeEnum.EXECUTE_SUCCESS.getMessage(), null, null, null, null); + } + + /** + * 操作成功响应结果， 自定义数据及信息 + * + * @param message 自定义信息 + * @param data 自定义数据 + * @param <T> 自定义数据类型 + * @return 响应结果 + */ + public static <T> BaseResult<T> success(final String message, final T data) { + return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), + true, message, null, null, null, data); + } + + /** + * 操作成功响应结果， 自定义数据及信息, 统计结果 + * + * @param message 自定义信息 + * @param data 自定义数据 + * @param <T> 自定义数据类型 + * @param statistics 统计结果 + * @return 响应结果 + */ + public static <T> BaseResult<T> success(final String message, final T data, final Map<String, Object> statistics) { + return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), + true, message, statistics, null, null, data); + } + + /** + * 操作成功响应结果，自定义数据，默认信息 + * + * @param data 自定义数据 + * @param <T> 自定义数据类型 + * @return 响应结果 + */ + public static <T> BaseResult<T> success(final T data) { + return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), true, + ResultCodeEnum.EXECUTE_SUCCESS.getMessage(), null, null, null, data); + } + + /** + * 操作成功响应结果，自定义信息，无数据 + * + * @param message 自定义信息 + * @return 响应结果 + */ + public static <T> BaseResult<T> success4Message(final String message) { + return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), + true, message, null, null, null, null); + } + + /** + * 操作失败响应结果， 默认结果 + * + * @return 操作成功的默认响应结果 + */ + public static <T> BaseResult<T> failure() { + return new BaseResult<>(ResultStatusEnum.FAIL.getCode(), + ResultCodeEnum.UNKNOW_ERROR.getCode(), false, ResultCodeEnum.UNKNOW_ERROR.getMessage(), null, null, null, null); + } + + /** + * 操作失败响应结果， 自定义错误编码及信息 + * + * @param status HTTP状态码 + * @param message 自定义信息 + * @return 响应结果 + */ + public static <T> BaseResult<T> failure(final int status, final String message) { + return new BaseResult<>(status, ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, null); + } + + /** + * 操作失败响应结果， 自定义错误编码及信息 + * + * @param status HTTP 状态码 + * @param code 返回业务编码 + * @param message 自定义信息 + * @return 响应结果 + */ + public static <T> BaseResult<T> failure(final int status, final String code, final String message) { + return new BaseResult<>(status, code, false, message, null, null, null, null); + } + + /** + * 操作失败响应结果， 自定义错误编码及信息 + * + * @param status HTTP 状态码 + * @param message 自定义信息 + * @return 响应结果 + */ + public static <T> BaseResult<T> failure(final int status, final String message, T data) { + return new BaseResult<>(status, ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, data); + } + + /** + * 操作失败响应结果，自定义错误编码 + * + * @param resultStatusEnum 自定义错误编码枚举 + * @return 响应结果 + */ + public static <T> BaseResult<T> failure(final ResultStatusEnum resultStatusEnum) { + return new BaseResult<>(resultStatusEnum.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, resultStatusEnum.getMessage(), null, null, null, null); + } + + /** + * 操作失败响应结果，自定义信息 + * + * @param message 自定义信息 + * @return 响应结果 + */ + public static <T> BaseResult<T> failure(final String message) { + return new BaseResult<>(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, null); + } + + /** + * 异常响应结果， 默认结果 + * + * @return 操作成功的默认响应结果 + */ + public static <T> BaseResult<T> error() { + return new BaseResult<>(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, ResultStatusEnum.SERVER_ERROR.getMessage(), null, null, null, null); + } + + /** + * 异常响应结果， 自定义错误编码及信息 + * + * @param code 自定义错误编码 + * @param message 自定义信息 + * @return 响应结果 + */ + public static <T> BaseResult<T> error(final int code, final String message) { + return new BaseResult<>(code, ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, null); + } + + /** + * 异常响应结果，自定义错误编码 + * + * @param resultStatusEnum 自定义错误编码枚举 + * @return 响应结果 + */ + public static <T> BaseResult<T> error(final ResultStatusEnum resultStatusEnum) { + return new BaseResult<>(resultStatusEnum.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, resultStatusEnum.getMessage(), null, null, null, null); + } + + /** + * 业务异常响应结果 + * + * @param be 业务异常 + * @return 响应结果 + */ + public static <T> BaseResult<T> error(final BusinessException be) { + return new BaseResult<>(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, be.getErrorMessage(), null, null, null, null); + } + + /** + * 异常响应结果，自定义信息 + * + * @param message 自定义信息 + * @return 响应结果 + */ + public static <T> BaseResult<T> error(final String message) { + return new BaseResult<>(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, null); + } + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/HttpClientUtil.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/HttpClientUtil.java new file mode 100644 index 0000000..b76ceae --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/HttpClientUtil.java @@ -0,0 +1,270 @@ +package com.mesalab.engine.common.util; + +import cn.hutool.core.util.StrUtil; +import com.mesalab.engine.common.exception.BusinessException; +import org.apache.http.*; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.conn.ConnectionKeepAliveStrategy; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicHeaderElementIterator; +import org.apache.http.protocol.HTTP; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLHandshakeException; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.Objects; + +public class HttpClientUtil { + /** + * 全局连接池对象 + */ + private static final PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); + + private static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class); + + /** + * 静态代码块配置连接池信息 + */ + static { + + // 设置最大连接数 + connManager.setMaxTotal(400); + // 设置每个连接的路由数 + connManager.setDefaultMaxPerRoute(80); + + } + + /** + * 获取Http客户端连接对象 + * + * @return Http客户端连接对象 + */ + public static CloseableHttpClient getHttpClient() { + // 创建Http请求配置参数 + RequestConfig requestConfig = RequestConfig.custom() + // 获取连接超时时间 + .setConnectionRequestTimeout(30000) + // 请求超时时间 + .setConnectTimeout(10000) + // 响应超时时间 + .setSocketTimeout(10000) + .build(); + + /** + * 测出超时重试机制为了防止超时不生效而设置 + * 如果直接放回false,不重试 + * 这里会根据情况进行判断是否重试 + */ + HttpRequestRetryHandler retry = (exception, executionCount, context) -> { + if (executionCount >= 3) {// 如果已经重试了3次，就放弃 + return false; + } + if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试 + return true; + } + if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常 + return false; + } + if (exception instanceof InterruptedIOException) {// 超时 + return true; + } + if (exception instanceof UnknownHostException) {// 目标服务器不可达 + return false; + } + if (exception instanceof ConnectTimeoutException) {// 连接被拒绝 + return false; + } + if (exception instanceof SSLException) {// ssl握手异常 + return false; + } + HttpClientContext clientContext = HttpClientContext.adapt(context); + HttpRequest request = clientContext.getRequest(); + // 如果请求是幂等的，就再次尝试 + if (!(request instanceof HttpEntityEnclosingRequest)) { + return true; + } + return false; + }; + + + ConnectionKeepAliveStrategy myStrategy = (response, context) -> { + HeaderElementIterator it = new BasicHeaderElementIterator + (response.headerIterator(HTTP.CONN_KEEP_ALIVE)); + while (it.hasNext()) { + HeaderElement he = it.nextElement(); + String param = he.getName(); + String value = he.getValue(); + if (value != null && param.equalsIgnoreCase + ("timeout")) { + return Long.parseLong(value) * 1000; + } + } + return 60 * 1000;//如果没有约定，则默认定义时长为60s + }; + + // 创建httpClient + return HttpClients.custom() + // 把请求相关的超时信息设置到连接客户端 + .setDefaultRequestConfig(requestConfig) + // 把请求重试设置到连接客户端 + .setRetryHandler(retry) + .setKeepAliveStrategy(myStrategy) + // 配置连接池管理对象 + .setConnectionManager(connManager) + .build(); + } + + + /** + * GET请求 + * + * @param url 请求地 + * @return + */ + public static String httpGet(String url, Header... headers) throws BusinessException { + String msg = "-1"; + + // 获取客户端连接对象 + CloseableHttpClient httpClient = getHttpClient(); + CloseableHttpResponse response = null; + + try { + + URL ul = new URL(url); + + URI uri = new URI(ul.getProtocol(), null, ul.getHost(), ul.getPort(), ul.getPath(), ul.getQuery(), null); + logger.info("http get uri {}", uri); + // 创建GET请求对象 + HttpGet httpGet = new HttpGet(uri); + if (Objects.nonNull(headers)) { + for (Header h : headers) { + httpGet.addHeader(h); + } + } + // 执行请求 + response = httpClient.execute(httpGet); + int statusCode = response.getStatusLine().getStatusCode(); + // 获取响应实体 + HttpEntity entity = response.getEntity(); + // 获取响应信息 + msg = EntityUtils.toString(entity, "UTF-8"); + + if (statusCode != HttpStatus.SC_OK) { + throw new BusinessException("Http get content is :" + msg); + } + + } catch (URISyntaxException e) { + logger.error("URI 转换错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } catch (ClientProtocolException e) { + logger.error("协议错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } catch (ParseException e) { + logger.error("解析错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } catch (IOException e) { + logger.error("IO错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } finally { + if (null != response) { + try { + EntityUtils.consume(response.getEntity()); + response.close(); + } catch (IOException e) { + logger.error("释放链接错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } + } + } + + return msg; + } + + /** + * POST 请求 + * + * @param url + * @param requestBody + * @return + */ + public static String httpPost(String url, String requestBody, Header... headers) throws BusinessException { + String msg = "-1"; + // 获取客户端连接对象 + CloseableHttpClient httpClient = getHttpClient(); + // 创建POST请求对象 + CloseableHttpResponse response = null; + try { + + URL ul = new URL(url); + + URI uri = new URI(ul.getProtocol(), null, ul.getHost(), ul.getPort(), ul.getPath(), ul.getQuery(), null); + + logger.info("http post uri:{}， http post body:{}", uri, requestBody); + + HttpPost httpPost = new HttpPost(uri); + httpPost.setHeader("Content-Type", "application/json"); + if (Objects.nonNull(headers)) { + for (Header h : headers) { + httpPost.addHeader(h); + } + } + + if (StrUtil.isNotBlank(requestBody)) { + httpPost.setEntity(new ByteArrayEntity(requestBody.getBytes("utf-8"))); + } + + response = httpClient.execute(httpPost); + int statusCode = response.getStatusLine().getStatusCode(); + // 获取响应实体 + HttpEntity entity = response.getEntity(); + // 获取响应信息 + msg = EntityUtils.toString(entity, "UTF-8"); + + if (statusCode != HttpStatus.SC_OK && statusCode != HttpStatus.SC_CREATED) { + throw new BusinessException(msg); + } + } catch (URISyntaxException e) { + logger.error("URI 转换错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } catch (ClientProtocolException e) { + logger.error("协议错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } catch (ParseException e) { + logger.error("解析错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } catch (IOException e) { + logger.error("IO错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } finally { + if (null != response) { + try { + EntityUtils.consumeQuietly(response.getEntity()); + response.close(); + } catch (IOException e) { + logger.error("释放链接错误: {}", e.getMessage()); + throw new BusinessException(e.getMessage()); + } + } + } + return msg; + } + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLRouter.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLRouter.java new file mode 100644 index 0000000..f112311 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLRouter.java @@ -0,0 +1,46 @@ +package com.mesalab.engine.component; + + +import com.mesalab.engine.common.enums.EngineTypeEnum; +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; +import com.mesalab.engine.common.pojo.DSLObject; +import com.mesalab.engine.service.ClickHouseEngineService; +import com.mesalab.engine.service.DruidEngineService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * @Date: 2020-07-20 18:11 + * @Author : liuyongqiang + * @ClassName : DSLRouter + * @Description : DSL路由执行,负责调用查询引擎执行查询 + */ +@Component +public class DSLRouter { + + @Autowired + private ClickHouseEngineService clickHouseEngineService; + + @Autowired + private DruidEngineService druidEngineService; + + /** + * @param dslObject: + * @Description: 查询引擎路由 + * @Author: liuyongqiang + * @Date: 2020/7/22 15:59 + * @return: com.mesalab.common.base.BaseResult + **/ + public BaseResult executeRouter(DSLObject dslObject) throws BusinessException { + switch (EngineTypeEnum.getByEngine(dslObject.getDataEngine())) { + case CACULATION_ENGINE: + return druidEngineService.dslQuery(dslObject); + case BUSINESS_ENGINE: + return clickHouseEngineService.dslQuery(dslObject); + default: + return null; + } + } + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLValidate.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLValidate.java new file mode 100644 index 0000000..3d5cde7 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLValidate.java @@ -0,0 +1,367 @@ +package com.mesalab.engine.component; + +import cn.hutool.core.util.StrUtil; +import com.mesalab.engine.common.enums.*; +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; +import com.mesalab.engine.common.pojo.DSLObject; +import com.mesalab.engine.service.ClickHouseEngineService; +import com.mesalab.engine.service.DruidEngineService; +import org.apache.http.HttpStatus; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * @Date: 2020-07-20 18:10 + * @Author : wangwei + * @ClassName : DSLValidate + * @Description : DSL格式校验 + */ +@Component +public class DSLValidate { + + public static Pattern periodOfPT = Pattern.compile("PT(\\d+)[SMH]", Pattern.CASE_INSENSITIVE); + public static Pattern periodOfP = Pattern.compile("P(\\d+)[DWMY]", Pattern.CASE_INSENSITIVE); + public static Pattern strFormatDateTime = Pattern.compile("\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}", Pattern.CASE_INSENSITIVE); + + @Autowired + DruidEngineService druidEngineService; + @Autowired + ClickHouseEngineService clickHouseEngineService; + @Value("${engine.maxCacheNum}") + int maxCacheNum; + + + /** + * 1.dbType字段必须在支持的范围之内(ArangoDB、ClickHouse、Druid) + * 2.dataSource数据表是否在库中存在 + * 3.limit不能大于100000 + * 4.时间格式校验 + * 5.intervals.fieldValues长度必须==2 + * 6.intervals.type暂时只支持between方式 + * 7.sort.type支持desc和asc + * 8.match.type在:["exactly"|"prefix"|"suffix"|"substring"]内 + * 9.range.type在:["gt"|"lt"|"eq"|"ge"|"le"|"ne"] + * 10.对表中的字段做是否存在的验证 + * 11.match.fieldValues中不允许出现以*开头$结束的值 + * 12.match.fieldKey字段类型不能为数值型 + */ + public void executeValidate(DSLObject dslObject) throws BusinessException { + if (Objects.isNull(dslObject)) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "DSLObject is invalid"); + } + if (isValidEngine(dslObject.getDataEngine())) { + if (!isValidDataSource(dslObject)) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "dataSource is invalid"); + } + } else { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "dataEngine is invalid"); + } + if (Objects.isNull(dslObject.getQuery())) { + return; + } + if (!isValidLimit(dslObject.getLimit())) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "Limit is invalid"); + } + if (!isValidGranularity(dslObject)) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "query.Granularity value is invalid"); + } + if (!isValidMatch(dslObject)) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "query.Match type is invalid"); + } + if (!isValidRange(dslObject)) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "query.Range type is invalid"); + } + if (!isValidIntervals(dslObject)) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "query.Intervals is invalid"); + } + if (!isValidSort(dslObject)) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), + "query.Sort type is invalid"); + } + } + + /** + * 校验dataSource + * 1.判定是否在{@link EngineTypeEnum}相应数据库中 + * + * @param dslObject + * @return + */ + private boolean isValidDataSource(DSLObject dslObject) { + if (StrUtil.isBlank(dslObject.getDataSource())) { + return false; + } + if (EngineTypeEnum.BUSINESS_ENGINE.getEngine().equals(dslObject.getDataEngine())) { + BaseResult baseResult = clickHouseEngineService.tablesQuery(dslObject); + return isDataSourceInResult(dslObject.getDataSource(), baseResult); + } else if (EngineTypeEnum.CACULATION_ENGINE.getEngine().equals(dslObject.getDataEngine())) { + BaseResult baseResult = druidEngineService.tablesQuery(dslObject); + return isDataSourceInResult(dslObject.getDataSource(), baseResult); + } + return false; + } + + + /** + * 获取表中所有列 + * + * @param dslObject + * @return + */ + private List getFields(DSLObject dslObject) { + if (EngineTypeEnum.BUSINESS_ENGINE.getEngine().equalsIgnoreCase(dslObject.getDataEngine())) { + BaseResult baseResult = clickHouseEngineService.schemaQuery(dslObject); + return fieldsInDataSource(baseResult); + } else if (EngineTypeEnum.CACULATION_ENGINE.getEngine().equalsIgnoreCase(dslObject.getDataEngine())) { + BaseResult baseResult = druidEngineService.schemaQuery(dslObject); + return fieldsInDataSource(baseResult); + } else if (EngineTypeEnum.ANALYSIS_ENGINE.getEngine().equalsIgnoreCase(dslObject.getDataEngine())) { + return new ArrayList(); + + } + throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.UNKNOW_ERROR.getCode(), "field is invalid"); + } + + /** + * 表中所有字段 + * + * @param baseResult + * @return + */ + private List fieldsInDataSource(BaseResult baseResult) { + if (baseResult.isSuccess()) { + List<Map> data = (List<Map>) baseResult.getData(); + List fields = new ArrayList<>(); + data.forEach(o -> { + fields.add(o.get("name")); + } + ); + return fields; + } else { + throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.UNKNOW_ERROR.getCode(), "get fields error: " + baseResult); + } + } + + /** + * 判断表是否在对应dataEngine对应库 + * + * @param dataSource + * @param baseResult + * @return + */ + private boolean isDataSourceInResult(String dataSource, BaseResult baseResult) { + if (baseResult.isSuccess()) { + List<Map> data = (List<Map>) baseResult.getData(); + for (Map datum : data) { + if (datum.get("name").equals(dataSource)) { + return true; + } + } + } else { + throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.UNKNOW_ERROR.getCode(), "get datasource error: " + baseResult); + } + return false; + } + + /** + * 时间粒度校验, 遵循ISO8601 durations 定义 + * + * @param dslObject + * @return + */ + private boolean isValidGranularity(DSLObject dslObject) { + String granularity = dslObject.getQuery().getGranularity(); + if (StrUtil.isBlank(granularity)) { + return true; + } + if (periodOfP.matcher(granularity).find() || periodOfPT.matcher(granularity).find()) { + return true; + } + return false; + } + + /** + * 校验range: + * 1.是否属于{@link RangeTypeEnum}支持的类型 + * + * @param dslObject + * @return + */ + private boolean isValidRange(DSLObject dslObject) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getRange())) { + return true; + } + for (DSLObject.QueryBean.RangeBean rangeBean : dslObject.getQuery().getRange()) { + if (RangeTypeEnum.getByType(rangeBean.getType()) == null) { + return false; + } + checkField(dslObject, rangeBean.getFieldKey()); + } + return true; + } + + /** + * 校验match: + * 1.是否属于{@link MatchTypeEnum}限定类型 + * 2.不能以*开始、或$结尾 + * + * @param dslObject + * @return + */ + private boolean isValidMatch(DSLObject dslObject) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getMatch())) { + return true; + } + for (DSLObject.QueryBean.MatchBean matchBean : dslObject.getQuery().getMatch()) { + if (MatchTypeEnum.getByType(matchBean.getType()) == null) { + return false; + } + for (String fieldValue : matchBean.getFieldValues()) { + if (fieldValue.startsWith("*") || fieldValue.endsWith("$")) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Match fieldValues cannot startWith '*' or endWith '$'"); + } + } + checkField(dslObject, matchBean.getFieldKey()); + } + return true; + } + + /** + * 校验sort: + * 1.排序类型只能是DESC或ASC + * 2.不限大小写 + * + * @param dslObject + * @return + */ + private boolean isValidSort(DSLObject dslObject) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getSort())) { + return true; + } + for (DSLObject.QueryBean.SortBean sortBean : dslObject.getQuery().getSort()) { + if (!"DESC".equalsIgnoreCase(sortBean.getType()) && !"ASC".equalsIgnoreCase(sortBean.getType())) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "sort type is illegal, must be asc or desc"); + } + checkField(dslObject, sortBean.getFieldKey()); + } + return true; + } + + /** + * 校验limit: + * 1.limit范围[0,100000] + * + * @param limitStr + * @return + */ + private boolean isValidLimit(String limitStr) { + if (StrUtil.isBlank(limitStr)) { + return true; + } + int limit; + String[] split = limitStr.split(","); + if (split.length > 2) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Limit format error"); + } else if (split.length == 2) { + limit = Integer.parseInt(split[0]) + Integer.parseInt(split[1]); + } else { + limit = Integer.parseInt(split[split.length - 1]); + } + if (0 > limit || maxCacheNum < limit) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Limit must be in the range [0, " + maxCacheNum + "]"); + } + return true; + } + + /** + * 校验Interval: + * 1.目前只支持between类型 + * 2.时间区间必须是 开始时间 < 结束时间 + * + * @param dslObject + */ + private boolean isValidIntervals(DSLObject dslObject) { + if (CollectionUtils.isEmpty(dslObject.getQuery().getIntervals())) { + return true; + } + for (DSLObject.QueryBean.IntervalsBean interval : dslObject.getQuery().getIntervals()) { + if (Objects.isNull(IntervalTypeEnum.getByType(interval.getType().toUpperCase()))) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Interval type not support " + interval.getType()); + } + if (interval.getFieldValues().size() != 2) { + return false; + } + for (String fieldValue : interval.getFieldValues()) { + if (!strFormatDateTime.matcher(fieldValue).find()) { + return false; + } + } + + try { + DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"); + if (dateTimeFormatter.parseMillis(interval.getFieldValues().get(1)) - dateTimeFormatter.parseMillis(interval.getFieldValues().get(0)) < 0) { + throw new RuntimeException("Interval fieldValue should be [start, end]"); + } + } catch (Exception e) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), e.getMessage()); + } + checkField(dslObject, interval.getFieldKey()); + } + return true; + } + + /** + * 校验dataEngine: + * 1.是否属于{@link EngineTypeEnum}支持的类型 + * + * @param dataEngine + * @return + */ + private boolean isValidEngine(String dataEngine) { + if (StrUtil.isBlank(dataEngine)) { + return false; + } + for (EngineTypeEnum engineTypeEnum : EngineTypeEnum.values()) { + if (engineTypeEnum.getEngine().equals(dataEngine)) { + return true; + } + } + return false; + } + + /** + * 校验field是否是相应表字段 + * + * @param dslObject + * @param fieldKey + */ + private void checkField(DSLObject dslObject, String fieldKey) { + //AnangoDB暂不校验 + if (EngineTypeEnum.ANALYSIS_ENGINE.getEngine().equalsIgnoreCase(dslObject.getDataEngine())) { + return; + } + if (!getFields(dslObject).contains(fieldKey)) { + throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), fieldKey + " is invalid"); + } + } + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/DslQueryController.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/DslQueryController.java new file mode 100644 index 0000000..620073a --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/DslQueryController.java @@ -0,0 +1,47 @@ +package com.mesalab.engine.controller; + +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; +import com.mesalab.engine.common.pojo.DSLObject; +import com.mesalab.engine.common.util.BaseResultUtil; +import com.mesalab.engine.component.DSLRouter; +import com.mesalab.engine.component.DSLValidate; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * @Date: 2020-08-17 10:55 + * @Author : liuyongqiang + * @ClassName : DslQueryController + * @Description : 基于DSL语言的单表数据查询 + */ +@Slf4j +@RestController +@RequestMapping("/v1/dsl/") +public class DslQueryController { + + @Autowired + protected DSLRouter dslRouter; + + @Autowired + protected DSLValidate dslValidate; + + @PostMapping("query") + public BaseResult query(@RequestBody DSLObject dslObject) { + BaseResult baseResult; + try { + dslValidate.executeValidate(dslObject); + baseResult = dslRouter.executeRouter(dslObject); + } catch (BusinessException e) { + return BaseResultUtil.failure( + e.getErrorStatus(), + e.getErrorCode(), + e.getErrorMessage()); + } + return baseResult; + } +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/ClickHouseEngineService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/ClickHouseEngineService.java new file mode 100644 index 0000000..d82cca0 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/ClickHouseEngineService.java @@ -0,0 +1,10 @@ +package com.mesalab.engine.service; + +/** + * @Date: 2020-07-20 10:00 + * @Author : liuyongqiang + * @ClassName : ClickHouseEngineService + * @Description : ClickHouse数据查询引擎接口 + */ +public interface ClickHouseEngineService extends CommonEngineService { +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/CommonEngineService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/CommonEngineService.java new file mode 100644 index 0000000..5abeac6 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/CommonEngineService.java @@ -0,0 +1,47 @@ +package com.mesalab.engine.service; + +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; +import com.mesalab.engine.common.pojo.DSLObject; + +/** + * @Date: 2020-07-20 18:39 + * @Author : liuyongqiang + * @ClassName : CommonEngineService + * @Description : 公共查询接口定义 + */ +public interface CommonEngineService { + + + /** + * @param dslObject: DSL查询对象 + * @Description: 单表数据查询 + * @Author: liuyongqiang + * @Date: 2020/7/21 9:41 + * @return: com.mesalab.common.base.BaseResult + **/ + BaseResult dslQuery(DSLObject dslObject) throws BusinessException; + + + /** + * @param dslObject: + * @Description: 查询数据源下的所有表 + * @Author: liuyongqiang + * @Date: 2020/7/23 15:16 + * @return: com.mesalab.common.base.BaseResult + **/ + BaseResult tablesQuery(DSLObject dslObject) throws BusinessException; + + + /** + * @param dslObject: + * @Description: 查询指定表中的所有字段 + * @Author: liuyongqiang + * @Date: 2020/7/23 11:08 + * @return: com.mesalab.common.base.BaseResult + **/ + BaseResult schemaQuery(DSLObject dslObject) throws BusinessException; + + +} + diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/DruidEngineService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/DruidEngineService.java new file mode 100644 index 0000000..d4ce603 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/DruidEngineService.java @@ -0,0 +1,11 @@ +package com.mesalab.engine.service; + +/** + * @Date: 2020-07-20 10:02 + * @Author : liuyongqiang + * @ClassName : DruidEngineService + * @Description : Druid数据查询引擎接口 + */ +public interface DruidEngineService extends CommonEngineService { + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/EngineExecuteService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/EngineExecuteService.java new file mode 100644 index 0000000..ae87d2c --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/EngineExecuteService.java @@ -0,0 +1,31 @@ +package com.mesalab.engine.service; + +import com.mesalab.engine.common.exception.BusinessException; + +/** + * @Date: 2020-07-22 16:11 + * @Author : liuyongqiang + * @ClassName : EngineExecuteService + * @Description : 引擎执行查询接口 + */ +public interface EngineExecuteService { + + /** + * @param sql: + * @Description: 执行Druid查询语句 + * @Author: liuyongqiang + * @Date: 2020/7/22 16:16 + * @return: java.lang.String + **/ + String executeDruidSql(String sql) throws BusinessException; + + /** + * @param sql: + * @Description: 执行ClickHouse查询语句 + * @Author: liuyongqiang + * @Date: 2020/7/22 16:16 + * @return: java.lang.String + **/ + String executeClickHouseSql(String sql) throws BusinessException; + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/ClickHouseEngineServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/ClickHouseEngineServiceImpl.java new file mode 100644 index 0000000..25e6106 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/ClickHouseEngineServiceImpl.java @@ -0,0 +1,91 @@ +package com.mesalab.engine.service.impl; + + +import com.google.gson.Gson; +import com.mesalab.engine.common.dto.ClickHouseResult; +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; +import com.mesalab.engine.common.pojo.ClickHouseParser; +import com.mesalab.engine.common.pojo.DSLObject; +import com.mesalab.engine.common.util.BaseResultUtil; +import com.mesalab.engine.service.ClickHouseEngineService; +import com.mesalab.engine.service.EngineExecuteService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.text.MessageFormat; + +/** + * @Date: 2020-07-20 10:04 + * @Author : liuyongqiang + * @ClassName : ClickHouseEngineServiceImpl + * @Description : ClickHouse数据查询引擎实现 + */ +@Slf4j +@Service("clickHouseEngineService") +public class ClickHouseEngineServiceImpl implements ClickHouseEngineService { + + @Autowired + ClickHouseParser clickHouseDSLParser; + + @Autowired + EngineExecuteService executeService; + + /** + * @param dslObject: DSL查询对象 + * @Description: 单表数据查询 + * @Author: liuyongqiang + * @Date: 2020/7/21 9:41 + * @return: com.mesalab.common.base.BaseResult + **/ + @Override + public BaseResult dslQuery(DSLObject dslObject) throws BusinessException { + String executeSql = clickHouseDSLParser.allQuery(dslObject); + String jsonResult = executeService.executeClickHouseSql(executeSql); + return assemble(jsonResult); + } + + /** + * @param dslObject: + * @Description: 查询数据源下的所有表 + * @Author: liuyongqiang + * @Date: 2020/7/23 15:16 + * @return: com.mesalab.common.base.BaseResult + **/ + @Override + public BaseResult tablesQuery(DSLObject dslObject) throws BusinessException { + String executeSql = ClickHouseParser.TABLES_QUERY; + String jsonResult = executeService.executeClickHouseSql(executeSql); + return assemble(jsonResult); + } + + /** + * @param dslObject: + * @Description: 查询指定表中的所有字段 + * @Author: liuyongqiang + * @Date: 2020/7/23 11:08 + * @return: com.mesalab.common.base.BaseResult + **/ + @Override + public BaseResult schemaQuery(DSLObject dslObject) throws BusinessException { + String dataSource = dslObject.getDataSource(); + String executeSql = MessageFormat.format(ClickHouseParser.SCHEAM_QUERY, dataSource); + return assemble(executeService.executeClickHouseSql(executeSql)); + } + + /** + * @param jsonResult: + * @Description: 组装返回结果 + * @Author: liuyongqiang + * @Date: 2020/7/23 16:52 + * @return: com.mesalab.common.base.BaseResult + **/ + private BaseResult assemble(String jsonResult) { + ClickHouseResult clickHouseResultDTO = new Gson().fromJson(jsonResult, ClickHouseResult.class); + BaseResult baseResult = BaseResultUtil.success(clickHouseResultDTO.getData()); + baseResult.setStatistics(clickHouseResultDTO.getStatistics()); + baseResult.setMeta(clickHouseResultDTO.getMeta()); + return baseResult; + } +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/DruidEngineServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/DruidEngineServiceImpl.java new file mode 100644 index 0000000..d9e7df0 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/DruidEngineServiceImpl.java @@ -0,0 +1,97 @@ +package com.mesalab.engine.service.impl; + +import com.google.gson.Gson; +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; +import com.mesalab.engine.common.pojo.DSLObject; +import com.mesalab.engine.common.pojo.DruidParser; +import com.mesalab.engine.common.util.BaseResultUtil; +import com.mesalab.engine.service.DruidEngineService; +import com.mesalab.engine.service.EngineExecuteService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.text.MessageFormat; +import java.util.List; +import java.util.Map; + +/** + * @Date: 2020-07-20 10:05 + * @Author : liuyongqiang + * @ClassName : DruidEngineServiceImpl + * @Description : Druid数据查询引擎实现 + */ +@Service("druidEngineService") +public class DruidEngineServiceImpl implements DruidEngineService { + + @Autowired + DruidParser druidDSLParse; + + @Autowired + EngineExecuteService executeService; + + /** + * @param dslObject: DSL查询对象 + * @Description: 单表数据查询 + * @Author: liuyongqiang + * @Date: 2020/7/21 9:41 + * @return: com.mesalab.common.base.BaseResult + **/ + @Override + public BaseResult dslQuery(DSLObject dslObject) throws BusinessException { + String executeSql = druidDSLParse.allQuery(dslObject); + String jsonResult = executeService.executeDruidSql(executeSql); + List<Map<String, Object>> results = new Gson().fromJson(jsonResult, List.class); + dataFormat(results); + return BaseResultUtil.success(results); + } + + + /** + * @param dslObject: + * @Description: 查询数据源下的所有表 + * @Author: liuyongqiang + * @Date: 2020/7/23 15:16 + * @return: com.mesalab.common.base.BaseResult + **/ + @Override + public BaseResult tablesQuery(DSLObject dslObject) throws BusinessException { + String executeSql = DruidParser.TABLES_QUERY; + String jsonResult = executeService.executeDruidSql(executeSql); + List<Map<String, Object>> results = new Gson().fromJson(jsonResult, List.class); + return BaseResultUtil.success(results); + } + + /** + * @param dslObject: + * @Description: 查询指定表中的所有字段 + * @Author: liuyongqiang + * @Date: 2020/7/23 11:08 + * @return: com.mesalab.common.base.BaseResult + **/ + @Override + public BaseResult schemaQuery(DSLObject dslObject) throws BusinessException { + String dataSource = dslObject.getDataSource(); + String executeSql = MessageFormat.format(DruidParser.SCHEAM_QUERY, dataSource); + String jsonResult = executeService.executeDruidSql(executeSql); + List<Map<String, Object>> results = new Gson().fromJson(jsonResult, List.class); + return BaseResultUtil.success(results); + } + + /** + * @param results: + * @Description: 数据格式化 + * @Author: liuyongqiang + * @Date: 2020/7/28 16:56 + * @return: java.util.List<java.util.Map < java.lang.String, java.lang.Object>> + **/ + private void dataFormat(List<Map<String, Object>> results) { + for (Map<String, Object> map : results) { + for (String key : map.keySet()) { + if (map.get(key) instanceof Double) { + map.put(key, ((Double) map.get(key)).longValue()); + } + } + } + } +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/EngineExecuteServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/EngineExecuteServiceImpl.java new file mode 100644 index 0000000..67627b8 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/EngineExecuteServiceImpl.java @@ -0,0 +1,62 @@ +package com.mesalab.engine.service.impl; + + +import com.google.gson.Gson; +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.util.HttpClientUtil; +import com.mesalab.engine.service.EngineExecuteService; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; + +/** + * @Date: 2020-07-22 16:15 + * @Author : liuyongqiang + * @ClassName : EngineExecuteServiceImpl + * @Description : 引擎执行查询实现 + */ +@Service("engineExecuteService") +public class EngineExecuteServiceImpl implements EngineExecuteService { + + final Header JSON_HEADER = new BasicHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); + + @Value("${druid.query.url}") + String druidQueryUrl; + @Value("${clickhouse.query.url}") + String clickHouseQueryUrl; + @Value("${clickhouse.dbname}") + String clickHouseDbName; + @Value("${clickhouse.real.time.username}") + String ckRealTimeUsername; + @Value("${clickhouse.real.time.password}") + String ckRealTimePassword; + + @Override + public String executeDruidSql(String sql) throws BusinessException { + Map<String, String> queryParams = new HashMap<>(); + queryParams.put("query", sql); + String jsonParam = new Gson().toJson(queryParams); + return HttpClientUtil.httpPost(druidQueryUrl, jsonParam, JSON_HEADER); + } + + @Override + public String executeClickHouseSql(String sql) throws BusinessException { + String url = clickHouseQueryUrl + .concat("/?user=") + .concat(ckRealTimeUsername) + .concat("&password=") + .concat(ckRealTimePassword) + .concat("&database=") + .concat(clickHouseDbName) + .concat("&query=").concat(sql) + .concat(" FORMAT JSON"); + return HttpClientUtil.httpGet(url); + } + +} @@ -31,8 +31,6 @@ </distributionManagement> <modules> - <!--公共模块--> - <module>galaxy-common</module> <!--查询网关--> <module>galaxy-gateway</module> <!--认证中心--> @@ -51,6 +49,7 @@ <module>galaxy-job-executor</module> <!--定时任务核心模块--> <module>galaxy-job-core</module> + <!--数据查询引擎--> <module>galaxy-query-engine</module> </modules> @@ -71,7 +70,7 @@ <maven.compiler.target>1.8</maven.compiler.target> <commons.lang3.version>3.5</commons.lang3.version> <calcite.core.version>1.22.0</calcite.core.version> - <spring.cloud.version>Hoxton.SR1</spring.cloud.version> + <spring.cloud.version>Finchley.RC2</spring.cloud.version> <spring.boot.version>2.0.2.RELEASE</spring.boot.version> <galaxy.common.version>1.0-SNAPSHOT</galaxy.common.version> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> @@ -84,13 +83,6 @@ <dependencyManagement> <dependencies> <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-dependencies</artifactId> - <version>${spring.cloud.version}</version> - <type>pom</type> - <scope>import</scope> - </dependency> - <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-dependencies</artifactId> <version>${spring.boot.version}</version> |