diff options
| author | liuyongqiang <[email protected]> | 2020-11-10 15:29:33 +0800 |
|---|---|---|
| committer | liuyongqiang <[email protected]> | 2020-11-10 15:29:33 +0800 |
| commit | e412d42c40fba667c1187c8ef91230a0772e43da (patch) | |
| tree | 1837470ad24ac55bf3f779f4c7328f8024b49bf4 | |
| parent | 45a53e650ccd07663a5ebc0971a6113b1fa9c736 (diff) | |
构建多模块工程完成
213 files changed, 242 insertions, 19190 deletions
@@ -12,8 +12,6 @@ **Spring Cloud Eureka** https://docs.spring.io/spring-cloud-netflix/docs/2.2.4.RELEASE/reference/html/ -**Spring Cloud Gateway** https://docs.spring.io/spring-cloud-gateway/docs/2.2.4.RELEASE/reference/html/ - **Spring Boot Admin 2.3.0** https://codecentric.github.io/spring-boot-admin/2.3.0/ ## 系统架构图  diff --git a/galaxy-admin-server/config/application.properties b/galaxy-admin-server/config/application.properties index ab87051..f0827a7 100644 --- a/galaxy-admin-server/config/application.properties +++ b/galaxy-admin-server/config/application.properties @@ -1,15 +1,6 @@ #application config server.port=8800 spring.application.name=galaxy-admin-server -#spring cloud eureka config -eureka.instance.prefer-ip-address=true -eureka.client.registry-fetch-interval-seconds=5 -eureka.instance.lease-renewal-interval-in-seconds=10 -eureka.instance.health-check-url-path=/actuator/health -eureka.instance.metadata-map.user.name=${spring.security.user.name} -eureka.instance.metadata-map.user.password=${spring.security.user.password} -eureka.instance.instance-id=${spring.cloud.client.ip-address}:${server.port} -eureka.client.serviceUrl.defaultZone=http://admin:[email protected]:8501/eureka/ #management config management.endpoints.web.exposure.include=* management.endpoint.health.show-details=always diff --git a/galaxy-admin-server/config/logback-spring.xml b/galaxy-admin-server/config/logback-spring.xml index 13f5549..a523140 100644 --- a/galaxy-admin-server/config/logback-spring.xml +++ b/galaxy-admin-server/config/logback-spring.xml @@ -1,66 +1,45 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<configuration scan="true" scanPeriod="60 seconds"> - <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" /> - <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" /> - <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" /> - - <property name="CONSOLE_LOG_PATTERN" value="${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/> - - <property name="FILE_LOG_PATTERN" value="${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/> - - <property name="TEST_FILE_PATH" value="/home/galaxy/galaxy-admin-server/logs" /> - - <property name="PROD_FILE_PATH" value="/home/galaxy/galaxy-admin-server/logs" /> +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + + <!--日志文件路径--> + <property name="LOG_PATH" value="/home/galaxy/galaxy-admin-server/logs"/> + <!--日志文件名称--> + <property name="LOG_FILE_NAME" value="galaxy-admin-server"/> + + <!--控制台日志， 控制台输出 --> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度,%msg：日志消息，%n是换行符--> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + </encoder> + </appender> - <property name="LOG_NAME" value="galaxy-admin-server" /> - <!-- dev env --> - <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <appender name="ALL_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <!-- 正在记录的日志文件的路径及文件名 --> + <file>${LOG_PATH}/${LOG_FILE_NAME}.log</file> + <!--日志文件输出格式--> <encoder> - <pattern>${CONSOLE_LOG_PATTERN}</pattern> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + <charset>UTF-8</charset> <!-- 设置字符集 --> </encoder> + <!-- 日志记录器的滚动策略，按日期，按大小记录 --> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- 日志归档 --> + <fileNamePattern>${LOG_PATH}/history/log-%d{yyyy-MM-dd}.%i.log</fileNamePattern> + <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> + <maxFileSize>100MB</maxFileSize> + </timeBasedFileNamingAndTriggeringPolicy> + <!--日志文件保留天数--> + <maxHistory>15</maxHistory> + </rollingPolicy> + <!-- 此日志文件只记录debug以上级别的 --> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>DEBUG</level> + </filter> </appender> - <root level="INFO"> - <appender-ref ref="CONSOLE" /> + <root level="info"> + <appender-ref ref="ALL_FILE" /> + <appender-ref ref="STDOUT" /> </root> - - <!-- test env --> - <springProfile name="test"> - <appender name="TEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <encoder> - <pattern>${FILE_LOG_PATTERN}</pattern> - </encoder> - <file>${TEST_FILE_PATH}${file.separator}${LOG_NAME}.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> - <!-- rollover daily --> - <fileNamePattern>${TEST_FILE_PATH}${file.separator}${LOG_NAME}-%d{yyyy-MM-dd}.%i.log</fileNamePattern> - <!-- each file should be at most 100MB, keep 30 days worth of history, but at most 20GB --> - <maxHistory>30</maxHistory> - <maxFileSize>100MB</maxFileSize> - </rollingPolicy> - </appender> - <root level="INFO"> - <appender-ref ref="TEST_FILE" /> - </root> - </springProfile> - - <!-- prod env --> - <springProfile name="prod"> - <appender name="PROD_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <encoder> - <pattern>${FILE_LOG_PATTERN}</pattern> - </encoder> - <file>${PROD_FILE_PATH}${file.separator}${LOG_NAME}.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> - <!-- rollover daily --> - <fileNamePattern>${PROD_FILE_PATH}${file.separator}${LOG_NAME}-%d{yyyy-MM-dd}.%i.log</fileNamePattern> - <!-- each file should be at most 100MB, keep 30 days worth of history, but at most 20GB --> - <maxHistory>30</maxHistory> - <maxFileSize>100MB</maxFileSize> - </rollingPolicy> - </appender> - <root level="INFO"> - <appender-ref ref="PROD_FILE" /> - </root> - </springProfile> -</configuration> +</configuration>
\ No newline at end of file diff --git a/galaxy-admin-server/src/main/resources/banner.txt b/galaxy-admin-server/src/main/resources/banner.txt deleted file mode 100644 index adbcaf3..0000000 --- a/galaxy-admin-server/src/main/resources/banner.txt +++ /dev/null @@ -1,17 +0,0 @@ -${AnsiColor.BRIGHT_GREEN} -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -${AnsiColor.BRIGHT_BLUE} - _ _ _ - | | | | (_) - __ _ __ _| | __ ___ ___ _ ______ __ _ __| |_ __ ___ _ _ __ ______ ___ ___ _ ____ _____ _ __ - / _` |/ _` | |/ _` \ \/ / | | |______/ _` |/ _` | '_ ` _ \| | '_ \______/ __|/ _ \ '__\ \ / / _ \ '__| - | (_| | (_| | | (_| |> <| |_| | | (_| | (_| | | | | | | | | | | \__ \ __/ | \ V / __/ | - \__, |\__,_|_|\__,_/_/\_\\__, | \__,_|\__,_|_| |_| |_|_|_| |_| |___/\___|_| \_/ \___|_| - __/ | __/ | - |___/ |___/ -${AnsiColor.BRIGHT_BLUE} -Spring Boot Version: ${spring-boot.version} -${AnsiColor.BRIGHT_GREEN} -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -${AnsiColor.DEFAULT} - diff --git a/galaxy-auth-center/config/application.properties b/galaxy-auth-center/config/application.properties index 1e93bad..18b9d4e 100644 --- a/galaxy-auth-center/config/application.properties +++ b/galaxy-auth-center/config/application.properties @@ -1,24 +1,8 @@ #application config -server.port=8601 +server.port=8801 spring.application.name=galaxy-auth-center -#spring cloud eureka config -eureka.instance.prefer-ip-address=true -eureka.client.registry-fetch-interval-seconds=5 -eureka.instance.lease-renewal-interval-in-seconds=10 -eureka.instance.health-check-url-path=/actuator/health -eureka.instance.metadata-map.user.name=${spring.security.user.name} -eureka.instance.metadata-map.user.password=${spring.security.user.password} -eureka.instance.instance-id=${spring.cloud.client.ip-address}:${server.port} -eureka.client.serviceUrl.defaultZone=http://admin:[email protected]:8501/eureka/ #management config management.endpoints.web.exposure.include=* management.endpoint.health.show-details=always #log file path config -logging.config=./config/logback-spring.xml - -#arango db config -arango.username=query -arango.password=ceiec2018 -arango.address=192.168.44.12:8529 -arango.authurl=http://${arango.address}/_db/tsg_galaxy_v3/_open/auth -arango.query.url=http://${arango.address}/_db/tsg_galaxy_v3/_api/cursor
\ No newline at end of file +logging.config=./config/logback-spring.xml
\ No newline at end of file diff --git a/galaxy-auth-center/config/logback-spring.xml b/galaxy-auth-center/config/logback-spring.xml index c8b1682..c0856ec 100644 --- a/galaxy-auth-center/config/logback-spring.xml +++ b/galaxy-auth-center/config/logback-spring.xml @@ -1,66 +1,45 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<configuration scan="true" scanPeriod="60 seconds"> - <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" /> - <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" /> - <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" /> - - <property name="CONSOLE_LOG_PATTERN" value="${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/> - - <property name="FILE_LOG_PATTERN" value="${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/> - - <property name="TEST_FILE_PATH" value="/home/galaxy/galaxy-auth-center/logs" /> - - <property name="PROD_FILE_PATH" value="/home/galaxy/galaxy-auth-center/logs" /> +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + + <!--日志文件路径--> + <property name="LOG_PATH" value="/home/galaxy/galaxy-auth-center/logs"/> + <!--日志文件名称--> + <property name="LOG_FILE_NAME" value="galaxy-admin-server"/> + + <!--控制台日志， 控制台输出 --> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度,%msg：日志消息，%n是换行符--> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + </encoder> + </appender> - <property name="LOG_NAME" value="galaxy-auth-center" /> - <!-- dev env --> - <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <appender name="ALL_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <!-- 正在记录的日志文件的路径及文件名 --> + <file>${LOG_PATH}/${LOG_FILE_NAME}.log</file> + <!--日志文件输出格式--> <encoder> - <pattern>${CONSOLE_LOG_PATTERN}</pattern> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + <charset>UTF-8</charset> <!-- 设置字符集 --> </encoder> + <!-- 日志记录器的滚动策略，按日期，按大小记录 --> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- 日志归档 --> + <fileNamePattern>${LOG_PATH}/history/log-%d{yyyy-MM-dd}.%i.log</fileNamePattern> + <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> + <maxFileSize>100MB</maxFileSize> + </timeBasedFileNamingAndTriggeringPolicy> + <!--日志文件保留天数--> + <maxHistory>15</maxHistory> + </rollingPolicy> + <!-- 此日志文件只记录debug以上级别的 --> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>DEBUG</level> + </filter> </appender> - <root level="INFO"> - <appender-ref ref="CONSOLE" /> + <root level="info"> + <appender-ref ref="ALL_FILE" /> + <appender-ref ref="STDOUT" /> </root> - - <!-- test env --> - <springProfile name="test"> - <appender name="TEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <encoder> - <pattern>${FILE_LOG_PATTERN}</pattern> - </encoder> - <file>${TEST_FILE_PATH}${file.separator}${LOG_NAME}.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> - <!-- rollover daily --> - <fileNamePattern>${TEST_FILE_PATH}${file.separator}${LOG_NAME}-%d{yyyy-MM-dd}.%i.log</fileNamePattern> - <!-- each file should be at most 100MB, keep 30 days worth of history, but at most 20GB --> - <maxHistory>30</maxHistory> - <maxFileSize>100MB</maxFileSize> - </rollingPolicy> - </appender> - <root level="INFO"> - <appender-ref ref="TEST_FILE" /> - </root> - </springProfile> - - <!-- prod env --> - <springProfile name="prod"> - <appender name="PROD_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <encoder> - <pattern>${FILE_LOG_PATTERN}</pattern> - </encoder> - <file>${PROD_FILE_PATH}${file.separator}${LOG_NAME}.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> - <!-- rollover daily --> - <fileNamePattern>${PROD_FILE_PATH}${file.separator}${LOG_NAME}-%d{yyyy-MM-dd}.%i.log</fileNamePattern> - <!-- each file should be at most 100MB, keep 30 days worth of history, but at most 20GB --> - <maxHistory>30</maxHistory> - <maxFileSize>100MB</maxFileSize> - </rollingPolicy> - </appender> - <root level="INFO"> - <appender-ref ref="PROD_FILE" /> - </root> - </springProfile> -</configuration> +</configuration>
\ No newline at end of file diff --git a/galaxy-auth-center/src/main/java/com/mesalab/auth/GalaxyAuthCenterApp.java b/galaxy-auth-center/src/main/java/com/mesalab/auth/GalaxyAuthCenterApp.java deleted file mode 100644 index ce9cb39..0000000 --- a/galaxy-auth-center/src/main/java/com/mesalab/auth/GalaxyAuthCenterApp.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.mesalab.auth; - -import org.springframework.boot.SpringApplication; -import org.springframework.cloud.client.SpringCloudApplication; -import org.springframework.cloud.client.loadbalancer.LoadBalanced; -import org.springframework.context.annotation.Bean; -import org.springframework.web.client.RestTemplate; - -@SpringCloudApplication -public class GalaxyAuthCenterApp { - - @Bean - //负载均衡注解 - @LoadBalanced - RestTemplate restTemplate() { - return new RestTemplate(); - } - - public static void main(String[] args) { - SpringApplication.run(GalaxyAuthCenterApp.class); - } - -} diff --git a/galaxy-auth-center/src/main/java/com/mesalab/auth/component/IDGenerator.java b/galaxy-auth-center/src/main/java/com/mesalab/auth/component/IDGenerator.java deleted file mode 100644 index 28403ed..0000000 --- a/galaxy-auth-center/src/main/java/com/mesalab/auth/component/IDGenerator.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.mesalab.auth.component; - -import cn.hutool.core.date.DatePattern; -import cn.hutool.core.date.DateTime; -import cn.hutool.core.lang.ObjectId; -import cn.hutool.core.lang.Snowflake; -import cn.hutool.core.net.NetUtil; -import cn.hutool.core.util.IdUtil; -import cn.hutool.core.util.RandomUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; - -/** - * @Date: 2020-07-30 14:04 - * @Author : liuyongqiang - * @ClassName : IdGenerator - * @Description : ID生成器 - */ -@Slf4j -@Component -public class IDGenerator { - - private long workerId = 0; - - @PostConstruct - void init() { - try { - workerId = NetUtil.ipv4ToLong(NetUtil.getLocalhostStr()); - log.info("Current machine workerId: {}", workerId); - } catch (Exception e) { - log.warn("Get machine id fail:", e); - workerId = NetUtil.getLocalhost().hashCode(); - log.info("Current machine workerId: {}", workerId); - } - } - - /** - * 获取一个批次号，形如 2019071015301361000101237 - * <p> - * 数据库使用 char(25) 存储 - * - * @param tenantId 租户ID，5 位 - * @param module 业务模块ID，2 位 - * @return 返回批次号 - */ - public synchronized String batchId(int tenantId, int module) { - String prefix = DateTime.now().toString(DatePattern.PURE_DATETIME_MS_PATTERN); - return prefix + tenantId + module + RandomUtil.randomNumbers(3); - } - - @Deprecated - public synchronized String getBatchId(int tenantId, int module) { - return batchId(tenantId, module); - } - - /** - * 生成的是不带-的字符串，类似于：b17f24ff026d40949c85a24f4f375d42 - * - * @return - */ - public String simpleUUID() { - return IdUtil.simpleUUID(); - } - - /** - * 生成的UUID是带-的字符串，类似于：a5c8a5e8-df2b-4706-bea4-08d0939410e3 - * - * @return - */ - public String randomUUID() { - return IdUtil.randomUUID(); - } - - private Snowflake snowflake = IdUtil.createSnowflake(workerId, 1); - - public synchronized long snowflakeId() { - return snowflake.nextId(); - } - - public synchronized long snowflakeId(long workerId, long dataCenterId) { - Snowflake snowflake = IdUtil.createSnowflake(workerId, dataCenterId); - return snowflake.nextId(); - } - - /** - * 生成类似：5b9e306a4df4f8c54a39fb0c - * <p> - * ObjectId 是 MongoDB 数据库的一种唯一 ID 生成策略， - * 是 UUID version1 的变种，详细介绍可见：服务化框架－分布式 Unique ID 的生成方法一览。 - * - * @return - */ - public String objectId() { - return ObjectId.next(); - } - -} - diff --git a/galaxy-auth-center/src/main/java/com/mesalab/auth/component/SysLogAspect.java b/galaxy-auth-center/src/main/java/com/mesalab/auth/component/SysLogAspect.java deleted file mode 100644 index 6e0db71..0000000 --- a/galaxy-auth-center/src/main/java/com/mesalab/auth/component/SysLogAspect.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.mesalab.auth.component; - -import com.google.gson.Gson; -import com.mesalab.common.util.LoggerCache; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Pointcut; -import org.aspectj.lang.reflect.MethodSignature; -import org.slf4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.lang.reflect.Method; - -/** - * @Date: 2020-07-30 11:54 - * @Author : liuyongqiang - * @ClassName : SysLogAspect - * @Description : 切面实现日志记录 - */ -@Aspect -@Component -public class SysLogAspect { - - @Autowired - IDGenerator idGenerator; - - @Pointcut("@annotation(com.mesalab.common.annotation.SysLog)") - public void log() { - } - - - /** - * 加入注解自动记录方法日志 - * - * @param joinPoint - * @return - * @throws Throwable - */ - @Around(value = "log()") - public Object logAround(ProceedingJoinPoint joinPoint) throws Throwable { - // 获取执行方法的类的名称（包名加类名） - String className = joinPoint.getTarget().getClass().getName(); - // 获取实例和方法 - MethodSignature signature = (MethodSignature) joinPoint.getSignature(); - Method method = signature.getMethod(); - // 从缓存中获取日志实例 - Logger log = LoggerCache.getLoggerByClassName(className); - // 记录日志 - long logId = idGenerator.snowflakeId(); - log.info("[" + logId + "]" + " - Execute:" + className + "." + method.getName()); - Object[] args = joinPoint.getArgs(); - log.info("[" + logId + "]" + " - Params:" + new Gson().toJson(args)); - // 执行方法获取返回值 - Object proceed = joinPoint.proceed(); - // 记录日志 - log.info("[" + logId + "]" + " - Returns:" + new Gson().toJson(proceed)); - // 返回 - return proceed; - } -} diff --git a/galaxy-auth-center/src/main/java/com/mesalab/auth/component/config/ArangoConfig.java b/galaxy-auth-center/src/main/java/com/mesalab/auth/component/config/ArangoConfig.java deleted file mode 100644 index 717b755..0000000 --- a/galaxy-auth-center/src/main/java/com/mesalab/auth/component/config/ArangoConfig.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.mesalab.auth.component.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @description: for http - * @author: zhq - * @create: 2020-07-07 - **/ -@Data -@Component -public class ArangoConfig { - - @Value("${arango.query.url}") - private String queryurl; - - @Value("${arango.username}") - private String userName; - - @Value("${arango.password}") - private String passWord; - - @Value("${arango.authurl}") - private String authUrl; - -} diff --git a/galaxy-auth-center/src/main/java/com/mesalab/auth/controller/AuthorizeController.java b/galaxy-auth-center/src/main/java/com/mesalab/auth/controller/AuthorizeController.java deleted file mode 100644 index 5bf050c..0000000 --- a/galaxy-auth-center/src/main/java/com/mesalab/auth/controller/AuthorizeController.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.mesalab.auth.controller; - -import com.mesalab.auth.service.AuthorizeService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * @Date: 2020-08-17 14:25 - * @Author : liuyongqiang - * @ClassName : JwtAuthorizeController - * @Description : JWT授权服务 - */ -@Slf4j -@RestController -@RequestMapping("/authorize/") -public class AuthorizeController { - - @Autowired - AuthorizeService jwtAuthorizeService; - - @PostMapping("arangoJwtLogin") - public String arangoJwtLogin(){ - return jwtAuthorizeService.arangoJwtLogin(); - } -} diff --git a/galaxy-auth-center/src/main/java/com/mesalab/auth/controller/GlobalExceptionHandler.java b/galaxy-auth-center/src/main/java/com/mesalab/auth/controller/GlobalExceptionHandler.java deleted file mode 100644 index a694ed7..0000000 --- a/galaxy-auth-center/src/main/java/com/mesalab/auth/controller/GlobalExceptionHandler.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.mesalab.auth.controller; - -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.RestControllerAdvice; - -import javax.servlet.http.HttpServletResponse; - -@Slf4j -@RestControllerAdvice -public class GlobalExceptionHandler { - - - @ExceptionHandler(Exception.class) - public BaseResult handleException(Exception e) { - return BaseResultUtil.failure(e.getMessage()); - } - - - @ExceptionHandler({BusinessException.class}) - public BaseResult handleBusinessException(BusinessException e, HttpServletResponse response) { - response.setStatus(e.getErrorStatus()); - log.error("Execute Query Error: {}", e.getMessage() + ";" + (e.getCause() != null ? e.getCause().getMessage() : "")); - return BaseResultUtil.failure(e.getErrorStatus(), e.getErrorCode(), - (e.getMessage() != null ? e.getMessage() : e.getErrorMessage()) + " " + (e.getCause() != null ? e.getCause().getMessage() : "")); - } - - -} diff --git a/galaxy-auth-center/src/main/java/com/mesalab/auth/service/AuthorizeService.java b/galaxy-auth-center/src/main/java/com/mesalab/auth/service/AuthorizeService.java deleted file mode 100644 index e04a974..0000000 --- a/galaxy-auth-center/src/main/java/com/mesalab/auth/service/AuthorizeService.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.mesalab.auth.service; - -import com.mesalab.common.exception.BusinessException; - -/** - * @Date: 2020-08-17 14:21 - * @Author : liuyongqiang - * @ClassName : AuthorizeService - * @Description : JWT授权服务 - */ -public interface AuthorizeService { - - /** - * @Description: ArangoDBJwt登录 - * @Author: liuyongqiang - * @Date: 2020/7/20 10:18 - * @return: java.lang.String - **/ - String arangoJwtLogin() throws BusinessException; - - /** - * @Description: 验证ArangoJwtToken是否有效 - * @Author: liuyongqiang - * @Date: 2020/8/27 18:07 - * @param token: - * @return: boolean - **/ - boolean tokenValidate(String token)throws BusinessException; -} diff --git a/galaxy-auth-center/src/main/java/com/mesalab/auth/service/impl/AuthorizeServiceImpl.java b/galaxy-auth-center/src/main/java/com/mesalab/auth/service/impl/AuthorizeServiceImpl.java deleted file mode 100644 index 7c768fe..0000000 --- a/galaxy-auth-center/src/main/java/com/mesalab/auth/service/impl/AuthorizeServiceImpl.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.mesalab.auth.service.impl; - -import com.google.common.collect.ImmutableMap; -import com.google.gson.Gson; -import com.mesalab.auth.component.config.ArangoConfig; -import com.mesalab.auth.service.AuthorizeService; -import com.mesalab.common.annotation.SysLog; -import com.mesalab.common.enums.CacheKeyEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.HttpClientUtil; -import com.mesalab.common.util.MemoryCache; -import com.mesalab.common.util.SysConstant; -import com.zdjizhi.utils.JsonMapper; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.Header; -import org.apache.http.message.BasicHeader; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpHeaders; -import org.springframework.stereotype.Service; - -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -/** - * @Date: 2020-08-17 14:22 - * @Author : liuyongqiang - * @ClassName : AuthorizeServiceImpl - * @Description : JWT授权服务 - */ -@Slf4j -@SuppressWarnings("unchecked") -@Service("jwtAuthorizeService") -public class AuthorizeServiceImpl implements AuthorizeService { - - @Autowired - ArangoConfig arangoConfig; - - /** - * @Description: ArangoDBJwt登录 - * @Author: liuyongqiang - * @Date: 2020/7/20 10:18 - * @return: java.lang.String - **/ - @SysLog - @Override - public String arangoJwtLogin() throws BusinessException { - - String cacheKey = CacheKeyEnum.ARONGO_JWT_KEY.getKey(); - Object cacheVal = MemoryCache.getInstance().get(cacheKey); - log.info("从缓存中获取的Token为:{}", cacheVal); - if (Objects.nonNull(cacheVal)) { - if (tokenValidate(String.valueOf(cacheVal))) { - log.info("从缓存中获取的Token未失效!"); - return String.valueOf(cacheVal); - } else { - log.info("从缓存中获取的Token已失效!"); - } - } - - Map<String, String> params = new HashMap<>(); - params.put("username", arangoConfig.getUserName()); - params.put("password", arangoConfig.getPassWord()); - String jwtStr; - - try { - String jsonRes = HttpClientUtil.httpPost(arangoConfig.getAuthUrl(), new Gson().toJson(params)); - params = new Gson().fromJson(jsonRes, Map.class); - jwtStr = params.get(SysConstant.ARANGO_JWT_STR); - log.info("请求arango授权接口获取的jwt:{}", jwtStr); - //将jwt字符串做缓存处理,过期时间为7天 - MemoryCache.getInstance().put(cacheKey, jwtStr, 604800); - } catch (BusinessException e) { - log.info("请求arango授权接口发生异常,异常信息:{}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } - - return jwtStr; - } - - /** - * @param token: - * @Description: 验证ArangoJwtToken是否有效 - * @Author: liuyongqiang - * @Date: 2020/8/27 18:07 - * @return: boolean - **/ - @Override - public boolean tokenValidate(String token) throws BusinessException { - Map queryMap = ImmutableMap.of("query", ""); - String jwtStr = SysConstant.ARANGO_JWT_PRE.concat(token); - Header header = new BasicHeader(HttpHeaders.AUTHORIZATION, jwtStr); - try { - HttpClientUtil.httpPost(arangoConfig.getQueryurl(), JsonMapper.toJsonString(queryMap), header); - } catch (BusinessException e) { - if (e.getErrorMessage() != null && e.getErrorMessage().contains("not authorized to execute this request")) { - return false; - } else { - return true; - } - } - return true; - } - -} diff --git a/galaxy-auth-center/src/main/resources/banner.txt b/galaxy-auth-center/src/main/resources/banner.txt deleted file mode 100644 index 5f87840..0000000 --- a/galaxy-auth-center/src/main/resources/banner.txt +++ /dev/null @@ -1,17 +0,0 @@ -${AnsiColor.BRIGHT_GREEN} -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -${AnsiColor.BRIGHT_BLUE} - _ _ _ _ - | | | | | | | | - __ _ __ _| | __ ___ ___ _ ______ __ _ _ _| |_| |__ ______ ___ ___ _ __ | |_ ___ _ __ - / _` |/ _` | |/ _` \ \/ / | | |______/ _` | | | | __| '_ \______/ __/ _ \ '_ \| __/ _ \ '__| - | (_| | (_| | | (_| |> <| |_| | | (_| | |_| | |_| | | | | (_| __/ | | | || __/ | - \__, |\__,_|_|\__,_/_/\_\\__, | \__,_|\__,_|\__|_| |_| \___\___|_| |_|\__\___|_| - __/ | __/ | - |___/ |___/ -${AnsiColor.BRIGHT_BLUE} -Spring Boot Version: ${spring-boot.version} -${AnsiColor.BRIGHT_GREEN} -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -${AnsiColor.DEFAULT} - diff --git a/galaxy-auth-center/src/test/java/com/mesalab/test/JwtAuthorizeServiceTest.java b/galaxy-auth-center/src/test/java/com/mesalab/api/test/JwtAuthorizeServiceTest.java index c60ca9e..953ec17 100644 --- a/galaxy-auth-center/src/test/java/com/mesalab/test/JwtAuthorizeServiceTest.java +++ b/galaxy-auth-center/src/test/java/com/mesalab/api/test/JwtAuthorizeServiceTest.java @@ -1,4 +1,4 @@ -package com.mesalab.test; +package com.mesalab.api.test; import com.mesalab.auth.GalaxyAuthCenterApp; import com.mesalab.auth.service.AuthorizeService; diff --git a/galaxy-business-api/config/application.properties b/galaxy-business-api/config/application.properties new file mode 100644 index 0000000..8858a8f --- /dev/null +++ b/galaxy-business-api/config/application.properties @@ -0,0 +1,8 @@ +#application config +server.port=8802 +spring.application.name=galaxy-business-api +#management config +management.endpoints.web.exposure.include=* +management.endpoint.health.show-details=always +#log file path config +logging.config=./config/logback-spring.xml
\ No newline at end of file diff --git a/galaxy-business-api/config/logback-spring.xml b/galaxy-business-api/config/logback-spring.xml new file mode 100644 index 0000000..82659ab --- /dev/null +++ b/galaxy-business-api/config/logback-spring.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + + <!--日志文件路径--> + <property name="LOG_PATH" value="/home/galaxy/galaxy-business-api/logs"/> + <!--日志文件名称--> + <property name="LOG_FILE_NAME" value="galaxy-business-api"/> + + <!--控制台日志， 控制台输出 --> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度,%msg：日志消息，%n是换行符--> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + </encoder> + </appender> + + + <appender name="ALL_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <!-- 正在记录的日志文件的路径及文件名 --> + <file>${LOG_PATH}/${LOG_FILE_NAME}.log</file> + <!--日志文件输出格式--> + <encoder> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + <charset>UTF-8</charset> <!-- 设置字符集 --> + </encoder> + <!-- 日志记录器的滚动策略，按日期，按大小记录 --> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- 日志归档 --> + <fileNamePattern>${LOG_PATH}/history/log-%d{yyyy-MM-dd}.%i.log</fileNamePattern> + <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> + <maxFileSize>100MB</maxFileSize> + </timeBasedFileNamingAndTriggeringPolicy> + <!--日志文件保留天数--> + <maxHistory>15</maxHistory> + </rollingPolicy> + <!-- 此日志文件只记录debug以上级别的 --> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>DEBUG</level> + </filter> + </appender> + <root level="info"> + <appender-ref ref="ALL_FILE" /> + <appender-ref ref="STDOUT" /> + </root> +</configuration>
\ No newline at end of file diff --git a/galaxy-business-api/docker/Dockerfile b/galaxy-business-api/docker/Dockerfile new file mode 100644 index 0000000..eb52dff --- /dev/null +++ b/galaxy-business-api/docker/Dockerfile @@ -0,0 +1,11 @@ +ARG JDK_IMAGE +FROM ${JDK_IMAGE} +MAINTAINER galaxy-data-platform +VOLUME /tmp +WORKDIR /home/galaxy/galaxy-business-api +COPY config config +ARG JAR_FILE +COPY ${JAR_FILE} galaxy-business-api.jar +ENV JAVA_OPTS=" -Xms1024m -Xmx2048m " +ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 +ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar galaxy-business-api.jar" ]
\ No newline at end of file diff --git a/galaxy-business-api/pom.xml b/galaxy-business-api/pom.xml index ec9f07d..ed2f4a4 100644 --- a/galaxy-business-api/pom.xml +++ b/galaxy-business-api/pom.xml @@ -12,5 +12,7 @@ <artifactId>galaxy-business-api</artifactId> <packaging>jar</packaging> - + <build> + <finalName>galaxy-business-api</finalName> + </build> </project>
\ No newline at end of file diff --git a/galaxy-common/pom.xml b/galaxy-common/pom.xml index 0cd2b05..72d01ca 100644 --- a/galaxy-common/pom.xml +++ b/galaxy-common/pom.xml @@ -12,71 +12,6 @@ <artifactId>galaxy-common</artifactId> <packaging>jar</packaging> - <dependencies> - <dependency> - <groupId>org.projectlombok</groupId> - <artifactId>lombok</artifactId> - </dependency> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>cn.hutool</groupId> - <artifactId>hutool-all</artifactId> - </dependency> - <dependency> - <groupId>com.github.jsqlparser</groupId> - <artifactId>jsqlparser</artifactId> - </dependency> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </dependency> - <dependency> - <groupId>org.apache.calcite</groupId> - <artifactId>calcite-core</artifactId> - </dependency> - <dependency> - <groupId>joda-time</groupId> - <artifactId>joda-time</artifactId> - </dependency> - <dependency> - <groupId>jakarta.validation</groupId> - <artifactId>jakarta.validation-api</artifactId> - <version>${jakarta.version}</version> - </dependency> - <dependency> - <groupId>org.nutz</groupId> - <artifactId>nutz</artifactId> - </dependency> - <dependency> - <groupId>com.zdjizhi</groupId> - <artifactId>galaxy</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> <build> <finalName>galaxy-common</finalName> diff --git a/galaxy-common/src/main/java/com/mesalab/common/annotation/SysLog.java b/galaxy-common/src/main/java/com/mesalab/common/annotation/SysLog.java deleted file mode 100644 index 39d70fe..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/annotation/SysLog.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.mesalab.common.annotation; - -import java.lang.annotation.*; - -/** - * @Date: 2020-07-30 11:52 - * @Author : liuyongqiang - * @ClassName : SysLog - * @Description : 日志注解 - */ -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface SysLog { -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/dto/params/ExecutorParam.java b/galaxy-common/src/main/java/com/mesalab/common/dto/params/ExecutorParam.java deleted file mode 100644 index caa3802..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/dto/params/ExecutorParam.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.mesalab.common.dto.params; - -import lombok.Data; - -@Data -public class ExecutorParam { - private int maxdays; -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/dto/params/NetworkParam.java b/galaxy-common/src/main/java/com/mesalab/common/dto/params/NetworkParam.java deleted file mode 100644 index 01b8fe4..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/dto/params/NetworkParam.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.mesalab.common.dto.params; - -import jakarta.validation.Valid; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - - -/** - * @author wangwei - * @version 1.0 - * @date 2020/6/30 1:52 下午 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class NetworkParam { - - private String clientId; - @Valid - private QueryParam query; -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/dto/params/QueryParam.java b/galaxy-common/src/main/java/com/mesalab/common/dto/params/QueryParam.java deleted file mode 100644 index 02bf4bf..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/dto/params/QueryParam.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.mesalab.common.dto.params; - -import jakarta.validation.constraints.NotBlank; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -import java.io.Serializable; -import java.util.List; - -/** - * @author wangwei - * @version 1.0 - * @date 2020/7/7 6:58 下午 - */ -@Data -@AllArgsConstructor -@NoArgsConstructor -public class QueryParam implements Serializable { - - @NotBlank - private String queryType; - @NotBlank - private String dataSource; - private String granularity; - private String parameters; - private Interval intervals; - - public void setIntervals(List<String> intervals) { - if (intervals == null || intervals.size() != 1) { - throw new IllegalArgumentException("intervals params is error"); - } - String str = intervals.get(0); - int separator = str.indexOf("/"); - if (separator < 0) { - throw new IllegalArgumentException("Format requires a '/' separator: " + str); - } - String leftStr = str.substring(0, separator); - if (leftStr.length() <= 0) { - throw new IllegalArgumentException("Format invalid: " + str); - } - String rightStr = str.substring(separator + 1); - if (rightStr.length() <= 0) { - throw new IllegalArgumentException("Format invalid: " + str); - } - DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"); - if (dateTimeFormatter.parseMillis(rightStr) - dateTimeFormatter.parseMillis(leftStr) <= 0) { - throw new IllegalArgumentException("Intervals should be [startTime/endTime], " + str); - } - this.intervals = new Interval(leftStr, rightStr); - } - - @Data - @AllArgsConstructor - @NoArgsConstructor - public class Interval implements Serializable { - private String startTime; - private String endTime; - } - -}
\ No newline at end of file diff --git a/galaxy-common/src/main/java/com/mesalab/common/dto/params/SchemaParam.java b/galaxy-common/src/main/java/com/mesalab/common/dto/params/SchemaParam.java deleted file mode 100644 index 834d9cc..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/dto/params/SchemaParam.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.mesalab.common.dto.params; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @Date: 2020-08-17 16:45 - * @Author : liuyongqiang - * @ClassName : SchemaParam - * @Description : SchemaParam - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class SchemaParam { - - private String type; - - private String name; - - private String tableName; - - private String key; - - public SchemaParam(String type, String name) { - this.type = type; - this.name = name; - } - - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/dto/results/ArangoCollectionResult.java b/galaxy-common/src/main/java/com/mesalab/common/dto/results/ArangoCollectionResult.java deleted file mode 100644 index 2bcdabc..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/dto/results/ArangoCollectionResult.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.mesalab.common.dto.results; - -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.util.List; - -/** - * @Date: 2020-07-27 10:49 - * @Author : liuyongqiang - * @ClassName : ArangoCollectionResult - * @Description : Arango集合名称查询结果封装 - */ -@Data -@AllArgsConstructor -public class ArangoCollectionResult { - - private boolean error; - private int code; - private List<ResultBean> result; - - @Data - public static class ResultBean { - private String id; - private String name; - private int status; - private int type; - private boolean isSystem; - private String globallyUniqueId; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/dto/results/ArangoCursorResult.java b/galaxy-common/src/main/java/com/mesalab/common/dto/results/ArangoCursorResult.java deleted file mode 100644 index b0d55cf..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/dto/results/ArangoCursorResult.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.mesalab.common.dto.results; - -import com.google.gson.annotations.SerializedName; -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.util.List; -import java.util.Map; - -/** - * @Date: 2020-07-27 10:13 - * @Author : liuyongqiang - * @ClassName : ArangoCursorResult - * @Description : Arango文档查询结果封装 - */ -@Data -@AllArgsConstructor -public class ArangoCursorResult { - - - private boolean hasMore; - private boolean cached; - private ExtraBean extra; - private boolean error; - private int code; - private int errorNum; - private List<Map<String,Object>> result; - - @Data - public static class ExtraBean { - - private StatsBean stats; - private ProfileBean profile; - private List<?> warnings; - - @Data - public static class StatsBean { - private int writesExecuted; - private int writesIgnored; - private int scannedFull; - private int scannedIndex; - private int filtered; - private int httpRequests; - private double executionTime; - private int peakMemoryUsage; - } - - @Data - public static class ProfileBean { - private double initializing; - private double parsing; - @SerializedName("optimizing ast") - private double _$OptimizingAst41; - @SerializedName("loading collections") - private double _$LoadingCollections53; - @SerializedName("instantiating plan") - private double _$InstantiatingPlan79; - @SerializedName("optimizing plan") - private double _$OptimizingPlan284; - private double executing; - private double finalizing; - - } - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/dto/results/BaseResult.java b/galaxy-common/src/main/java/com/mesalab/common/dto/results/BaseResult.java deleted file mode 100644 index 10e000e..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/dto/results/BaseResult.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.mesalab.common.dto.results; - -import com.mesalab.common.enums.ResultStatusEnum; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.Map; - -/** - * 响应结果 - * @author dazzlzy - * @date 2018/3/21 - */ -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class BaseResult<T> implements Serializable { - - private Integer status; - - private String code; - - private boolean success; - - private String message; - - private Map<String, Object> statistics; - - private String formatType; - - private T meta; - - private T data; - - - /** - * 判断是否是成功结果 - * JsonIgnore使之不在json序列化结果当中 - * - * @return 是否为成功结果 - */ - public boolean isSuccess() { - return ResultStatusEnum.SUCCESS.getCode() == this.status; - } - -}
\ No newline at end of file diff --git a/galaxy-common/src/main/java/com/mesalab/common/dto/results/ClickHouseResult.java b/galaxy-common/src/main/java/com/mesalab/common/dto/results/ClickHouseResult.java deleted file mode 100644 index 3ae19ba..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/dto/results/ClickHouseResult.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.mesalab.common.dto.results; - -import lombok.AllArgsConstructor; -import lombok.Data; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -/** - * @Date: 2020-07-23 16:21 - * @Author : liuyongqiang - * @ClassName : ClickHouseResult - * @Description : ClickHouse查询结果封装 - */ -@Data -@AllArgsConstructor -public class ClickHouseResult implements Serializable { - - private int rows; - private List<MetaBean> meta; - private List<Map<String,Object>> data; - private Map<String,Object> statistics; - - @Data - public static class MetaBean { - private String name; - private String type; - } - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/dto/results/DemoResult.java b/galaxy-common/src/main/java/com/mesalab/common/dto/results/DemoResult.java deleted file mode 100644 index 734ec18..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/dto/results/DemoResult.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.mesalab.common.dto.results; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @Date: 2020-08-05 14:03 - * @Author : liuyongqiang - * @ClassName : DemoParam - * @Description : DemoParam - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class DemoResult { - - public String userName; - public String passWord; - public String address; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/BaseResultEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/BaseResultEnum.java deleted file mode 100644 index 3934e78..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/BaseResultEnum.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.mesalab.common.enums; - -/** - * @Date: 2020-08-05 14:18 - * @Author : liuyongqiang - * @ClassName : BaseResultEnum - * @Description : BaseResults枚举类 - */ -public enum BaseResultEnum { - - SUCCESS(200,"成功"), - BAD_REQUEST(400 ,"错误请求"), - FORBID_REQUEST(403 ,"禁止请求"), - NOT_FOUND(404 ,"未找到"), - UNKNOWN_ERROR(500 ,"未知错误"); - - private Integer code; - private String message; - - BaseResultEnum(Integer code, String message) { - this.code = code; - this.message = message; - } - - public Integer getCode() { - return code; - } - - public String getMessage() { - return message; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/CacheKeyEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/CacheKeyEnum.java deleted file mode 100644 index a565ef3..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/CacheKeyEnum.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @Date: 2020-07-22 14:05 - * @Author : liuyongqiang - * @ClassName : CacheKeyEnum - * @Description : 缓存Key定义 - */ -@Getter -@AllArgsConstructor -public enum CacheKeyEnum { - - ARONGO_JWT_KEY("arongo_jwt_key"); - - private String key; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/DBTypeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/DBTypeEnum.java deleted file mode 100644 index 418ef80..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/DBTypeEnum.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.Getter; - -@Getter -public enum DBTypeEnum { - CLICKHOUSE("clickHouse"), - DRUID("druid"), - HBASE("hbase"),; - private String value; - DBTypeEnum(String value) {this.value = value;} -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/EngineTypeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/EngineTypeEnum.java deleted file mode 100644 index f66eafc..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/EngineTypeEnum.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @Date: 2020-07-20 18:45 - * @Author : liuyongqiang - * @ClassName : EngineTypeEnum - * @Description : 查询引擎映射枚举类 - */ -@Getter -@AllArgsConstructor -public enum EngineTypeEnum { - - //AnalysisEngine-AnangoDB 分析、学习引擎 - //CaculationEngine-Druid 保存、计算引擎 - //BusinessEngine-ClickHouse 业务数据引擎 - - ANALYSIS_ENGINE("AnalysisEngine","AnangoDB"), - CACULATION_ENGINE("CaculationEngine","Druid"), - BUSINESS_ENGINE("BusinessEngine","ClickHouse"); - - private String engine;//引擎名称 - private String dbtype;//数据库类型 - - public static EngineTypeEnum getByEngine(String engine){ - for (EngineTypeEnum constants : values()) { - if (constants.engine.equalsIgnoreCase(engine)) { - return constants; - } - } - return null; - } - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/EnvironmentEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/EnvironmentEnum.java deleted file mode 100644 index 421fb63..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/EnvironmentEnum.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.Getter; - -/** - * 运行环境枚举 - * - * @author dazzlzy - * @date 2018/5/26 - */ -@Getter -public enum EnvironmentEnum { - - /** - * 开发环境 - */ - DEV("dev"), - /** - * 生产环境 - */ - PROD("prod"),; - - private String name; - - EnvironmentEnum(String name) { - this.name = name; - } - - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/EnvironmentGroupEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/EnvironmentGroupEnum.java deleted file mode 100644 index 5e433e0..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/EnvironmentGroupEnum.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.Getter; - -/** - * 运行环境组枚举 - * - * @author dazzlzy - * @date 2018/5/26 - */ -@Getter -public enum EnvironmentGroupEnum { - - /** - * RUNTIME运行环境组： - * 1. DEV(开发环境) - * 2. PROD(生产环境) - */ - RUNTIME(new EnvironmentEnum[]{EnvironmentEnum.DEV, EnvironmentEnum.PROD}),; - - /** - * 运行环境 - */ - private EnvironmentEnum[] environments; - - EnvironmentGroupEnum(EnvironmentEnum[] environments) { - this.environments = environments; - } - - /** - * 是否是runtime运行环境组 - * - * @param s 环境名 - * @return boolean - */ - public static boolean isRuntime(String s) { - EnvironmentEnum[] environmentEnums = RUNTIME.getEnvironments(); - for (EnvironmentEnum environmentEnum : environmentEnums) { - if (environmentEnum.getName().equals(s)) { - return true; - } - } - return false; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/IntervalTypeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/IntervalTypeEnum.java deleted file mode 100644 index 24ac85b..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/IntervalTypeEnum.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @Date: 2020-07-29 15:45 - * @Author : wangwei - * @ClassName : IntervalTypeEnum - * @Description : 区间查询类型枚举 - */ -@Getter -@AllArgsConstructor -public enum IntervalTypeEnum { - EXACTLY("BETWEEN", "闭区间间隔"); - - private String type; - private String name; - - public static IntervalTypeEnum getByType(String type) { - for (IntervalTypeEnum constants : values()) { - if (constants.type.equalsIgnoreCase(type)) { - return constants; - } - } - return null; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/JobHandlerEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/JobHandlerEnum.java deleted file mode 100644 index 8c184b7..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/JobHandlerEnum.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.Getter; - - -@Getter -public enum JobHandlerEnum { - - DELETE_TRAFFIC_DATA_JOB_HANDLER("deleteTrafficDataJobHandler"), - DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER("deleteReportAndMetricsDataJobHandler"), - DELETE_FILES_JOB_HANDLER("deleteFilesJobHandler"), - DELETE_ALL_TRAFFIC_DATA_JOB_HANDLER("deleteAllTrafficDataJobHandler"), - DELETE_ALL_REPORT_AND_METRICS_DATA_JOB_HANDLER("deleteAllReportAndMetricsDataJobHandler"), - DELETE_ALL_FILES_JOB_HANDLER("deleteAllFilesJobHandler"); - private String value; - - JobHandlerEnum(String value) { - this.value = value; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/MatchTypeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/MatchTypeEnum.java deleted file mode 100644 index 189d374..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/MatchTypeEnum.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @Date: 2020-07-22 17:45 - * @Author : liuyongqiang - * @ClassName : MatchTypeEnum - * @Description : 匹配查询类型枚举 - */ -@Getter -@AllArgsConstructor -public enum MatchTypeEnum { - - EXACTLY("exactly","完全匹配"), - PREFIX("prefix","前缀匹配"), - SUFFIX("suffix","后缀匹配"), - SUBSTRING("substring","字串匹配"), - REGEX("regex","正则匹配"); - - private String type; - private String name; - - public static MatchTypeEnum getByType(String type){ - for (MatchTypeEnum constants : values()) { - if (constants.type.equalsIgnoreCase(type)) { - return constants; - } - } - return null; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/QueryFormatEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/QueryFormatEnum.java deleted file mode 100644 index 741139c..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/QueryFormatEnum.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.Getter; - -@Getter -public enum QueryFormatEnum { - JSON("json"), - CSV("csv"),; - private String value; - QueryFormatEnum(String value) {this.value = value;} -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/QueryOptionEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/QueryOptionEnum.java deleted file mode 100644 index 7d0923d..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/QueryOptionEnum.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.Getter; - -@Getter -public enum QueryOptionEnum { - REAL_TIME("real-time"), - LONG_TERM("long-term"), - SYNTAX_CHECK("syntax-check"); - private String value; - QueryOptionEnum(String value) {this.value = value;} -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/QueryParamEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/QueryParamEnum.java deleted file mode 100644 index b9718eb..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/QueryParamEnum.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.Getter; - -@Getter -public enum QueryParamEnum { - QUERY("query"), - FORMAT("format"), - OPTION("option"),; - private String value; - QueryParamEnum(String value) {this.value = value;} -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/QueryTypeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/QueryTypeEnum.java deleted file mode 100644 index 04c0391..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/QueryTypeEnum.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @description: for queryType - * @author: zhq - * @create: 2020-07-16 - **/ -@Getter -@AllArgsConstructor -public enum QueryTypeEnum { - - IPLEARNING("iplearning"), - IPPOOL("ippool"), - SUBSCRIBERIDPOOL("subscriberidpool"); - - private final String type; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/RangeTypeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/RangeTypeEnum.java deleted file mode 100644 index 28eb929..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/RangeTypeEnum.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @Date: 2020-07-22 17:44 - * @Author : liuyongqiang - * @ClassName : RangeTypeEnum - * @Description : 范围查询类型枚举 - */ -@Getter -@AllArgsConstructor -public enum RangeTypeEnum { - - GT("gt"," > "), - LT("lt"," < "), - EQ("eq"," = "), - GE("ge"," >= "), - LE("le"," <= "), - NE("ne"," != "); - - private String type; - private String expr; - - public static RangeTypeEnum getByType(String type){ - for (RangeTypeEnum constants : values()) { - if (constants.type.equalsIgnoreCase(type)) { - return constants; - } - } - return null; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/ResultCodeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/ResultCodeEnum.java deleted file mode 100644 index 767bb80..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/ResultCodeEnum.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * BaseResult的业务状态码 - */ -@Getter -@AllArgsConstructor -public enum ResultCodeEnum { - - EXECUTE_SUCCESS("200666", "成功"), - PARAM_SYNTAX_ERROR("400001", "参数检查异常"), - SQL_SYNTAX_ERROR("400010", "SQL语句检查异常"), - SQL_EXECUTION_ERROR("500001", "SQL 执行异常"), - ENGINE_STATISTICS_ERROR("500010", "引擎计算异常"), - UNKNOW_ERROR("500999", "执行失败"); - - private String code; - private String message; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/ResultStatusEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/ResultStatusEnum.java deleted file mode 100644 index bd31151..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/ResultStatusEnum.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * BaseResult的http状态编码枚举 - */ -@Getter -@AllArgsConstructor -public enum ResultStatusEnum { - - /** - * SUCCESS: 200 成功 - * FAIL: 400 失败 - * NOT_FOUND： 404 不存在 - * SERVER_ERROR: 500 网络服务异常 - */ - SUCCESS(200, "成功"), - FAIL(400, "失败"), - NOT_FOUND(404, "不存在"), - REQ_FORBIDDEN(403, "重复请求"), - SERVER_ERROR(500, "服务异常"); - - private int code; - private String message; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/SchemaTypeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/SchemaTypeEnum.java deleted file mode 100644 index 59d19d9..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/SchemaTypeEnum.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.mesalab.common.enums; - -import lombok.Data; -import lombok.Getter; - -/** - * @Date: 2020-08-17 16:53 - * @Author : liuyongqiang - * @ClassName : SchemaTypeEnum - * @Description : SchemaTypeEnum - */ -@Getter -public enum SchemaTypeEnum { - - TABLES("tables"), - FIELDS("fields"); - private String value; - SchemaTypeEnum(String value) {this.value = value;} -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/DataSourceEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/DataSourceEnum.java deleted file mode 100644 index e0b425a..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/DataSourceEnum.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.mesalab.common.enums.knowledge; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @description: for datasource - * @author: zhq - * @create: 2020-07-06 - **/ -@Getter -@AllArgsConstructor -public enum DataSourceEnum { - - - IP_LEARNING_VIEW("IP_LEARNING_VIEW"), - - IP_VIEW("IP_VIEW"), - - SUBSCRIBER_ID_VIEW("SUBSCRIBER_ID_VIEW"); - - private String view; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/MatchEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/MatchEnum.java deleted file mode 100644 index 4efc8e3..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/MatchEnum.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.mesalab.common.enums.knowledge; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * 匹配查询 - * <p> - * type:匹配方式，["exactly"|"prefix"|"suffix"|"substring"|"regex"] - * <p> - * exactly:完全匹配 - * prefix:前缀匹配 - * suffix:后缀匹配 - * substring:子串匹配 - * regex:正则符号匹配 - * fieldKey:属性名称， - * <p> - * fieldValues:属性值 - * <p> - * 注：当type=regex时 - * <p> - * fieldValues值匹配方式符合下列规则，数组内多个值为“或”的关系 - * <p> - * 匹配方式转义 - * <p> - * 以*结尾，不以*或者$开始，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示左匹配(前缀匹配），例如aaaa*bbb$ccc*$* - * 以*开始，不以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示右匹配（后缀匹配），例如*$*aaa*bbb$ccc - * 以*开始，以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示子串匹配，例如*aaa$bbb*ccc* - * 不以*或者$开头，不以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示子串匹配，例如aaa*bbb$ccc*$ - * 以$开头，不以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示完整匹配，例如$aaa*bbb$或者$aaa*$bbb - */ -@Getter -@AllArgsConstructor -public enum MatchEnum { - - EXACTLY("exactly", "\"{0}\""), - PREFIX("prefix", "\"{0}%\""), - SUFFIX("suffix", "\"%{0}\""), - SUBSTRING("substring", "\"%{0}%\""), - REGEX("regex", "\"{0}\""); - - private final String type; - private final String matchExp; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/QueryTypeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/QueryTypeEnum.java deleted file mode 100644 index 96e54f8..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/QueryTypeEnum.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.mesalab.common.enums.knowledge; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * @description: for queryType - * @author: zhq - * @create: 2020-07-16 - **/ -@Getter -@AllArgsConstructor -public enum QueryTypeEnum { - - - IPLEARNING("iplearning"), - IPPOOL("ippool"), - SUBSCRIBERIDPOOL("subscriberidpool"); - - - private final String type; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/RangeEnum.java b/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/RangeEnum.java deleted file mode 100644 index e3b21d2..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/enums/knowledge/RangeEnum.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.mesalab.common.enums.knowledge; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -/** - * 范围查询 - * <p> - * type:["gt"|"lt"|"eq"|"ge"|"le"|"ne"] - * <p> - * gt: > 大于（greater than） - * lt: < 小于（less than） - * eq: = 等于（equal to） - * ge: >= 大于等于（greater than or equal） - * le: <= 小于等于（less than or equal） - * ne: != 不等于（not equal） - */ -@Getter -@AllArgsConstructor -public enum RangeEnum { - - - EQ("eq", "==", true), - NE("ne", "<>", true), - LT("lt", "<", false), - LE("le", "<=", false), - GT("gt", ">", false), - GE("ge", ">=", false); - - private final String type; - private final String symbol; - private final boolean symmetric; - - - /** - * Returns if this operation is symmetric with respect to its parameters. - * More formally returns true if: A oper B is true if and only if B oper A is - * true. - * - * @return if this operation is symmetric with respect to its parameters - */ - public boolean isSymmetric() { - return symmetric; - } - -}
\ No newline at end of file diff --git a/galaxy-common/src/main/java/com/mesalab/common/exception/BusinessException.java b/galaxy-common/src/main/java/com/mesalab/common/exception/BusinessException.java deleted file mode 100644 index ecaf184..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/exception/BusinessException.java +++ /dev/null @@ -1,48 +0,0 @@ -package com.mesalab.common.exception; - -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import lombok.*; - -/** - * 业务异常 - * @author dazzlzy - * @date 2018/3/22 - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(callSuper = false) -public class BusinessException extends RuntimeException { - - /** - * 返回HTTP状态码 - */ - private int errorStatus = ResultStatusEnum.SERVER_ERROR.getCode(); - - /** - * 内部执行错误码 - */ - private String errorCode = ResultCodeEnum.UNKNOW_ERROR.getCode(); - - /** - * 异常信息 - */ - private String errorMessage; - - - public BusinessException(String errorMessage) { - this.errorMessage = errorMessage; - } - - - public BusinessException(String errorMessage, Throwable e) { - super(errorMessage, e); - } - - public BusinessException(int errorStatus, String errorCode, String errorMessage, Throwable e) { - super(errorMessage, e); - this.errorStatus = errorStatus; - this.errorCode = errorCode; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/knowledge/KnowDslObject.java b/galaxy-common/src/main/java/com/mesalab/common/knowledge/KnowDslObject.java deleted file mode 100644 index 3d3e1c1..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/knowledge/KnowDslObject.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.mesalab.common.knowledge; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class KnowDslObject implements Serializable { - - private static final long serialVersionUID = 4506414702067122671L; - - private Integer clientId; - - private DSLQuery query; - - @Data - public static class DSLQuery { - private String queryType; - private String dataSource; - private Parameters parameters; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/knowledge/Match.java b/galaxy-common/src/main/java/com/mesalab/common/knowledge/Match.java deleted file mode 100644 index 05a95a7..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/knowledge/Match.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.mesalab.common.knowledge; - -import cn.hutool.core.collection.CollectionUtil; -import com.google.common.base.CharMatcher; -import com.mesalab.common.enums.knowledge.MatchEnum; -import lombok.Data; -import org.apache.commons.lang3.EnumUtils; -import java.text.MessageFormat; -import java.util.List; -import java.util.function.Predicate; -import static java.util.stream.Collectors.joining; - -/** - * @description: 查询单位字符串 - * @author: zhq - * @create: 2020-07-02 - **/ -@Data -public class Match extends QueryExp { - - - public static String like(List<Match> matches, Predicate<Match> predicate) { - - return CollectionUtil.isEmpty(matches) ? "" : matches.stream().filter(m -> predicate.test(m)).map(m -> { - - MatchEnum match = EnumUtils.getEnum(MatchEnum.class, m.getType().toUpperCase()); - return m.getFieldValues().stream().filter(mv -> mv != null).map(mv -> { - String matchExp = match.getMatchExp(); - String value = String.valueOf(mv); - if (MatchEnum.REGEX.getType().equals(m.getType())) { - if (value.startsWith("$")) { - matchExp = MatchEnum.EXACTLY.getMatchExp(); - value = CharMatcher.anyOf("$").trimLeadingFrom(value); - } else if (value.startsWith("*") && !value.endsWith("*")) { - matchExp = MatchEnum.SUFFIX.getMatchExp(); - value = CharMatcher.anyOf("*").trimLeadingFrom(value); - } else if (value.endsWith("*") && !value.startsWith("*")) { - matchExp = MatchEnum.PREFIX.getMatchExp(); - value = CharMatcher.anyOf("*").trimTrailingFrom(value); - } else if (value.startsWith("*") && value.endsWith("*")) { - matchExp = MatchEnum.SUBSTRING.getMatchExp(); - value = CharMatcher.anyOf("*").trimFrom(value); - } else { - matchExp = MatchEnum.SUBSTRING.getMatchExp(); - } - } - return MessageFormat.format(matchExp, value); - }).collect(joining(" or doc._key like ", " ( doc._key like ", ") ")); - - }).collect(joining(" and ", " filter ", " ")); - } - -}
\ No newline at end of file diff --git a/galaxy-common/src/main/java/com/mesalab/common/knowledge/Parameters.java b/galaxy-common/src/main/java/com/mesalab/common/knowledge/Parameters.java deleted file mode 100644 index b98e2c5..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/knowledge/Parameters.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.mesalab.common.knowledge; - -import lombok.Data; -import java.util.List; - -@Data -public class Parameters { - - private List<Match> match; - - private List<Range> range; - - private List<String> intervals; - - private List<Sort> sort; - - private String limit; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/knowledge/QueryExp.java b/galaxy-common/src/main/java/com/mesalab/common/knowledge/QueryExp.java deleted file mode 100644 index e98465d..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/knowledge/QueryExp.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.mesalab.common.knowledge; - -import lombok.Data; - -import java.util.List; - -@Data -public class QueryExp { - - protected String type; - protected String fieldKey; - protected List<Object> fieldValues; - protected String operator; - protected String qlSegment; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/knowledge/Range.java b/galaxy-common/src/main/java/com/mesalab/common/knowledge/Range.java deleted file mode 100644 index b7dd729..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/knowledge/Range.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.mesalab.common.knowledge; - -import lombok.Data; - -/** - * @description: 查询单位范围 - * @author: zhq - * @create: 2020-07-02 - **/ -@Data -public class Range extends QueryExp { - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/knowledge/Sort.java b/galaxy-common/src/main/java/com/mesalab/common/knowledge/Sort.java deleted file mode 100644 index 84487c4..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/knowledge/Sort.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.common.knowledge; - -import lombok.Data; - -/** - * @description: 排序 - * @author: zhq - * @create: 2020-07-03 - **/ -@Data -public class Sort extends QueryExp { -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/DSLObject.java b/galaxy-common/src/main/java/com/mesalab/common/model/DSLObject.java deleted file mode 100644 index bf378f4..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/DSLObject.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.mesalab.common.model; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; - -/** - * @Date: 2020-07-20 09:48 - * @Author : liuyongqiang - * @ClassName : DSLObject - * @Description : 标准服务接口模型 - */ -@Data -public class DSLObject implements Serializable { - - private static final long serialVersionUID = 4956257906217545258L; - - private String limit;//数据条数 - private String dataEngine;//数据引擎 - private String dataSource;//数据来源 - private QueryBean query;//查询结构 - - @Data - public static class QueryBean { - private String granularity;//时间粒度 - private List<MatchBean> match;//匹配查询 - private List<RangeBean> range;//区间查询 - private List<IntervalsBean> intervals;//时间范围 - private List<SortBean> sort;//排序方式 - - @Data - public static class MatchBean { - private String type; - private String fieldKey; - private List<String> fieldValues; - } - - @Data - public static class RangeBean { - private String type; - private String fieldKey; - private List<String> fieldValues; - } - - @Data - public static class IntervalsBean { - private String type; - private String fieldKey; - private List<String> fieldValues; - } - - @Data - public static class SortBean { - private String type; - private String fieldKey; - } - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/IConvertor.java b/galaxy-common/src/main/java/com/mesalab/common/model/IConvertor.java deleted file mode 100644 index 0a86c69..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/IConvertor.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.mesalab.common.model; - -/** - * 转换器接口，可以将Source转换为Target - * - * @author dazzlzy - * @date 2018/5/9 - */ -public interface IConvertor<Source, Target> { - - /** - * 将Source转换为Target - * - * @param source 需要转换的对象 - * @return 转换后的对象 - */ - Target convert(Source source); - -}
\ No newline at end of file diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/ProtocolTree.java b/galaxy-common/src/main/java/com/mesalab/common/model/ProtocolTree.java deleted file mode 100644 index f3197be..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/ProtocolTree.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.mesalab.common.model; - -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; -import java.util.Map; -import java.util.Objects; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class ProtocolTree { - private String id; - private String name; - private String parentId; - private List<ProtocolTree> childrens = Lists.newArrayList(); - private Map<String, Object> metrics = Maps.newLinkedHashMap(); - private long sentBytes; - private long receivedBytes; - private static final String HIERARCCY_FLAG = "/"; - - - public ProtocolTree(String id, String name, String parentId) { - this.id = id; - this.name = name; - this.parentId = parentId; - } - - public ProtocolTree(String id, String name, String parentId, long sentBytes, long receivedBytes) { - this.id = id; - this.name = name; - this.parentId = parentId; - this.sentBytes = sentBytes; - this.receivedBytes = receivedBytes; - } - - - public String getParentId() { - return id.lastIndexOf(HIERARCCY_FLAG) > 0 ? - id.substring(0, id.lastIndexOf(HIERARCCY_FLAG)) : null; - } - - public void setParentId(String parentId) { - this.parentId = parentId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ProtocolTree that = (ProtocolTree) o; - return Objects.equals(id, that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id, name); - } - - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/SchemaBase.java b/galaxy-common/src/main/java/com/mesalab/common/model/SchemaBase.java deleted file mode 100644 index 713c06e..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/SchemaBase.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.mesalab.common.model; - - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class SchemaBase implements Serializable { - - private String type; - private String name; - private String namespace; - private List<Map> fields; - - public String getNamespace() { - return namespace = "com.zdjizhi"; - } - - public String getType() { - return type = "record"; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/StorageDeletion.java b/galaxy-common/src/main/java/com/mesalab/common/model/StorageDeletion.java deleted file mode 100644 index 66eb373..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/StorageDeletion.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.mesalab.common.model; - - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class StorageDeletion { - private String logType; - private Integer maxDays; -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/XxlJobInfo.java b/galaxy-common/src/main/java/com/mesalab/common/model/XxlJobInfo.java deleted file mode 100644 index 0c8f7ab..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/XxlJobInfo.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.mesalab.common.model; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class XxlJobInfo { - private int id; // 主键ID - private int jobGroup; // 执行器主键ID - private String jobCron; // 任务执行CRON表达式 - private String jobDesc; - private String author; // 负责人 - private String alarmEmail; // 报警邮件 - private String executorRouteStrategy; // 执行器路由策略 - private String executorHandler; // 执行器，任务Handler名称 - private String executorParam; // 执行器，任务参数 - private String executorBlockStrategy; // 阻塞处理策略 - private String childJobId; // 子任务ID，多个逗号分隔 -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/arango/BaseDocument.java b/galaxy-common/src/main/java/com/mesalab/common/model/arango/BaseDocument.java deleted file mode 100644 index d89df40..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/arango/BaseDocument.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.mesalab.common.model.arango; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.io.Serializable; - -/** - * @author Mark Vollmary - */ -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class BaseDocument implements Serializable { - - private static final long serialVersionUID = -1824742667228719116L; - - @JsonProperty(value = "_id") - protected String id; - @JsonProperty(value = "_key") - protected String key; - @JsonProperty(value = "_rev") - protected String revision; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/arango/BaseEdgeDocument.java b/galaxy-common/src/main/java/com/mesalab/common/model/arango/BaseEdgeDocument.java deleted file mode 100644 index 492d466..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/arango/BaseEdgeDocument.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.mesalab.common.model.arango; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -/** - * @author Mark Vollmary - */ -@Data -public class BaseEdgeDocument { - - private static final long serialVersionUID = 6904923804449368783L; - - @JsonProperty(value = "_from") - private String from; - - @JsonProperty(value = "_to") - private String to; - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/arango/IpLearningPath.java b/galaxy-common/src/main/java/com/mesalab/common/model/arango/IpLearningPath.java deleted file mode 100644 index a6a339a..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/arango/IpLearningPath.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.mesalab.common.model.arango; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -/** - * @description: for iplearning - * @author: zhq - * @create: 2020-07-15 - **/ -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class IpLearningPath { - - private List<IplearningEdges> edges; - - @Data - @JsonIgnoreProperties(ignoreUnknown = true) - public static class IplearningEdges extends BaseEdgeDocument { - - @JsonProperty(value = "FIRST_FOUND_TIME") - private Long firstFoundTime; - - @JsonProperty(value = "LAST_FOUND_TIME") - private Long lastFoundTime; - - @JsonProperty(value = "HTTP_CNT_RECENT") - private List<Long> httpCntRecent; - - @JsonProperty(value = "HTTP_CNT_TOTAL") - private Long httpCntTotal; - - @JsonProperty(value = "DNS_CNT_RECENT") - private List<Long> dnsCntRecent; - - @JsonProperty(value = "DNS_CNT_TOTAL") - private Long dnsCntTotal; - - @JsonProperty(value = "TLS_CNT_RECENT") - private List<Long> tlsCntRecent; - - @JsonProperty(value = "TLS_CNT_TOTAL") - private Long tlsCntTotal; - - @JsonProperty(value = "DIST_CIP_TS") - private List<Long> distCipTs; - - @JsonProperty(value = "DIST_CIP") - private List<String> distCip; - - } - - @Data - @JsonIgnoreProperties(ignoreUnknown = true) - public static class IpLearningVertices { - - @JsonProperty(value = "FQDN_NAME") - private String fqdnName; - - @JsonProperty(value = "FIRST_FOUND_TIME") - private String firstFoundTime; - - @JsonProperty(value = "LAST_FOUND_TIME") - private String lastFoundTime; - - @JsonProperty(value = "IP") - private String ip; - - @JsonProperty(value = "COMMON_LINK_INFO") - private String commonLinkInfo; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/model/arango/SubscriberIdPath.java b/galaxy-common/src/main/java/com/mesalab/common/model/arango/SubscriberIdPath.java deleted file mode 100644 index b48a4d5..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/model/arango/SubscriberIdPath.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.mesalab.common.model.arango; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -import java.util.List; - -/** - * @description: for path - * @author: zhq - * @create: 2020-07-17 - **/ -@Data -@JsonIgnoreProperties(ignoreUnknown = true) -public class SubscriberIdPath { - - private List<SubscriberIdVertices> vertices; - - @Data - @JsonIgnoreProperties(ignoreUnknown = true) - public static class SubscriberIdVertices extends BaseDocument { - - @JsonProperty(value = "SUBSCRIBER_ID") - private String fqdnName; - - @JsonProperty(value = "FIRST_FOUND_TIME") - private String firstFoundTime; - - @JsonProperty(value = "LAST_FOUND_TIME") - private String lastFoundTime; - - @JsonProperty(value = "IP") - private String ip; - - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/BaseResultUtil.java b/galaxy-common/src/main/java/com/mesalab/common/util/BaseResultUtil.java deleted file mode 100644 index 86a7c4b..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/BaseResultUtil.java +++ /dev/null @@ -1,206 +0,0 @@ -package com.mesalab.common.util; - - -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; - -import java.util.Map; - -/** - * BaseResult生成器 - * @author dazzlzy - * @date 2018/4/1 - */ -public class BaseResultUtil { - - /** - * 生成返回结果 - * - * @param status 返回HTTP响应码 - * @param code 返回业务编码 - * @param message 返回消息 - * @param data 返回数据 - * @param formatType 返回数据格式 JSON/CSV - * @param <T> 返回数据类型 - * @return 返回结果 - */ - public static <T> BaseResult<T> generate(final int status, final String code, final String message, T data, T meta, final Map<String,Object> statistics, final String formatType) { - return new BaseResult<>(status, code, false, message, statistics, formatType, meta, data); - } - - /** - * 操作成功响应结果， 默认结果 - * - * @return 操作成功的默认响应结果 - */ - public static <T> BaseResult<T> success() { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), - true, ResultCodeEnum.EXECUTE_SUCCESS.getMessage(),null, null, null, null); - } - - /** - * 操作成功响应结果， 自定义数据及信息 - * - * @param message 自定义信息 - * @param data 自定义数据 - * @param <T> 自定义数据类型 - * @return 响应结果 - */ - public static <T> BaseResult<T> success(final String message, final T data) { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), - true, message, null,null, null, data); - } - - /** - * 操作成功响应结果， 自定义数据及信息, 统计结果 - * - * @param message 自定义信息 - * @param data 自定义数据 - * @param <T> 自定义数据类型 - * @param statistics 统计结果 - * @return 响应结果 - */ - public static <T> BaseResult<T> success(final String message, final T data, final Map<String,Object> statistics) { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), - true, message, statistics,null, null, data); - } - - /** - * 操作成功响应结果，自定义数据，默认信息 - * - * @param data 自定义数据 - * @param <T> 自定义数据类型 - * @return 响应结果 - */ - public static <T> BaseResult<T> success(final T data) { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), true, - ResultCodeEnum.EXECUTE_SUCCESS.getMessage(), null,null, null, data); - } - - /** - * 操作成功响应结果，自定义信息，无数据 - * - * @param message 自定义信息 - * @return 响应结果 - */ - public static <T> BaseResult<T> success4Message(final String message) { - return new BaseResult<>(ResultStatusEnum.SUCCESS.getCode(), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), - true, message, null, null, null, null); - } - - /** - * 操作失败响应结果， 默认结果 - * - * @return 操作成功的默认响应结果 - */ - public static <T> BaseResult<T> failure() { - return new BaseResult<>(ResultStatusEnum.FAIL.getCode(), - ResultCodeEnum.UNKNOW_ERROR.getCode(), false, ResultCodeEnum.UNKNOW_ERROR.getMessage(), null, null, null, null); - } - - /** - * 操作失败响应结果， 自定义错误编码及信息 - * - * @param status HTTP状态码 - * @param message 自定义信息 - * @return 响应结果 - */ - public static <T> BaseResult<T> failure(final int status, final String message) { - return new BaseResult<>(status, ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, null); - } - - /** - * 操作失败响应结果， 自定义错误编码及信息 - * @param status HTTP 状态码 - * @param code 返回业务编码 - * @param message 自定义信息 - * @return 响应结果 - */ - public static <T> BaseResult<T> failure(final int status, final String code, final String message) { - return new BaseResult<>(status, code, false, message, null, null, null, null); - } - - /** - * 操作失败响应结果， 自定义错误编码及信息 - * - * @param status HTTP 状态码 - * @param message 自定义信息 - * @return 响应结果 - */ - public static <T> BaseResult<T> failure(final int status, final String message, T data) { - return new BaseResult<>(status, ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, data); - } - - /** - * 操作失败响应结果，自定义错误编码 - * - * @param resultStatusEnum 自定义错误编码枚举 - * @return 响应结果 - */ - public static <T> BaseResult<T> failure(final ResultStatusEnum resultStatusEnum) { - return new BaseResult<>(resultStatusEnum.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, resultStatusEnum.getMessage(), null, null, null, null); - } - - /** - * 操作失败响应结果，自定义信息 - * - * @param message 自定义信息 - * @return 响应结果 - */ - public static <T> BaseResult<T> failure(final String message) { - return new BaseResult<>(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, null); - } - - /** - * 异常响应结果， 默认结果 - * - * @return 操作成功的默认响应结果 - */ - public static <T> BaseResult<T> error() { - return new BaseResult<>(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, ResultStatusEnum.SERVER_ERROR.getMessage(), null, null, null, null); - } - - /** - * 异常响应结果， 自定义错误编码及信息 - * - * @param code 自定义错误编码 - * @param message 自定义信息 - * @return 响应结果 - */ - public static <T> BaseResult<T> error(final int code, final String message) { - return new BaseResult<>(code, ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, null); - } - - /** - * 异常响应结果，自定义错误编码 - * - * @param resultStatusEnum 自定义错误编码枚举 - * @return 响应结果 - */ - public static <T> BaseResult<T> error(final ResultStatusEnum resultStatusEnum) { - return new BaseResult<>(resultStatusEnum.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, resultStatusEnum.getMessage(), null, null, null, null); - } - - /** - * 业务异常响应结果 - * - * @param be 业务异常 - * @return 响应结果 - */ - public static <T> BaseResult<T> error(final BusinessException be) { - return new BaseResult<>(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, be.getErrorMessage(), null, null, null, null); - } - - /** - * 异常响应结果，自定义信息 - * - * @param message 自定义信息 - * @return 响应结果 - */ - public static <T> BaseResult<T> error(final String message) { - return new BaseResult<>(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), false, message, null, null, null, null); - } - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/ConvertUtil.java b/galaxy-common/src/main/java/com/mesalab/common/util/ConvertUtil.java deleted file mode 100644 index a0f093d..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/ConvertUtil.java +++ /dev/null @@ -1,353 +0,0 @@ -package com.mesalab.common.util; - -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.mesalab.common.model.IConvertor; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.StringUtil; -import org.apache.commons.lang3.StringUtils; -import org.nutz.lang.Lang; - -import java.util.*; - -/** - * 转换器工具类 - * - * @author dazzlzy - * @date 2018/5/9 - */ -@SuppressWarnings("unchecked") -public class ConvertUtil { - - public static final String GROUP_SEPRATOR = "|G|"; - - enum Fill { - - NONE(""), NULL("NULL"), ZERO("0"), PREVIOUS("previous"), NEXT("next"); - - private final String value; - - Fill(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - - } - - - /** - * 将来源collection转换成目标list. - * - * @param sources 来源 - * @param convertor 转换器 - * @param ignoreNull 是否忽略空值 - * @param duplicateRemoval 是否去除重复 - * @param <Source> 来源 - * @param <Target> 目标对象 - * @return 转换后的List - */ - public static <Source, Target> List<Target> convert(Collection<Source> sources, IConvertor<Source, Target> convertor, - boolean ignoreNull, boolean duplicateRemoval) { - // 返回目标值 - Collection<Target> targets; - - //去重使用HashSet，不去冲使用ArrayList - targets = duplicateRemoval ? new HashSet<>() : new ArrayList<>(); - - // 来源不为空 - if (sources != null && sources.size() > 0) { - for (Source source : sources) { - Target target = convertor.convert(source); - // 不忽略空值或者目标不为空 - if (!ignoreNull || target != null) { - // 添加目标 - targets.add(convertor.convert(source)); - } - } - } - // 转换成list - return Lang.collection2list(targets); - } - - /** - * 将来源collection转换成目标list. - * - * @param sources 来源 - * @param convertor 转换器 - * @param <Source> 来源 - * @param <Target> 目标对象 - * @return 转换后的List - */ - public static <Source, Target> List<Target> convert(Collection<Source> sources, IConvertor<Source, Target> convertor) { - return convert(sources, convertor, false, false); - } - - /** - * 将来源按转换器的方式转换成Map<Key, Obj>的方式. - * - * @param sources 来源 - * @param convertor 转换器 - * @param <Key> 返回的Map的key的类型 - * @param <Obj> 返回的Map的value的类型，即来源集合中的类型 - * @return 转换后的对象类型 - */ - public static <Key, Obj> Map<Key, Obj> map(Collection<Obj> sources, IConvertor<Obj, Key> convertor) { - if (sources == null) { - return new HashMap<>(16); - } - Map<Key, Obj> mapping = new HashMap<>(sources.size()); - // 来源不为空 - if (sources.size() > 0) { - for (Obj obj : sources) { - Key key = convertor.convert(obj); - mapping.put(key, obj); - } - } - return mapping; - } - - /** - * 将来源按转换器的方式转换成Map<Key,List<Obj>>的方式. - * - * @param sources 来源 - * @param convertor 转换器 - * @param <Key> 返回的Map的key的类型 - * @param <Obj> 返回的Map的value的List包含类型 - * @return 转换后的对象类型 - */ - public static <Key, Obj> Map<Key, List<Obj>> maplist(Collection<Obj> sources, IConvertor<Obj, Key> convertor) { - Map<Key, List<Obj>> mapping = new HashMap<>(16); - // 来源不为空 - if (sources != null && sources.size() > 0) { - for (Obj obj : sources) { - Key key = convertor.convert(obj); - List<Obj> list = mapping.get(key); - - if (list == null) { - mapping.put(key, Lang.list(obj)); - } else { - list.add(obj); - } - } - } - return mapping; - } - - /** - * 将时序特征的数据，转换成层级结构； - * 例如：Map<groupKey, timeseries Map> - * Map<timeGran,Metrics Map> - * Map<metric,value> - * - * @param sources - * @param groupLabel - * @param timeGranLabel - * @return - */ - public static Map<String, Map<String, Map<String, Object>>> convertResultListToTimeseries(Collection<Object> sources, String groupLabel, String timeGranLabel) { - - - Map<String, Map<String, Map<String, Object>>> resultMap = Maps.newLinkedHashMap(); - - for (Object source : sources) { - - Map<String, Object> sourceMap = ((Map<String, Object>) source); - - String groupKey = null; - if (StringUtil.isBlank(groupLabel)) { - groupKey = "default"; - - } else { - for(String group : StringUtils.split(groupLabel, GROUP_SEPRATOR)) { - groupKey = Joiner.on(GROUP_SEPRATOR).skipNulls().join(groupKey, sourceMap.get(group)); - sourceMap.remove(group); - } - } - - - String timeGran = sourceMap.get(timeGranLabel).toString(); - sourceMap.remove(timeGranLabel); - - Map groupMap = resultMap.get(groupKey); - - if (StringUtil.isEmpty(groupMap)) { - groupMap = Maps.newTreeMap(); - groupMap.put(timeGran, sourceMap); - resultMap.put(groupKey,groupMap); - - } else { - - groupMap.put(timeGran, sourceMap); - } - - - } - - return resultMap; - - } - - - - public static List<Map<String, Object>> convertTimeseriesToList(Map<String, Map<String, Map<String, Object>>> target, String groupLabel, String timeGranLabel) { - List<Map<String, Object>> results = Lists.newArrayList(); - String[] grouLabelArray; - String[] groupKeyArray; - for(String groupKey : target.keySet()) { - - Map<String, Map<String, Object>> groupMap = target.get(groupKey); - - for (Map.Entry<String, Map<String,Object>> entry: groupMap.entrySet()) { - - Map<String, Object> resultMap = Maps.newLinkedHashMap(); - resultMap.put(timeGranLabel, entry.getKey()); - if (StringUtil.isNotBlank(groupLabel)) { - grouLabelArray = StringUtils.split(groupLabel, GROUP_SEPRATOR); - groupKeyArray = StringUtils.splitByWholeSeparatorPreserveAllTokens(groupKey, GROUP_SEPRATOR); - for (int i = 0; i < grouLabelArray.length; i++) { - resultMap.put(grouLabelArray[i], StringUtil.isBlank(groupKey) ? StringUtil.EMPTY : groupKeyArray[i]); - } - - } - //metrics - resultMap.putAll(entry.getValue()); - results.add(resultMap); - - } - } - - target.clear(); - - return results; - } - - - - - - /** - * 时序数据补全操作 - * @param sources TreeMap<timeGran, metricMap> - * @param dateList - * @param fill - * @return - */ - public static Map<String, Map<String, Object>> completeTimeseries(TreeMap<String, Map<String, Object>> sources, - List<Date> dateList, String fill) { - - Map<String, Object> defaultElement = Maps.newHashMap(sources.firstEntry().getValue()); - - - if (fill.equalsIgnoreCase(Fill.ZERO.name())){ - for(String key: defaultElement.keySet()){ - defaultElement.put(key, Fill.ZERO.getValue()); - } - - } else if (fill.equalsIgnoreCase(Fill.NULL.name())){ - for(String key: defaultElement.keySet()) { - defaultElement.put(key, Fill.NULL.value); - } - } else if (fill.equalsIgnoreCase(Fill.NONE.name())) { - for(String key: defaultElement.keySet()) { - defaultElement.put(key, Fill.NONE.value); - } - } else { - defaultElement.clear(); - } - - TreeMap<String, Map<String, Object>> target = Maps.newTreeMap(sources); - - boolean isTimestamp = StringUtil.isNumeric(sources.firstKey()); - for(Date date : dateList) { - - String timeKey; - if (isTimestamp) { - timeKey = String.valueOf(date.getTime()/1000); - } else { - timeKey = DateUtils.getFormatDate(date, DateUtils.YYYY_MM_DD_HH24_MM_SS); - } - - Map<String, Object> metricsMap = sources.get(timeKey); - if (StringUtil.isEmpty(metricsMap)) { - target.put(timeKey, defaultElement); - } - } - - - - if (StringUtil.isEmpty(defaultElement)) { - - defaultElement = getDefaultElement(sources.get(sources.firstKey())); - - for (String key : target.keySet()) { - - if (StringUtil.isEmpty(target.get(key))) { - - Map<String, Object> element= getPositionElement(target, key, fill); - - if (StringUtil.isEmpty(element)) { - target.put(key, defaultElement); - } else { - target.put(key, element); - } - - - } - - - } - - } - - - return target; - } - - - - - - private static Map<String, Object> getPositionElement(TreeMap<String, Map<String, Object>> target, - String key, String fill) { - Map<String, Object> element = Maps.newHashMap(); - - if (fill.equalsIgnoreCase(Fill.PREVIOUS.name())) { - if (StringUtil.isNotEmpty(target.lowerEntry(key))) { - element = target.lowerEntry(key).getValue(); - } - - - } else if (fill.equalsIgnoreCase(Fill.NEXT.name())) { - if (StringUtil.isNotEmpty(target.higherEntry(key))) { - element = target.higherEntry(key).getValue(); - } - - - } - return element; - } - - - - private static Map<String, Object> getDefaultElement(Map<String, Object> firstElement) { - Map<String, Object> element = Maps.newHashMap(firstElement); - - for (String key : element.keySet()) { - element.put(key, Fill.NONE.getValue()); - } - - return element; - } - - - - - - - - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/DataTypeUtil.java b/galaxy-common/src/main/java/com/mesalab/common/util/DataTypeUtil.java deleted file mode 100644 index 6dc178a..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/DataTypeUtil.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.mesalab.common.util; - -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Table; -import org.apache.calcite.sql.type.SqlTypeName; - -import java.math.BigDecimal; -import java.sql.Date; -import java.util.Set; - -/** - * 这里使用了GUAVA的table 作为存SQL和JAVA数据类型的数据结构 - * 这并不是一个好的设计，而是为了让大家更容易理解而做的设计 - */ -public class DataTypeUtil { - - public static Table<String, SqlTypeName, Class> TYPEMAPPING = HashBasedTable.create(); - public static final String CHAR = "char"; - public static final String VARCHAR = "varchar"; - public static final String BOOLEAN = "boolean"; - public static final String DATE = "date"; - public static final String INTEGER = "integer"; - public static final String INT = "int"; - public static final String TINYINT = "tinyint"; - public static final String SMALLINT = "smallint"; - public static final String BIGINT = "bigint"; - public static final String DECIMAL = "decimal"; - public static final String NUMERIC = "numeric"; - public static final String FLOAT = "float"; - public static final String REAL = "real"; - public static final String DOUBLE = "double"; - public static final String TIME = "time"; - public static final String TIMESTAMP = "timestamp"; - public static final String ANY = "any"; - - static { - TYPEMAPPING.put(DataTypeUtil.CHAR, SqlTypeName.CHAR, Character.class); - TYPEMAPPING.put(DataTypeUtil.VARCHAR, SqlTypeName.VARCHAR, String.class); - TYPEMAPPING.put(DataTypeUtil.BOOLEAN, SqlTypeName.BOOLEAN, Boolean.class); - TYPEMAPPING.put(DataTypeUtil.DATE, SqlTypeName.DATE, Date.class); - TYPEMAPPING.put(DataTypeUtil.INTEGER, SqlTypeName.INTEGER, Integer.class); - TYPEMAPPING.put(DataTypeUtil.INT, SqlTypeName.TIMESTAMP, String.class); - TYPEMAPPING.put(DataTypeUtil.TINYINT, SqlTypeName.TINYINT, Integer.class); - TYPEMAPPING.put(DataTypeUtil.SMALLINT, SqlTypeName.SMALLINT, Integer.class); - TYPEMAPPING.put(DataTypeUtil.BIGINT, SqlTypeName.BIGINT, Long.class); - TYPEMAPPING.put(DataTypeUtil.DECIMAL, SqlTypeName.DECIMAL, BigDecimal.class); - TYPEMAPPING.put(DataTypeUtil.NUMERIC, SqlTypeName.DECIMAL, Long.class); - TYPEMAPPING.put(DataTypeUtil.FLOAT, SqlTypeName.FLOAT, Float.class); - TYPEMAPPING.put(DataTypeUtil.REAL, SqlTypeName.REAL, Double.class); - TYPEMAPPING.put(DataTypeUtil.DOUBLE, SqlTypeName.DOUBLE, Double.class); - TYPEMAPPING.put(DataTypeUtil.TIME, SqlTypeName.TIME, Date.class); - TYPEMAPPING.put(DataTypeUtil.TIMESTAMP, SqlTypeName.TIMESTAMP, Long.class); - TYPEMAPPING.put(DataTypeUtil.ANY, SqlTypeName.ANY, String.class); - } - - /** - * 根据名字获取，对应的java类型 - */ - public static Class getJavaClassByName(String name) { - Set<Table.Cell<String, SqlTypeName, Class>> table = TYPEMAPPING.cellSet(); - for (Table.Cell<String, SqlTypeName, Class> it : table) { - if (it.getRowKey().equals(name)) { - return it.getValue(); - } - } - return null; - } - - public static SqlTypeName getSqlTypeByName(String name) { - for (Table.Cell<String, SqlTypeName, Class> it : TYPEMAPPING.cellSet()) { - if (it.getRowKey().equals(name)) { - return it.getColumnKey(); - } - } - return null; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/HttpClientUtil.java b/galaxy-common/src/main/java/com/mesalab/common/util/HttpClientUtil.java deleted file mode 100644 index 918139d..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/HttpClientUtil.java +++ /dev/null @@ -1,269 +0,0 @@ -package com.mesalab.common.util; - -import cn.hutool.core.util.StrUtil; -import com.mesalab.common.exception.BusinessException; -import org.apache.http.*; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.message.BasicHeaderElementIterator; -import org.apache.http.protocol.HTTP; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLException; -import javax.net.ssl.SSLHandshakeException; -import java.io.IOException; -import java.io.InterruptedIOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.UnknownHostException; -import java.util.Objects; - -public class HttpClientUtil { - /** - * 全局连接池对象 - */ - private static final PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); - - private static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class); - - /** - * 静态代码块配置连接池信息 - */ - static { - - // 设置最大连接数 - connManager.setMaxTotal(400); - // 设置每个连接的路由数 - connManager.setDefaultMaxPerRoute(80); - - } - - /** - * 获取Http客户端连接对象 - * - * @return Http客户端连接对象 - */ - public static CloseableHttpClient getHttpClient() { - // 创建Http请求配置参数 - RequestConfig requestConfig = RequestConfig.custom() - // 获取连接超时时间 - .setConnectionRequestTimeout(30000) - // 请求超时时间 - .setConnectTimeout(10000) - // 响应超时时间 - .setSocketTimeout(10000) - .build(); - - /** - * 测出超时重试机制为了防止超时不生效而设置 - * 如果直接放回false,不重试 - * 这里会根据情况进行判断是否重试 - */ - HttpRequestRetryHandler retry = (exception, executionCount, context) -> { - if (executionCount >= 3) {// 如果已经重试了3次，就放弃 - return false; - } - if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试 - return true; - } - if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常 - return false; - } - if (exception instanceof InterruptedIOException) {// 超时 - return true; - } - if (exception instanceof UnknownHostException) {// 目标服务器不可达 - return false; - } - if (exception instanceof ConnectTimeoutException) {// 连接被拒绝 - return false; - } - if (exception instanceof SSLException) {// ssl握手异常 - return false; - } - HttpClientContext clientContext = HttpClientContext.adapt(context); - HttpRequest request = clientContext.getRequest(); - // 如果请求是幂等的，就再次尝试 - if (!(request instanceof HttpEntityEnclosingRequest)) { - return true; - } - return false; - }; - - - ConnectionKeepAliveStrategy myStrategy = (response, context) -> { - HeaderElementIterator it = new BasicHeaderElementIterator - (response.headerIterator(HTTP.CONN_KEEP_ALIVE)); - while (it.hasNext()) { - HeaderElement he = it.nextElement(); - String param = he.getName(); - String value = he.getValue(); - if (value != null && param.equalsIgnoreCase - ("timeout")) { - return Long.parseLong(value) * 1000; - } - } - return 60 * 1000;//如果没有约定，则默认定义时长为60s - }; - - // 创建httpClient - return HttpClients.custom() - // 把请求相关的超时信息设置到连接客户端 - .setDefaultRequestConfig(requestConfig) - // 把请求重试设置到连接客户端 - .setRetryHandler(retry) - .setKeepAliveStrategy(myStrategy) - // 配置连接池管理对象 - .setConnectionManager(connManager) - .build(); - } - - - /** - * GET请求 - * - * @param url 请求地 - * @return - */ - public static String httpGet(String url,Header... headers) throws BusinessException { - String msg = "-1"; - - // 获取客户端连接对象 - CloseableHttpClient httpClient = getHttpClient(); - CloseableHttpResponse response = null; - - try { - - URL ul = new URL(url); - - URI uri = new URI(ul.getProtocol(), null, ul.getHost(), ul.getPort(), ul.getPath(), ul.getQuery(), null); - logger.info("http get uri {}", uri); - // 创建GET请求对象 - HttpGet httpGet = new HttpGet(uri); - if (Objects.nonNull(headers)) { - for (Header h : headers) { - httpGet.addHeader(h); - } - } - // 执行请求 - response = httpClient.execute(httpGet); - int statusCode = response.getStatusLine().getStatusCode(); - // 获取响应实体 - HttpEntity entity = response.getEntity(); - // 获取响应信息 - msg = EntityUtils.toString(entity, "UTF-8"); - - if (statusCode != HttpStatus.SC_OK) { - throw new BusinessException( "Http get content is :" + msg); - } - - } catch (URISyntaxException e) { - logger.error("URI 转换错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } catch (ClientProtocolException e) { - logger.error("协议错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } catch (ParseException e) { - logger.error("解析错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } catch (IOException e) { - logger.error("IO错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } finally { - if (null != response) { - try { - EntityUtils.consume(response.getEntity()); - response.close(); - } catch (IOException e) { - logger.error("释放链接错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } - } - } - - return msg; - } - - /** - * POST 请求 - * @param url - * @param requestBody - * @return - */ - public static String httpPost(String url, String requestBody, Header... headers)throws BusinessException { - String msg = "-1"; - // 获取客户端连接对象 - CloseableHttpClient httpClient = getHttpClient(); - // 创建POST请求对象 - CloseableHttpResponse response = null; - try { - - URL ul = new URL(url); - - URI uri = new URI(ul.getProtocol(), null, ul.getHost(), ul.getPort(), ul.getPath(), ul.getQuery(), null); - - logger.info("http post uri:{}， http post body:{}", uri, requestBody); - - HttpPost httpPost = new HttpPost(uri); - httpPost.setHeader("Content-Type", "application/json"); - if (Objects.nonNull(headers)) { - for (Header h : headers) { - httpPost.addHeader(h); - } - } - - if (StrUtil.isNotBlank(requestBody)) { - httpPost.setEntity(new ByteArrayEntity(requestBody.getBytes("utf-8"))); - } - - response = httpClient.execute(httpPost); - int statusCode = response.getStatusLine().getStatusCode(); - // 获取响应实体 - HttpEntity entity = response.getEntity(); - // 获取响应信息 - msg = EntityUtils.toString(entity, "UTF-8"); - - if (statusCode != HttpStatus.SC_OK&&statusCode!=HttpStatus.SC_CREATED) { - throw new BusinessException(msg); - } - } catch (URISyntaxException e) { - logger.error("URI 转换错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } catch (ClientProtocolException e) { - logger.error("协议错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } catch (ParseException e) { - logger.error("解析错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } catch (IOException e) { - logger.error("IO错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } finally { - if (null != response) { - try { - EntityUtils.consumeQuietly(response.getEntity()); - response.close(); - } catch (IOException e) { - logger.error("释放链接错误: {}", e.getMessage()); - throw new BusinessException(e.getMessage()); - } - } - } - return msg; - } - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/JdbcConstant.java b/galaxy-common/src/main/java/com/mesalab/common/util/JdbcConstant.java deleted file mode 100644 index 961381f..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/JdbcConstant.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.mesalab.common.util; - -/** - * @Date: 2020-08-18 17:47 - * @Author : liuyongqiang - * @ClassName : JdbcConstant - * @Description : JdbcConstant - */ -public interface JdbcConstant { - - String MYSQL = "mysql"; - - String HIVE = "hive"; -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/LoggerCache.java b/galaxy-common/src/main/java/com/mesalab/common/util/LoggerCache.java deleted file mode 100644 index 91c1157..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/LoggerCache.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.mesalab.common.util; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; - -/** - * @Date: 2020-07-30 11:53 - * @Author : liuyongqiang - * @ClassName : LoggerCache - * @Description : 日志实例缓存 - */ -public class LoggerCache { - - /** - * 日志实例记录在内存中 - */ - private static HashMap<String, Logger> LOGERS = new HashMap<String, Logger>(); - - /** - * 根据类名获取缓存的日志实例 - * @param className 包名加类名 this.getClass().getName(); - * @return - */ - public static Logger getLoggerByClassName(String className) { - // 从静态map中获取日志实例 - Logger logger = LOGERS.get(className); - // 如果没取到 - if (logger == null) { - // 创建一个日志实例 - logger = LoggerFactory.getLogger(className); - // 加入到静态map中 - LOGERS.put(className, logger); - } - // 返回 - return logger; - } -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/MemoryCache.java b/galaxy-common/src/main/java/com/mesalab/common/util/MemoryCache.java deleted file mode 100644 index 8c1aa23..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/MemoryCache.java +++ /dev/null @@ -1,209 +0,0 @@ -package com.mesalab.common.util; - -import java.util.*; - -/** - * @Date: 2020-07-20 10:46 - * @Author : liuyongqiang - * @ClassName : CommonController - * @Description : 基于Java内存的简单缓存实现 - */ -public class MemoryCache<K, V> extends HashMap<K, V> { - - private static final long serialVersionUID = 1L; - - /** - * default expiry time one day - */ - private long EXPIRY = 86400; - - private HashMap<K, Long> expiryMap = new HashMap<>(); - - /** 缓存实例对象 */ - private volatile static MemoryCache<String, String> SameUrlMap; - - /** - * 采用单例模式获取实例 - * @return - */ - public static MemoryCache getInstance() { - //第一次判空,提高效率 - if (null == SameUrlMap) { - //保证线程安全 - synchronized (MemoryCache.class) { - //第二次判空,保证单例对象的唯一性,防止第一次有多个线程进入第一个if判断 - if (null == SameUrlMap) { - SameUrlMap = new MemoryCache<>(); - } - } - } - return SameUrlMap; - } - - public MemoryCache(){ - super(); - } - - public MemoryCache(long defaultExpiryTime){ - this(1 << 4, defaultExpiryTime); - } - - public MemoryCache(int initialCapacity, long defaultExpiryTime){ - super(initialCapacity); - this.EXPIRY = defaultExpiryTime; - } - - @Override - public V put(K key, V value) { - expiryMap.put(key, System.currentTimeMillis() + EXPIRY); - return super.put(key, value); - } - - @Override - public boolean containsKey(Object key) { - return !checkExpiry(key, true) && super.containsKey(key); - } - /** - * @param key - * @param value - * @param expiryTime 键值对有效期 毫秒 - * @return - */ - public V put(K key, V value, long expiryTime) { - expiryMap.put(key, System.currentTimeMillis() + expiryTime); - return super.put(key, value); - } - - @Override - public int size() { - return entrySet().size(); - } - - @Override - public boolean isEmpty() { - return entrySet().size() == 0; - } - - @Override - public boolean containsValue(Object value) { - if (value == null) { - return Boolean.FALSE; - } - Set<Entry<K, V>> set = super.entrySet(); - Iterator<Entry<K, V>> iterator = set.iterator(); - while (iterator.hasNext()) { - Entry<K, V> entry = iterator.next(); - if(value.equals(entry.getValue())){ - if(checkExpiry(entry.getKey(), false)) { - iterator.remove(); - return Boolean.FALSE; - }else { - return Boolean.TRUE; - } - } - } - return Boolean.FALSE; - } - - @Override - public Collection<V> values() { - - Collection<V> values = super.values(); - - if(values == null || values.size() < 1) { - return values; - } - - Iterator<V> iterator = values.iterator(); - - while (iterator.hasNext()) { - V next = iterator.next(); - if(!containsValue(next)) { - iterator.remove(); - } - } - return values; - } - - @Override - public V get(Object key) { - if (key == null) { - return null; - } - if(checkExpiry(key, true)) { - return null; - } - return super.get(key); - } - /** - * - * @Description: 是否过期 - * @param key - * @return null:不存在或key为null -1:过期 存在且没过期返回value 因为过期的不是实时删除，所以稍微有点作用 - */ - public Object isInvalid(Object key) { - if (key == null) { - return null; - } - if(!expiryMap.containsKey(key)){ - return null; - } - long expiryTime = expiryMap.get(key); - - boolean flag = System.currentTimeMillis() > expiryTime; - - if(flag){ - super.remove(key); - expiryMap.remove(key); - return -1; - } - return super.get(key); - } - - @Override - public void putAll(Map<? extends K, ? extends V> m) { - for (Entry<? extends K, ? extends V> e : m.entrySet()) { - expiryMap.put(e.getKey(), System.currentTimeMillis() + EXPIRY); - } - super.putAll(m); - } - - @Override - public Set<Entry<K,V>> entrySet() { - Set<Entry<K, V>> set = super.entrySet(); - Iterator<Entry<K, V>> iterator = set.iterator(); - while (iterator.hasNext()) { - Entry<K, V> entry = iterator.next(); - if(checkExpiry(entry.getKey(), false)) { - iterator.remove(); - } - } - - return set; - } - /** - * - * @Description: 是否过期 - * @param key - * @param isRemoveSuper - * @return - */ - private boolean checkExpiry(Object key, boolean isRemoveSuper){ - - if(!expiryMap.containsKey(key)){ - return Boolean.FALSE; - } - long expiryTime = expiryMap.get(key); - - boolean flag = System.currentTimeMillis() > expiryTime; - - if(flag){ - if(isRemoveSuper) { - super.remove(key); - } - expiryMap.remove(key); - } - return flag; - } - -}
\ No newline at end of file diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/SqlHelper.java b/galaxy-common/src/main/java/com/mesalab/common/util/SqlHelper.java deleted file mode 100644 index e7eec9c..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/SqlHelper.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.mesalab.common.util; - -import com.mesalab.common.enums.DBTypeEnum; -import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.expression.LongValue; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.select.Limit; -import net.sf.jsqlparser.statement.select.PlainSelect; -import net.sf.jsqlparser.statement.select.Select; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -@Slf4j -public class SqlHelper { - - public static Pattern pOrder = Pattern.compile("order\\s*by[\\w|\\W|\\s|\\S]*", Pattern.CASE_INSENSITIVE); - - - /** - * 去除qlString的select子句。 - * - * @param qlString - * @return - */ - public static String removeSelect(String qlString) { - int beginPos = qlString.toLowerCase().indexOf("from"); - return qlString.substring(beginPos); - } - - /** - * 去除orderBy子句。 - * - * @param sql - * @return - */ - public static String removeOrders(String sql) { - Matcher m = pOrder.matcher(sql); - StringBuffer sb = new StringBuffer(); - while (m.find()) { - m.appendReplacement(sb, ""); - } - m.appendTail(sb); - return sb.toString(); - } - - - /** - * 为当前SQL最外层增加Limit限制 - * - * @param sql 原始SQL语句 - * @param offset 开始条数 - * @param rowCount 每页显示多少纪录条数 - * @return limit限制后SQL语句 - */ - public static String getLimitString(String sql, int offset, int rowCount) { - try { - Select selectStatement = (Select) CCJSqlParserUtil.parse(sql); - PlainSelect select = (PlainSelect) selectStatement.getSelectBody(); - Limit limit = new Limit(); - if (offset > 0) { - limit.setOffset(new LongValue(offset)); - } - limit.setRowCount(new LongValue(rowCount)); - select.setLimit(limit); - sql = String.valueOf(selectStatement.getSelectBody()); - } catch (Exception e) { - log.error("Set limit error, SQL is:{} , Error Message is :{} ", sql, e); - } - - return sql; - } - - - /** - * 获取当前数据库预定义的方言 - * - * @param dbType - * @return - */ - public static String getDialectDBType(String dbType) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - return JdbcConstant.MYSQL; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - return JdbcConstant.HIVE; - } else if (DBTypeEnum.HBASE.getValue().equals(dbType)) { - return JdbcConstant.MYSQL; - } else { - return null; - } - - } - - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/SqlKeywords.java b/galaxy-common/src/main/java/com/mesalab/common/util/SqlKeywords.java deleted file mode 100644 index f6fdc9b..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/SqlKeywords.java +++ /dev/null @@ -1,71 +0,0 @@ -package com.mesalab.common.util; - -/** - * @Date: 2020-07-22 19:41 - * @Author : liuyongqiang - * @ClassName : SqlKeywords - * @Description : SQL关键字常量 - */ -public class SqlKeywords { - - public static final String SELECT = " select "; - - public static final String AND = " and "; - - public static final String WHERE = " where "; - - public static final String LIKE = " like "; - - public static final String OR = " or "; - - public static final String FROM = " from "; - - public static final String BWTWEEN = " between "; - - public static final String ORDER_BY = " order by "; - - public static final String LIMIT = " limit "; - - public static final String D_TIME = " __time "; - - public static final String D_TIME_FLOOR = " time_floor(__time,''{0}'') "; - - public static final String C_TO_DATE_TIME = "toDateTime(''{0}'')"; - - public static final String A_SORT ="SORT"; - - public static final String A_TIME = "_TIME"; - - public static final String A_FILTER = "FILTER"; - - public static final String A_AND = "AND"; - - public static final String A_LIKE = "LIKE"; - - public static final String A_OR = "OR"; - - public static final String A_SPACE = " "; - - public static final String A_SPOT = "."; - - public static final String A_LIMIT = "LIMIT" ; - - public static final String A_RETURN = "RETURN"; - - public static final String C_SECOND = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} SECOND)))) as granularity"; - - public static final String C_MINUTE = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} MINUTE)))) as granularity"; - - public static final String C_HOUR = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} HOUR)))) as granularity"; - - public static final String C_DAY = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} DAY)))) as granularity"; - - public static final String C_WEEK = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} WEEK)))) as granularity"; - - public static final String C_MONTH = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} MONTH)))) as granularity"; - - public static final String C_YEAR = "toDateTime(toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(common_recv_time),INTERVAL {0} YEAR)))) as granularity"; - - - -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/SysConstant.java b/galaxy-common/src/main/java/com/mesalab/common/util/SysConstant.java deleted file mode 100644 index 48aa0fc..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/SysConstant.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.mesalab.common.util; - -/** - * @Date: 2020-07-22 15:35 - * @Author : liuyongqiang - * @ClassName : SysConstant - * @Description : 常量定义 - */ -public class SysConstant { - - /** - * AnangoDB获取Jwt字符常量 - * */ - public static final String ARANGO_JWT_STR = "jwt"; - - /** - * AnangoDB获取Jwt字符前缀 - * */ - public static final String ARANGO_JWT_PRE = "bearer "; - - /** - * Double数据类型格式化 - * */ - public static final String DOUBLE_FORMAT = "%.0f"; -} diff --git a/galaxy-common/src/main/java/com/mesalab/common/util/TreeUtil.java b/galaxy-common/src/main/java/com/mesalab/common/util/TreeUtil.java deleted file mode 100644 index d9c6b2d..0000000 --- a/galaxy-common/src/main/java/com/mesalab/common/util/TreeUtil.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.mesalab.common.util; - - -import cn.hutool.core.collection.CollectionUtil; -import lombok.extern.slf4j.Slf4j; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.Stack; -import java.util.function.BiConsumer; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -public class TreeUtil { - - - /** - * 遍历树结构 - * - * @param root 节点树 - * @param getChildrenNode 获取当前节点的子节点列表；函数，接收当前节点对象 - * @param behavior 当前节点操作函数；定义遍历当前节点的操作行为 - * @param <T> 树的节点对象 - */ - public static <T> void traversalTree(List<T> root, Function<T, List<T>> getChildrenNode, - Consumer<T> behavior) { - - Stack<T> stack = new Stack<>(); - root.forEach(stack::push); - while (!stack.isEmpty()) { - T o = stack.pop(); - behavior.accept(o); - List<T> childrens = getChildrenNode.apply(o); - if (CollectionUtil.isNotEmpty(childrens)) { - childrens.forEach(stack::push); - } - } - - } - - /** - * 平铺树结构 - * - * @param root 节点树 - * @param getChildrenNode 获取当前节点的子节点列表；函数，接收当前节点对象 - * @param <T> 树的节点对象 - * @return 平铺后的节点结构 - */ - public static <T> List<T> flatTree(List<T> root, - Function<T, List<T>> getChildrenNode) { - List<T> list = new ArrayList<>(); - traversalTree(root, getChildrenNode, list::add); - return list; - } - - - /** - * 聚合为树结构 - * - * @param list 节点列表数据 - * @param loadKey 节点的ID；函数，接收一个节点获取唯一ID - * @param loadParentKey 节点的父亲ID；函数，接收一个节点获取父亲ID - * @param write 节点Children写入函数；函数，接收当前节点和子节点列表，然后将子节点项写入当前节点。 - * @param <T> 当前节点对象 - * @param <R> 请求节点返回的属性信息 - * @return 树结构列表 - */ - public static <T, R> List<T> mergeTree(List<T> list, - Function<T, R> loadKey, - Function<T, R> loadParentKey, - BiConsumer<T, List<T>> write) { - List<T> root = list.stream().filter(o -> Objects.isNull(loadParentKey.apply(o))).collect(Collectors.toList()); - Stack<T> stack = new Stack<T>(); - root.forEach(stack::push); - while (!stack.isEmpty()) { - T o = stack.pop(); - R key = loadKey.apply(o); - List<T> childrens = list.stream().filter(k -> key.equals(loadParentKey.apply(k))).collect(Collectors.toList()); - write.accept(o, childrens); - if (childrens.size() > 0) { - childrens.forEach(stack::push); - } - - } - return root; - } - - -} diff --git a/galaxy-gateway/config/logback-spring.xml b/galaxy-gateway/config/logback-spring.xml index f23fe46..10a07ca 100644 --- a/galaxy-gateway/config/logback-spring.xml +++ b/galaxy-gateway/config/logback-spring.xml @@ -1,66 +1,45 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<configuration scan="true" scanPeriod="60 seconds"> - <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" /> - <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" /> - <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" /> - - <property name="CONSOLE_LOG_PATTERN" value="${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/> - - <property name="FILE_LOG_PATTERN" value="${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/> - - <property name="TEST_FILE_PATH" value="/home/galaxy/galaxy-gateway/logs" /> - - <property name="PROD_FILE_PATH" value="/home/galaxy/galaxy-gateway/logs" /> +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + + <!--日志文件路径--> + <property name="LOG_PATH" value="/home/galaxy/galaxy-gateway/logs"/> + <!--日志文件名称--> + <property name="LOG_FILE_NAME" value="galaxy-gateway"/> + + <!--控制台日志， 控制台输出 --> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度,%msg：日志消息，%n是换行符--> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + </encoder> + </appender> - <property name="LOG_NAME" value="galaxy-gateway" /> - <!-- dev env --> - <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <appender name="ALL_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <!-- 正在记录的日志文件的路径及文件名 --> + <file>${LOG_PATH}/${LOG_FILE_NAME}.log</file> + <!--日志文件输出格式--> <encoder> - <pattern>${CONSOLE_LOG_PATTERN}</pattern> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + <charset>UTF-8</charset> <!-- 设置字符集 --> </encoder> + <!-- 日志记录器的滚动策略，按日期，按大小记录 --> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- 日志归档 --> + <fileNamePattern>${LOG_PATH}/history/log-%d{yyyy-MM-dd}.%i.log</fileNamePattern> + <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> + <maxFileSize>100MB</maxFileSize> + </timeBasedFileNamingAndTriggeringPolicy> + <!--日志文件保留天数--> + <maxHistory>15</maxHistory> + </rollingPolicy> + <!-- 此日志文件只记录debug以上级别的 --> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>DEBUG</level> + </filter> </appender> - <root level="INFO"> - <appender-ref ref="CONSOLE" /> + <root level="info"> + <appender-ref ref="ALL_FILE" /> + <appender-ref ref="STDOUT" /> </root> - - <!-- test env --> - <springProfile name="test"> - <appender name="TEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <encoder> - <pattern>${FILE_LOG_PATTERN}</pattern> - </encoder> - <file>${TEST_FILE_PATH}${file.separator}${LOG_NAME}.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> - <!-- rollover daily --> - <fileNamePattern>${TEST_FILE_PATH}${file.separator}${LOG_NAME}-%d{yyyy-MM-dd}.%i.log</fileNamePattern> - <!-- each file should be at most 100MB, keep 30 days worth of history, but at most 20GB --> - <maxHistory>30</maxHistory> - <maxFileSize>100MB</maxFileSize> - </rollingPolicy> - </appender> - <root level="INFO"> - <appender-ref ref="TEST_FILE" /> - </root> - </springProfile> - - <!-- prod env --> - <springProfile name="prod"> - <appender name="PROD_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <encoder> - <pattern>${FILE_LOG_PATTERN}</pattern> - </encoder> - <file>${PROD_FILE_PATH}${file.separator}${LOG_NAME}.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> - <!-- rollover daily --> - <fileNamePattern>${PROD_FILE_PATH}${file.separator}${LOG_NAME}-%d{yyyy-MM-dd}.%i.log</fileNamePattern> - <!-- each file should be at most 100MB, keep 30 days worth of history, but at most 20GB --> - <maxHistory>30</maxHistory> - <maxFileSize>100MB</maxFileSize> - </rollingPolicy> - </appender> - <root level="INFO"> - <appender-ref ref="PROD_FILE" /> - </root> - </springProfile> -</configuration> +</configuration>
\ No newline at end of file diff --git a/galaxy-gateway/pom.xml b/galaxy-gateway/pom.xml index 4964e57..e5fe68a 100644 --- a/galaxy-gateway/pom.xml +++ b/galaxy-gateway/pom.xml @@ -48,7 +48,7 @@ </executions> <configuration> <includeSystemScope>true</includeSystemScope> - <mainClass>com.mesalab.gateway.GalaxyGatewayApp</mainClass> + <mainClass>com.mesalab.api.gateway.GalaxyGatewayApp</mainClass> </configuration> </plugin> <plugin> diff --git a/galaxy-gateway/src/main/java/com/mesalab/gateway/GalaxyGatewayApp.java b/galaxy-gateway/src/main/java/com/mesalab/api/gateway/GalaxyGatewayApp.java index c7ca109..661d03a 100644 --- a/galaxy-gateway/src/main/java/com/mesalab/gateway/GalaxyGatewayApp.java +++ b/galaxy-gateway/src/main/java/com/mesalab/api/gateway/GalaxyGatewayApp.java @@ -1,4 +1,4 @@ -package com.mesalab.gateway; +package com.mesalab.api.gateway; import org.springframework.boot.SpringApplication; import org.springframework.cloud.client.SpringCloudApplication; diff --git a/galaxy-query-engine/config/application.properties b/galaxy-query-engine/config/application.properties index b03dc02..d884eee 100644 --- a/galaxy-query-engine/config/application.properties +++ b/galaxy-query-engine/config/application.properties @@ -1,58 +1,8 @@ #application config -server.port=8701 -spring.application.name=galaxy-data-engine -#spring cloud eureka config -eureka.instance.prefer-ip-address=true -eureka.client.registry-fetch-interval-seconds=5 -eureka.instance.lease-renewal-interval-in-seconds=10 -eureka.instance.health-check-url-path=/actuator/health -eureka.instance.metadata-map.user.name=${spring.security.user.name} -eureka.instance.metadata-map.user.password=${spring.security.user.password} -eureka.instance.instance-id=${spring.cloud.client.ip-address}:${server.port} -eureka.client.serviceUrl.defaultZone=http://admin:[email protected]:8501/eureka/ +server.port=8803 +spring.application.name=galaxy-query-engine #management config management.endpoints.web.exposure.include=* management.endpoint.health.show-details=always #log file path config -logging.config=./config/logback-spring.xml - -#httpclient config -httpclient.connectTimeout=1000 -httpclient.socketTimeout=21700000 -httpclient.maxConnection=500 -httpclient.maxPerRoute=500 -#job admin config -xxl.job.admin.add=192.168.44.12:8181 -xxl.job.admin.url=http://${xxl.job.admin.add}/xxl-job-admin -xxl.job.admin.username=admin -xxl.job.admin.password=123456 -#engine config -engine.maxCacheNum=500000 -engine.defaultResultNum=300000 -#hbase config -hbase.url=192.168.44.12:8084 -hbase.dbname=tsg -hbase.tableName=report_result -hbase.columnFamily=response -hbase.columnName=result -#druid config -druid.address=192.168.44.12:8082 -druid.dbname=druid -druid.sqlTimeZone=Asia/Shanghai -druid.query.url=http://${druid.address}/druid/v2/sql -#arango config -arango.address=192.168.44.12:8529 -arango.maxrows=10000 -arango.login=http://galaxy-auth-center/authorize/arangoJwtLogin -arango.query.url=http://${arango.address}/_db/tsg_galaxy_v3/_api/cursor -arango.collection=http://${arango.address}/_db/tsg_galaxy_v3/_api/collection -#clickhouse config -clickhouse.address=192.168.44.12:8123 -clickhouse.query.url=http://${clickhouse.address} -clickhouse.dbname=tsg_galaxy_v3 -clickhouse.real.time.username=tsg_query -clickhouse.real.time.password=ceiec2018 -clickhouse.long.term.username=tsg_report -clickhouse.long.term.password=ceiec2019 -#iplearning config -iplearning.uniqciprecent=-24
\ No newline at end of file +logging.config=./config/logback-spring.xml
\ No newline at end of file diff --git a/galaxy-query-engine/config/logback-spring.xml b/galaxy-query-engine/config/logback-spring.xml index 07bad98..b4e4240 100644 --- a/galaxy-query-engine/config/logback-spring.xml +++ b/galaxy-query-engine/config/logback-spring.xml @@ -1,66 +1,45 @@ -<?xml version="1.0" encoding="UTF-8" ?> -<configuration scan="true" scanPeriod="60 seconds"> - <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" /> - <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" /> - <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" /> - - <property name="CONSOLE_LOG_PATTERN" value="${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/> - - <property name="FILE_LOG_PATTERN" value="${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/> - - <property name="TEST_FILE_PATH" value="/home/galaxy/galaxy-data-engine/logs" /> - - <property name="PROD_FILE_PATH" value="/home/galaxy/galaxy-data-engine/logs" /> +<?xml version="1.0" encoding="UTF-8"?> +<configuration> + + <!--日志文件路径--> + <property name="LOG_PATH" value="/home/galaxy/galaxy-query-engine/logs"/> + <!--日志文件名称--> + <property name="LOG_FILE_NAME" value="galaxy-query-engine"/> + + <!--控制台日志， 控制台输出 --> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度,%msg：日志消息，%n是换行符--> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + </encoder> + </appender> - <property name="LOG_NAME" value="galaxy-data-engine" /> - <!-- dev env --> - <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <appender name="ALL_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <!-- 正在记录的日志文件的路径及文件名 --> + <file>${LOG_PATH}/${LOG_FILE_NAME}.log</file> + <!--日志文件输出格式--> <encoder> - <pattern>${CONSOLE_LOG_PATTERN}</pattern> + <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n</pattern> + <charset>UTF-8</charset> <!-- 设置字符集 --> </encoder> + <!-- 日志记录器的滚动策略，按日期，按大小记录 --> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <!-- 日志归档 --> + <fileNamePattern>${LOG_PATH}/history/log-%d{yyyy-MM-dd}.%i.log</fileNamePattern> + <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> + <maxFileSize>100MB</maxFileSize> + </timeBasedFileNamingAndTriggeringPolicy> + <!--日志文件保留天数--> + <maxHistory>15</maxHistory> + </rollingPolicy> + <!-- 此日志文件只记录debug以上级别的 --> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>DEBUG</level> + </filter> </appender> - <root level="INFO"> - <appender-ref ref="CONSOLE" /> + <root level="info"> + <appender-ref ref="ALL_FILE" /> + <appender-ref ref="STDOUT" /> </root> - - <!-- test env --> - <springProfile name="test"> - <appender name="TEST_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <encoder> - <pattern>${FILE_LOG_PATTERN}</pattern> - </encoder> - <file>${TEST_FILE_PATH}${file.separator}${LOG_NAME}.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> - <!-- rollover daily --> - <fileNamePattern>${TEST_FILE_PATH}${file.separator}${LOG_NAME}-%d{yyyy-MM-dd}.%i.log</fileNamePattern> - <!-- each file should be at most 100MB, keep 30 days worth of history, but at most 20GB --> - <maxHistory>30</maxHistory> - <maxFileSize>100MB</maxFileSize> - </rollingPolicy> - </appender> - <root level="INFO"> - <appender-ref ref="TEST_FILE" /> - </root> - </springProfile> - - <!-- prod env --> - <springProfile name="prod"> - <appender name="PROD_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <encoder> - <pattern>${FILE_LOG_PATTERN}</pattern> - </encoder> - <file>${PROD_FILE_PATH}${file.separator}${LOG_NAME}.log</file> - <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> - <!-- rollover daily --> - <fileNamePattern>${PROD_FILE_PATH}${file.separator}${LOG_NAME}-%d{yyyy-MM-dd}.%i.log</fileNamePattern> - <!-- each file should be at most 100MB, keep 30 days worth of history, but at most 20GB --> - <maxHistory>30</maxHistory> - <maxFileSize>100MB</maxFileSize> - </rollingPolicy> - </appender> - <root level="INFO"> - <appender-ref ref="PROD_FILE" /> - </root> - </springProfile> -</configuration> +</configuration>
\ No newline at end of file diff --git a/galaxy-query-engine/docker/Dockerfile b/galaxy-query-engine/docker/Dockerfile index 92acb05..febd304 100644 --- a/galaxy-query-engine/docker/Dockerfile +++ b/galaxy-query-engine/docker/Dockerfile @@ -2,12 +2,10 @@ ARG JDK_IMAGE FROM ${JDK_IMAGE} MAINTAINER galaxy-data-platform VOLUME /tmp -WORKDIR /home/galaxy/galaxy-data-engine +WORKDIR /home/galaxy/galaxy-query-engine COPY config config -COPY mmdb mmdb -COPY schema schema ARG JAR_FILE -COPY ${JAR_FILE} galaxy-data-engine.jar +COPY ${JAR_FILE} galaxy-query-engine.jar ENV JAVA_OPTS=" -Xms1024m -Xmx2048m " ENV LANG=en_US.UTF-8 LANGUAGE=en_US:en LC_ALL=en_US.UTF-8 -ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar galaxy-data-engine.jar" ]
\ No newline at end of file +ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar galaxy-query-engine.jar" ]
\ No newline at end of file diff --git a/galaxy-query-engine/mmdb/asn_v4.mmdb b/galaxy-query-engine/mmdb/asn_v4.mmdb Binary files differdeleted file mode 100644 index 4508ea2..0000000 --- a/galaxy-query-engine/mmdb/asn_v4.mmdb +++ /dev/null diff --git a/galaxy-query-engine/mmdb/asn_v6.mmdb b/galaxy-query-engine/mmdb/asn_v6.mmdb Binary files differdeleted file mode 100644 index a779312..0000000 --- a/galaxy-query-engine/mmdb/asn_v6.mmdb +++ /dev/null diff --git a/galaxy-query-engine/mmdb/ip_private_v4.mmdb b/galaxy-query-engine/mmdb/ip_private_v4.mmdb Binary files differdeleted file mode 100644 index 445f2aa..0000000 --- a/galaxy-query-engine/mmdb/ip_private_v4.mmdb +++ /dev/null diff --git a/galaxy-query-engine/mmdb/ip_private_v6.mmdb b/galaxy-query-engine/mmdb/ip_private_v6.mmdb Binary files differdeleted file mode 100644 index cf82ebe..0000000 --- a/galaxy-query-engine/mmdb/ip_private_v6.mmdb +++ /dev/null diff --git a/galaxy-query-engine/mmdb/ip_v4.mmdb b/galaxy-query-engine/mmdb/ip_v4.mmdb Binary files differdeleted file mode 100644 index 842cce9..0000000 --- a/galaxy-query-engine/mmdb/ip_v4.mmdb +++ /dev/null diff --git a/galaxy-query-engine/mmdb/ip_v6.mmdb b/galaxy-query-engine/mmdb/ip_v6.mmdb Binary files differdeleted file mode 100644 index cf82ebe..0000000 --- a/galaxy-query-engine/mmdb/ip_v6.mmdb +++ /dev/null diff --git a/galaxy-query-engine/pom.xml b/galaxy-query-engine/pom.xml index c95671c..caea719 100644 --- a/galaxy-query-engine/pom.xml +++ b/galaxy-query-engine/pom.xml @@ -9,126 +9,7 @@ </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>galaxy-data-engine</artifactId> + <artifactId>galaxy-query-engine</artifactId> <packaging>jar</packaging> - <dependencies> - <dependency> - <groupId>com.mesalab</groupId> - <artifactId>galaxy-common</artifactId> - </dependency> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - </dependency> - <dependency> - <groupId>org.projectlombok</groupId> - <artifactId>lombok</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-web</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.cloud</groupId> - <artifactId>spring-cloud-starter-netflix-hystrix</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-actuator</artifactId> - </dependency> - <dependency> - <groupId>cn.hutool</groupId> - <artifactId>hutool-all</artifactId> - </dependency> - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro</artifactId> - </dependency> - <dependency> - <groupId>com.github.java-json-tools</groupId> - <artifactId>json-schema-validator</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-test</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.zdjizhi</groupId> - <artifactId>galaxy</artifactId> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - - <build> - <finalName>galaxy-data-engine</finalName> - <plugins> - <plugin> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-maven-plugin</artifactId> - <version>2.0.1.RELEASE</version> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>repackage</goal> - </goals> - </execution> - </executions> - <configuration> - <includeSystemScope>true</includeSystemScope> - <mainClass>com.mesalab.engine.GalaxyDataEngineApp</mainClass> - </configuration> - </plugin> - <plugin> - <groupId>com.spotify</groupId> - <artifactId>docker-maven-plugin</artifactId> - <version>1.0.0</version> - <configuration> - <serverId>153-docker-repo</serverId> - <registryUrl>${docker.registry}:${docker.registry.port}</registryUrl> - <pushImage>true</pushImage> - <imageName>${docker.registry}:${docker.registry.port}/${docker.image.prefix}/${project.artifactId} - </imageName> - <forceTags>true</forceTags> - <dockerHost>http://192.168.40.153:2375</dockerHost> - <dockerDirectory>docker</dockerDirectory> - <buildArgs> - <JDK_IMAGE>192.168.40.153:9080/common/jdk:1.8.0_73</JDK_IMAGE> - <JAR_FILE>${project.build.finalName}.jar</JAR_FILE> - </buildArgs> - <resources> - <resource> - <targetPath>/</targetPath> - <directory>${project.build.directory}</directory> - <include>${project.build.finalName}.jar</include> - </resource> - <resource> - <targetPath>/mmdb</targetPath> - <directory>mmdb</directory> - </resource> - <resource> - <targetPath>/config</targetPath> - <directory>config</directory> - </resource> - <resource> - <targetPath>/schema</targetPath> - <directory>schema</directory> - </resource> - </resources> - </configuration> - </plugin> - </plugins> - </build> - </project>
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/active_defence_event_log.avsc b/galaxy-query-engine/schema/clickhouse/active_defence_event_log.avsc deleted file mode 100644 index d21aca3..0000000 --- a/galaxy-query-engine/schema/clickhouse/active_defence_event_log.avsc +++ /dev/null @@ -1,133 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "active_defence_event_log", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"schema_query\":{\"dimensions\":[\"common_policy_id\",\"ad_target_ip\",\"ad_cc_target_url\"],\"metrics\":[\"ad_target_ip\",\"ad_sent_byte_num\",\"ad_sent_pkt_num\",\"ad_cc_initiate_connection_num\",\"ad_cc_established_connection_num\",\"ad_cc_rejected_connection_num\"],\"filters\":[\"common_policy_id\",\"ad_target_ip\",\"ad_target_port\",\"ad_protocol\",\"common_address_type\",\"ad_sent_byte_num\",\"ad_sent_pkt_num\",\"ad_cc_initiate_connection_num\",\"ad_cc_established_connection_num\",\"ad_cc_rejected_connection_num\"]},\"schema_query\":{\"dimensions\":[\"common_policy_id\",\"ad_target_ip\",\"ad_cc_target_url\"],\"metrics\":[\"ad_target_ip\",\"ad_sent_byte_num\",\"ad_sent_pkt_num\",\"ad_cc_initiate_connection_num\",\"ad_cc_established_connection_num\",\"ad_cc_rejected_connection_num\"],\"filters\":[\"common_policy_id\",\"ad_target_ip\",\"ad_target_port\",\"ad_protocol\",\"common_address_type\",\"ad_sent_byte_num\",\"ad_sent_pkt_num\",\"ad_cc_initiate_connection_num\",\"ad_cc_established_connection_num\",\"ad_cc_rejected_connection_num\"]},\"schema_type\":{\"REFLECTION\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_address_type\",\"common_device_id\",\"common_link_id\",\"common_entrance_id\",\"common_user_region\",\"ad_method\",\"ad_protocol\",\"ad_target_ip\",\"ad_target_port\",\"ad_target_ip_location\",\"ad_target_ip_asn\",\"ad_reflector_profile_id\",\"ad_sent_pkt_num\",\"ad_sent_byte_num\",\"ad_generate_time\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"ad_target_ip\",\"ad_target_port\",\"ad_reflector_profile_id\",\"ad_sent_pkt_num\",\"ad_sent_byte_num\"]},\"FLOOD\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_address_type\",\"common_device_id\",\"common_link_id\",\"common_entrance_id\",\"common_user_region\",\"ad_method\",\"ad_protocol\",\"ad_target_ip\",\"ad_target_port\",\"ad_target_ip_location\",\"ad_target_ip_asn\",\"ad_claimed_src_ip_profile_id\",\"ad_sent_pkt_num\",\"ad_sent_byte_num\",\"ad_generate_time\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"ad_target_ip\",\"ad_target_port\",\"ad_claimed_src_ip_profile_id\",\"ad_protocol\"]},\"CC\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_address_type\",\"common_device_id\",\"common_link_id\",\"common_entrance_id\",\"common_user_region\",\"ad_method\",\"ad_protocol\",\"ad_cc_target_url\",\"ad_claimed_src_ip_profile_id\",\"ad_cc_initiate_connection_num\",\"ad_cc_established_connection_num\",\"ad_cc_rejected_connection_num\",\"ad_generate_time\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"ad_cc_target_url\",\"ad_claimed_src_ip_profile_id\",\"ad_protocol\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"ad_target_ip\",\"ad_target_port\",\"ad_cc_target_url\"]}}", - "fields": [ - { - "name": "common_recv_time", - "label": "Receive Time", - "type": "int", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}}" - }, - { - "name": "common_log_id", - "label": "Log ID", - "type": "long", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "label": "Policy ID", - "type": "int", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_address_type", - "label": "Address Type", - "type": "int", - "doc": "{\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_entrance_id", - "label": "Entrance ID", - "type": "int", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_device_id", - "type": "string", - "label": "Device ID" - }, - { - "name": "common_link_id", - "type": "int", - "label": "Link ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_user_region", - "label": "User Region", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "ad_target_ip", - "label": "Target IP", - "type": "string", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_country,geo_asn\",\"appendTo\":\"ad_target_ip_location,ad_target_ip_asn\"}}" - }, - { - "name": "ad_target_port", - "label": "Target Port", - "type": "int" - }, - { - "name": "ad_cc_target_url", - "type": "string", - "label": "Target URL", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "ad_target_ip_location", - "label": "Target Location", - "type": "string" - }, - { - "name": "ad_target_ip_asn", - "label": "Target ASN", - "type": "string" - }, - { - "name": "ad_protocol", - "label": "Protocol", - "type": "string" - }, - { - "name": "ad_method", - "label": "Method", - "type": "string" - }, - { - "name": "ad_claimed_src_ip_profile_id", - "label": "Claimed Profile ID", - "type": "int" - }, - { - "name": "ad_reflector_profile_id", - "label": "Reflector Profile ID", - "type": "int" - }, - { - "name": "ad_sent_pkt_num", - "label": "Packets Sent", - "type": "int" - }, - { - "name": "ad_sent_byte_num", - "label": "Bytes Sent", - "type": "int" - }, - { - "name": "ad_cc_initiate_connection_num", - "label": "Initiate Numbers", - "type": "int" - }, - { - "name": "ad_cc_established_connection_num", - "label": "Established Numbers", - "type": "int" - }, - { - "name": "ad_cc_rejected_connection_num", - "label": "Rejected Numbers", - "type": "int" - }, - { - "name": "ad_generate_time", - "label": "Generate Time", - "type": "int", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/connection_record_log.avsc b/galaxy-query-engine/schema/clickhouse/connection_record_log.avsc deleted file mode 100644 index 3687b50..0000000 --- a/galaxy-query-engine/schema/clickhouse/connection_record_log.avsc +++ /dev/null @@ -1,742 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "connection_record_log", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"index_table\":\"connection_record_log_common_client_ip,connection_record_log_common_server_ip,connection_record_log_common_subscriber_id,connection_record_log_http_domain\",\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_sled_ip\",\"common_server_location\",\"common_subscriber_id\",\"common_server_port\",\"http_domain\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_sessions\"],\"filters\":[\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_server_port\",\"common_server_location\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_l4_protocol\",\"http_domain\",\"ssl_sni\"]},\"schema_type\":{\"BASE\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\"]},\"HTTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"http_url\",\"http_host\",\"http_domain\",\"http_request_line\",\"http_response_line\",\"http_request_header\",\"http_response_header\",\"http_request_body\",\"http_response_body\",\"http_request_body_key\",\"http_response_body_key\",\"http_proxy_flag\",\"http_sequence\",\"http_snapshot\",\"http_cookie\",\"http_referer\",\"http_user_agent\",\"http_content_length\",\"http_content_type\",\"http_set_cookie\",\"http_version\",\"http_response_lantency_ms\",\"http_session_duration_ms\",\"http_action_file_size\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"http_url\",\"common_server_port\"]},\"MAIL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"mail_protocol_type\",\"mail_account\",\"mail_from_cmd\",\"mail_to_cmd\",\"mail_from\",\"mail_to\",\"mail_cc\",\"mail_bcc\",\"mail_subject\",\"mail_subject_charset\",\"mail_content\",\"mail_content_charset\",\"mail_attachment_name\",\"mail_attachment_name_charset\",\"mail_attachment_content\",\"mail_eml_file\",\"mail_snapshot\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"mail_from\",\"mail_to\",\"mail_subject\"]},\"DNS\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"dns_message_id\",\"dns_qr\",\"dns_opcode\",\"dns_aa\",\"dns_tc\",\"dns_rd\",\"dns_ra\",\"dns_rcode\",\"dns_qdcount\",\"dns_ancount\",\"dns_nscount\",\"dns_arcount\",\"dns_qname\",\"dns_qtype\",\"dns_qclass\",\"dns_cname\",\"dns_sub\",\"dns_rr\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_client_ip\",\"dns_qr\",\"dns_qname\",\"dns_qtype\"]},\"SSL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ssl_sni\",\"ssl_san\",\"ssl_cn\",\"ssl_pinningst\",\"ssl_intercept_state\",\"ssl_server_side_latency\",\"ssl_client_side_latency\",\"ssl_server_side_version\",\"ssl_client_side_version\",\"ssl_cert_verify\",\"ssl_error\",\"ssl_con_latency_ms\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ssl_sni\",\"common_server_ip\",\"common_server_port\"]},\"QUIC\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"quic_version\",\"quic_sni\",\"quic_user_agent\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"quic_sni\",\"common_server_ip\",\"common_server_port\"]},\"FTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ftp_account\",\"ftp_url\",\"ftp_content\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"ftp_url\",\"common_server_ip\",\"common_server_port\"]},\"BGP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"bgp_type\",\"bgp_as_num\",\"bgp_route\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"bgp_type\",\"bgp_as_num\",\"common_server_ip\",\"common_server_port\"]},\"VOIP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"voip_calling_account\",\"voip_called_account\",\"voip_calling_number\",\"voip_called_number\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"voip_calling_account\",\"voip_called_account\",\"common_server_ip\",\"common_server_port\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"common_schema_type\"]}", - "fields": [ - { - "name": "common_recv_time", - "type": "int", - "label": "Receive Time", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}}" - }, - { - "name": "common_log_id", - "type": "long", - "label": "Log ID", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "type": "int", - "label": "Policy ID", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_subscriber_id", - "type": "string", - "label": "Subscriber ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "type": "string", - "label": "Client IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_detail,geo_asn,radius_match\", \"appendTo\":\"common_client_location,common_client_asn,common_subscriber_id\"}}" - }, - { - "name": "common_client_port", - "type": "int", - "label": "Client Port" - }, - { - "name": "common_l4_protocol", - "type": "string", - "label": "L4 Protocol" - }, - { - "name": "common_address_type", - "type": "int", - "label": "Address Type", - "doc": "{\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "type": "string", - "label": "Server IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_country,geo_asn\", \"appendTo\":\"common_server_location,common_server_asn\"}}" - }, - { - "name": "common_server_port", - "type": "int", - "label": "Server Port", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_action", - "type": "int", - "label": "Action", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"128\",\"value\":\"Allow\"}]}" - }, - { - "name": "common_direction", - "type": "int", - "label": "Direction", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"outbound\"},{\"code\":\"1\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "type": "int", - "label": "Entrance ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "type": "string", - "label": "Sled IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "type": "string", - "label": "Client Location" - }, - { - "name": "common_client_asn", - "type": "string", - "label": "Client ASN" - }, - { - "name": "common_server_location", - "type": "string", - "label": "Server Location" - }, - { - "name": "common_server_asn", - "type": "string", - "label": "Server ASN" - }, - { - "name": "common_sessions", - "type": "int", - "label": "Sessions" - }, - { - "name": "common_c2s_pkt_num", - "type": "long", - "label": "Packets Sent" - }, - { - "name": "common_s2c_pkt_num", - "type": "long", - "label": "Packets Received" - }, - { - "name": "common_c2s_byte_num", - "type": "long", - "label": "Bytes Sent" - }, - { - "name": "common_s2c_byte_num", - "type": "long", - "label": "Bytes Received" - }, - { - "name": "common_service", - "type": "int", - "label": "Service", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "type": "string", - "label": "Schema Type", - "doc": "{\"data\":[{\"code\":\"BASE\",\"value\":\"BASE\"},{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"SSL\",\"value\":\"SSL\"}],\"allow_query\":\"true\"}" - }, - { - "name": "common_user_tags", - "type": "string", - "label": "User Tags", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "type": "string", - "label": "Sub Action" - }, - { - "name": "common_user_region", - "type": "string", - "label": "User Region", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "type": "string", - "label": "Device ID" - }, - { - "name": "common_link_id", - "type": "int", - "label": "Link ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "type": "string", - "label": "ISP", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_encapsulation", - "type": "int", - "label": "Encapsulation", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_label", - "type": "string", - "label": "Application Label", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_protocol_label", - "type": "string", - "label": "Protocol Label", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_start_time", - "type": "int", - "label": "Start Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_end_time", - "type": "int", - "label": "End Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_establish_latency_ms", - "type": "int", - "label": "Establish Latency(ms)" - }, - { - "name": "common_con_duration_ms", - "type": "int", - "label": "Duration(ms)" - }, - { - "name": "common_stream_dir", - "type": "int", - "label": "Stream Direction", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "type": "string", - "label": "Address List", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_has_dup_traffic", - "type": "int", - "label": "Duplication Traffic", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "type": "string", - "label": "Stream Error", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_trace_id", - "type": "long", - "label": "Session ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_url", - "type": "string", - "label": "Http.URL" - }, - { - "name": "http_host", - "type": "string", - "label": "Http.Host", - "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "http_domain", - "type": "string", - "doc": "{\"allow_query\":\"true\"}", - "label": "Http.Domain" - }, - { - "name": "http_request_line", - "type": "string", - "label": "Http.Request Line", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_line", - "type": "string", - "label": "Http.Response Line", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_header", - "type": "string", - "label": "Http.Request Headers", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_header", - "type": "string", - "label": "Http.Response Headers", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_body", - "type": "string", - "label": "Http.Request Body", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_response_body", - "type": "string", - "label": "Http.Response Body", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_request_body_key", - "type": "string", - "label": "Http.Request Body Key", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_response_body_key", - "type": "string", - "label": "Http.Response Body Key", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_proxy_flag", - "type": "int", - "label": "Http.Proxy Flag", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_sequence", - "type": "int", - "label": "Http.Sequence", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_snapshot", - "type": "string", - "label": "Http.Snapshot", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_cookie", - "type": "string", - "label": "Http.Cookie" - }, - { - "name": "http_referer", - "type": "string", - "label": "Http.Referer" - }, - { - "name": "http_user_agent", - "type": "string", - "label": "Http.User Agent" - }, - { - "name": "http_content_length", - "type": "string", - "label": "Http.Content Length" - }, - { - "name": "http_content_type", - "type": "string", - "label": "Http.Content Type" - }, - { - "name": "http_set_cookie", - "type": "string", - "label": "Http.Set Cookie" - }, - { - "name": "http_version", - "type": "string", - "label": "Http.Version" - }, - { - "name": "http_response_lantency_ms", - "type": "int", - "label": "Http.Response Latency(ms)" - }, - { - "name": "http_session_duration_ms", - "type": "int", - "label": "Http.Session Duration(ms)" - }, - { - "name": "http_action_file_size", - "type": "int", - "label": "Http.Action File Size" - }, - { - "name": "mail_protocol_type", - "type": "string", - "label": "Mail.Protocol Type" - }, - { - "name": "mail_account", - "type": "string", - "label": "Mail.Account" - }, - { - "name": "mail_from_cmd", - "type": "string", - "label": "Mail.From CMD" - }, - { - "name": "mail_to_cmd", - "type": "string", - "label": "Mail.To CMD" - }, - { - "name": "mail_from", - "doc": "{\"constraints\":{\"type\":\"email\"}}", - "type": "string", - "label": "Mail.From" - }, - { - "name": "mail_to", - "doc": "{\"constraints\":{\"type\":\"email\"}}", - "type": "string", - "label": "Mail.To" - }, - { - "name": "mail_cc", - "type": "string", - "label": "Mail.CC" - }, - { - "name": "mail_bcc", - "type": "string", - "label": "Mail.BCC" - }, - { - "name": "mail_subject", - "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"mail_subject_charset\"}}", - "type": "string", - "label": "Mail.Subject" - }, - { - "name": "mail_subject_charset", - "type": "string", - "label": "Mail.Subject Charset", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "mail_content", - "type": "string", - "label": "Mail.Content", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_content_charset", - "type": "string", - "label": "Mail.Content Charset", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_attachment_name", - "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"mail_attachment_name_charset\"}}", - "type": "string", - "label": "Mail.Attachment" - }, - { - "name": "mail_attachment_name_charset", - "type": "string", - "label": "Mail.Attachment Charset", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_attachment_content", - "type": "string", - "label": "Mail.Attachment Content", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_eml_file", - "doc": "{\"constraints\":{\"type\":\"file\"}}", - "type": "string", - "label": "Mail.EML File" - }, - { - "name": "mail_snapshot", - "type": "string", - "label": "Mail.Snapshot", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "dns_message_id", - "type": "int", - "label": "Dns.Message ID" - }, - { - "name": "dns_qr", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"REESPONSE\"}]}", - "type": "int", - "label": "Dns.QR" - }, - { - "name": "dns_opcode", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"IQUERY\"},{\"code\":\"2\",\"value\":\"STATUS\"},{\"code\":\"5\",\"value\":\"UPDATE\"}]}", - "type": "int", - "label": "Dns.OPCODE" - }, - { - "name": "dns_aa", - "type": "int", - "label": "Dns.AA" - }, - { - "name": "dns_tc", - "type": "int", - "label": "Dns.TC" - }, - { - "name": "dns_rd", - "type": "int", - "label": "Dns.RD" - }, - { - "name": "dns_ra", - "type": "int", - "label": "Dns.RA" - }, - { - "name": "dns_rcode", - "type": "int", - "label": "Dns.RCODE" - }, - { - "name": "dns_qdcount", - "type": "int", - "label": "Dns.QDCOUNT" - }, - { - "name": "dns_ancount", - "type": "int", - "label": "Dns.ANCOUNT" - }, - { - "name": "dns_nscount", - "type": "int", - "label": "Dns.NSCOUNT" - }, - { - "name": "dns_arcount", - "type": "int", - "label": "Dns.ARCOUNT" - }, - { - "name": "dns_qname", - "type": "string", - "label": "Dns.QNAME" - }, - { - "name": "dns_qtype", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"A\"},{\"code\":\"2\",\"value\":\"NS\"},{\"code\":\"5\",\"value\":\"CNAME\"},{\"code\":\"6\",\"value\":\"SOA\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"12\",\"value\":\"PTR\"},{\"code\":\"13\",\"value\":\"HINFO\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"15\",\"value\":\"MX\"},{\"code\":\"28\",\"value\":\"AAAA\"}]}", - "type": "int", - "label": "Dns.QTYPE" - }, - { - "name": "dns_qclass", - "type": "int", - "label": "Dns.QCLASS" - }, - { - "name": "dns_cname", - "type": "string", - "label": "Dns.CNAME" - }, - { - "name": "dns_sub", - "type": "int", - "label": "Dns.SUB", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"DNS\"},{\"code\":\"2\",\"value\":\"DNSSEC\"}]}" - }, - { - "name": "dns_rr", - "type": "string", - "label": "Dns.RR" - }, - { - "name": "ssl_version", - "type": "string", - "label": "SSL.Version", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "ssl_sni", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}", - "type": "string", - "label": "SSL.SNI" - }, - { - "name": "ssl_san", - "type": "string", - "label": "SSL.SAN" - }, - { - "name": "ssl_cn", - "type": "string", - "label": "SSL.CN" - }, - { - "name": "ssl_pinningst", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Not Pinning\"},{\"code\":\"1\",\"value\":\"Pinning\"},{\"code\":\"2\",\"value\":\"Maybe Pinning\"}]}", - "type": "int", - "label": "SSL.Pinning" - }, - { - "name": "ssl_intercept_state", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Passthrough\"},{\"code\":\"1\",\"value\":\"Intercept\"},{\"code\":\"2\",\"value\":\"Shutdown\"}]}", - "type": "int", - "label": "SSL.Intercept State" - }, - { - "name": "ssl_server_side_latency", - "type": "int", - "label": "SSL.Server Side Latency(ms)" - }, - { - "name": "ssl_client_side_latency", - "type": "int", - "label": "SSL.Client Side Latency(ms)" - }, - { - "name": "ssl_server_side_version", - "type": "string", - "label": "SSL.Server Side Version" - }, - { - "name": "ssl_client_side_version", - "type": "string", - "label": "SSL.Client Side Version" - }, - { - "name": "ssl_cert_verify", - "type": "int", - "label": "SSL.Certificate Verify", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}]}" - }, - { - "name": "ssl_error", - "type": "string", - "label": "SSL.Error" - }, - { - "name": "ssl_con_latency_ms", - "type": "int", - "label": "SSL.Connection Latency(ms)", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "quic_version", - "type": "string", - "label": "Quic.Version" - }, - { - "name": "quic_sni", - "type": "string", - "label": "Quic.SNI", - "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "quic_user_agent", - "type": "string", - "label": "Quic.User Agent" - }, - { - "name": "ftp_account", - "type": "string", - "label": "Ftp.Account" - }, - { - "name": "ftp_url", - "type": "string", - "label": "Ftp.URL" - }, - { - "name": "ftp_content", - "type": "string", - "label": "Ftp.Content" - }, - { - "name": "bgp_type", - "type": "int", - "label": "BGP.Type", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "bgp_as_num", - "type": "string", - "label": "BGP.AS Number", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "bgp_route", - "type": "string", - "label": "BGP.Route", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_calling_account", - "type": "string", - "label": "Voip.Calling Account", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_called_account", - "type": "string", - "label": "Voip.Called Account", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_calling_number", - "type": "string", - "label": "Voip.Calling Number", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_called_number", - "type": "string", - "label": "Voip.Called Number", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "streaming_media_url", - "type": "string", - "label": "Streaming.Media URL", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "streaming_media_protocol", - "type": "string", - "label": "Streaming.Media Protocol", - "doc": "{\"visibility\":\"disabled\"}" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/connection_record_log_common_client_ip.avsc b/galaxy-query-engine/schema/clickhouse/connection_record_log_common_client_ip.avsc deleted file mode 100644 index 76eb947..0000000 --- a/galaxy-query-engine/schema/clickhouse/connection_record_log_common_client_ip.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "connection_record_log_common_client_ip", - "doc": "{\"primary_key\":\"common_log_id\",\"index_key\":\"common_client_ip\"}", - "fields": [ - { - "name": "common_log_id", - "type": "long" - }, - { - "name": "common_recv_time", - "type": "int" - }, - { - "name": "common_policy_id", - "type": "int" - }, - { - "name": "common_action", - "type": "int" - }, - { - "name": "common_server_ip", - "type": "string" - }, - { - "name": "common_client_ip", - "type": "string" - }, - { - "name": "common_sled_ip", - "type": "string" - }, - { - "name": "common_entrance_id", - "type": "int" - }, - { - "name": "common_subscriber_id", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "type": "long" - }, - { - "name": "http_domain", - "type": "string" - }, - { - "name": "ssl_sni", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/connection_record_log_common_server_ip.avsc b/galaxy-query-engine/schema/clickhouse/connection_record_log_common_server_ip.avsc deleted file mode 100644 index 9e717ee..0000000 --- a/galaxy-query-engine/schema/clickhouse/connection_record_log_common_server_ip.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "connection_record_log_common_server_ip", - "doc": "{\"primary_key\":\"common_log_id\",\"index_key\":\"common_server_ip\"}", - "fields": [ - { - "name": "common_log_id", - "type": "long" - }, - { - "name": "common_recv_time", - "type": "int" - }, - { - "name": "common_policy_id", - "type": "int" - }, - { - "name": "common_action", - "type": "int" - }, - { - "name": "common_server_ip", - "type": "string" - }, - { - "name": "common_client_ip", - "type": "string" - }, - { - "name": "common_sled_ip", - "type": "string" - }, - { - "name": "common_entrance_id", - "type": "int" - }, - { - "name": "common_subscriber_id", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "type": "long" - }, - { - "name": "http_domain", - "type": "string" - }, - { - "name": "ssl_sni", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/connection_record_log_common_subscriber_id.avsc b/galaxy-query-engine/schema/clickhouse/connection_record_log_common_subscriber_id.avsc deleted file mode 100644 index f8b0167..0000000 --- a/galaxy-query-engine/schema/clickhouse/connection_record_log_common_subscriber_id.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "connection_record_log_common_subscriber_id", - "doc": "{\"primary_key\":\"common_log_id\",\"index_key\":\"common_subscriber_id\"}", - "fields": [ - { - "name": "common_log_id", - "type": "long" - }, - { - "name": "common_recv_time", - "type": "int" - }, - { - "name": "common_policy_id", - "type": "int" - }, - { - "name": "common_action", - "type": "int" - }, - { - "name": "common_server_ip", - "type": "string" - }, - { - "name": "common_client_ip", - "type": "string" - }, - { - "name": "common_sled_ip", - "type": "string" - }, - { - "name": "common_entrance_id", - "type": "int" - }, - { - "name": "common_subscriber_id", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "type": "long" - }, - { - "name": "http_domain", - "type": "string" - }, - { - "name": "ssl_sni", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/connection_record_log_http_domain.avsc b/galaxy-query-engine/schema/clickhouse/connection_record_log_http_domain.avsc deleted file mode 100644 index 5d6ba28..0000000 --- a/galaxy-query-engine/schema/clickhouse/connection_record_log_http_domain.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "connection_record_log_http_domain", - "doc": "{\"primary_key\":\"common_log_id\",\"index_key\":\"http_domain\"}", - "fields": [ - { - "name": "common_log_id", - "type": "long" - }, - { - "name": "common_recv_time", - "type": "int" - }, - { - "name": "common_policy_id", - "type": "int" - }, - { - "name": "common_action", - "type": "int" - }, - { - "name": "common_server_ip", - "type": "string" - }, - { - "name": "common_client_ip", - "type": "string" - }, - { - "name": "common_sled_ip", - "type": "string" - }, - { - "name": "common_entrance_id", - "type": "int" - }, - { - "name": "common_subscriber_id", - "type": "string" - }, - { - "name": "common_stream_trace_id", - "type": "long" - }, - { - "name": "http_domain", - "type": "string" - }, - { - "name": "ssl_sni", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/processes.avsc b/galaxy-query-engine/schema/clickhouse/processes.avsc deleted file mode 100644 index 75d74a9..0000000 --- a/galaxy-query-engine/schema/clickhouse/processes.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace": "system", - "type": "record", - "name": "processes", - "fields": [ - { - "name": "query_id", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/proxy_event_log.avsc b/galaxy-query-engine/schema/clickhouse/proxy_event_log.avsc deleted file mode 100644 index 834e3b2..0000000 --- a/galaxy-query-engine/schema/clickhouse/proxy_event_log.avsc +++ /dev/null @@ -1,551 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "proxy_event_log", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_policy_id\",\"common_sub_action\",\"common_sled_ip\",\"common_server_location\",\"common_subscriber_id\",\"common_server_port\",\"http_domain\",\"http_url\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_subscriber_id\",\"common_c2s_byte_num\",\"common_s2c_byte_num\"],\"filters\":[\"common_policy_id\",\"common_sub_action\",\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_server_port\",\"common_server_location\",\"common_subscriber_id\",\"common_l4_protocol\",\"http_domain\",\"http_url\",\"http_content_type\"]},\"schema_type\":{\"HTTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"http_url\",\"http_host\",\"http_domain\",\"http_request_line\",\"http_response_line\",\"http_request_header\",\"http_response_header\",\"http_request_body\",\"http_response_body\",\"http_request_body_key\",\"http_response_body_key\",\"http_proxy_flag\",\"http_sequence\",\"http_snapshot\",\"http_cookie\",\"http_referer\",\"http_user_agent\",\"http_content_length\",\"http_content_type\",\"http_set_cookie\",\"http_version\",\"http_response_lantency_ms\",\"http_session_duration_ms\",\"http_action_file_size\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"http_url\",\"common_sub_action\"]},\"DOH\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"doh_url\",\"doh_host\",\"doh_request_line\",\"doh_response_line\",\"doh_cookie\",\"doh_referer\",\"doh_user_agent\",\"doh_content_length\",\"doh_content_type\",\"doh_set_cookie\",\"doh_version\",\"doh_message_id\",\"doh_qr\",\"doh_opcode\",\"doh_aa\",\"doh_tc\",\"doh_rd\",\"doh_ra\",\"doh_rcode\",\"doh_qdcount\",\"doh_ancount\",\"doh_nscount\",\"doh_arcount\",\"doh_qname\",\"doh_qtype\",\"doh_qclass\",\"doh_cname\",\"doh_sub\",\"doh_rr\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_client_ip\",\"doh_url\",\"doh_qname\",\"common_server_port\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"common_sub_action\",\"common_schema_type\"]}", - "fields": [ - { - "name": "common_recv_time", - "type": "int", - "label": "Receive Time", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}}" - }, - { - "name": "common_log_id", - "type": "long", - "label": "Log ID", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "type": "int", - "label": "Policy ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_subscriber_id", - "type": "string", - "label": "Subscriber ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "type": "string", - "label": "Client IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_detail,geo_asn,radius_match\", \"appendTo\":\"common_client_location,common_client_asn,common_subscriber_id\"}}" - }, - { - "name": "common_client_port", - "type": "int", - "label": "Client Port" - }, - { - "name": "common_l4_protocol", - "type": "string", - "label": "L4 Protocol" - }, - { - "name": "common_address_type", - "type": "int", - "label": "Address Type", - "doc": "{\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "type": "string", - "label": "Server IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_country,geo_asn\", \"appendTo\":\"common_server_location,common_server_asn\"}}" - }, - { - "name": "common_server_port", - "type": "int", - "label": "Server Port", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_action", - "type": "int", - "label": "Action", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"48\",\"value\":\"Manipulation\"},{\"code\":\"128\",\"value\":\"Allow\"}]}" - }, - { - "name": "common_direction", - "type": "int", - "label": "Direction", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"outbound\"},{\"code\":\"1\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "type": "int", - "label": "Entrance ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "type": "string", - "label": "Sled IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "type": "string", - "label": "Client Location" - }, - { - "name": "common_client_asn", - "type": "string", - "label": "Client ASN" - }, - { - "name": "common_server_location", - "type": "string", - "label": "Server Location" - }, - { - "name": "common_server_asn", - "type": "string", - "label": "Server ASN" - }, - { - "name": "common_sessions", - "type": "int", - "label": "Sessions", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_pkt_num", - "type": "long", - "label": "Packets Sent", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_s2c_pkt_num", - "type": "long", - "label": "Packets Received", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_byte_num", - "type": "long", - "label": "Bytes Sent" - }, - { - "name": "common_s2c_byte_num", - "type": "long", - "label": "Bytes Received" - }, - { - "name": "common_service", - "type": "int", - "label": "Service", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "type": "string", - "label": "Schema Type", - "doc": "{\"allow_query\":\"true\",\"data\":[{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"DOH\",\"value\":\"DOH\"}]}" - }, - { - "name": "common_user_tags", - "type": "string", - "label": "User Tags", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "type": "string", - "label": "Sub Action", - "doc": "{\"allow_query\":\"true\",\"data\":[{\"code\":\"allow\",\"value\":\"allow\"},{\"code\":\"deny\",\"value\":\"deny\"},{\"code\":\"monitor\",\"value\":\"monitor\"},{\"code\":\"replace\",\"value\":\"replace\"},{\"code\":\"redirect\",\"value\":\"redirect\"},{\"code\":\"insert\",\"value\":\"insert\"},{\"code\":\"hijack\",\"value\":\"hijack\"}]}" - }, - { - "name": "common_user_region", - "type": "string", - "label": "User Region", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "type": "string", - "label": "Device ID" - }, - { - "name": "common_link_id", - "type": "int", - "label": "Link ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "type": "string", - "label": "ISP", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_encapsulation", - "type": "int", - "label": "Encapsulation", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_label", - "type": "string", - "label": "Application Label", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_protocol_label", - "type": "string", - "label": "Protocol Label", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_start_time", - "type": "int", - "label": "Start Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_end_time", - "type": "int", - "label": "End Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_establish_latency_ms", - "type": "int", - "label": "Establish Latency(ms)" - }, - { - "name": "common_con_duration_ms", - "type": "int", - "label": "Duration(ms)" - }, - { - "name": "common_stream_dir", - "type": "int", - "label": "Stream Direction", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "type": "string", - "label": "Address List", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_has_dup_traffic", - "type": "int", - "label": "Duplication Traffic", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "type": "string", - "label": "Stream Error", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_trace_id", - "type": "long", - "label": "Session ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_url", - "type": "string", - "label": "Http.URL" - }, - { - "name": "http_host", - "type": "string", - "label": "Http.Host", - "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "http_domain", - "type": "string", - "label": "Http.Domain", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "http_request_line", - "type": "string", - "label": "Http.Request Line", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_line", - "type": "string", - "label": "Http.Response Line", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_header", - "type": "string", - "label": "Http.Request Header", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_header", - "type": "string", - "label": "Http.Response Header", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_body", - "type": "string", - "label": "Http.Request Body", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_response_body", - "type": "string", - "label": "Http.Response Body", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_request_body_key", - "type": "string", - "label": "Http.Request Body Key", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_response_body_key", - "type": "string", - "label": "Http.Response Body Key", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_proxy_flag", - "type": "int", - "label": "Http.Proxy Flag", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_sequence", - "type": "int", - "label": "Http.Sequence", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_snapshot", - "type": "string", - "label": "Http.Snapshot", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_cookie", - "type": "string", - "label": "Http.Cookie" - }, - { - "name": "http_referer", - "type": "string", - "label": "Http.Referer" - }, - { - "name": "http_user_agent", - "type": "string", - "label": "Http.User Agent" - }, - { - "name": "http_content_length", - "type": "string", - "label": "Http.Content Length" - }, - { - "name": "http_content_type", - "type": "string", - "label": "Http.Content Type" - }, - { - "name": "http_set_cookie", - "type": "string", - "label": "Http.Set Cookie" - }, - { - "name": "http_version", - "type": "string", - "label": "Http.Version" - }, - { - "name": "http_response_lantency_ms", - "type": "int", - "label": "Http.Response Latency(ms)" - }, - { - "name": "http_session_duration_ms", - "type": "int", - "label": "Http.Session Duration(ms)" - }, - { - "name": "http_action_file_size", - "type": "int", - "label": "Http.Action File Size" - }, - { - "name": "doh_url", - "label": "DoH.URL", - "type": "string" - }, - { - "name": "doh_host", - "label": "DoH.Host", - "type": "string" - }, - { - "name": "doh_request_line", - "label": "DoH.Request Line", - "doc": "{\"visibility\":\"hidden\"}", - "type": "string" - }, - { - "name": "doh_response_line", - "doc": "{\"visibility\":\"hidden\"}", - "label": "DoH.Response Line", - "type": "string" - }, - { - "name": "doh_cookie", - "label": "DoH.Cookie", - "type": "string" - }, - { - "name": "doh_referer", - "label": "DoH.Referer", - "type": "string" - }, - { - "name": "doh_user_agent", - "label": "DoH.User Agent", - "type": "string" - }, - { - "name": "doh_content_length", - "label": "DoH.Content Length", - "type": "string" - }, - { - "name": "doh_content_type", - "label": "DoH.Content Type", - "type": "string" - }, - { - "name": "doh_set_cookie", - "label": "DoH.Set Cookie", - "type": "string" - }, - { - "name": "doh_version", - "label": "DoH.Version", - "type": "string" - }, - { - "name": "doh_message_id", - "label": "DoH.Message ID", - "type": "int" - }, - { - "name": "doh_qr", - "label": "DoH.QR", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"REESPONSE\"}]}", - "type": "int" - }, - { - "name": "doh_opcode", - "label": "DoH.OPCODE", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"IQUERY\"},{\"code\":\"2\",\"value\":\"STATUS\"},{\"code\":\"5\",\"value\":\"UPDATE\"}]}", - "type": "int" - }, - { - "name": "doh_aa", - "label": "DoH.AA", - "type": "int" - }, - { - "name": "doh_tc", - "label": "DoH.TC", - "type": "int" - }, - { - "name": "doh_rd", - "label": "DoH.RD", - "type": "int" - }, - { - "name": "doh_ra", - "label": "DoH.RA", - "type": "int" - }, - { - "name": "doh_rcode", - "label": "DoH.RCODE", - "type": "int" - }, - { - "name": "doh_qdcount", - "label": "DoH.QDCOUNT", - "type": "int" - }, - { - "name": "doh_ancount", - "label": "DoH.ANCOUNT", - "type": "int" - }, - { - "name": "doh_nscount", - "label": "DoH.NSCOUNT", - "type": "int" - }, - { - "name": "doh_arcount", - "label": "DoH.ARCOUNT", - "type": "int" - }, - { - "name": "doh_qname", - "label": "DoH.QNAME", - "type": "string" - }, - { - "name": "doh_qtype", - "label": "DoH.QTYPE", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"A\"},{\"code\":\"2\",\"value\":\"NS\"},{\"code\":\"5\",\"value\":\"CNAME\"},{\"code\":\"6\",\"value\":\"SOA\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"12\",\"value\":\"PTR\"},{\"code\":\"13\",\"value\":\"HINFO\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"15\",\"value\":\"MX\"},{\"code\":\"28\",\"value\":\"AAAA\"}]}", - "type": "int" - }, - { - "name": "doh_qclass", - "label": "DoH.QCLASS", - "type": "int" - }, - { - "name": "doh_cname", - "label": "DoH.CNAME", - "type": "string" - }, - { - "name": "doh_sub", - "label": "DoH.SUB", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"DNS\"},{\"code\":\"2\",\"value\":\"DNSSEC\"}]}", - "type": "int" - }, - { - "name": "doh_rr", - "label": "DoH.RR", - "type": "string" - } - ] -} diff --git a/galaxy-query-engine/schema/clickhouse/proxy_ip_info.avsc b/galaxy-query-engine/schema/clickhouse/proxy_ip_info.avsc deleted file mode 100644 index 87797bd..0000000 --- a/galaxy-query-engine/schema/clickhouse/proxy_ip_info.avsc +++ /dev/null @@ -1,22 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "proxy_ip_info", - "fields": [ - { - "name": "stat_time", - "type": "int" - }, - { - "name": "policy_id", - "type": "int" - }, - { - "name": "ip_list", - "type": { - "type": "array", - "items": "string" - } - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/query_log.avsc b/galaxy-query-engine/schema/clickhouse/query_log.avsc deleted file mode 100644 index 4f5e8d5..0000000 --- a/galaxy-query-engine/schema/clickhouse/query_log.avsc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "namespace": "system", - "type": "record", - "name": "query_log", - "fields": [ - { - "name": "query_id", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/radius_onff_log.avsc b/galaxy-query-engine/schema/clickhouse/radius_onff_log.avsc deleted file mode 100644 index 0e475c4..0000000 --- a/galaxy-query-engine/schema/clickhouse/radius_onff_log.avsc +++ /dev/null @@ -1,37 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "radius_onff_log", - "fields": [ - { - "name": "event_timestamp", - "type": "int", - "label": "Event Time" - }, - { - "name": "account", - "type": "string", - "label": "Account" - }, - { - "name": "framed_ip", - "type": "string", - "label": "Framed IP" - }, - { - "name": "acct_session_id", - "type": "string", - "label": "Acct Session ID" - }, - { - "name": "acct_status_type", - "type": "int", - "label": "Acct Status Type" - }, - { - "name": "acct_session_time", - "type": "int", - "label": "Acct Session Time" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/radius_record_log.avsc b/galaxy-query-engine/schema/clickhouse/radius_record_log.avsc deleted file mode 100644 index 981a031..0000000 --- a/galaxy-query-engine/schema/clickhouse/radius_record_log.avsc +++ /dev/null @@ -1,409 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "radius_record_log", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"schema_query\":{\"dimensions\":[\"radius_nas_ip\",\"radius_framed_ip\",\"common_subscriber_id\"],\"metrics\":[\"radius_framed_ip\",\"radius_event_timestamp\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\"],\"filters\":[\"radius_framed_ip\",\"common_subscriber_id\",\"radius_packet_type\",\"radius_acct_session_id\",\"radius_acct_multi_session_id\",\"radius_acct_status_type\"]},\"schema_type\":{\"RADIUS\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"radius_packet_type\",\"radius_nas_ip\",\"radius_framed_ip\",\"radius_account\",\"radius_session_timeout\",\"radius_idle_timeout\",\"radius_acct_status_type\",\"radius_acct_terminate_cause\",\"radius_event_timestamp\",\"radius_nas_port\",\"radius_service_type\",\"radius_framed_protocol\",\"radius_callback_number\",\"radius_callback_id\",\"radius_termination_action\",\"radius_called_station_id\",\"radius_calling_station_id\",\"radius_acct_delay_time\",\"radius_acct_session_id\",\"radius_acct_multi_session_id\",\"radius_acct_input_octets\",\"radius_acct_output_octets\",\"radius_acct_input_packets\",\"radius_acct_output_packets\",\"radius_acct_session_time\",\"radius_acct_link_count\",\"radius_acct_interim_interval\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"radius_nas_ip\",\"radius_framed_ip\",\"radius_acct_status_type\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_subscriber_id\",\"radius_nas_ip\",\"radius_framed_ip\",\"radius_acct_status_type\"]}", - "fields": [ - { - "name": "common_recv_time", - "type": "int", - "label": "Receive Time", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}}" - }, - { - "name": "common_log_id", - "type": "long", - "label": "Log ID", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "type": "int", - "label": "Policy ID", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_subscriber_id", - "type": "string", - "label": "Subscriber ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "type": "string", - "label": "Client IP", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_client_port", - "type": "int", - "label": "Client Port", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_l4_protocol", - "type": "string", - "label": "L4 Protocol" - }, - { - "name": "common_address_type", - "type": "int", - "label": "Address Type", - "doc": "{\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "type": "string", - "label": "Server IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_country,geo_asn\", \"appendTo\":\"common_server_location,common_server_asn\"}}" - }, - { - "name": "common_server_port", - "type": "int", - "label": "Server Port", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_action", - "type": "int", - "label": "Action", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"48\",\"value\":\"Manipulation\"},{\"code\":\"128\",\"value\":\"Allow\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_direction", - "type": "int", - "label": "Direction", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"outbound\"},{\"code\":\"1\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "type": "int", - "label": "Entrance ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "type": "string", - "label": "Sled IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "type": "string", - "label": "Client Location" - }, - { - "name": "common_client_asn", - "type": "string", - "label": "Client ASN" - }, - { - "name": "common_server_location", - "type": "string", - "label": "Server Location" - }, - { - "name": "common_server_asn", - "type": "string", - "label": "Server ASN" - }, - { - "name": "common_sessions", - "type": "int", - "label": "Sessions", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_pkt_num", - "type": "long", - "label": "Packets Sent" - }, - { - "name": "common_s2c_pkt_num", - "type": "long", - "label": "Packets Received" - }, - { - "name": "common_c2s_byte_num", - "type": "long", - "label": "Bytes Sent" - }, - { - "name": "common_s2c_byte_num", - "type": "long", - "label": "Bytes Received" - }, - { - "name": "common_service", - "type": "int", - "label": "Service", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "type": "string", - "label": "Schema Type", - "doc": "{\"data\":[{\"code\":\"BASE\",\"value\":\"BASE\"},{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"MAIL\",\"value\":\"MAIL\"},{\"code\":\"DNS\",\"value\":\"DNS\"},{\"code\":\"SSL\",\"value\":\"SSL\"},{\"code\":\"FTP\",\"value\":\"FTP\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_user_tags", - "type": "string", - "label": "User Tags", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "type": "string", - "label": "Sub Action", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_user_region", - "type": "string", - "label": "User Region", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "type": "string", - "label": "Device ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_link_id", - "type": "int", - "label": "Link ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "type": "string", - "label": "ISP", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_encapsulation", - "type": "int", - "label": "Encapsulation", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_label", - "type": "string", - "label": "Application Label", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_protocol_label", - "type": "string", - "label": "Protocol Label", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_start_time", - "type": "int", - "label": "Start Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"visibility\":\"hidden\"}" - }, - { - "name": "common_end_time", - "type": "int", - "label": "End Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"visibility\":\"hidden\"}" - }, - { - "name": "common_establish_latency_ms", - "type": "int", - "label": "Establish Latency(ms)", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_con_duration_ms", - "type": "int", - "label": "Duration(ms)", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_dir", - "type": "int", - "label": "Stream Direction", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "type": "string", - "label": "Address List", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_has_dup_traffic", - "type": "int", - "label": "Duplication Traffic", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "type": "string", - "label": "Stream Error", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_trace_id", - "type": "long", - "label": "Session ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "radius_packet_type", - "type": "int", - "label": "Packet Type", - "doc": "{\"data\":[{\"code\":\"1\",\"value\": \"Access-Request\"},{\"code\": \"2\",\"value\": \"Access-Accept\"},{\"code\": \"3\",\"value\": \"Access-Reject\"},{\"code\": \"4\",\"value\": \"Accounting-Request\"},{\"code\": \"5\",\"value\": \"Accounting-Response\"},{\"code\": \"11\",\"value\": \"Access-Challenge\"}]}" - }, - { - "name": "radius_account", - "type": "string", - "label": "Account", - "doc": "{\"format\":{\"functions\":\"get_value\",\"appendTo\":\"common_subscriber_id\"}}" - }, - { - "name": "radius_nas_ip", - "type": "string", - "label": "Nas IP" - }, - { - "name": "radius_framed_ip", - "type": "string", - "label": "Framed IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_detail,geo_asn\", \"appendTo\":\"common_client_location,common_client_asn\"}}" - }, - { - "name": "radius_session_timeout", - "type": "int", - "label": "Session Timeout" - }, - { - "name": "radius_idle_timeout", - "type": "int", - "label": "Idle Timeout" - }, - { - "name": "radius_acct_status_type", - "type": "int", - "label": "ACC Status Type", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"Start\"},{\"code\":\"2\",\"value\":\"Stop\"},{\"code\":\"3\",\"value\":\"Interim-Update\"},{\"code\":\"7\",\"value\":\"Accounting-On\"},{\"code\":\"8\",\"value\":\"Accounting-Off\"}]}" - }, - { - "name": "radius_acct_terminate_cause", - "type": "int", - "label": "Acct Terminate Cause", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"User Request\"},{\"code\":\"2\",\"value\":\"Lost Carrier\"},{\"code\":\"3\",\"value\":\"Lost Service\"},{\"code\":\"4\",\"value\":\"Idle Timeout\"},{\"code\":\"5\",\"value\":\"Session Timeout\"},{\"code\":\"6\",\"value\":\"Admin Reset\"},{\"code\":\"7\",\"value\":\"Admin Reboot\"},{\"code\":\"8\",\"value\":\"Port Error\"},{\"code\":\"9\",\"value\":\"NAS Error\"},{\"code\":\"10\",\"value\":\"NAS Request\"},{\"code\":\"11\",\"value\":\"NAS Reboot\"},{\"code\":\"12\",\"value\":\"Port Unneeded\"},{\"code\":\"13\",\"value\":\"Port Preempted\"},{\"code\":\"14\",\"value\":\"Port Suspended\"},{\"code\":\"15\",\"value\":\"Service Unavailable\"},{\"code\":\"16\",\"value\":\"Callback\"},{\"code\":\"17\",\"value\":\"User Error\"},{\"code\":\"18\",\"value\":\"Host Request\"}]}" - }, - { - "name": "radius_event_timestamp", - "type": "int", - "label": "Event Timestamp" - }, - { - "name": "radius_service_type", - "type": "int", - "label": "Service Type" - }, - { - "name": "radius_nas_port", - "type": "int", - "label": "Nas Port" - }, - { - "name": "radius_framed_protocol", - "type": "int", - "label": "Framed Protocol" - }, - { - "name": "radius_callback_number", - "type": "string", - "label": "Callback Number" - }, - { - "name": "radius_callback_id", - "type": "string", - "label": "Callback ID" - }, - { - "name": "radius_termination_action", - "type": "int", - "label": "Termination Action" - }, - { - "name": "radius_called_station_id", - "type": "string", - "label": "Called Station Id" - }, - { - "name": "radius_calling_station_id", - "type": "string", - "label": "Calling Station Id" - }, - { - "name": "radius_acct_delay_time", - "type": "int", - "label": "Acct Delay Time" - }, - { - "name": "radius_acct_session_id", - "type": "string", - "label": "Acct Session ID" - }, - { - "name": "radius_acct_multi_session_id", - "type": "string", - "label": "Acct Multi Session ID" - }, - { - "name": "radius_acct_input_octets", - "type": "int", - "label": "Acct Input Octets" - }, - { - "name": "radius_acct_output_octets", - "type": "int", - "label": "Acct Output Octets" - }, - { - "name": "radius_acct_input_packets", - "type": "int", - "label": "Acct Input Packets" - }, - { - "name": "radius_acct_output_packets", - "type": "int", - "label": "Acct Output Packets" - }, - { - "name": "radius_acct_session_time", - "type": "int", - "label": "Acct Session Time" - }, - { - "name": "radius_acct_link_count", - "type": "int", - "label": "Acct Link Count" - }, - { - "name": "radius_acct_interim_interval", - "type": "int", - "label": "Acct Interim Interval" - } - ] -} diff --git a/galaxy-query-engine/schema/clickhouse/security_event_log.avsc b/galaxy-query-engine/schema/clickhouse/security_event_log.avsc deleted file mode 100644 index 0a58bca..0000000 --- a/galaxy-query-engine/schema/clickhouse/security_event_log.avsc +++ /dev/null @@ -1,744 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "security_event_log", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\",\"schema_query\":{\"dimensions\":[\"common_server_ip\",\"common_client_ip\",\"common_policy_id\",\"common_action\",\"common_sled_ip\",\"common_server_location\",\"common_subscriber_id\",\"common_server_port\",\"http_domain\",\"ssl_sni\"],\"metrics\":[\"common_server_ip\",\"common_client_ip\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\"],\"filters\":[\"common_policy_id\",\"common_action\",\"common_address_type\",\"common_server_ip\",\"common_client_ip\",\"common_server_port\",\"common_server_location\",\"common_subscriber_id\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_l4_protocol\",\"http_domain\",\"http_content_type\",\"ssl_sni\",\"ssl_pinningst\",\"ssl_intercept_state\"]},\"schema_type\":{\"BASE\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\"]},\"HTTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"http_url\",\"http_host\",\"http_domain\",\"http_request_line\",\"http_response_line\",\"http_request_header\",\"http_response_header\",\"http_request_body\",\"http_response_body\",\"http_request_body_key\",\"http_response_body_key\",\"http_proxy_flag\",\"http_sequence\",\"http_snapshot\",\"http_cookie\",\"http_referer\",\"http_user_agent\",\"http_content_length\",\"http_content_type\",\"http_set_cookie\",\"http_version\",\"http_response_lantency_ms\",\"http_session_duration_ms\",\"http_action_file_size\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"http_url\",\"common_server_port\"]},\"MAIL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"mail_protocol_type\",\"mail_account\",\"mail_from_cmd\",\"mail_to_cmd\",\"mail_from\",\"mail_to\",\"mail_cc\",\"mail_bcc\",\"mail_subject\",\"mail_subject_charset\",\"mail_content\",\"mail_content_charset\",\"mail_attachment_name\",\"mail_attachment_name_charset\",\"mail_attachment_content\",\"mail_eml_file\",\"mail_snapshot\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"mail_from\",\"mail_to\",\"mail_subject\"]},\"DNS\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"dns_message_id\",\"dns_qr\",\"dns_opcode\",\"dns_aa\",\"dns_tc\",\"dns_rd\",\"dns_ra\",\"dns_rcode\",\"dns_qdcount\",\"dns_ancount\",\"dns_nscount\",\"dns_arcount\",\"dns_qname\",\"dns_qtype\",\"dns_qclass\",\"dns_cname\",\"dns_sub\",\"dns_rr\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_client_ip\",\"dns_qr\",\"dns_qname\",\"dns_qtype\"]},\"SSL\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ssl_sni\",\"ssl_san\",\"ssl_cn\",\"ssl_pinningst\",\"ssl_intercept_state\",\"ssl_server_side_latency\",\"ssl_client_side_latency\",\"ssl_server_side_version\",\"ssl_client_side_version\",\"ssl_cert_verify\",\"ssl_error\",\"ssl_con_latency_ms\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"ssl_sni\",\"common_server_ip\",\"common_server_port\"]},\"QUIC\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"quic_version\",\"quic_sni\",\"quic_user_agent\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"quic_sni\",\"common_server_ip\",\"common_server_port\"]},\"FTP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"ftp_account\",\"ftp_url\",\"ftp_content\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"ftp_url\",\"common_server_ip\",\"common_server_port\"]},\"BGP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"bgp_type\",\"bgp_as_num\",\"bgp_route\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"bgp_type\",\"bgp_as_num\",\"common_server_ip\",\"common_server_port\"]},\"VOIP\":{\"columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_client_port\",\"common_l4_protocol\",\"common_address_type\",\"common_server_ip\",\"common_server_port\",\"common_action\",\"common_direction\",\"common_entrance_id\",\"common_sled_ip\",\"common_client_location\",\"common_client_asn\",\"common_server_location\",\"common_server_asn\",\"common_sessions\",\"common_c2s_pkt_num\",\"common_s2c_pkt_num\",\"common_c2s_byte_num\",\"common_s2c_byte_num\",\"common_service\",\"common_schema_type\",\"common_user_tags\",\"common_sub_action\",\"common_user_region\",\"common_device_id\",\"common_link_id\",\"common_isp\",\"common_encapsulation\",\"common_app_label\",\"common_protocol_label\",\"common_start_time\",\"common_end_time\",\"common_establish_latency_ms\",\"common_con_duration_ms\",\"common_stream_dir\",\"common_address_list\",\"common_has_dup_traffic\",\"common_stream_error\",\"common_stream_trace_id\",\"common_link_info_c2s\",\"common_link_info_s2c\",\"voip_calling_account\",\"voip_called_account\",\"voip_calling_number\",\"voip_called_number\"],\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"voip_calling_account\",\"voip_called_account\",\"common_server_ip\",\"common_server_port\"]}},\"default_columns\":[\"common_recv_time\",\"common_log_id\",\"common_policy_id\",\"common_subscriber_id\",\"common_client_ip\",\"common_server_ip\",\"common_server_port\",\"common_schema_type\"]}", - "fields": [ - { - "name": "common_recv_time", - "type": "int", - "label": "Receive Time", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}}" - }, - { - "name": "common_log_id", - "type": "long", - "label": "Log ID", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "type": "int", - "label": "Policy ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_subscriber_id", - "type": "string", - "label": "Subscriber ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "type": "string", - "label": "Client IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_detail,geo_asn,radius_match\", \"appendTo\":\"common_client_location,common_client_asn,common_subscriber_id\"}}" - }, - { - "name": "common_client_port", - "type": "int", - "label": "Client Port" - }, - { - "name": "common_l4_protocol", - "type": "string", - "label": "L4 Protocol" - }, - { - "name": "common_address_type", - "type": "int", - "label": "Address Type", - "doc": "{\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "type": "string", - "label": "Server IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"},\"format\":{\"functions\":\"geo_ip_country,geo_asn\", \"appendTo\":\"common_server_location,common_server_asn\"}}" - }, - { - "name": "common_server_port", - "type": "int", - "label": "Server Port", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_action", - "type": "int", - "label": "Action", - "doc": "{\"allow_query\":\"true\",\"data\":[{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"128\",\"value\":\"Allow\"}]}" - }, - { - "name": "common_direction", - "type": "int", - "label": "Direction", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"outbound\"},{\"code\":\"1\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "type": "int", - "label": "Entrance ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "type": "string", - "label": "Sled IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "type": "string", - "label": "Client Location" - }, - { - "name": "common_client_asn", - "type": "string", - "label": "Client ASN" - }, - { - "name": "common_server_location", - "type": "string", - "label": "Server Location" - }, - { - "name": "common_server_asn", - "type": "string", - "label": "Server ASN" - }, - { - "name": "common_sessions", - "type": "int", - "label": "Sessions", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_c2s_pkt_num", - "type": "long", - "label": "Packets Sent" - }, - { - "name": "common_s2c_pkt_num", - "type": "long", - "label": "Packets Received" - }, - { - "name": "common_c2s_byte_num", - "type": "long", - "label": "Bytes Sent" - }, - { - "name": "common_s2c_byte_num", - "type": "long", - "label": "Bytes Received" - }, - { - "name": "common_service", - "type": "int", - "label": "Service", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "type": "string", - "label": "Schema Type", - "doc": "{\"data\":[{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"MAIL\",\"value\":\"MAIL\"},{\"code\":\"DNS\",\"value\":\"DNS\"},{\"code\":\"SSL\",\"value\":\"SSL\"},{\"code\":\"QUIC\",\"value\":\"QUIC\"},{\"code\":\"FTP\",\"value\":\"FTP\"}],\"allow_query\":\"true\"}" - }, - { - "name": "common_user_tags", - "type": "string", - "label": "User Tags", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "type": "string", - "label": "Sub Action" - }, - { - "name": "common_user_region", - "type": "string", - "label": "User Region", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "type": "string", - "label": "Device ID" - }, - { - "name": "common_link_id", - "type": "int", - "label": "Link ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "type": "string", - "label": "ISP", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_encapsulation", - "type": "int", - "label": "Encapsulation", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_app_label", - "type": "string", - "label": "Application Label", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_protocol_label", - "type": "string", - "label": "Protocol Label", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_start_time", - "type": "int", - "label": "Start Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_end_time", - "type": "int", - "label": "End Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"}}" - }, - { - "name": "common_establish_latency_ms", - "type": "int", - "label": "Establish Latency(ms)" - }, - { - "name": "common_con_duration_ms", - "type": "int", - "label": "Duration(ms)" - }, - { - "name": "common_stream_dir", - "type": "int", - "label": "Stream Direction", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "type": "string", - "label": "Address List", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_has_dup_traffic", - "type": "int", - "label": "Duplication Traffic", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "type": "string", - "label": "Stream Error", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_trace_id", - "type": "long", - "label": "Session ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_url", - "type": "string", - "label": "Http.URL", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "http_host", - "type": "string", - "label": "Http.Host", - "doc": "{\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}" - }, - { - "name": "http_domain", - "type": "string", - "label": "Http.Domain", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "http_request_line", - "type": "string", - "label": "Http.Request Line", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_line", - "type": "string", - "label": "Http.Response Line", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_header", - "type": "string", - "label": "Http.Request Header", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_response_header", - "type": "string", - "label": "Http.Response Header", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_request_body", - "type": "string", - "label": "Http.Request Body", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_response_body", - "type": "string", - "label": "Http.Response Body", - "doc": "{\"constraints\":{\"type\":\"file\"}}" - }, - { - "name": "http_request_body_key", - "type": "string", - "label": "Http.Request Body Key", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_response_body_key", - "type": "string", - "label": "Http.Response Body Key", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "http_proxy_flag", - "type": "int", - "label": "Http.Proxy Flag", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_sequence", - "type": "int", - "label": "Http.Sequence", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_snapshot", - "type": "string", - "label": "Http.Snapshot", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "http_cookie", - "type": "string", - "label": "Http.Cookie" - }, - { - "name": "http_referer", - "type": "string", - "label": "Http.Referer" - }, - { - "name": "http_user_agent", - "type": "string", - "label": "Http.User Agent" - }, - { - "name": "http_content_length", - "type": "string", - "label": "Http.Content Length" - }, - { - "name": "http_content_type", - "type": "string", - "label": "Http.Content Type" - }, - { - "name": "http_set_cookie", - "type": "string", - "label": "Http.Set Cookie" - }, - { - "name": "http_version", - "type": "string", - "label": "Http.Version" - }, - { - "name": "http_response_lantency_ms", - "type": "int", - "label": "Http.Response Latency(ms)" - }, - { - "name": "http_action_file_size", - "type": "int", - "label": "Http.Action File Size" - }, - { - "name": "http_session_duration_ms", - "type": "int", - "label": "Http.Session Duration(ms)" - }, - { - "name": "mail_protocol_type", - "type": "string", - "label": "Mail.Protocol Type" - }, - { - "name": "mail_account", - "type": "string", - "label": "Mail.Account" - }, - { - "name": "mail_from_cmd", - "type": "string", - "label": "Mail.From CMD" - }, - { - "name": "mail_to_cmd", - "type": "string", - "label": "Mail.To CMD" - }, - { - "name": "mail_from", - "doc": "{\"constraints\":{\"type\":\"email\"}}", - "type": "string", - "label": "Mail.From" - }, - { - "name": "mail_to", - "doc": "{\"constraints\":{\"type\":\"email\"}}", - "type": "string", - "label": "Mail.To" - }, - { - "name": "mail_cc", - "type": "string", - "label": "Mail.CC" - }, - { - "name": "mail_bcc", - "type": "string", - "label": "Mail.BCC" - }, - { - "name": "mail_subject", - "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"mail_subject_charset\"}}", - "type": "string", - "label": "Mail.Subject" - }, - { - "name": "mail_subject_charset", - "type": "string", - "label": "Mail.Subject Charset", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "mail_content", - "type": "string", - "label": "Mail.Content", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_content_charset", - "type": "string", - "label": "Mail.Content Charset", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_attachment_name", - "doc": "{\"format\":{\"functions\":\"decode_of_base64\",\"param\":\"mail_attachment_name_charset\"}}", - "type": "string", - "label": "Mail.Attachment" - }, - { - "name": "mail_attachment_name_charset", - "type": "string", - "label": "Mail.Attachment Charset", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_attachment_content", - "type": "string", - "label": "Mail.Attachment Content", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "mail_eml_file", - "doc": "{\"constraints\":{\"type\":\"file\"}}", - "type": "string", - "label": "Mail.EML File" - }, - { - "name": "mail_snapshot", - "type": "string", - "label": "Mail.Snapshot", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "dns_message_id", - "type": "int", - "label": "Dns.Message ID" - }, - { - "name": "dns_qr", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"REESPONSE\"}]}", - "type": "int", - "label": "Dns.QR" - }, - { - "name": "dns_opcode", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"QUERY\"},{\"code\":\"1\",\"value\":\"IQUERY\"},{\"code\":\"2\",\"value\":\"STATUS\"},{\"code\":\"5\",\"value\":\"UPDATE\"}]}", - "type": "int", - "label": "Dns.OPCODE" - }, - { - "name": "dns_aa", - "type": "int", - "label": "Dns.AA" - }, - { - "name": "dns_tc", - "type": "int", - "label": "Dns.TC" - }, - { - "name": "dns_rd", - "type": "int", - "label": "Dns.RD" - }, - { - "name": "dns_ra", - "type": "int", - "label": "Dns.RA" - }, - { - "name": "dns_rcode", - "type": "int", - "label": "Dns.RCODE" - }, - { - "name": "dns_qdcount", - "type": "int", - "label": "Dns.QDCOUNT" - }, - { - "name": "dns_ancount", - "type": "int", - "label": "Dns.ANCOUNT" - }, - { - "name": "dns_nscount", - "type": "int", - "label": "Dns.NSCOUNT" - }, - { - "name": "dns_arcount", - "type": "int", - "label": "Dns.ARCOUNT" - }, - { - "name": "dns_qname", - "type": "string", - "label": "Dns.QNAME" - }, - { - "name": "dns_qtype", - "type": "int", - "label": "Dns.QTYPE", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"A\"},{\"code\":\"2\",\"value\":\"NS\"},{\"code\":\"5\",\"value\":\"CNAME\"},{\"code\":\"6\",\"value\":\"SOA\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"12\",\"value\":\"PTR\"},{\"code\":\"13\",\"value\":\"HINFO\"},{\"code\":\"11\",\"value\":\"WKS\"},{\"code\":\"15\",\"value\":\"MX\"},{\"code\":\"28\",\"value\":\"AAAA\"}]}" - }, - { - "name": "dns_qclass", - "type": "int", - "label": "Dns.QCLASS" - }, - { - "name": "dns_cname", - "type": "string", - "label": "Dns.CNAME" - }, - { - "name": "dns_sub", - "type": "int", - "label": "Dns.SUB", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"DNS\"},{\"code\":\"2\",\"value\":\"DNSSEC\"}]}" - }, - { - "name": "dns_rr", - "type": "string", - "label": "Dns.RR" - }, - { - "name": "ssl_version", - "type": "string", - "label": "SSL.Version", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "ssl_sni", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"sub_domain\",\"appendTo\":\"http_domain\"}}", - "type": "string", - "label": "SSL.SNI" - }, - { - "name": "ssl_san", - "type": "string", - "label": "SSL.SAN" - }, - { - "name": "ssl_cn", - "type": "string", - "label": "SSL.CN" - }, - { - "name": "ssl_pinningst", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Not Pinning\"},{\"code\":\"1\",\"value\":\"Pinning\"},{\"code\":\"2\",\"value\":\"Maybe Pinning\"}]}", - "type": "int", - "label": "SSL.Pinning" - }, - { - "name": "ssl_intercept_state", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Passthrough\"},{\"code\":\"1\",\"value\":\"Intercept\"},{\"code\":\"2\",\"value\":\"Shutdown\"}]}", - "type": "int", - "label": "SSL.Intercept State" - }, - { - "name": "ssl_server_side_latency", - "type": "int", - "label": "SSL.Server Side Latency(ms)" - }, - { - "name": "ssl_client_side_latency", - "type": "int", - "label": "SSL.Client Side Latency(ms)" - }, - { - "name": "ssl_server_side_version", - "type": "string", - "label": "SSL.Server Side Version" - }, - { - "name": "ssl_client_side_version", - "type": "string", - "label": "SSL.Client Side Version" - }, - { - "name": "ssl_cert_verify", - "type": "int", - "label": "SSL.Certificate Verify", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}]}" - }, - { - "name": "ssl_error", - "type": "string", - "label": "SSL.Error" - }, - { - "name": "ssl_con_latency_ms", - "type": "int", - "label": "SSL.Connection Latency(ms)", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "quic_version", - "type": "string", - "label": "Quic.Version" - }, - { - "name": "quic_sni", - "type": "string", - "label": "Quic.SNI", - "doc": "{\"format\":{\"functions\":\"sub_domain\", \"appendTo\":\"http_domain\"}}" - }, - { - "name": "quic_user_agent", - "type": "string", - "label": "Quic.User Agent" - }, - { - "name": "ftp_account", - "type": "string", - "label": "Ftp.Account" - }, - { - "name": "ftp_url", - "type": "string", - "label": "Ftp.URL" - }, - { - "name": "ftp_content", - "type": "string", - "label": "Ftp.Content" - }, - { - "name": "bgp_type", - "type": "int", - "label": "BGP.Type", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "bgp_as_num", - "type": "string", - "label": "BGP.AS Number", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "bgp_route", - "type": "string", - "label": "BGP.Route", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_calling_account", - "type": "string", - "label": "Voip.Calling Account", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_called_account", - "type": "string", - "label": "Voip.Called Account", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_calling_number", - "type": "string", - "label": "Voip.Calling Number", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "voip_called_number", - "type": "string", - "label": "Voip.Called Number", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "streaming_media_url", - "type": "string", - "label": "Streaming.Media URL", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "streaming_media_protocol", - "type": "string", - "label": "Streaming.Media Protocol", - "doc": "{\"visibility\":\"disabled\"}" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/security_ip_info.avsc b/galaxy-query-engine/schema/clickhouse/security_ip_info.avsc deleted file mode 100644 index e4d8ad9..0000000 --- a/galaxy-query-engine/schema/clickhouse/security_ip_info.avsc +++ /dev/null @@ -1,33 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "security_ip_info", - "fields": [ - { - "name": "stat_time", - "type": "int" - }, - { - "name": "policy_id", - "type": "int" - }, - { - "name": "ip", - "type": "string" - }, - { - "name": "domain_list", - "type": { - "type": "array", - "items": "string" - } - }, - { - "name": "port_list", - "type": { - "type": "array", - "items": "int" - } - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/security_website_domain_info.avsc b/galaxy-query-engine/schema/clickhouse/security_website_domain_info.avsc deleted file mode 100644 index be084e6..0000000 --- a/galaxy-query-engine/schema/clickhouse/security_website_domain_info.avsc +++ /dev/null @@ -1,47 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "security_website_domain_info", - "fields": [ - { - "name": "stat_time", - "type": "int" - }, - { - "name": "policy_id", - "type": "int" - }, - { - "name": "domain", - "type": "string" - }, - { - "name": "ip_list", - "type": { - "type": "array", - "items": "string" - } - }, - { - "name": "cdn_list", - "type": { - "type": "array", - "items": "string" - } - }, - { - "name": "protocol_type_list", - "type": { - "type": "array", - "items": "string" - } - }, - { - "name": "port_list", - "type": { - "type": "array", - "items": "int" - } - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/clickhouse/sys_packet_capture_log.avsc b/galaxy-query-engine/schema/clickhouse/sys_packet_capture_log.avsc deleted file mode 100644 index 73d3c42..0000000 --- a/galaxy-query-engine/schema/clickhouse/sys_packet_capture_log.avsc +++ /dev/null @@ -1,291 +0,0 @@ -{ - "namespace": "tsg_galaxy_v3", - "type": "record", - "name": "sys_packet_capture_log", - "doc": "{\"primary_key\":\"common_log_id\",\"partition_key\":\"common_recv_time\"}", - "fields": [ - { - "name": "common_recv_time", - "type": "int", - "label": "Receive Time", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"timestamp\"},\"format\":{\"functions\":\"current_timestamp\"}}}" - }, - { - "name": "common_log_id", - "type": "long", - "label": "Log ID", - "doc": "{\"allow_query\":\"true\",\"format\":{\"functions\":\"snowflake_id\"}}" - }, - { - "name": "common_policy_id", - "type": "int", - "label": "Policy ID", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_subscriber_id", - "type": "string", - "label": "Subscriber ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_client_ip", - "type": "string", - "label": "Client IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_port", - "type": "int", - "label": "Client Port" - }, - { - "name": "common_l4_protocol", - "type": "string", - "label": "L4 Protocol" - }, - { - "name": "common_address_type", - "type": "int", - "label": "Address Type", - "doc": "{\"data\":[{\"code\":\"4\",\"value\":\"ipv4\"},{\"code\":\"6\",\"value\":\"ipv6\"}]}" - }, - { - "name": "common_server_ip", - "type": "string", - "label": "Server IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_server_port", - "type": "int", - "label": "Server Port", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_action", - "type": "int", - "label": "Action", - "doc": "{\"allow_query\":\"true\",\"data\":[{\"code\":\"0\",\"value\":\"None\"},{\"code\":\"1\",\"value\":\"Monitor\"},{\"code\":\"2\",\"value\":\"Intercept\"},{\"code\":\"16\",\"value\":\"Deny\"},{\"code\":\"128\",\"value\":\"Allow\"}]}" - }, - { - "name": "common_direction", - "type": "int", - "label": "Direction", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"outbound\"},{\"code\":\"1\",\"value\":\"inbound\"}]}" - }, - { - "name": "common_entrance_id", - "type": "int", - "label": "Entrance ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sled_ip", - "type": "string", - "label": "Sled IP", - "doc": "{\"allow_query\":\"true\",\"constraints\":{\"type\":\"ip\"}}" - }, - { - "name": "common_client_location", - "type": "string", - "label": "Client Location" - }, - { - "name": "common_client_asn", - "type": "string", - "label": "Client ASN" - }, - { - "name": "common_server_location", - "type": "string", - "label": "Server Location" - }, - { - "name": "common_server_asn", - "type": "string", - "label": "Server ASN" - }, - { - "name": "common_sessions", - "type": "int", - "label": "Sessions" - }, - { - "name": "common_c2s_pkt_num", - "type": "long", - "label": "Packets Sent" - }, - { - "name": "common_s2c_pkt_num", - "type": "long", - "label": "Packets Received" - }, - { - "name": "common_c2s_byte_num", - "type": "long", - "label": "Bytes Sent" - }, - { - "name": "common_s2c_byte_num", - "type": "long", - "label": "Bytes Received" - }, - { - "name": "common_service", - "type": "int", - "label": "Service", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_schema_type", - "type": "string", - "label": "Schema Type", - "doc": "{\"data\":[{\"code\":\"BASE\",\"value\":\"BASE\"},{\"code\":\"HTTP\",\"value\":\"HTTP\"},{\"code\":\"MAIL\",\"value\":\"MAIL\"},{\"code\":\"DNS\",\"value\":\"DNS\"},{\"code\":\"SSL\",\"value\":\"SSL\"},{\"code\":\"FTP\",\"value\":\"FTP\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_user_tags", - "type": "string", - "label": "User Tags", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_sub_action", - "type": "string", - "label": "Sub Action", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_user_region", - "type": "string", - "label": "User Region", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_device_id", - "type": "string", - "label": "Device ID" - }, - { - "name": "common_link_id", - "type": "int", - "label": "Link ID", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_isp", - "type": "string", - "label": "ISP", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_encapsulation", - "type": "int", - "label": "Encapsulation", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"Ethernet\"},{\"code\":\"8\",\"value\":\"PPP\"},{\"code\":\"12\",\"value\":\"CiscoHDLC\"}]}" - }, - { - "name": "common_app_label", - "type": "string", - "label": "Application Label", - "doc": "{\"visibility\":\"disabled\"}" - }, - { - "name": "common_protocol_label", - "type": "string", - "label": "Protocol Label", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_start_time", - "type": "int", - "label": "Start Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"visibility\":\"hidden\"}" - }, - { - "name": "common_end_time", - "type": "int", - "label": "End Time", - "doc": "{\"constraints\":{\"type\":\"timestamp\"},\"visibility\":\"hidden\"}" - }, - { - "name": "common_establish_latency_ms", - "type": "int", - "label": "Establish Latency(ms)", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_con_duration_ms", - "type": "int", - "label": "Duration(ms)", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_dir", - "type": "int", - "label": "Stream Direction", - "doc": "{\"data\":[{\"code\":\"1\",\"value\":\"c2s\"},{\"code\":\"2\",\"value\":\"s2c\"},{\"code\":\"3\",\"value\":\"double\"}]}" - }, - { - "name": "common_address_list", - "type": "string", - "label": "Address List", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_has_dup_traffic", - "type": "int", - "label": "Duplication Traffic", - "doc": "{\"data\":[{\"code\":\"0\",\"value\":\"No\"},{\"code\":\"1\",\"value\":\"Yes\"}],\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_error", - "type": "string", - "label": "Stream Error", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_stream_trace_id", - "type": "long", - "label": "Session ID", - "doc": "{\"allow_query\":\"true\"}" - }, - { - "name": "common_link_info_c2s", - "label": "Link Info(c2s)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "common_link_info_s2c", - "label": "Link Info(s2c)", - "type": "string", - "doc": "{\"visibility\":\"hidden\"}" - }, - { - "name": "nic_name", - "label": "Nic Name", - "type": "string" - }, - { - "name": "origin_source_mac", - "label": "Origin Source Mac", - "type": "string" - }, - { - "name": "origin_dest_mac", - "label": "Origin Dest Mac", - "type": "string" - }, - { - "name": "packet_url", - "type": "string", - "label": "Packet URL" - }, - { - "name": "pcap_storage_task_id", - "type": "int", - "label": "Task ID" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/injection_metrics_log.avsc b/galaxy-query-engine/schema/druid/injection_metrics_log.avsc deleted file mode 100644 index 9e932bf..0000000 --- a/galaxy-query-engine/schema/druid/injection_metrics_log.avsc +++ /dev/null @@ -1,24 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "injection_metrics_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "bytes", - "type": "long" - }, - { - "name": "service_type", - "type": "string" - }, - { - "name": "device_id", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/proxy_event_hits_log.avsc b/galaxy-query-engine/schema/druid/proxy_event_hits_log.avsc deleted file mode 100644 index f6f700c..0000000 --- a/galaxy-query-engine/schema/druid/proxy_event_hits_log.avsc +++ /dev/null @@ -1,56 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "proxy_event_hits_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "action", - "type": "long" - }, - { - "name": "isp", - "type": "string" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "hits", - "type": "long" - }, - { - "name": "policy_id", - "type": "long" - }, - { - "name": "sub_action", - "type": "string" - }, - { - "name": "country", - "type": "string" - }, - { - "name": "location", - "type": "string" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "ip_object", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/security_event_hits_log.avsc b/galaxy-query-engine/schema/druid/security_event_hits_log.avsc deleted file mode 100644 index 740c124..0000000 --- a/galaxy-query-engine/schema/druid/security_event_hits_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "security_event_hits_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "action", - "type": "long" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "hits", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "isp", - "type": "string" - }, - { - "name": "policy_id", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/source_country_ip_num_log.avsc b/galaxy-query-engine/schema/druid/source_country_ip_num_log.avsc deleted file mode 100644 index 908fa08..0000000 --- a/galaxy-query-engine/schema/druid/source_country_ip_num_log.avsc +++ /dev/null @@ -1,20 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "source_country_ip_num_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "country", - "type": "string" - }, - { - "name": "ip_object", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/sys_storage_log.avsc b/galaxy-query-engine/schema/druid/sys_storage_log.avsc deleted file mode 100644 index 1ab7b1c..0000000 --- a/galaxy-query-engine/schema/druid/sys_storage_log.avsc +++ /dev/null @@ -1,32 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "sys_storage_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "log_type", - "type": "string" - }, - { - "name": "max_size", - "type": "long" - }, - { - "name": "used_size", - "type": "long" - }, - { - "name": "aggregate_size", - "type": "long" - }, - { - "name": "last_storage", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/top_domain_recommend_daily_log.avsc b/galaxy-query-engine/schema/druid/top_domain_recommend_daily_log.avsc deleted file mode 100644 index 76766b5..0000000 --- a/galaxy-query-engine/schema/druid/top_domain_recommend_daily_log.avsc +++ /dev/null @@ -1,32 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_domain_recommend_daily_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "domain", - "type": "string" - }, - { - "name": "ip_num", - "type": "long" - }, - { - "name": "order_num", - "type": "long" - }, - { - "name": "protocol_type", - "type": "string" - }, - { - "name": "session_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/top_external_host_log.avsc b/galaxy-query-engine/schema/druid/top_external_host_log.avsc deleted file mode 100644 index 700e774..0000000 --- a/galaxy-query-engine/schema/druid/top_external_host_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_external_host_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "destination", - "type": "string" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/top_internal_host_log.avsc b/galaxy-query-engine/schema/druid/top_internal_host_log.avsc deleted file mode 100644 index 8ecf953..0000000 --- a/galaxy-query-engine/schema/druid/top_internal_host_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_internal_host_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "source", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/top_urls_log.avsc b/galaxy-query-engine/schema/druid/top_urls_log.avsc deleted file mode 100644 index 44b5dbd..0000000 --- a/galaxy-query-engine/schema/druid/top_urls_log.avsc +++ /dev/null @@ -1,20 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_urls_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "url", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/top_user_log.avsc b/galaxy-query-engine/schema/druid/top_user_log.avsc deleted file mode 100644 index 1f8dc0f..0000000 --- a/galaxy-query-engine/schema/druid/top_user_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_user_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "subscriber_id", - "type": "string" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/top_website_domain_log.avsc b/galaxy-query-engine/schema/druid/top_website_domain_log.avsc deleted file mode 100644 index b6bfc7f..0000000 --- a/galaxy-query-engine/schema/druid/top_website_domain_log.avsc +++ /dev/null @@ -1,40 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_website_domain_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "domain", - "type": "string" - }, - { - "name": "order_by", - "type": "string" - }, - { - "name": "s2c_byte_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/top_website_urls_daily_log.avsc b/galaxy-query-engine/schema/druid/top_website_urls_daily_log.avsc deleted file mode 100644 index 5679d14..0000000 --- a/galaxy-query-engine/schema/druid/top_website_urls_daily_log.avsc +++ /dev/null @@ -1,28 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "top_website_urls_daily_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "url", - "type": "string" - }, - { - "name": "ip_num", - "type": "long" - }, - { - "name": "session_num", - "type": "long" - }, - { - "name": "content_length", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/traffic_metrics_log.avsc b/galaxy-query-engine/schema/druid/traffic_metrics_log.avsc deleted file mode 100644 index f6df6d1..0000000 --- a/galaxy-query-engine/schema/druid/traffic_metrics_log.avsc +++ /dev/null @@ -1,220 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "traffic_metrics_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "allow_conn_num", - "type": "long" - }, - { - "name": "allow_in_bytes", - "type": "long" - }, - { - "name": "allow_in_packets", - "type": "long" - }, - { - "name": "allow_out_bytes", - "type": "long" - }, - { - "name": "allow_out_packets", - "type": "long" - }, - { - "name": "close_conn_num", - "type": "long" - }, - { - "name": "default_conn_num", - "type": "long" - }, - { - "name": "default_in_bytes", - "type": "long" - }, - { - "name": "default_in_packets", - "type": "long" - }, - { - "name": "default_out_bytes", - "type": "long" - }, - { - "name": "default_out_packets", - "type": "long" - }, - { - "name": "deny_conn_num", - "type": "long" - }, - { - "name": "deny_in_bytes", - "type": "long" - }, - { - "name": "deny_in_packets", - "type": "long" - }, - { - "name": "deny_out_bytes", - "type": "long" - }, - { - "name": "deny_out_packets", - "type": "long" - }, - { - "name": "device_id", - "type": "string" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "intercept_conn_num", - "type": "long" - }, - { - "name": "intercept_in_bytes", - "type": "long" - }, - { - "name": "intercept_in_packets", - "type": "long" - }, - { - "name": "intercept_out_bytes", - "type": "long" - }, - { - "name": "intercept_out_packets", - "type": "long" - }, - { - "name": "established_conn_num", - "type": "long" - }, - { - "name": "monitor_conn_num", - "type": "long" - }, - { - "name": "monitor_in_bytes", - "type": "long" - }, - { - "name": "monitor_in_packets", - "type": "long" - }, - { - "name": "monitor_out_bytes", - "type": "long" - }, - { - "name": "monitor_out_packets", - "type": "long" - }, - { - "name": "new_conn_num", - "type": "long" - }, - { - "name": "total_in_bytes", - "type": "long" - }, - { - "name": "total_in_packets", - "type": "long" - }, - { - "name": "total_out_bytes", - "type": "long" - }, - { - "name": "total_out_packets", - "type": "long" - }, - { - "name": "alert_bytes", - "type": "long" - }, - { - "name": "flood_bytes", - "type": "long" - }, - { - "name": "hijack_bytes", - "type": "long" - }, - { - "name": "insert_bytes", - "type": "long" - }, - { - "name": "intercept_allow_conn_num", - "type": "long" - }, - { - "name": "intercept_deny_conn_num", - "type": "long" - }, - { - "name": "intercept_hijack_conn_num", - "type": "long" - }, - { - "name": "intercept_insert_conn_num", - "type": "long" - }, - { - "name": "intercept_monitor_conn_num", - "type": "long" - }, - { - "name": "intercept_redirect_conn_num", - "type": "long" - }, - { - "name": "intercept_replace_conn_num", - "type": "long" - }, - { - "name": "maybe_pinning_num", - "type": "long" - }, - { - "name": "not_pinning_num", - "type": "long" - }, - { - "name": "pinning_num", - "type": "long" - }, - { - "name": "reflection_bytes", - "type": "long" - }, - { - "name": "ad_cc_bytes", - "type": "long" - }, - { - "name": "ad_flood_bytes", - "type": "long" - }, - { - "name": "ad_reflection_bytes", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/traffic_protocol_stat_log.avsc b/galaxy-query-engine/schema/druid/traffic_protocol_stat_log.avsc deleted file mode 100644 index 9eaf478..0000000 --- a/galaxy-query-engine/schema/druid/traffic_protocol_stat_log.avsc +++ /dev/null @@ -1,44 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "traffic_protocol_stat_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "protocol_id", - "type": "string" - }, - { - "name": "isp", - "type": "string" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "sessions", - "type": "long" - }, - { - "name": "c2s_pkt_num", - "type": "long" - }, - { - "name": "s2c_pkt_num", - "type": "long" - }, - { - "name": "c2s_byte_num", - "type": "long" - }, - { - "name": "s2c_byte_num", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/druid/traffic_summary_log.avsc b/galaxy-query-engine/schema/druid/traffic_summary_log.avsc deleted file mode 100644 index c50c5fe..0000000 --- a/galaxy-query-engine/schema/druid/traffic_summary_log.avsc +++ /dev/null @@ -1,32 +0,0 @@ -{ - "namespace": "druid", - "type": "record", - "name": "traffic_summary_log", - "doc": "{\"partition_key\":\"__time\"}", - "fields": [ - { - "name": "__time", - "type": "long" - }, - { - "name": "isp", - "type": "string" - }, - { - "name": "entrance_id", - "type": "long" - }, - { - "name": "ip_object", - "type": "string" - }, - { - "name": "one_sided_connections", - "type": "long" - }, - { - "name": "uncategorized_bytes", - "type": "long" - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/schema/memory-schema.json b/galaxy-query-engine/schema/memory-schema.json deleted file mode 100644 index 4880e0d..0000000 --- a/galaxy-query-engine/schema/memory-schema.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version": "1.0", - "defaultSchema": "memorySchema", - "schemas": [ - { - "type": "custom", - "name": "memorySchema", - "factory": "com.mesalab.engine.component.calcite.InMemorySchemaFactory", - "operand": { - } - } - ] -}
\ No newline at end of file diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/GalaxyDataEngineApp.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/GalaxyDataEngineApp.java deleted file mode 100644 index 708e09e..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/GalaxyDataEngineApp.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.mesalab.engine; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; -import org.springframework.boot.web.servlet.server.ServletWebServerFactory; -import org.springframework.cloud.client.SpringCloudApplication; -import org.springframework.cloud.client.loadbalancer.LoadBalanced; -import org.springframework.context.annotation.Bean; -import org.springframework.web.client.RestTemplate; - -@SpringCloudApplication -public class GalaxyDataEngineApp { - - @Bean - @LoadBalanced - RestTemplate restTemplate() { - return new RestTemplate(); - } - - public static void main(String[] args) { - SpringApplication.run(GalaxyDataEngineApp.class); - } - - @Bean - public ServletWebServerFactory webServerFactory() { - TomcatServletWebServerFactory factory = new TomcatServletWebServerFactory(); - factory.addConnectorCustomizers(connector -> connector.setProperty("relaxedPathChars", "\"<>[\\]^`{|}")); - return factory; - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/ArangoParser.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/ArangoParser.java deleted file mode 100644 index 4776439..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/ArangoParser.java +++ /dev/null @@ -1,205 +0,0 @@ -package com.mesalab.engine.component; - -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.enums.RangeTypeEnum; -import com.mesalab.common.util.SqlKeywords; -import com.zdjizhi.utils.DateUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang.StringUtils; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.text.MessageFormat; -import java.util.List; -import java.util.Objects; - -/** - * @Date: 2020-07-22 18:02 - * @Author : liuyongqiang - * @ClassName : ArangoDSLParser - * @Description : ArangoDB DSL语法解析器 - */ -@Slf4j -@Component -public class ArangoParser extends DSLParser { - - public final static String ALL_QUERY = "FOR {0} IN {1} RETURN {2}"; - - @Override - public String allQuery(DSLObject dslObject) { - String dataSource = dslObject.getDataSource(); - String aql = MessageFormat.format( - ArangoParser.ALL_QUERY, - dataSource.toLowerCase(), - dataSource.toUpperCase(), - dataSource.toLowerCase()); - aql = parseQuery(dslObject,aql); - return aql; - } - - @Override - public String parseQuery(DSLObject dslObject, String aql) { - if(!Objects.isNull(dslObject.getQuery())){ - aql = parseGranularity(dslObject,aql); - aql = parseMatch(dslObject,aql); - aql = parseRange(dslObject,aql); - aql = parseIntervals(dslObject,aql); - aql = parseSort(dslObject,aql); - } - aql = parseLimit(dslObject,aql); - log.info("Parse Query AQL:{}",aql); - return aql; - } - - @Override - public String parseGranularity(DSLObject dslObject, String aql) { - return aql; - } - - @Override - public String parseLimit(DSLObject dslObject, String aql) { - - if(StringUtils.isEmpty(dslObject.getLimit())){ - return aql; - } - - StringBuilder stringBuilder = new StringBuilder(aql); - int index = stringBuilder.indexOf(SqlKeywords.A_RETURN); - stringBuilder.insert(index, SqlKeywords.A_LIMIT - .concat(SqlKeywords.A_SPACE).concat(dslObject.getLimit()).concat(SqlKeywords.A_SPACE)); - - return stringBuilder.toString(); - } - - @Override - public String parseRange(DSLObject dslObject, String aql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getRange())){ - return aql; - } - - StringBuilder builder1 = new StringBuilder(aql); - StringBuilder builder2; - String dataSource = dslObject.getDataSource(); - List<DSLObject.QueryBean.RangeBean> rangeBeanList = dslObject.getQuery().getRange(); - - for (DSLObject.QueryBean.RangeBean rangeBean:rangeBeanList){ - String type = rangeBean.getType(); - String fieldKey = rangeBean.getFieldKey(); - List<String> values = rangeBean.getFieldValues(); - if(CollectionUtils.isEmpty(values)) break; - - String expr = RangeTypeEnum.getByType(type).getExpr(); - String join = aql.contains(SqlKeywords.A_FILTER)? SqlKeywords.A_AND: SqlKeywords.A_FILTER; - builder1.insert(getIndex(aql,builder1),join.concat(SqlKeywords.A_SPACE).concat(dataSource.toLowerCase()).concat(".") - .concat(fieldKey).concat(expr).concat(values.get(0)).concat(SqlKeywords.A_SPACE)); - builder2 = new StringBuilder(builder1.toString()); - - if(values.size()>1){ - for (int i=1;i<values.size();i++){ - builder2.insert(getIndex(builder1.toString(),builder2), - SqlKeywords.A_OR.concat(SqlKeywords.A_SPACE) - .concat(dataSource.toLowerCase()).concat(SqlKeywords.A_SPOT).concat(fieldKey) - .concat(expr).concat(values.get(i)).concat(SqlKeywords.A_SPACE)); - } - } - aql = builder2.toString(); - } - return aql; - } - - @Override - public String parseSort(DSLObject dslObject, String aql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getSort())){ - return aql; - } - StringBuilder stringBuilder = new StringBuilder(aql); - int index; - if(aql.contains(SqlKeywords.A_LIMIT)){ - index = stringBuilder.indexOf(SqlKeywords.A_LIMIT); - }else{ - index = stringBuilder.indexOf(SqlKeywords.A_RETURN); - } - List<DSLObject.QueryBean.SortBean> sortBeans = dslObject.getQuery().getSort(); - stringBuilder.insert(index, SqlKeywords.A_SORT.concat(SqlKeywords.A_SPACE) - .concat(dslObject.getDataSource().toLowerCase()) - .concat(SqlKeywords.A_SPOT) - .concat(sortBeans.get(0).getFieldKey()) - .concat(SqlKeywords.A_SPACE) - .concat(sortBeans.get(0).getType()) - .concat(SqlKeywords.A_SPACE)); - return stringBuilder.toString(); - } - - @Override - public String parseIntervals(DSLObject dslObject, String aql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getIntervals())){ - return aql; - } - StringBuilder builder1 = new StringBuilder(aql); - String dataSource = dslObject.getDataSource(); - List<DSLObject.QueryBean.IntervalsBean> intervalsBeanList = dslObject.getQuery().getIntervals(); - for (DSLObject.QueryBean.IntervalsBean intervalsBean : intervalsBeanList){ - String fieldKey = intervalsBean.getFieldKey(); - List<String> values = intervalsBean.getFieldValues(); - if(CollectionUtils.isEmpty(values)) break; - - String join = aql.contains(SqlKeywords.A_FILTER)? SqlKeywords.A_AND: SqlKeywords.A_FILTER; - builder1.insert(getIndex(aql,builder1),join.concat(SqlKeywords.A_SPACE) - .concat(dataSource.toLowerCase()).concat(SqlKeywords.A_SPOT).concat(fieldKey).concat(SqlKeywords.A_SPACE).concat(">=") - .concat(SqlKeywords.A_SPACE).concat(String.valueOf(DateUtils.convertStringToTimestamp(values.get(0),DateUtils.YYYY_MM_DD_HH24_MM_SS))) - .concat(SqlKeywords.A_SPACE).concat(SqlKeywords.A_AND).concat(SqlKeywords.A_SPACE).concat(dataSource.toLowerCase()).concat(SqlKeywords.A_SPOT) - .concat(fieldKey).concat(SqlKeywords.A_SPACE).concat("<").concat(SqlKeywords.A_SPACE) - .concat(String.valueOf(DateUtils.convertStringToTimestamp(values.get(1),DateUtils.YYYY_MM_DD_HH24_MM_SS))).concat(SqlKeywords.A_SPACE)); - aql = builder1.toString(); - } - return aql; - } - - @Override - public String parseMatch(DSLObject dslObject, String aql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getMatch())){ - return aql; - } - StringBuilder builder1 = new StringBuilder(aql); - StringBuilder builder2; - String dataSource = dslObject.getDataSource(); - List<DSLObject.QueryBean.MatchBean> matchBeanList = dslObject.getQuery().getMatch(); - for (DSLObject.QueryBean.MatchBean matchBean : matchBeanList){ - String type = matchBean.getType(); - String fieldKey = matchBean.getFieldKey(); - List<String> values = matchBean.getFieldValues(); - if(CollectionUtils.isEmpty(values)) break; - - String join = aql.contains(SqlKeywords.A_FILTER)? SqlKeywords.A_AND: SqlKeywords.A_FILTER; - builder1.insert(getIndex(aql,builder1),join.concat(SqlKeywords.A_SPACE) - .concat(dataSource.toLowerCase()).concat(SqlKeywords.A_SPOT) - .concat(fieldKey).concat(SqlKeywords.A_SPACE) - .concat(SqlKeywords.A_LIKE).concat(SqlKeywords.A_SPACE) - .concat(parseLike(values.get(0),type)).concat(SqlKeywords.A_SPACE)); - - builder2 = new StringBuilder(builder1.toString()); - if(values.size()>1){ - for (int i=1;i<values.size();i++){ - builder2.insert(getIndex(builder1.toString(),builder2), SqlKeywords.A_OR - .concat(SqlKeywords.A_SPACE).concat(dataSource.toLowerCase()).concat(SqlKeywords.A_SPOT) - .concat(fieldKey).concat(SqlKeywords.A_SPACE).concat(SqlKeywords.A_LIKE).concat(SqlKeywords.A_SPACE) - .concat(parseLike(values.get(i),type)).concat(SqlKeywords.A_SPACE)); - } - } - aql = builder2.toString(); - } - return aql; - } - - private int getIndex(String str,StringBuilder builder){ - int index; - if(str.contains(SqlKeywords.A_SORT)){ - index = builder.indexOf(SqlKeywords.A_SORT); - }else if(str.contains(SqlKeywords.A_LIMIT)){ - index = builder.indexOf(SqlKeywords.A_LIMIT); - }else{ - index = builder.indexOf(SqlKeywords.A_RETURN); - } - return index; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/ClickHouseParser.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/ClickHouseParser.java deleted file mode 100644 index 43358f8..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/ClickHouseParser.java +++ /dev/null @@ -1,123 +0,0 @@ -package com.mesalab.engine.component; - -import cn.hutool.core.util.StrUtil; -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.util.SqlKeywords; -import com.mesalab.engine.service.ClickHouseEngineService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.text.MessageFormat; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @Date: 2020-07-22 18:00 - * @Author : liuyongqiang - * @ClassName : ClickHouseDSLParser - * @Description : ClickHouse DSL语法解析器 - */ -@Slf4j -@Component -public class ClickHouseParser extends DSLParser { - - @Autowired - ClickHouseEngineService clickHouseEngineService; - - //表字段和数据类型查询SQL - public final static String SCHEAM_QUERY = " desc {0} "; - - //查询所有表 - public final static String TABLES_QUERY = " show tables "; - - public static Pattern periodOfPT = Pattern.compile("PT(\\d+)(\\w+)", Pattern.CASE_INSENSITIVE); - - public static Pattern periodOfP = Pattern.compile("P(\\d+)(\\w+)", Pattern.CASE_INSENSITIVE); - - - @Override - public String parseGranularity(DSLObject dslObject, String sql) { - if(StrUtil.isEmpty(dslObject.getQuery().getGranularity())){ - return sql; - } - String granularity = dslObject.getQuery().getGranularity(); - Matcher matcherPT = periodOfPT.matcher(granularity); - Matcher matcherP = periodOfP.matcher(granularity); - String format = ""; - - if(matcherPT.find()){ - String PTnum = matcherPT.group(1); - String PTunit = matcherPT.group(2); - if ("S".equalsIgnoreCase(PTunit)) { - format = MessageFormat.format(SqlKeywords.C_SECOND,PTnum); - } else if ("M".equalsIgnoreCase(PTunit)) { - format = MessageFormat.format(SqlKeywords.C_MINUTE,PTnum); - } else if ("H".equalsIgnoreCase(PTunit)) { - format = MessageFormat.format(SqlKeywords.C_HOUR,PTnum); - } - } - - if(matcherP.find()){ - String Pnum = matcherP.group(1); - String Punit = matcherP.group(2); - if ("D".equalsIgnoreCase(Punit)) { - format = MessageFormat.format(SqlKeywords.C_DAY,Pnum); - } else if ("W".equalsIgnoreCase(Punit)) { - format = MessageFormat.format(SqlKeywords.C_WEEK,Pnum); - } else if ("M".equalsIgnoreCase(Punit)) { - format = MessageFormat.format(SqlKeywords.C_MONTH,Pnum); - } else if ("Y".equalsIgnoreCase(Punit)) { - format = MessageFormat.format(SqlKeywords.C_YEAR,Pnum); - } - } - sql = sql.replace(SqlKeywords.SELECT, SqlKeywords.SELECT.concat(format).concat(",")); - return sql; - } - - @Override - public String parseSort(DSLObject dslObject, String sql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getSort())){ - return sql; - } - String sortType = dslObject.getQuery().getSort().get(0).getType(); - sql = sql.concat(SqlKeywords.ORDER_BY - .concat(dslObject.getQuery().getSort().get(0).getFieldKey())) - .concat(" ") - .concat(sortType); - if(dslObject.getQuery().getSort().size()>1){ - List<DSLObject.QueryBean.SortBean> sortBeans = dslObject.getQuery().getSort(); - for (int i=1;i<sortBeans.size();i++){ - sql = sql.concat(" , ") - .concat(sortBeans.get(i).getFieldKey()) - .concat(" ") - .concat(sortBeans.get(i).getType()); - } - } - return sql; - } - - @Override - public String parseIntervals(DSLObject dslObject, String sql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getIntervals())){ - return sql; - } - - List<DSLObject.QueryBean.IntervalsBean> intervalsBeanList = dslObject.getQuery().getIntervals(); - for (DSLObject.QueryBean.IntervalsBean intervalsBean : intervalsBeanList){ - String fieldKey = intervalsBean.getFieldKey(); - List<String> values = intervalsBean.getFieldValues(); - if(CollectionUtils.isEmpty(values)) break; - sql = sql.concat(sql.contains(SqlKeywords.WHERE)? SqlKeywords.AND: SqlKeywords.WHERE) - .concat(fieldKey) - .concat(SqlKeywords.BWTWEEN) - .concat(MessageFormat.format(SqlKeywords.C_TO_DATE_TIME,values.get(0))) - .concat(SqlKeywords.AND) - .concat(MessageFormat.format(SqlKeywords.C_TO_DATE_TIME,values.get(1))); - } - return sql; - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLParser.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLParser.java deleted file mode 100644 index 9923c08..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLParser.java +++ /dev/null @@ -1,289 +0,0 @@ -package com.mesalab.engine.component; - -import cn.hutool.core.util.StrUtil; -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.EngineTypeEnum; -import com.mesalab.common.enums.MatchTypeEnum; -import com.mesalab.common.enums.RangeTypeEnum; -import com.mesalab.common.util.SqlKeywords; -import com.mesalab.engine.service.ClickHouseEngineService; -import com.mesalab.engine.service.DruidEngineService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.text.MessageFormat; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * @Date: 2020-07-22 17:35 - * @Author : liuyongqiang - * @ClassName : DslParser - * @Description : DSLParse解析抽象类 - */ -@Slf4j -@Component -@SuppressWarnings("unchecked") -public abstract class DSLParser { - - //默认数据大小 - private final static String DEFAULT_LIMIT = "100000"; - - @Autowired - DruidEngineService druidEngine; - - @Autowired - ClickHouseEngineService clickHouseEngine; - - - /** - * 解析granularity字段 - * */ - public abstract String parseGranularity(DSLObject dslObject, String sql); - - /** - * 解析 Sort对象 - * */ - public abstract String parseSort(DSLObject dslObject, String sql); - - /** - * 解析 Intervals对象 - * */ - public abstract String parseIntervals(DSLObject dslObject, String sql); - - /** - * @Description: 单表全列查询 - * @Author: liuyongqiang - * @Date: 2020/7/23 18:23 - * @param dslObject: - * @return: java.lang.String - **/ - public String allQuery(DSLObject dslObject) { - String selectSql = this.allColumn(dslObject); - selectSql = parseQuery(dslObject,selectSql); - return selectSql; - } - - /** - * @Description: 查询表中的所有列 - * @Author: liuyongqiang - * @Date: 2020/7/23 17:24 - * @param dslObject: - * @return: java.lang.String - **/ - public String allColumn(DSLObject dslObject){ - BaseResult baseResult = null; - - switch (EngineTypeEnum.getByEngine(dslObject.getDataEngine())){ - case CACULATION_ENGINE: - baseResult = druidEngine.schemaQuery(dslObject); - break; - case BUSINESS_ENGINE: - baseResult = clickHouseEngine.schemaQuery(dslObject); - break; - } - - List<Map<String,Object>> results = (List<Map<String, Object>>) baseResult.getData(); - String column = ""; - for (Map<String,Object> map : results){ - String name = String.valueOf(map.get("name")); - //ClickHouse数据表中的时间字段做日期转换 - if(dslObject.getDataEngine().equals(EngineTypeEnum.BUSINESS_ENGINE.getEngine())){ - if(name.lastIndexOf("_time")>0){ - name = MessageFormat.format(" toDateTime({0}) as c_{1} ",name,name) ; - } - } - //Druid数据表中的时间字段做日期转换 - if(dslObject.getDataEngine().equals(EngineTypeEnum.CACULATION_ENGINE.getEngine())){ - if(name.equalsIgnoreCase("__time")){ - name = "TIME_FORMAT(__time,'yyyy-MM-dd HH:mm:ss') as c_time"; - } - } - //拼接表中的列名称 - column = column.concat(name).concat(","); - } - //去掉最后一个逗号 - column = column.substring(0,column.length()-1); - return SqlKeywords.SELECT - .concat(column) - .concat(SqlKeywords.FROM) - .concat(dslObject.getDataSource()); - } - - - - /** - * @Description: 解析query对象 - * @Author: liuyongqiang - * @Date: 2020/7/23 14:29 - * @param dslObject: - * @param sql: - * @return: java.lang.String - **/ - public String parseQuery(DSLObject dslObject, String sql) { - if(Objects.nonNull(dslObject.getQuery())){ - sql = parseGranularity(dslObject,sql);//解析Granularity - sql = parseMatch(dslObject,sql);//解析Match - sql = parseRange(dslObject,sql);//range解析 - sql = parseIntervals(dslObject,sql);//解析Intervals - sql = parseSort(dslObject,sql);//解析Sort - } - sql = parseLimit(dslObject,sql);//limit解析 - log.info("Parse Query SQL:{}",sql); - return sql; - } - - /** - * @Description: 解析 Range对象 - * @Author: liuyongqiang - * @Date: 2020/7/23 17:47 - * @param dslObject: - * @param sql: - * @return: java.lang.String - **/ - public String parseRange(DSLObject dslObject, String sql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getRange())){ - return sql; - } - List<DSLObject.QueryBean.RangeBean> rangeBeanList = dslObject.getQuery().getRange(); - for (DSLObject.QueryBean.RangeBean rangeBean:rangeBeanList){ - String type = rangeBean.getType(); - String fieldKey = rangeBean.getFieldKey(); - List<String> values = rangeBean.getFieldValues(); - - if(CollectionUtils.isEmpty(values)) break; - - String join = sql.contains(SqlKeywords.WHERE)? SqlKeywords.AND: SqlKeywords.WHERE; - String expr = RangeTypeEnum.getByType(type).getExpr(); - sql = sql.concat(join).concat(fieldKey).concat(expr).concat(values.get(0)); - - if(values.size()>1){ - for (int i=1;i<values.size();i++){ - sql = sql.concat(SqlKeywords.OR) - .concat(fieldKey) - .concat(expr) - .concat(values.get(i)); - } - } - } - return sql; - } - - /** - * @Description: 解析 Match对象 - * @Author: liuyongqiang - * @Date: 2020/7/23 17:45 - * @param dslObject: - * @param sql: - * @return: java.lang.String - **/ - public String parseMatch(DSLObject dslObject, String sql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getMatch())){ - return sql; - } - List<DSLObject.QueryBean.MatchBean> matchBeanList = dslObject.getQuery().getMatch(); - int index = 0; - for (DSLObject.QueryBean.MatchBean matchBean : matchBeanList){ - index++; - String type = matchBean.getType(); - String fieldKey = matchBean.getFieldKey(); - List<String> values = matchBean.getFieldValues(); - - if(CollectionUtils.isEmpty(values)) break; - - String join = index==1 ? SqlKeywords.WHERE : SqlKeywords.AND; - sql = sql.concat(join) - .concat(fieldKey) - .concat(SqlKeywords.LIKE) - .concat(parseLike(values.get(0),type)); - - if(values.size()>1){ - for (int i=1;i<values.size();i++){ - sql = sql.concat(SqlKeywords.OR) - .concat(fieldKey) - .concat(SqlKeywords.LIKE) - .concat(parseLike(values.get(i),type)); - } - } - } - return sql; - } - - - /** - * @Description: 解析Limit对象 - * @Author: liuyongqiang - * @Date: 2020/7/23 17:44 - * @param dslObject: - * @param sql: - * @return: java.lang.String - **/ - public String parseLimit(DSLObject dslObject, String sql) { - sql = sql.concat(SqlKeywords.LIMIT); - if(StrUtil.isNotEmpty(dslObject.getLimit())){ - sql = sql.concat(dslObject.getLimit()); - }else{ - sql = sql.concat(DEFAULT_LIMIT); - } - return sql; - } - - /** - * @Description: match.type转换 - * @Author: liuyongqiang - * @Date: 2020/7/23 11:47 - * @param value: - * @param type: - * @return: java.lang.String - **/ - public String parseLike(String value,String type){ - switch (MatchTypeEnum.getByType(type)){ - case REGEX: - return parseRegex(value);//正则匹配 - case PREFIX: - return "'"+value+"%'";//前缀匹配 - case SUFFIX: - return "'%"+value+"'";//后缀匹配 - case EXACTLY: - return "'"+value+"'";//完全匹配 - case SUBSTRING: - return "'%"+value+"%'";//子串匹配 - } - return ""; - } - - /** - * @Description: parseRegex - * @Author: liuyongqiang - * @Date: 2020/7/23 11:49 - * @param value: - * @return: java.lang.String - **/ - public String parseRegex(String value) { - if(value.startsWith("$")){ - //以$开头，不以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示完整匹配 - value = value.replaceAll("\\$",""); - return "'"+value+"'"; - }else if(value.startsWith("*")&&value.endsWith("*")){ - //以*开始，以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示子串匹配 - value = value.replaceAll("\\*",""); - return "'%"+value+"%'"; - }else if(value.startsWith("*")){ - //以*开始，不以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示右匹配（后缀匹配） - value = value.replaceAll("\\*",""); - return "'%"+value+"'"; - }else if(value.endsWith("*")){ - //以*结尾，不以*或者$开始，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示左匹配(前缀匹配) - value = value.replaceAll("\\*",""); - return "'"+value+"%'"; - }else{ - //不以*或者$开头，不以*结尾，无论关键字其他位置是否包含表示匹配方式的字符*和$，即表示子串匹配 - return "'%"+value+"%'"; - } - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLRouter.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLRouter.java deleted file mode 100644 index fb917d6..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLRouter.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.mesalab.engine.component; - -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.EngineTypeEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.engine.service.ArangoEngineService; -import com.mesalab.engine.service.ClickHouseEngineService; -import com.mesalab.engine.service.DruidEngineService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * @Date: 2020-07-20 18:11 - * @Author : liuyongqiang - * @ClassName : DSLRouter - * @Description : DSL路由执行,负责调用查询引擎执行查询 - */ -@Component -public class DSLRouter { - - @Autowired - private ArangoEngineService arangoEngineService; - - @Autowired - private ClickHouseEngineService clickHouseEngineService; - - @Autowired - private DruidEngineService druidEngineService; - - /** - * @Description: 查询引擎路由 - * @Author: liuyongqiang - * @Date: 2020/7/22 15:59 - * @param dslObject: - * @return: com.mesalab.common.base.BaseResult - **/ - public BaseResult router(DSLObject dslObject) throws BusinessException { - switch (EngineTypeEnum.getByEngine(dslObject.getDataEngine())){ - case ANALYSIS_ENGINE: - return arangoEngineService.dslQuery(dslObject); - case CACULATION_ENGINE: - return druidEngineService.dslQuery(dslObject); - case BUSINESS_ENGINE: - return clickHouseEngineService.dslQuery(dslObject); - default: - return null; - } - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLValidate.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLValidate.java deleted file mode 100644 index 579e8f8..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DSLValidate.java +++ /dev/null @@ -1,377 +0,0 @@ -package com.mesalab.engine.component; - -import cn.hutool.core.util.StrUtil; -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.*; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.engine.service.ArangoEngineService; -import com.mesalab.engine.service.ClickHouseEngineService; -import com.mesalab.engine.service.DruidEngineService; -import org.apache.http.HttpStatus; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.regex.Pattern; - -/** - * @Date: 2020-07-20 18:10 - * @Author : wangwei - * @ClassName : DSLValidate - * @Description : DSL格式校验 - */ -@Component -@SuppressWarnings("unchecked") -public class DSLValidate { - - public static Pattern periodOfPT = Pattern.compile("PT(\\d+)[SMH]", Pattern.CASE_INSENSITIVE); - public static Pattern periodOfP = Pattern.compile("P(\\d+)[DWMY]", Pattern.CASE_INSENSITIVE); - public static Pattern strFormatDateTime = Pattern.compile("\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}", Pattern.CASE_INSENSITIVE); - - @Autowired - DruidEngineService druidEngineService; - @Autowired - ClickHouseEngineService clickHouseEngineService; - @Autowired - ArangoEngineService arangoDBEngineService; - @Value("${engine.maxCacheNum}") - int maxCacheNum; - - - /** - * 1.dbType字段必须在支持的范围之内(ArangoDB、ClickHouse、Druid) - * 2.dataSource数据表是否在库中存在 - * 3.limit不能大于100000 - * 4.时间格式校验 - * 5.intervals.fieldValues长度必须==2 - * 6.intervals.type暂时只支持between方式 - * 7.sort.type支持desc和asc - * 8.match.type在:["exactly"|"prefix"|"suffix"|"substring"]内 - * 9.range.type在:["gt"|"lt"|"eq"|"ge"|"le"|"ne"] - * 10.对表中的字段做是否存在的验证 - * 11.match.fieldValues中不允许出现以*开头$结束的值 - * 12.match.fieldKey字段类型不能为数值型 - * */ - public void executeValidate(DSLObject dslObject) throws BusinessException { - if (Objects.isNull(dslObject)) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), - "DSLObject is invalid"); - } - if (isValidEngine(dslObject.getDataEngine())) { - if (!isValidDataSource(dslObject)) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), - "dataSource is invalid"); - } - } else { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), - "dataEngine is invalid"); - } - if (Objects.isNull(dslObject.getQuery())) { - return; - } - if (!isValidLimit(dslObject.getLimit())) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), - "Limit is invalid"); - } - if (!isValidGranularity(dslObject)) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), - "query.Granularity value is invalid"); - } - if (!isValidMatch(dslObject)) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), - "query.Match type is invalid"); - } - if (!isValidRange(dslObject)) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), - "query.Range type is invalid"); - } - if (!isValidIntervals(dslObject)) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), - "query.Intervals is invalid"); - } - if (!isValidSort(dslObject)) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), - "query.Sort type is invalid"); - } - } - - /** - * 校验dataSource - * 1.判定是否在{@link EngineTypeEnum}相应数据库中 - * - * @param dslObject - * @return - */ - private boolean isValidDataSource(DSLObject dslObject) { - if (StrUtil.isBlank(dslObject.getDataSource())) { - return false; - } - if (EngineTypeEnum.ANALYSIS_ENGINE.getEngine().equals(dslObject.getDataEngine())) { - BaseResult baseResult = arangoDBEngineService.tablesQuery(dslObject); - return isDataSourceInResult(dslObject.getDataSource(), baseResult); - } else if (EngineTypeEnum.BUSINESS_ENGINE.getEngine().equals(dslObject.getDataEngine())) { - BaseResult baseResult = clickHouseEngineService.tablesQuery(dslObject); - return isDataSourceInResult(dslObject.getDataSource(), baseResult); - } else if (EngineTypeEnum.CACULATION_ENGINE.getEngine().equals(dslObject.getDataEngine())) { - BaseResult baseResult = druidEngineService.tablesQuery(dslObject); - return isDataSourceInResult(dslObject.getDataSource(), baseResult); - } - return false; - } - - - /** - * 获取表中所有列 - * - * @param dslObject - * @return - */ - private List getFields(DSLObject dslObject) { - if (EngineTypeEnum.BUSINESS_ENGINE.getEngine().equalsIgnoreCase(dslObject.getDataEngine())) { - BaseResult baseResult = clickHouseEngineService.schemaQuery(dslObject); - return fieldsInDataSource(baseResult); - } else if (EngineTypeEnum.CACULATION_ENGINE.getEngine().equalsIgnoreCase(dslObject.getDataEngine())) { - BaseResult baseResult = druidEngineService.schemaQuery(dslObject); - return fieldsInDataSource(baseResult); - } else if (EngineTypeEnum.ANALYSIS_ENGINE.getEngine().equalsIgnoreCase(dslObject.getDataEngine())) { - return new ArrayList(); - - } - throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.UNKNOW_ERROR.getCode(), - "field is invalid"); - } - - /** - * 表中所有字段 - * - * @param baseResult - * @return - */ - private List fieldsInDataSource(BaseResult baseResult) { - if (baseResult.isSuccess()) { - List<Map> data = (List<Map>) baseResult.getData(); - List fields = new ArrayList<>(); - data.forEach(o -> { - fields.add(o.get("name")); - } - ); - return fields; - } else { - throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.UNKNOW_ERROR.getCode(), - "get fields error: " + baseResult); - } - } - - /** - * 判断表是否在对应dataEngine对应库 - * - * @param dataSource - * @param baseResult - * @return - */ - private boolean isDataSourceInResult(String dataSource, BaseResult baseResult) { - if (baseResult.isSuccess()) { - List<Map> data = (List<Map>) baseResult.getData(); - for (Map datum : data) { - if (datum.get("name").equals(dataSource)) { - return true; - } - } - } else { - throw new BusinessException(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.UNKNOW_ERROR.getCode(), - "get datasource error: " + baseResult); - } - return false; - } - - /** - * 时间粒度校验, 遵循ISO8601 durations 定义 - * - * @param dslObject - * @return - */ - private boolean isValidGranularity(DSLObject dslObject) { - String granularity = dslObject.getQuery().getGranularity(); - if (StrUtil.isBlank(granularity)) { - return true; - } - if (periodOfP.matcher(granularity).find() || periodOfPT.matcher(granularity).find()) { - return true; - } - return false; - } - - /** - * 校验range: - * 1.是否属于{@link RangeTypeEnum}支持的类型 - * - * @param dslObject - * @return - */ - private boolean isValidRange(DSLObject dslObject) { - if (CollectionUtils.isEmpty(dslObject.getQuery().getRange())) { - return true; - } - for (DSLObject.QueryBean.RangeBean rangeBean : dslObject.getQuery().getRange()) { - if (RangeTypeEnum.getByType(rangeBean.getType()) == null) { - return false; - } - checkField(dslObject, rangeBean.getFieldKey()); - } - return true; - } - - /** - * 校验match: - * 1.是否属于{@link MatchTypeEnum}限定类型 - * 2.不能以*开始、或$结尾 - * - * @param dslObject - * @return - */ - private boolean isValidMatch(DSLObject dslObject) { - if (CollectionUtils.isEmpty(dslObject.getQuery().getMatch())) { - return true; - } - for (DSLObject.QueryBean.MatchBean matchBean : dslObject.getQuery().getMatch()) { - if (MatchTypeEnum.getByType(matchBean.getType()) == null) { - return false; - } - for (String fieldValue : matchBean.getFieldValues()) { - if (fieldValue.startsWith("*") || fieldValue.endsWith("$")) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Match fieldValues cannot startWith '*' or endWith '$'"); - } - } - checkField(dslObject, matchBean.getFieldKey()); - } - return true; - } - - /** - * 校验sort: - * 1.排序类型只能是DESC或ASC - * 2.不限大小写 - * - * @param dslObject - * @return - */ - private boolean isValidSort(DSLObject dslObject) { - if (CollectionUtils.isEmpty(dslObject.getQuery().getSort())) { - return true; - } - for (DSLObject.QueryBean.SortBean sortBean : dslObject.getQuery().getSort()) { - if (!"DESC".equalsIgnoreCase(sortBean.getType()) && !"ASC".equalsIgnoreCase(sortBean.getType())) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "sort type is illegal, must be asc or desc"); - } - checkField(dslObject, sortBean.getFieldKey()); - } - return true; - } - - /** - * 校验limit: - * 1.limit范围[0,100000] - * - * @param limitStr - * @return - */ - private boolean isValidLimit(String limitStr) { - if (StrUtil.isBlank(limitStr)) { - return true; - } - int limit; - String[] split = limitStr.split(","); - if (split.length > 2) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Limit format error"); - } else if (split.length == 2) { - limit = Integer.parseInt(split[0]) + Integer.parseInt(split[1]); - } else { - limit = Integer.parseInt(split[split.length - 1]); - } - if (0 > limit || maxCacheNum < limit) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Limit must be in the range [0, " + maxCacheNum + "]"); - } - return true; - } - - /** - * 校验Interval: - * 1.目前只支持between类型 - * 2.时间区间必须是 开始时间 < 结束时间 - * - * @param dslObject - */ - private boolean isValidIntervals(DSLObject dslObject) { - if (CollectionUtils.isEmpty(dslObject.getQuery().getIntervals())) { - return true; - } - for (DSLObject.QueryBean.IntervalsBean interval : dslObject.getQuery().getIntervals()) { - if (Objects.isNull(IntervalTypeEnum.getByType(interval.getType().toUpperCase()))) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Interval type not support " + interval.getType()); - } - if (interval.getFieldValues().size() != 2) { - return false; - } - for (String fieldValue : interval.getFieldValues()) { - if (!strFormatDateTime.matcher(fieldValue).find()) { - return false; - } - } - - try { - DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"); - if (dateTimeFormatter.parseMillis(interval.getFieldValues().get(1)) - dateTimeFormatter.parseMillis(interval.getFieldValues().get(0)) < 0) { - throw new RuntimeException("Interval fieldValue should be [start, end]"); - } - } catch (Exception e) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), e.getMessage()); - } - checkField(dslObject, interval.getFieldKey()); - } - return true; - } - - /** - * 校验dataEngine: - * 1.是否属于{@link EngineTypeEnum}支持的类型 - * - * @param dataEngine - * @return - */ - private boolean isValidEngine(String dataEngine) { - if (StrUtil.isBlank(dataEngine)) { - return false; - } - for (EngineTypeEnum engineTypeEnum : EngineTypeEnum.values()) { - if (engineTypeEnum.getEngine().equals(dataEngine)) { - return true; - } - } - return false; - } - - /** - * 校验field是否是相应表字段 - * - * @param dslObject - * @param fieldKey - */ - private void checkField(DSLObject dslObject, String fieldKey) { - //AnangoDB暂不校验 - if (EngineTypeEnum.ANALYSIS_ENGINE.getEngine().equalsIgnoreCase(dslObject.getDataEngine())) { - return; - } - if (!getFields(dslObject).contains(fieldKey)) { - throw new BusinessException(HttpStatus.SC_BAD_REQUEST, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), fieldKey + " is invalid"); - } - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DruidParser.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DruidParser.java deleted file mode 100644 index 4c672c1..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/DruidParser.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.mesalab.engine.component; - -import cn.hutool.core.util.StrUtil; -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.util.SqlKeywords; -import com.mesalab.engine.service.DruidEngineService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - -import java.text.MessageFormat; -import java.util.List; - -/** - * @Date: 2020-07-22 17:00 - * @Author : liuyongqiang - * @ClassName : DruidDSLParse - * @Description : Druid DSL语法解析器 - */ -@Slf4j -@Component -public class DruidParser extends DSLParser { - - @Autowired - DruidEngineService druidEngineService; - - //表字段和数据类型查询SQL - public final static String SCHEAM_QUERY = "SELECT COLUMN_NAME as name, DATA_TYPE as type FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ''{0}''"; - - //查询所有表 - public final static String TABLES_QUERY = "SELECT TABLE_NAME AS name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'TABLE'"; - - - @Override - public String parseGranularity(DSLObject dslObject, String sql) { - if(StrUtil.isEmpty(dslObject.getQuery().getGranularity())){ - return sql; - } - String granularity = dslObject.getQuery().getGranularity(); - sql = sql.replace(SqlKeywords.SELECT, SqlKeywords.SELECT - .concat(MessageFormat.format(SqlKeywords.D_TIME_FLOOR,granularity)) - .concat(",")); - return sql; - } - - - - - @Override - public String parseSort(DSLObject dslObject, String sql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getSort())){ - return sql; - } - String sortType = dslObject.getQuery().getSort().get(0).getType(); - sql = sql.concat(SqlKeywords.ORDER_BY - .concat(SqlKeywords.D_TIME)) - .concat(sortType); - return sql; - } - - @Override - public String parseIntervals(DSLObject dslObject, String sql) { - if(CollectionUtils.isEmpty(dslObject.getQuery().getIntervals())){ - return sql; - } - List<DSLObject.QueryBean.IntervalsBean> intervalsBeanList = dslObject.getQuery().getIntervals(); - for (DSLObject.QueryBean.IntervalsBean intervalsBean : intervalsBeanList){ - String fieldKey = intervalsBean.getFieldKey(); - List<String> values = intervalsBean.getFieldValues(); - - if(CollectionUtils.isEmpty(values)) break; - - String join = sql.contains(SqlKeywords.WHERE)? SqlKeywords.AND: SqlKeywords.WHERE; - - sql = sql.concat(join).concat(fieldKey) - .concat(SqlKeywords.BWTWEEN) - .concat("'"+values.get(0)+"'") - .concat(SqlKeywords.AND) - .concat("'"+values.get(1)+"'"); - } - return sql; - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/IDGenerator.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/IDGenerator.java deleted file mode 100644 index 94549e2..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/IDGenerator.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.mesalab.engine.component; - -import cn.hutool.core.date.DatePattern; -import cn.hutool.core.date.DateTime; -import cn.hutool.core.lang.ObjectId; -import cn.hutool.core.lang.Snowflake; -import cn.hutool.core.net.NetUtil; -import cn.hutool.core.util.IdUtil; -import cn.hutool.core.util.RandomUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; - -/** - * @Date: 2020-07-30 14:04 - * @Author : liuyongqiang - * @ClassName : IdGenerator - * @Description : ID生成器 - */ -@Slf4j -@Component -public class IDGenerator { - - private long workerId = 0; - - @PostConstruct - void init() { - try { - workerId = NetUtil.ipv4ToLong(NetUtil.getLocalhostStr()); - log.info("Current machine workerId: {}", workerId); - } catch (Exception e) { - log.warn("Get machine id fail:", e); - workerId = NetUtil.getLocalhost().hashCode(); - log.info("Current machine workerId: {}", workerId); - } - } - - /** - * 获取一个批次号，形如 2019071015301361000101237 - * <p> - * 数据库使用 char(25) 存储 - * - * @param tenantId 租户ID，5 位 - * @param module 业务模块ID，2 位 - * @return 返回批次号 - */ - public synchronized String batchId(int tenantId, int module) { - String prefix = DateTime.now().toString(DatePattern.PURE_DATETIME_MS_PATTERN); - return prefix + tenantId + module + RandomUtil.randomNumbers(3); - } - - @Deprecated - public synchronized String getBatchId(int tenantId, int module) { - return batchId(tenantId, module); - } - - /** - * 生成的是不带-的字符串，类似于：b17f24ff026d40949c85a24f4f375d42 - * - * @return - */ - public String simpleUUID() { - return IdUtil.simpleUUID(); - } - - /** - * 生成的UUID是带-的字符串，类似于：a5c8a5e8-df2b-4706-bea4-08d0939410e3 - * - * @return - */ - public String randomUUID() { - return IdUtil.randomUUID(); - } - - private Snowflake snowflake = IdUtil.createSnowflake(workerId, 1); - - public synchronized long snowflakeId() { - return snowflake.nextId(); - } - - public synchronized long snowflakeId(long workerId, long dataCenterId) { - Snowflake snowflake = IdUtil.createSnowflake(workerId, dataCenterId); - return snowflake.nextId(); - } - - /** - * 生成类似：5b9e306a4df4f8c54a39fb0c - * <p> - * ObjectId 是 MongoDB 数据库的一种唯一 ID 生成策略， - * 是 UUID version1 的变种，详细介绍可见：服务化框架－分布式 Unique ID 的生成方法一览。 - * - * @return - */ - public String objectId() { - return ObjectId.next(); - } - -} - diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/IPConverter.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/IPConverter.java deleted file mode 100644 index c4f2fd6..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/IPConverter.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.mesalab.engine.component; - -import com.zdjizhi.utils.IpLookup; - -import javax.servlet.http.HttpServletRequest; - -/** - * IP工具类 - * - * @author dazzlzy - * @date 2018/4/1 - */ -public class IPConverter { - - private static final String UNKNOWN = "unknown"; - private static final String DEFAULT_SEPARATOR = ","; - private static final int DEFAULT_IP_LENGTH = 15; - private static IpLookup ipLookup = new IpLookup.Builder(false) - .loadDataFileV4("mmdb/ip_v4.mmdb") - .loadDataFileV6("mmdb/ip_v6.mmdb") - .loadDataFilePrivateV4("mmdb/ip_private_v4.mmdb") - .loadDataFilePrivateV4("mmdb/ip_private_v6.mmdb") - .loadAsnDataFileV4("mmdb/asn_v4.mmdb") - .loadAsnDataFileV6("mmdb/asn_v6.mmdb") - .build(); - - public static String getGeo(String ip) { - return ipLookup.latLngLookup(ip); - } - - public static String getCityDetail(String ip) { - return ipLookup.cityLookupDetail(ip); - } - - public static String getCountry(String ip) { - return ipLookup.countryLookup(ip); - } - - - - /** - * 从http request中获取真实IP - * - * @param request HttpServletRequest - * @return 真实IP - */ - public static String parseIP(HttpServletRequest request) { - String ip = request.getHeader("x-forwarded-for"); - if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { - ip = request.getHeader("Proxy-Client-IP"); - } - if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { - // 代理IP - ip = request.getHeader("WL-Proxy-Client-IP"); - } - if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { - // 网宿cdn的真实ip - ip = request.getHeader("Cdn-Src-Ip"); - } - if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { - // 蓝讯cdn的真实ip - ip = request.getHeader("HTTP_CLIENT_IP"); - } - if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { - ip = request.getHeader("HTTP_X_FORWARDED_FOR"); - } - if (ip == null || ip.length() == 0 || UNKNOWN.equalsIgnoreCase(ip)) { - // 真实IP - ip = request.getRemoteAddr(); - } - // 如果是多级代理，那么取第一个ip为客户端ip - if (ip != null && ip.length() > DEFAULT_IP_LENGTH && ip.contains(DEFAULT_SEPARATOR)) { - String[] ips = ip.split(DEFAULT_SEPARATOR); - for (String strIp : ips) { - if (!UNKNOWN.equalsIgnoreCase(strIp)) { - ip = strIp; - break; - } - } - } - return ip; - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/LoggerCache.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/LoggerCache.java deleted file mode 100644 index 5f6ff33..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/LoggerCache.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.mesalab.engine.component; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import java.util.HashMap; - -/** - * @Date: 2020-07-30 11:53 - * @Author : liuyongqiang - * @ClassName : LoggerCache - * @Description : 日志实例缓存 - */ -@Component -public class LoggerCache { - - /** - * 日志实例记录在内存中 - */ - private HashMap<String, Logger> LOGERS = new HashMap<String, Logger>(); - - /** - * 根据类名获取缓存的日志实例 - * @param className 包名加类名 this.getClass().getName(); - * @return - */ - public Logger getLoggerByClassName(String className) { - // 从静态map中获取日志实例 - Logger logger = LOGERS.get(className); - // 如果没取到 - if (logger == null) { - // 创建一个日志实例 - logger = LoggerFactory.getLogger(className); - // 加入到静态map中 - LOGERS.put(className, logger); - } - // 返回 - return logger; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/ReportCache.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/ReportCache.java deleted file mode 100644 index 4a2ef84..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/ReportCache.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.mesalab.engine.component; - -import com.google.common.collect.Maps; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.QueryFormatEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.engine.component.config.HbaseConfig; -import com.mesalab.engine.service.HttpClientService; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.http.HttpStatus; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; - -/** - * @author wangwei - * @version 1.0 - * @date 2020/6/5 2:04 下午 - */ -@Slf4j -@SuppressWarnings("unchecked") -public class ReportCache { - - private static HbaseConfig hbaseConfig = (HbaseConfig) SpringContext.getBean("hbaseConfig"); - private static HttpClientService httpClientService = (HttpClientService) SpringContext.getBean("httpClientService"); - - public static BaseResult<Object> get(ApiQueryBean param) { - BaseResult<Object> baseResult; - StringBuilder urlBuilder = new StringBuilder("http://") - .append(hbaseConfig.getUrl()).append("/"); - - StringBuilder queryParamBuilder = new StringBuilder(hbaseConfig.getDbName()) - .append(":").append(hbaseConfig.getTableName()).append("/") - .append(getQueryId(param.getResultId(), param.getQuery())).append("/") - .append(hbaseConfig.getColumnFamily()).append(":").append(hbaseConfig.getColumnName()); - - Map<String, String> headers = Maps.newHashMap(); - headers.put("Accept", "application/json"); - Map<String, String> results = httpClientService.httpGet(urlBuilder.toString() + queryParamBuilder.toString(), headers); - - if (results.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - Map result = (Map) JsonMapper.fromJsonString(decryptHbaseResult(results.get("result")), Map.class); - baseResult = BaseResultUtil.generate(Integer.parseInt(results.get("status")), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), String.valueOf(result.get("message")), - result.get("data"), result.get("meta"), (Map<String, Object>) (result.get("statistics")), QueryFormatEnum.JSON.getValue()); - } else { - baseResult = BaseResultUtil.generate(Integer.parseInt(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), null, - results.get("result"), null, null, param.getFormat()); - } - return baseResult; - } - - - /** - * 解析json并将结果进行base64解密 - * - * @param hbaseResult - * @return - */ - private static String decryptHbaseResult(String hbaseResult) { - String result = StringUtil.EMPTY; - try { - Map<String, Object> maps = (Map<String, Object>) JsonMapper.fromJsonString(hbaseResult, Map.class); - List<Object> rowList = (ArrayList) maps.get("Row"); - if (StringUtil.isNotEmpty(rowList)) { - Map<String, Object> keyMap = (Map<String, Object>) rowList.get(0); - String encryptResult = ((Map<String, Object>) ((ArrayList) keyMap.get("Cell")).get(0)).get("$").toString(); - result = new String(Base64.getDecoder().decode(encryptResult), StandardCharsets.UTF_8); - } - } catch (Exception e) { - log.error("Parser HBase Result error: {}", e.getMessage()); - } - return result; - } - - /** - * 通过SQL生成查询唯一kEY - * - * @param sql - * @param resultId - * @return - */ - public static String getQueryId(String resultId, String sql) { - return DigestUtils.md5Hex(resultId + sql.trim()); - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SchemaCache.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SchemaCache.java deleted file mode 100644 index 2922929..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SchemaCache.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.mesalab.engine.component; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.CacheStats; -import com.google.common.cache.LoadingCache; -import com.mesalab.engine.service.MetadataService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.concurrent.TimeUnit; - -@Slf4j -@Component -public class SchemaCache { - - @Autowired - MetadataService metadataService; - - LoadingCache<String, Object> loadingCache; - - @PostConstruct - void initSchema() { - loadingCache = CacheBuilder.newBuilder() - .maximumSize(1000) - .expireAfterWrite(30, TimeUnit.MINUTES) - .recordStats() - .build(new CacheLoader<String, Object>() { - @Override - public Object load(String key) throws Exception { - Object value = metadataService.getAllSchema(); - return value; - } - - }); - log.info("SchemaCache Inited"); - } - - - public Object get(String key) { - try { - return loadingCache.get(key); - } catch (Exception e) { - log.error(" Schema Cache ,get key error ", e); - return null; - } - } - - - public void put(String key, Object value) { - loadingCache.put(key, value); - } - - public void remove(String key) { - loadingCache.invalidate(key); - } - - public void removeAll() { - loadingCache.invalidateAll(); - } - - public CacheStats getStats() { - return loadingCache.stats(); - } - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SpringContext.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SpringContext.java deleted file mode 100644 index d0d7418..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SpringContext.java +++ /dev/null @@ -1,94 +0,0 @@ -package com.mesalab.engine.component; - -import org.springframework.beans.BeansException; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -import org.springframework.stereotype.Component; - -import java.util.Map; - -/** - * Spring上下文工具类 - * @author dazzlzy - * @date 2018/4/25 - */ -@Component -public class SpringContext implements ApplicationContextAware { - - private static ApplicationContext context = null; - - private SpringContext() { - super(); - } - - /** - * 根据名称获取bean - * - * @param beanName bean的名字 - * @return bean对象 - */ - public static Object getBean(String beanName) { - return context.getBean(beanName); - } - - /** - * 根据bean名称获取指定类型bean - * - * @param beanName bean名称 - * @param clazz 返回的bean类型,若类型不匹配,将抛出异常 - */ - public static <T> T getBean(String beanName, Class<T> clazz) { - return context.getBean(beanName, clazz); - } - - /** - * 根据类型获取bean - * - * @param clazz bean类型 - * @return 指定类型的bean - */ - public static <T> T getBean(Class<T> clazz) { - T t = null; - Map<String, T> map = context.getBeansOfType(clazz); - for (Map.Entry<String, T> entry : map.entrySet()) { - t = entry.getValue(); - } - return t; - } - - /** - * 是否包含bean - * - * @param beanName bean名字 - * @return boolean，是否包含指定名字的bean - */ - public static boolean containsBean(String beanName) { - return context.containsBean(beanName); - } - - /** - * 是否是单例 - * - * @param beanName bean名字 - * @return boolean， 是否是单例 - */ - public static boolean isSingleton(String beanName) { - return context.isSingleton(beanName); - } - - /** - * bean的类型 - * - * @param beanName bean名字 - * @return bean的类型 - */ - public static Class getType(String beanName) { - return context.getType(beanName); - } - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - context = applicationContext; - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SqlFunction.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SqlFunction.java deleted file mode 100644 index fa21b7f..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SqlFunction.java +++ /dev/null @@ -1,1455 +0,0 @@ -package com.mesalab.engine.component; - -import com.google.common.base.Splitter; -import com.google.common.collect.Maps; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.engine.component.config.ClickHouseConfig; -import com.mesalab.engine.component.config.DruidConfig; -import com.mesalab.engine.service.HttpClientService; -import com.zdjizhi.utils.Encodes; -import com.zdjizhi.utils.JsonMapper; -import lombok.Data; -import org.apache.commons.lang.StringUtils; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; - -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author wangwei - * @data 2019/8/1 10:13 - */ -@SuppressWarnings("unchecked") -public class SqlFunction { - - private static DruidConfig druidConfig = (DruidConfig) SpringContext.getBean("druidConfig"); - private static ClickHouseConfig clickHouseConfig = (ClickHouseConfig) SpringContext.getBean("clickHouseConfig"); - private static HttpClientService httpClientService = (HttpClientService) SpringContext.getBean("httpClientService"); - - public static final String TIME_FLOOR_WITH_FILL = "TIME_FLOOR_WITH_FILL"; - public static final String IP_TO_GEO = "IP_TO_GEO"; - public static final String IP_TO_CITY = "IP_TO_CITY"; - public static final String IP_TO_COUNTRY = "IP_TO_COUNTRY"; - - /** - * 自定义函数 - */ - public static Pattern pTimeFloorWithFill = Pattern.compile("\\b" + TIME_FLOOR_WITH_FILL + "\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pIpToGeo = Pattern.compile("\\b" + IP_TO_GEO + "\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pIpToCity = Pattern.compile("\\b" + IP_TO_CITY + "\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pIpToCountry = Pattern.compile("\\b" + IP_TO_COUNTRY + "\\(", Pattern.CASE_INSENSITIVE); - - public static Pattern periodOfPT = Pattern.compile("PT(\\d+)(\\w+)", Pattern.CASE_INSENSITIVE); - public static Pattern periodOfP = Pattern.compile("P(\\d+)(\\w+)", Pattern.CASE_INSENSITIVE); - public static final HashMap<String, String> functions = Maps.newHashMap(); - - static { - functions.put(TIME_FLOOR_WITH_FILL, null); - functions.put(IP_TO_GEO, null); - functions.put(IP_TO_CITY, null); - functions.put(IP_TO_COUNTRY, null); - } - - /** - * 标准函数 - */ - public static Pattern pDateFormat = Pattern.compile("\\W(FROM_UNIXTIME|DATE_FORMAT|STR_TO_DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pGroupByFormat = Pattern.compile("(SELECT\\s+|GROUP\\s*BY\\s+|,\\s*)(FROM_UNIXTIME|DATE_FORMAT|STR_TO_DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pDateRelative = Pattern.compile("\\W(ADDDATE|DATE_ADD|DATE_SUB|SUBDATE)\\s*\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pIntervalNumUnit = Pattern.compile("INTERVAL(.*?)(YEAR|QUARTER|MONTH|WEEK|DAY|HOUR|MINUTE|SECOND)$", Pattern.CASE_INSENSITIVE); - public static Pattern pUnixTime = Pattern.compile("\\WUNIX_TIMESTAMP\\s*\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pDate = Pattern.compile("\\W(DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pDateCalculateNum = Pattern.compile("\\(\\s*(DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pUnitFuncGetNum = Pattern.compile("\\W(YEAR|QUARTER|MONTH|DAY|HOUR|MINUTE|SECOND|DAYOFYEAR|DAYOFMONTH|DAYOFWEEK)\\s*\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pMarkDate = Pattern.compile("\\W(MAKEDATE)\\s*\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pLastDateFunc = Pattern.compile("\\W(LAST_DAY)\\s*\\(", Pattern.CASE_INSENSITIVE); - public static Pattern pCKTimeStamp = Pattern.compile("\\s+TIMESTAMP\\s+([^'])", Pattern.CASE_INSENSITIVE); - public static Pattern pNow = Pattern.compile("NOW\\(\\s*\\)", Pattern.CASE_INSENSITIVE); - public static Pattern pStrDateTime = Pattern.compile("(<|>|=|between|and)\\s*('\\d{4}-\\d{2}-\\d{2}\\s*\\d{2}:\\d{2}:\\d{2}')", Pattern.CASE_INSENSITIVE); - public static Pattern pMergeFunction = Pattern.compile("\\WDATE_FORMAT\\(FROM_UNIXTIME\\s*\\(\\s*FLOOR\\s*\\(\\s*UNIX_TIMESTAMP\\s*\\(", Pattern.CASE_INSENSITIVE); - - - /** - * 解析mysql函数替换为对应数据库类型函数 - * - * @param sql 要解析的sql - * @param dbType 数据库类型 - * @return - */ - public static String generateDateFunction(String sql, String dbType) { - sql = translateUDF(sql, dbType); - sql = parseMergepFunction(sql, dbType); - sql = parseMarkDate(sql, dbType); - sql = parseDateCalculateNum(sql, dbType); - sql = parseLastDayFunc(sql, dbType); - sql = parseGroupByFormat(sql, dbType); - sql = parseDateFormat(sql, dbType); - sql = parseDateRelative(sql, dbType); - sql = parseUnixTime(sql, dbType); - sql = parseDate(sql, dbType); - sql = parseUnitFuncGetNum(sql, dbType); - sql = specialDisposeEnd(sql, dbType); - sql = replaceMark(sql); - return sql; - } - - /** - * 转义自定义函数 - * - * @param sql - * @param dbType - * @return - */ - public static String translateUDF(String sql, String dbType) { - String resultSql = sql; - resultSql = translateByPattern(resultSql, dbType, pTimeFloorWithFill); - resultSql = translateByPattern(resultSql, dbType, pIpToGeo); - resultSql = translateByPattern(resultSql, dbType, pIpToCity); - resultSql = translateByPattern(resultSql, dbType, pIpToCountry); - return resultSql; - } - - /** - * 查询数据源获取函数实际值 - * - * @param query - * @param dbType - * @return - */ - public static String getQueryValue(String query, String dbType) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - query = generateDateFunction("select " + query + " as actualValue", dbType); - query = Encodes.urlEncode(query); - Map<String, String> results = executeHttpGetOfCK(query); - Map<String, Object> maps = (Map<String, Object>) JsonMapper.fromJsonString(results.get("result"), Map.class); - List<Map<String, Object>> dates = (List<Map<String, Object>>) maps.get("data"); - Map<String, Object> data = dates.get(0); - Object actualValue = data.get("actualValue"); - return actualValue.toString(); - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - query = generateDateFunction("select TIME_FORMAT(" + query + ", 'yyyy-MM-dd HH:mm:ss') as actualValue", dbType); - Map<String, String> results = executeHttpPostOfDruid(query); - List<Object> maps = (List<Object>) JsonMapper.fromJsonString(results.get("result"), Object.class); - Map<String, Object> data = (Map<String, Object>) maps.get(0); - Object actualValue = data.get("actualValue"); - return actualValue.toString(); - } else { - return query; - } - } - - /** - * post查询druid - * - * @param query - * @return - */ - public static Map<String, String> executeHttpPostOfDruid(String query) { - DruidQueryParam druidQueryParam = new DruidQueryParam(); - druidQueryParam.setQuery(query); - druidQueryParam.getContext().put("sqlTimeZone", druidConfig.getSqlTimeZone()); - druidQueryParam.getContext().put("skipEmptyBuckets", druidConfig.getSkipEmptyBuckets()); - druidQueryParam.setResultFormat("object"); - Map<String, String> stringStringMap = httpClientService.httpPost(druidConfig.getQueryUrl(), JsonMapper.toJsonString(druidQueryParam)); - return stringStringMap; - - } - - /** - * get查询clickHouse - * - * @param sql - * @return - */ - private static Map<String, String> executeHttpGetOfCK(String sql) { - StringBuilder urlBuilder = new StringBuilder(clickHouseConfig.getQueryUrl()).append("/?"); - StringBuilder queryparamBuilder = new StringBuilder("user=") - .append(clickHouseConfig.getRealTimeUserName()).append("&") - .append("password=").append(clickHouseConfig.getRealTimePassword()).append("&") - .append("database=").append(clickHouseConfig.getDbName()).append("&") - .append("query=").append(sql) - .append(" FORMAT JSON;"); - List<NameValuePair> values = URLEncodedUtils.parse(queryparamBuilder.toString(), Charset.forName("UTF-8")); - Map<String, String> stringStringMap = httpClientService.httpGet(urlBuilder.toString() + URLEncodedUtils.format(values, "utf-8")); - return stringStringMap; - } - - /** - * 获取第一个括号对应右括号的索引以及包含括号的个数,向后匹配 - * - * @param str - * @param bracketL 支持：( OR [ OR { - * @param bool 是否将传入的字符串全部匹配完成,若不需要则在匹配到第一个括号匹配完成时结束 - * @return int[0]:匹配括号的个数,int[1]:第一个括号对应右括号的索引;若无匹配括号返回{0,-1};若有其中一个不成对则返回{-1,index} - */ - public static int[] getBracketsMatch(String str, String bracketL, boolean bool) { - int[] result = {0, -1}; - if (str == null) { - return result; - } - String bracketR = ")"; - if (bracketL == null) { - bracketL = "("; - } else if (bracketL.equals("[")) { - bracketR = "]"; - } else if (bracketL.equals("{")) { - bracketR = "}"; - } - StringBuffer sb = new StringBuffer(str); - int countL = 0, countR = 0; - for (int a = 0; a < sb.length(); a++) { - if (sb.indexOf(bracketR) == a) { - ++countR; - sb.replace(a, a + 1, "$"); - } else if (sb.indexOf(bracketL) == a) { - ++countL; - sb.replace(a, a + 1, "$"); - } - if (countR > countL) { - result[0] = -1; - result[1] = -1; - return result; - } - result[0] = countL; - result[1] = a; - if (countL != 0 && (countL == countR) && !bool) { - return result; - } - } - if (countR != countL) { - result[0] = -1; - } - return result; - } - - - /** - * 匹配连接函数: DATE_FORMAT(FROM_UNIXTIME(UNIX_TIMESTAMP( - * - * @param sql - * @param dbType - * @return - */ - private static String parseMergepFunction(String sql, String dbType) { - int count = 0; - while (true) { - Matcher matcher = pMergeFunction.matcher(sql); - if (!matcher.find() || count++ >= 40) { - return sql; - } - int start = matcher.start() + 1; - String sqlParse = sql.substring(start); - int[] bracketsMatch = getBracketsMatch(sqlParse, "(", false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String str = MergeFromUnixToTimeHavaMark(sqlParse, bracketsMatch[0], dbType); - sql = sql.substring(0, start) + str; - } - } - - - /** - * 匹配范围：FROM_UNIXTIME(、DATE_FORMAT(、STR_TO_DATE( - * - * @param sql 需要解析的sql - * @param dbType - * @return - */ - public static String parseDateFormat(String sql, String dbType) { - int count = 0; - while (true) { - Matcher matcher = pDateFormat.matcher(sql); - if (!matcher.find() || count++ >= 40) { - return sql; - } - int start = matcher.start() + 1; - String sqlParse = sql.substring(start); - int[] bracketsMatch = getBracketsMatch(sqlParse, "(", false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String str = dateFormatHavaMark(sqlParse, bracketsMatch[0], false, dbType); - sql = sql.substring(0, start) + str; - } - } - - /** - * 匹配格式化 FROM_UNIXTIME(、DATE_FORMAT(、STR_TO_DATE( - * - * @param sql - * @param dbType - * @return - */ - public static String parseGroupByFormat(String sql, String dbType) { - int count = 0; - while (true) { - Matcher matcher = pGroupByFormat.matcher(sql); - if (!matcher.find() || count++ >= 30) { - return sql; - } - int start = matcher.start(); - String sqlParse = sql.substring(start); - int[] bracketsMatch = getBracketsMatch(sqlParse, "(", false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String str = dateFormatHavaMark(sqlParse, bracketsMatch[0], true, dbType); - sql = sql.substring(0, start) + str; - } - } - - /** - * 匹配 SUBDATE(、SUBDATE(、SUBDATE(、ADDDATE(、DATE_SUB(、DATE_ADD( - * - * @param sql - * @param dbType - * @return - */ - public static String parseDateRelative(String sql, String dbType) { - int count = 0; - while (true) { - Matcher matcher = pDateRelative.matcher(sql); - if (!matcher.find() || count++ >= 30) { - return sql; - } - int start = matcher.start() + 1; - String sqlParse = sql.substring(start); - int[] bracketsMatch = getBracketsMatch(sqlParse, null, false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String str = dateRelativeHaveMark(sqlParse, bracketsMatch[0], dbType); - sql = sql.substring(0, start) + str; - } - } - - /** - * 匹配 UNIX_TIMESTAMP( - * - * @param sql - * @param dbType - * @return - */ - public static String parseUnixTime(String sql, String dbType) { - int count = 0; - while (true) { - Matcher matcher = pUnixTime.matcher(sql); - if (!matcher.find() || count++ == 30) { - return sql; - } - int start = matcher.start() + 1; - String sqlParse = sql.substring(start); - int[] bracketsMatch = getBracketsMatch(sqlParse, null, false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String str = parseUnixTimeHavaMark(sqlParse, bracketsMatch[0], dbType); - sql = sql.substring(0, start) + str; - } - } - - /** - * 匹配 DATE( - * - * @param sql - * @param dbType - * @return - */ - public static String parseDate(String sql, String dbType) { - int count = 0; - while (true) { - Matcher matcher = pDate.matcher(sql); - if (!matcher.find() || count++ >= 30) { - return sql; - } - int start = matcher.start() + 1; - String sqlParse = sql.substring(start); - int[] bracketsMatch = getBracketsMatch(sqlParse, "(", false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String str = parseDateHavaMark(sqlParse, bracketsMatch[0], dbType); - sql = sql.substring(0, start) + str; - } - } - - /** - * 匹配 YEAR(、QUARTER(、MONTH(、DAY(、HOUR(、MINUTE(、SECOND、DAYOFYEAR(、DAYOFMONTH(、DAYOFWEEK( - * - * @param sql - * @param dbType - * @return - */ - public static String parseUnitFuncGetNum(String sql, String dbType) { - int count = 0; - while (true) { - Matcher matcher = pUnitFuncGetNum.matcher(sql); - if (!matcher.find() || count++ >= 30) { - return sql; - } - int start = matcher.start() + 1; - String sqlParse = sql.substring(start); - int[] bracketsMatch = getBracketsMatch(sqlParse, "(", false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String str = parseUnitFuncGetNumHaveMark(sqlParse, bracketsMatch[0], dbType); - sql = sql.substring(0, start) + str; - } - } - - /** - * 匹配 pMarkDate( - * - * @param sql - * @param dbType - * @return - */ - public static String parseMarkDate(String sql, String dbType) { - int count = 0; - while (true) { - Matcher matcher = pMarkDate.matcher(sql); - if (!matcher.find() || count++ >= 30) { - return sql; - } - int start = matcher.start() + 1; - String sqlParse = sql.substring(start); - int[] bracketsMatch = getBracketsMatch(sqlParse, "(", false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String str = parseMarkDateHaveMark(sqlParse, bracketsMatch[0], dbType); - sql = sql.substring(0, start) + str; - } - } - - /** - * 匹配 LAST_DAY( - * - * @param sql - * @param dbType - * @return - */ - public static String parseLastDayFunc(String sql, String dbType) { - int count = 0; - while (true) { - Matcher matcher = pLastDateFunc.matcher(sql); - if (!matcher.find() || count++ >= 30) { - return sql; - } - int start = matcher.start() + 1; - String sqlParse = sql.substring(start); - int[] bracketsMatch = getBracketsMatch(sqlParse, "(", false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String str = parseLastDayFuncHaveMark(sqlParse, bracketsMatch[0], dbType); - sql = sql.substring(0, start) + str; - } - } - - /** - * 匹配 涉及DATE()-、DATE()+ - * - * @param sql - * @param dbType - * @return - */ - public static String parseDateCalculateNum(String sql, String dbType) { - int count = 0; - String sqlTmp = ""; - while (true) { - Matcher matcher = pDateCalculateNum.matcher(sql); - if (!matcher.find() || count++ >= 30) { - sql = sql.replace("##", "("); - return sql; - } - String sqlParse = sql.substring(matcher.start()); - if (sqlTmp.equals(sqlParse)) { - sqlParse = "##" + sqlParse.substring(1); - sql = sql.substring(0, matcher.start()) + sqlParse; - continue; - } - sqlTmp = sqlParse; - int[] bracketsMatch = getBracketsMatch(sqlParse, "(", false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - String sqlParse2 = sql.substring(matcher.start() + 1); - int[] bracketsMatch2 = getBracketsMatch(sqlParse2, "(", false); - if (bracketsMatch2[0] >= 1) { - --bracketsMatch2[0]; - } - String str = parseDateCalculateNumHaveMark(sqlParse, bracketsMatch[0], bracketsMatch2[0], dbType); - sql = sql.substring(0, matcher.start()) + str; - } - } - - - /** - * 替换连接函数: DATE_FOERMAT(FROM_UNIXTIME(UNIX_TIMESTAMP(...) - * - * @param sql - * @param num - * @param dbType - * @return - */ - private static String MergeFromUnixToTimeHavaMark(String sql, int num, String dbType) { - Pattern p = Pattern.compile("DATE_FORMAT\\s*\\(\\s*FROM_UNIXTIME\\s*\\(\\s*FLOOR\\s*\\(\\s*UNIX_TIMESTAMP\\s*(\\((.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); - StringBuffer sb = new StringBuffer(); - Matcher m = p.matcher(sql); - if (m.find()) { - String group = m.group(1); - int[] bracketsMatch = getBracketsMatch(group, "(", false); - String param = group.substring(1, bracketsMatch[1]); - String var = group.substring(bracketsMatch[1] + 1); - var = var.replaceAll(" ", ""); - String replaceValue = m.group(0); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - switch (var) { - case "/60)*60),'%Y-%m-%d%H:%i:%s'": - replaceValue = "toStartOfMinute#[toDateTime#[" + param + "]]"; - break; - case "/3600)*3600),'%Y-%m-%d%H:%i:%s'": - replaceValue = "toStartOfHour#[toDateTime#[" + param + "]]"; - break; - } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - if (param.matches("'\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}'")) { - param = "TIMESTAMP " + param; - } - switch (var) { - case "/30)*30),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_FLOOR#[" + param + ",'PT30S'],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/30)*30-30),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_SHIFT#[TIME_FLOOR#[" + param + ",'PT30S'],'PT30S',-1],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/30)*30+30),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_SHIFT#[TIME_FLOOR#[" + param + ",'PT30S'],'PT30S',1],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/60)*60),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_FLOOR#[" + param + ",'PT1M'],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/60)*60-60),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_SHIFT#[TIME_FLOOR#[" + param + ",'PT1M'],'PT1M',-1],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/60)*60+60),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_SHIFT#[TIME_FLOOR#[" + param + ",'PT1M'],'PT1M',1],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/300)*300),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_FLOOR#[" + param + ",'PT5M'],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/300)*300-300),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_SHIFT#[TIME_FLOOR#[" + param + ",'PT5M'],'PT5M',-1],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/300)*300+300),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_SHIFT#[TIME_FLOOR#[" + param + ",'PT5M'],'PT5M',1],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/3600)*3600),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_FLOOR#[" + param + ",'PT1H'],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/3600)*3600-3600),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_SHIFT#[TIME_FLOOR#[" + param + ",'PT1H'],'PT1H',-1],'yyyy-MM-dd HH:mm:ss']"; - break; - case "/3600)*3600+3600),'%Y-%m-%d%H:%i:%s'": - replaceValue = "TIME_FORMAT#[TIME_SHIFT#[TIME_FLOOR#[" + param + ",'PT1H'],'PT1H',1],'yyyy-MM-dd HH:mm:ss']"; - break; - } - } - m.appendReplacement(sb, replaceValue); - } - m.appendTail(sb); - return sb.toString(); - } - - - /** - * 替换 涉及DATE()-、DATE()+ - * - * @param sqlParse - * @param num 括号总个数 - * @param num2 date函数内括号数 - * @param dbType - * @return - */ - private static String parseDateCalculateNumHaveMark(String sqlParse, int num, int num2, String dbType) { - Pattern pDateCalculateNumParse = Pattern.compile("(DATE)\\s*\\((.*?(.*?\\).*?){" + num2 + "})\\)(.*?(.*?\\).*?){" + (num - num2 - 1) + "})\\)", Pattern.CASE_INSENSITIVE); - StringBuffer sb = new StringBuffer(); - Matcher mDateCalculateNumParse = pDateCalculateNumParse.matcher(sqlParse); - if (mDateCalculateNumParse.find() && (mDateCalculateNumParse.group(4).trim().startsWith("+") || mDateCalculateNumParse.group(4).trim().startsWith("-"))) { - String group0 = mDateCalculateNumParse.group(0); - String group2 = mDateCalculateNumParse.group(2); - String group4 = mDateCalculateNumParse.group(4); - if (getBracketsMatch(group0.substring(0, group0.length() - 1), "(", true)[0] == num) { - String[] split = group4.split(","); - String replaceValue = ""; - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - if (split.length == 1) { - replaceValue = "toStartOfDay#[toDateTime#[" + group2 + "]]+[" + group4 + "]*86400 )"; - } else if (split.length == 2) { - replaceValue = "toStartOfDay#[toDateTime#[" + group2 + "]]+[" + split[0] + "]*86400 ," + split[1] + ")"; - } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - if (split.length == 1) { - replaceValue = "TIME_SHIFT#[FLOOR#[ TIMESTAMP " + group2 + " to day],'P1D'," + group4 + "])"; - } else if (split.length == 2) { - replaceValue = "TIME_SHIFT#[FLOOR#[ TIMESTAMP " + group2 + "to day],'P1D'," + split[0] + "]," + split[1] + ")"; - } - } - mDateCalculateNumParse.appendReplacement(sb, replaceValue); - } - } - mDateCalculateNumParse.appendTail(sb); - return sb.toString(); - } - - - /** - * 替换 LAST_DAY(date) - * - * @param sqlParse - * @param num 包含括号个数 - * @param dbType - * @return - */ - public static String parseLastDayFuncHaveMark(String sqlParse, int num, String dbType) { - Pattern pLastDayParse = Pattern.compile("(LAST_DAY)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); - StringBuffer sb = new StringBuffer(); - Matcher mLastDayParse = pLastDayParse.matcher(sqlParse); - if (mLastDayParse.find()) { - String replaceValue = lastDayRealize(mLastDayParse.group(1), mLastDayParse.group(2), dbType); - mLastDayParse.appendReplacement(sb, replaceValue); - } - mLastDayParse.appendTail(sb); - return sb.toString(); - } - - /** - * 实现替换：LAST_DAY(date) - * - * @param unit - * @param param - * @param dbType - * @return - */ - private static String lastDayRealize(String unit, String param, String dbType) { - String replaceValue = null; - if ("LAST_DAY".equals(unit.toUpperCase())) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - replaceValue = "addDays#[addMonths#[toStartOfMonth#[toDateTime#[" + param + "]],1],-1]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - replaceValue = "TIME_SHIFT#[FLOOR#[TIME_SHIFT#[ TIMESTAMP " + param + ", 'P1M', 1] to month], 'P1D', -1]"; - } - } - return replaceValue; - } - - /** - * 替换：MAKEDATE(year,dayofyear) - * - * @param sqlParse - * @param num 函数参数包含括号的个数 - * @param dbType - * @return - */ - public static String parseMarkDateHaveMark(String sqlParse, int num, String dbType) { - Pattern pMarkDateParse = Pattern.compile("(MAKEDATE)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); - StringBuffer sb = new StringBuffer(); - Matcher mMarkDateParse = pMarkDateParse.matcher(sqlParse); - if (mMarkDateParse.find()) { - String replaceValue = markDateRealize(mMarkDateParse.group(1), mMarkDateParse.group(2), dbType); - mMarkDateParse.appendReplacement(sb, replaceValue); - } - mMarkDateParse.appendTail(sb); - return sb.toString(); - } - - /** - * 实现替换：MAKEDATE(year,dayofyear) - * - * @param func - * @param expr - * @param dbType - * @return - */ - private static String markDateRealize(String func, String expr, String dbType) { - String replaceValue = null; - List<String> params = diviParam(expr, ","); - if ("MAKEDATE".equals(func.toUpperCase()) && params.size() == 2) { - Pattern pYearFunNum = Pattern.compile("(YEAR)\\s*\\(", Pattern.CASE_INSENSITIVE); - Matcher matcher = pYearFunNum.matcher(expr); - if (matcher.find() && "YEAR".equals(matcher.group(1).toUpperCase())) { - String temp = expr.substring(matcher.start()); - int[] bracketsMatch = getBracketsMatch(temp, "(", false); - if (bracketsMatch[0] >= 1) { - --bracketsMatch[0]; - } - Pattern pYear = Pattern.compile("^(.*?)(YEAR)\\s*\\((.*?(.*?\\).*?){" + bracketsMatch[0] + "})\\)(.*?)$", Pattern.CASE_INSENSITIVE); - Matcher mYear = pYear.matcher(params.get(0)); - if (mYear.find()) { - String group1 = mYear.group(1); - String group5 = mYear.group(5); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - if (group1 != null && !"".equals(group1.trim()) && group5 != null && !"".equals(group5.trim())) { - replaceValue = "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "]]," + group1 + " " + group5 + "]," + params.get(1) + " -1 ]]"; - } else if (group5 != null && !"".equals(group5.trim())) { - replaceValue = "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "]]," + group5 + "]," + params.get(1) + " -1 ]]"; - } else if (group1 != null && !"".equals(group1.trim())) { - replaceValue = "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "]]," + group1 + "0 ]," + params.get(1) + " -1 ]]"; - } else { - replaceValue = "toDateTime#[addDays#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "]]," + params.get(1) + " -1 ]]"; - } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - if (group1 != null && !"".equals(group1.trim()) && group5 != null && !"".equals(group5.trim())) { - replaceValue = "TIME_SHIFT#[TIME_SHIFT#[FLOOR#[ TIMESTAMP " + mYear.group(3) + " to year],'P1Y'," + group1 + " " + group5 + " ],'P1D'," + params.get(1) + " -1 ]"; - } else if (group5 != null && !"".equals(group5.trim())) { - replaceValue = "TIME_SHIFT#[TIME_SHIFT#[FLOOR#[ TIMESTAMP " + mYear.group(3) + " to year],'P1Y'," + group5 + "],'P1D'," + params.get(1) + " -1 ]"; - } else if (group1 != null && !"".equals(group1.trim())) { - replaceValue = "TIME_SHIFT#[TIME_SHIFT#[FLOOR#[ TIMESTAMP " + mYear.group(3) + " to year],'P1Y'," + group1 + "0 ],'P1D'," + params.get(1) + " -1 ]"; - } else { - replaceValue = "TIME_SHIFT#[FLOOR#[ TIMESTAMP " + mYear.group(3) + " to year],'P1D'," + params.get(1) + "-1 ]"; - } - } - } - } - } - return replaceValue; - } - - /** - * 替换 YEAR(expr)、QUARTER(expr)、MONTH(expr)、DAY(expr)、HOUR(expr)、MINUTE(expr)、SECONDexpr)、DAYOFYEAR(expr)、DAYOFMONTH(expr)、DAYOFWEEK(expr) - * - * @param sqlParse - * @param num - * @param dbType - * @return - */ - private static String parseUnitFuncGetNumHaveMark(String sqlParse, int num, String dbType) { - Pattern pUnitFuncParse = Pattern.compile("(YEAR|QUARTER|MONTH|DAY|HOUR|MINUTE|SECOND|DAYOFYEAR|DAYOFMONTH|DAYOFWEEK)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); - StringBuffer sb = new StringBuffer(); - Matcher mUnitFuncParse = pUnitFuncParse.matcher(sqlParse); - if (mUnitFuncParse.find()) { - String replaceValue = unitFuncGetNumRealize(mUnitFuncParse.group(1), mUnitFuncParse.group(2), dbType); - mUnitFuncParse.appendReplacement(sb, replaceValue); - } - mUnitFuncParse.appendTail(sb); - return sb.toString(); - } - - /** - * 实现替换：YEAR(expr)、QUARTER(expr)、MONTH(expr)、DAY(expr)、HOUR(expr)、MINUTE(expr)、SECONDexpr)、DAYOFYEAR(expr)、DAYOFMONTH(expr)、DAYOFWEEK(expr) - * - * @param unit - * @param expr - * @param dbType - * @return - */ - private static String unitFuncGetNumRealize(String unit, String expr, String dbType) { - String replaceValue = null; - unit = unit.toUpperCase(); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - switch (unit) { - case "YEAR": - replaceValue = "toYear#[toDateTime#[" + expr + "]]"; - break; - case "QUARTER": - replaceValue = "toQuarter#[toDateTime#[" + expr + "]]"; - break; - case "MONTH": - replaceValue = "toMonth#[toDateTime#[" + expr + "]]"; - break; - case "DAY": - replaceValue = "toDayOfMonth#[toDateTime#[" + expr + "]]"; - break; - case "HOUR": - replaceValue = "toHour#[toDateTime#[" + expr + "]]"; - break; - case "MINUTE": - replaceValue = "toMinute#[toDateTime#[" + expr + "]]"; - break; - case "SECOND": - replaceValue = "toSecond#[toDateTime#[" + expr + "]]"; - break; - case "DAYOFYEAR": - replaceValue = "toDayOfYear#[toDateTime#[" + expr + "]]"; - break; - case "DAYOFMONTH": - replaceValue = "toDayOfMonth#[toDateTime#[" + expr + "]]"; - break; - case "DAYOFWEEK": - replaceValue = "toDayOfWeek#[addDays#[toDateTime#[" + expr + "],1]]"; - break; - default: - replaceValue = unit + "#[" + expr + "]"; - } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - switch (unit) { - case "YEAR": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'YEAR' ]"; - break; - case "QUARTER": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'QUARTER' ]"; - break; - case "MONTH": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'MONTH' ]"; - break; - case "DAY": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'DAY' ]"; - break; - case "HOUR": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'HOUR' ]"; - break; - case "MINUTE": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'MINUTE' ]"; - break; - case "SECOND": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'SECOND' ]"; - break; - case "DAYOFYEAR": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'DOY' ]"; - break; - case "DAYOFMONTH": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'DAY' ]"; - break; - case "DAYOFWEEK": - replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'P1D',1], 'DOW' ]"; - break; - default: - replaceValue = unit + "#[" + expr + "]"; - } - } - return replaceValue; - } - - /** - * 实现相对时间相关替换： - * - * @param params - * @param sign ADD相关为true,SUB相关为false; - * @param dbType - * @return - */ - private static String dateRelative(List<String> params, Boolean sign, String dbType) { - String replaceValue = ""; - if (params.size() == 2 && params.get(1).toUpperCase().contains("INTERVAL")) { - String param1 = params.get(0); - String param2 = params.get(1); - Matcher matcher = pIntervalNumUnit.matcher(param2); - if (matcher.find()) { - param2 = matcher.group(1); - if (!sign) { - param2 = "- #[" + param2 + "]"; - } - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - switch (matcher.group(2).toUpperCase()) { - case "SECOND": - replaceValue = "toDateTime#[" + param1 + "]" + param2; - break; - case "MINUTE": - replaceValue = "toDateTime#[" + param1 + "]" + param2 + "*60"; - break; - case "HOUR": - replaceValue = "toDateTime#[" + param1 + "]" + param2 + "*3600"; - break; - case "DAY": - replaceValue = "toDateTime#[" + param1 + "]" + param2 + "*86400"; - break; - case "WEEK": - replaceValue = "toDateTime#[" + param1 + "]" + param2 + "*604800"; - break; - case "MONTH": - replaceValue = "addMonths#[toDateTime#[" + param1 + "] ," + param2 + "]"; - break; - case "QUARTER": - replaceValue = "addQuarter#[toDateTime#[" + param1 + "]," + param2 + "]"; - break; - case "YEAR": - replaceValue = "addYears#[toDateTime#[" + param1 + "]," + param2 + "]"; - break; - default: - replaceValue = param1 + param2; - break; - } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - switch (matcher.group(2).toUpperCase()) { - case "SECOND": - replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'PT1S'," + param2 + "]"; - break; - case "MINUTE": - replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'PT1M'," + param2 + "]"; - break; - case "HOUR": - replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'PT1H'," + param2 + "]"; - break; - case "DAY": - replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1D'," + param2 + "]"; - break; - case "WEEK": - replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1W'," + param2 + "]"; - break; - case "MONTH": - replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1M'," + param2 + "]"; - break; - case "QUARTER": - replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1M'," + param2 + "]"; - break; - case "YEAR": - replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1Y'," + param2 + "]"; - break; - default: - replaceValue = param1 + param2; - break; - } - } - return replaceValue; - } - } else if (params.size() == 2) { - String param1 = params.get(0); - String param2 = params.get(1); - if (!sign) { - param2 = "- #[" + param2 + "]"; - } - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - replaceValue = "addDays#[toDateTime#[" + param1 + "]," + param2 + "]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1D'," + param2 + "]"; - } - } - return replaceValue; - } - - /** - * 实现替换：FROM_UNIXTIME(unix_timestamp)、FROM_UNIXTIME(date,format)、DATE_FORMAT(date,format)、STR_TO_DATE(date,format) - * - * @param func - * @param param - * @param dbType - * @return - */ - private static String dateFormatRealize(String func, String param, boolean bool, String dbType) { - List<String> params = diviParam(param, ","); - String param1 = params.get(0); - String replaceValue = null; - if ("FROM_UNIXTIME".equals(func.toUpperCase()) && params.size() == 1) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - replaceValue = "toDateTime#[" + param + "]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && !bool) { - replaceValue = "MILLIS_TO_TIMESTAMP#[ 1000 * [" + param + "]]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && bool) { - replaceValue = "TIME_FORMAT#[MILLIS_TO_TIMESTAMP#[ 1000 * [" + param + "]],'YYYY-MM-dd HH:mm:ss']"; - } - } else if (func.contains("FROM_UNIXTIME") && DBTypeEnum.DRUID.getValue().equals(dbType) && "%Y-%m-%d %H:%i:%s".equals(params.get(1).replaceAll("\\'|\\\"", "").trim())) { - if (!bool) { - replaceValue = "MILLIS_TO_TIMESTAMP#[ 1000*[" + param1 + "]]"; - } else if (bool) { - replaceValue = "TIME_FORMAT#[MILLIS_TO_TIMESTAMP#[ 1000 * [" + param + "]],'YYYY-MM-dd HH:mm:ss']"; - } - } else { - String param2 = params.get(1).replaceAll("\\'|\\\"", "").trim(); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType) && bool) { - switch (param2) { - case "%Y-%m-%d %H:%i:%s": - replaceValue = "toDateTime#[" + param1 + "]"; - break; - case "%Y-%m-%d %H:%i:00": - replaceValue = "toStartOfMinute#[toDateTime#[" + param1 + "]]"; - break; - case "%Y-%m-%d %H:00:00": - replaceValue = "toStartOfHour#[toDateTime#[" + param1 + "]]"; - break; - case "%Y-%m-%d 00:00:00": - replaceValue = "toStartOfDay#[toDateTime#[" + param1 + "]]"; - break; - case "%Y-%m-01 00:00:00": - replaceValue = "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + "]]]"; - break; - case "%Y-01-01 00:00:00": - replaceValue = "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + "]]]"; - break; - case "%Y-%m-%d": - replaceValue = "formatDateTime#[toDateTime#[" + param1 + "], '%Y-%m-%d]"; - break; - case "%Y-%m-01": - replaceValue = "formatDateTime#[toDateTime#[" + param1 + "], '%Y-%m-01]"; - break; - case "%Y-01-01": - replaceValue = "formatDateTime#[toDateTime#[" + param1 + "], '%Y-01-01]"; - break; - default: - replaceValue = "toDateTime#[" + param1 + "]"; - } - } else if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType) && !bool) { - switch (param2) { - case "%Y-%m-%d %H:%i:%s": - replaceValue = "toDateTime#[" + param1 + "]"; - break; - case "%Y-%m-%d %H:%i:00": - replaceValue = "toStartOfMinute#[toDateTime#[" + param1 + "]]"; - break; - case "%Y-%m-%d %H:00:00": - replaceValue = "toStartOfHour#[toDateTime#[" + param1 + "]]"; - break; - case "%Y-%m-%d 00:00:00": - replaceValue = "toStartOfDay#[toDateTime#[" + param1 + "]]"; - break; - case "%Y-%m-01 00:00:00": - replaceValue = "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + "]]]"; - break; - case "%Y-01-01 00:00:00": - replaceValue = "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + "]]]"; - break; - case "%Y-%m-%d": - replaceValue = "toStartOfDay#[toDateTime#[" + param1 + "]]"; - break; - case "%Y-%m-01": - replaceValue = "toStartOfMonth#[toDateTime#[" + param1 + "]]"; - break; - case "%Y-01-01": - replaceValue = "toStartOfYear#[toDateTime#[" + param1 + "]]"; - break; - default: - replaceValue = "toDateTime#[" + param1 + "]"; - } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && bool) { - switch (param2) { - case "%Y-%m-%d %H:%i:%s": - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:mm:ss']"; - break; - case "%Y-%m-%d %H:%i:00": - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:mm:00']"; - break; - case "%Y-%m-%d %H:00:00": - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:00:00']"; - break; - case "%Y-%m-%d 00:00:00": - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd 00:00:00']"; - break; - case "%Y-%m-01 00:00:00": - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-01 00:00:00']"; - break; - case "%Y-01-01 00:00:00": - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-01-01 00:00:00']"; - break; - case "%Y-%m-%d": - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd']"; - break; - case "%Y-%m-01": - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-01']"; - break; - case "%Y-01-01": - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-01-01']"; - break; - default: - replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:mm:ss']"; - } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && !bool) { - switch (param2) { - case "%Y-%m-%d %H:%i:%s": - replaceValue = " TIMESTAMP " + param1; - break; - case "%Y-%m-%d %H:%i:00": - replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to minute]"; - break; - case "%Y-%m-%d %H:00:00": - replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to hour]"; - break; - case "%Y-%m-%d 00:00:00": - replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to day]"; - break; - case "%Y-%m-01 00:00:00": - replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to month]"; - break; - case "%Y-01-01 00:00:00": - replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to year]"; - break; - case "%Y-%m-%d": - replaceValue = "TIME_FLOOR#[ TIMESTAMP " + param1 + ",'P1D']"; - break; - case "%Y-%m-01": - replaceValue = "TIME_FLOOR#[ TIMESTAMP " + param1 + ",'P1M']"; - break; - case "%Y-01-01": - replaceValue = "TIME_FLOOR#[ TIMESTAMP " + param1 + ",'P1Y']"; - break; - default: - replaceValue = " TIMESTAMP " + param1; - break; - } - } - } - return replaceValue; - } - - /** - * 解析之后的sql处理 - * - * @param sql - * @param dbType - * @return - */ - public static String specialDisposeEnd(String sql, String dbType) { - if (sql == null) { - return sql; - } - StringBuffer sb = new StringBuffer(); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - Matcher mStrDateTime = pStrDateTime.matcher(sql); - while (mStrDateTime.find()) { - String innerValue = mStrDateTime.group(1) + " toDateTime#[" + mStrDateTime.group(2) + "]"; - mStrDateTime.appendReplacement(sb, innerValue); - } - mStrDateTime.appendTail(sb); - sql = sb.toString(); - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - - Matcher mNowFun = pNow.matcher(sql); - while (mNowFun.find()) { - String innerValue = "CURRENT_TIMESTAMP"; - mNowFun.appendReplacement(sb, innerValue); - } - mNowFun.appendTail(sb); - sql = sb.toString(); - - Matcher mStrDateTime = pStrDateTime.matcher(sql); - sb = new StringBuffer(); - while (mStrDateTime.find()) { - String innerValue = mStrDateTime.group(1) + " TIMESTAMP " + mStrDateTime.group(2); - mStrDateTime.appendReplacement(sb, innerValue); - } - mStrDateTime.appendTail(sb); - sql = sb.toString(); - - - Matcher matcher = pCKTimeStamp.matcher(sql); - sb = new StringBuffer(); - while (matcher.find()) { - if (matcher.groupCount() == 1) { - matcher.appendReplacement(sb, " " + matcher.group(1)); - } - } - matcher.appendTail(sb); - sql = sb.toString(); - } - return sql; - } - - /** - * 替换解析时候加入的[、]、# - * - * @param sql - * @return - */ - public static String replaceMark(String sql) { - sql = sql.replace("[", "("); - sql = sql.replace("]", ")"); - sql = sql.replace("#", ""); - return sql; - } - - /** - * 替换：FROM_UNIXTIME(unix_timestamp)、FROM_UNIXTIME(date,format)、DATE_FORMAT(date,format)、STR_TO_DATE(date,format) - * - * @param sqlParse - * @param num 包含括号的个数 - * @param bool 应用的分组中为true 条件范围中则为false - * @param dbType - * @return - */ - public static String dateFormatHavaMark(String sqlParse, int num, boolean bool, String dbType) { - Pattern pDateFormatMark = Pattern.compile("(FROM_UNIXTIME|DATE_FORMAT|STR_TO_DATE)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); - Matcher mDateFormatMark = pDateFormatMark.matcher(sqlParse); - StringBuffer sb = new StringBuffer(); - if (mDateFormatMark.find()) { - String group2 = mDateFormatMark.group(2); - String replaceValue = group2; - if (getBracketsMatch(group2, "(", false)[0] >= 0) { - replaceValue = dateFormatRealize(mDateFormatMark.group(1), group2, bool, dbType); - } - mDateFormatMark.appendReplacement(sb, replaceValue); - } - mDateFormatMark.appendTail(sb); - return sb.toString(); - } - - /** - * 替换SUBDATE(expr,days)、SUBDATE(expr,days)、SUBDATE(date,INTERVAL expr dbType)、ADDDATE(date,INTERVAL expr dbType)、DATE_SUB(date,INTERVAL expr dbType)、DATE_ADD(date,INTERVAL expr dbType) - * - * @param sqlParse - * @param num 包含的括号个数 - * @param dbType - * @return - */ - public static String dateRelativeHaveMark(String sqlParse, int num, String dbType) { - Pattern pDateRelativeParse = Pattern.compile("(ADDDATE|DATE_ADD|SUBDATE|DATE_SUB)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); - Matcher matcherParse = pDateRelativeParse.matcher(sqlParse); - String innerValue = ""; - StringBuffer sb = new StringBuffer(); - if (matcherParse.find()) { - Boolean sign = matcherParse.group(1).toUpperCase().contains("ADD"); - if (getBracketsMatch(matcherParse.group(2), null, false)[0] >= 0) { - List<String> params = diviParam(matcherParse.group(2), ","); - innerValue = dateRelative(params, sign, dbType); - } - matcherParse.appendReplacement(sb, innerValue); - } - matcherParse.appendTail(sb); - return sb.toString(); - } - - /** - * 实现替换：UNIX_TIMESTAMP()、UNIX_TIMESTAMP(expr) - * - * @param sqlParse - * @param num 包含的括号个数 - * @param dbType - * @return - */ - public static String parseUnixTimeHavaMark(String sqlParse, int num, String dbType) { - Pattern pUnixTimeParse = Pattern.compile("(UNIX_TIMESTAMP)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); - Matcher matcherParse = pUnixTimeParse.matcher(sqlParse); - StringBuffer sb = new StringBuffer(); - String innerValue = null; - if (matcherParse.find()) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { - innerValue = "toUnixTimestamp#[now#[]]"; - } else { - innerValue = "toUnixTimestamp#[ " + matcherParse.group(2) + "]"; - } - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { - innerValue = "0.001 * TIMESTAMP_TO_MILLIS[CURRENT_TIMESTAMP]"; - } else { - innerValue = "0.001 * TIMESTAMP_TO_MILLIS#[ TIMESTAMP " + matcherParse.group(2) + "]"; - } - } - matcherParse.appendReplacement(sb, innerValue); - } - matcherParse.appendTail(sb); - return sb.toString(); - } - - /** - * 实现替换：DATE(expr) - * - * @param sqlParse - * @param num 包含的括号个数 - * @param dbType - * @return - */ - public static String parseDateHavaMark(String sqlParse, int num, String dbType) { - Pattern pDateParse = Pattern.compile("(DATE)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); - Matcher matcherParse = pDateParse.matcher(sqlParse); - StringBuffer sb = new StringBuffer(); - String innerValue = null; - if (matcherParse.find()) { - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - innerValue = "toStartOfDay#[toDateTime#[" + matcherParse.group(2) + "]]"; - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - innerValue = "TIME_FLOOR#[ TIMESTAMP " + matcherParse.group(2) + ",'P1D']"; - } - matcherParse.appendReplacement(sb, innerValue); - } - matcherParse.appendTail(sb); - return sb.toString(); - } - - /** - * 获取函数参数 - * - * @param str - * @param div 参数分隔符 - * @return - */ - public static List<String> diviParam(String str, String div) { - if (str == null) { - return null; - } - List<String> result = new ArrayList<>(); - String[] split = str.split(div); - String resultTemp = ""; - for (int i = 0; i < split.length; i++) { - resultTemp += split[i]; - if (getBracketsMatch(resultTemp, "(", true)[0] >= 0 - && getBracketsMatch(resultTemp, "[", true)[0] >= 0) { - result.add(resultTemp); - resultTemp = ""; - continue; - } - resultTemp += div; - } - return result; - } - - - /** - * 公共，通过pattern匹配转义 - * - * @param sql - * @param dbType - * @param pattern - * @return - */ - private static String translateByPattern(String sql, String dbType, Pattern pattern) { - int count = 0; - while (true) { - Matcher matcher = pattern.matcher(sql); - if (!matcher.find() || count++ >= 40) { - return sql; - } - String sqlParse = sql.substring(matcher.start()); - int[] bracketsMatch = getBracketsMatch(sqlParse, "(", false); - - if (bracketsMatch[0] >= 1) { - int start = StringUtils.indexOf(sqlParse, "("); - Function function = new Function(); - function.setName(sqlParse.substring(0, start)); - int end = bracketsMatch[1]; - List<String> strings = Splitter.on(",").trimResults().omitEmptyStrings().splitToList(sqlParse.substring(start + 1, end)); - StringBuffer sb = new StringBuffer(); - for (String str : strings) { - sb.append(str); - int[] a = getBracketsMatch(sb.toString(), "(", true); - if (a[0] != -1) { - function.getParamsList().add(sb.toString()); - sb = new StringBuffer(); - continue; - } else { - sb.append(","); - } - } - - String replace = translateFunction(function, dbType); - sql = sql.substring(0, matcher.start()) + replace + sql.substring(matcher.start() + 1 + end); - } - } - } - - /** - * 转义函数 - * - * @param fun - * @param dbType - * @return - */ - private static String translateFunction(Function fun, String dbType) { - String function = null; - if (TIME_FLOOR_WITH_FILL.equals(fun.getName())) { - function = translateTimeFloorWithFill(fun, dbType); - } else if (IP_TO_GEO.equals(fun.getName())) { - function = translateIpToGeo(fun); - } else if (IP_TO_CITY.equals(fun.getName())) { - function = translateIpToCity(fun); - } else if (IP_TO_COUNTRY.equals(fun.getName())) { - function = translateIpToCountry(fun); - } - return function; - } - - /** - * 转义IP_TO_GEO - * - * @param fun - * @return - */ - private static String translateIpToCountry(Function fun) { - return fun.paramsList.get(0); - } - - /** - * 转义IP_TO_CITY - * - * @param fun - * @return - */ - private static String translateIpToCity(Function fun) { - return fun.paramsList.get(0); - } - - /** - * 转义IP_TO_COUNTRY - * - * @param fun - * @return - */ - private static String translateIpToGeo(Function fun) { - return fun.paramsList.get(0); - } - - /** - * 转义time_floor_with_fill - * - * @param function - * @param dbType - * @return - */ - private static String translateTimeFloorWithFill(Function function, String dbType) { - String timestamp = function.getParamsList().get(0); - String period = function.getParamsList().get(1); - - Matcher matcherPT = periodOfPT.matcher(period); - if (matcherPT.find()) { - String num = matcherPT.group(1); - String unit = matcherPT.group(2); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - String format = null; - if ("S".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s SECOND)))"; - } else if ("M".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s MINUTE)))"; - } else if ("H".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s HOUR)))"; - } - return String.format(format, timestamp, num); - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - String format = "TIMESTAMP_TO_MILLIS(time_floor(%s * 1000,%s))/1000"; - return String.format(format, timestamp, period); - } - } - - Matcher matcherP = periodOfP.matcher(period); - if (matcherP.find()) { - String num = matcherP.group(1); - String unit = matcherP.group(2); - if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { - String format = null; - if ("D".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s DAY)))"; - } else if ("W".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s WEEK)))"; - } else if ("M".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s MONTH)))"; - } else if ("Y".equalsIgnoreCase(unit)) { - format = "toUnixTimestamp(toDateTime(toStartOfInterval(toDateTime(%s),INTERVAL %s YEAR)))"; - } - return String.format(format, timestamp, num); - } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { - String format = "TIMESTAMP_TO_MILLIS(time_floor(%s * 1000,%s))/1000"; - return String.format(format, timestamp, period); - } - } - return function.toString(); - } - - @Data - static class Function { - private String name; - private List<String> paramsList = new ArrayList<>(); - } - - @Data - static class DruidQueryParam { - private String query; - private Map<String, String> context = Maps.newHashMap(); - private String resultFormat; - - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SysLogAspect.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SysLogAspect.java deleted file mode 100644 index d7f499c..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/SysLogAspect.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.mesalab.engine.component; - -import com.google.gson.Gson; -import org.aspectj.lang.ProceedingJoinPoint; -import org.aspectj.lang.annotation.Around; -import org.aspectj.lang.annotation.Aspect; -import org.aspectj.lang.annotation.Pointcut; -import org.aspectj.lang.reflect.MethodSignature; -import org.slf4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.lang.reflect.Method; - -/** - * @Date: 2020-07-30 11:54 - * @Author : liuyongqiang - * @ClassName : SysLogAspect - * @Description : 切面实现日志记录 - */ -@Aspect -@Component -public class SysLogAspect { - - @Autowired - IDGenerator idGenerator; - - @Autowired - LoggerCache loggerCache; - - @Pointcut("@annotation(com.mesalab.common.annotation.SysLog)") - public void log() { - } - - - /** - * 加入注解自动记录方法日志 - * - * @param joinPoint - * @return - * @throws Throwable - */ - @Around(value = "log()") - public Object logAround(ProceedingJoinPoint joinPoint) throws Throwable { - // 获取执行方法的类的名称（包名加类名） - String className = joinPoint.getTarget().getClass().getName(); - // 获取实例和方法 - MethodSignature signature = (MethodSignature) joinPoint.getSignature(); - Method method = signature.getMethod(); - // 从缓存中获取日志实例 - Logger log = loggerCache.getLoggerByClassName(className); - // 记录日志 - long logId = idGenerator.snowflakeId(); - log.info("["+logId+"]"+" - Execute:"+className + "." + method.getName()); - Object[] args = joinPoint.getArgs(); - log.info("["+logId+"]"+" - Params:" + new Gson().toJson(args)); - // 执行方法获取返回值 - Object proceed = joinPoint.proceed(); - // 记录日志 - log.info("["+logId+"]"+" - Returns:" + new Gson().toJson(proceed)); - // 返回 - return proceed; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/bean/ApiQueryBean.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/bean/ApiQueryBean.java deleted file mode 100644 index 97a792b..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/bean/ApiQueryBean.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.mesalab.engine.component.bean; - -import cn.hutool.core.util.StrUtil; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -public class ApiQueryBean implements Serializable { - - private String query; - private String option; - private String format; - private String schema; - private String tableName; - private String dbType;// 数据库类型 - private String dialectDBType; //方言数据库类型，用于SQL格式解析校验 - private String reportId;//离线统计报告唯一ID - private String resultId;//离线统计报告结果ID - - private String sql;//当前执行的SQL - - private SqlQueryBean engineQuerySource; //网关引擎处理SQL对象 - - private SqlQueryBean dbQuerySource; //下沉数据源引擎的SQL对象 - - - public String getResultId() { - - return StrUtil.isNotBlank(reportId) ? reportId : resultId; - } - - public void setResultId(String resultId) { - this.resultId = resultId; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/bean/SqlQueryBean.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/bean/SqlQueryBean.java deleted file mode 100644 index fb153eb..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/bean/SqlQueryBean.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.mesalab.engine.component.bean; - -import cn.hutool.core.util.StrUtil; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.mesalab.engine.component.udf.UDF; -import lombok.Data; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.statement.select.GroupByElement; -import net.sf.jsqlparser.statement.select.OrderByElement; -import net.sf.jsqlparser.statement.select.SubSelect; - -import java.util.List; -import java.util.Map; -import java.util.Set; - - -@Data -public class SqlQueryBean { - - private String sqlBody; - private String tableName; - private String partitionKey; - /** - * 分区键，多用于检索查询首要过滤列；例如timestamp - **/ - private String fromItem; - private String selectItems; - private String expr; - private String orderBy; - private String limit; - private SubSelect subSelect; - private Expression whereExpression; - /** - * where 对象 - **/ - private GroupByElement groupByElement; - private List<OrderByElement> listOrderElement; - private Map<String, String> aliasFields = Maps.newHashMap(); - /** - * k=alias,v=column - **/ - private Set<UDF> udfSet = Sets.newHashSet(); - private Map<String, String> groupDimension = Maps.newHashMap(); - private List<SqlQueryBean> subSqlQuerySources = Lists.newArrayList(); - private boolean isEnableLimit; - - - public boolean isEnableLimit() { - if (StrUtil.isNotBlank(limit) && limit.equalsIgnoreCase("NO LIMIT")) { - return false; - } else { - return true; - } - } - - - public Map<String, String> getGroupDimension() { - if (groupByElement == null) { - return null; - } - for (Expression groupByExpression : groupByElement.getGroupByExpressions()) { - - String groupBy = groupByExpression.toString(); - if (groupBy.startsWith("\"") && groupBy.endsWith("\"")) { - groupBy = groupBy.substring(1, groupBy.length() - 1); - } - if (StrUtil.isNotBlank(aliasFields.get(groupBy))) { - groupDimension.put(groupBy, aliasFields.get(groupBy)); - } else { - //非别名，确定实际的别名 - for (String key : aliasFields.keySet()) { - if (aliasFields.get(key).equalsIgnoreCase(groupBy)) { - groupDimension.put(key, groupBy); - break; - } - } - } - - } - return groupDimension; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/CalciteMemory.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/CalciteMemory.java deleted file mode 100644 index 88e7f66..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/CalciteMemory.java +++ /dev/null @@ -1,165 +0,0 @@ -package com.mesalab.engine.component.calcite; - -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.engine.component.calcite.storage.DataTypeMapping; -import com.mesalab.engine.component.calcite.storage.Storage; -import lombok.extern.slf4j.Slf4j; -import org.apache.calcite.jdbc.CalciteConnection; - -import java.io.File; -import java.sql.*; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -@Slf4j -@SuppressWarnings("unchecked") -public class CalciteMemory { - - public static Map<String, Object> executeMemoryQuery(String tableName, List<Map<String, String>> meta, - List<Map<String, Object>> data, String sql) { - Map<String, Object> result = Maps.newHashMap(); - if (meta.isEmpty() || data.isEmpty()) { - data.clear(); - meta.clear(); - result.put("meta", meta); - result.put("data", data); - return result; - } - ResultSet resultSet = null; - Statement statement = null; - Connection connection = null; - try { - Class.forName("org.apache.calcite.jdbc.Driver"); - Properties info = new Properties(); - info.setProperty("lex", "MYSQL_ANSI"); - String jsonModel = new File("").getCanonicalPath() + "/schema/memory-schema.json"; - connection = DriverManager.getConnection("jdbc:calcite:model=" + jsonModel, info); - CalciteConnection calciteConn = connection.unwrap(CalciteConnection.class); - - log.debug("Calcite: 加载数据的数量: " + data.size()); - //将数据存入缓存表 - Storage.DummyTable table = buildDummyTable(meta, tableName); - table = loadData(table, data); - InMemoryTable inMemoryTable = new InMemoryTable(tableName, table); - - //将缓存表植入rootSchema - calciteConn.getRootSchema().add(tableName, inMemoryTable); - statement = connection.createStatement(); - resultSet = statement.executeQuery(sql); - ResultSetMetaData metaData = resultSet.getMetaData(); - meta.clear(); - data.clear(); - - for (int i = 1; i <= metaData.getColumnCount(); i++) { - Map<String, String> metaMap = Maps.newLinkedHashMap(); - metaMap.put("name", metaData.getColumnLabel(i)); - metaMap.put("type", metaData.getColumnTypeName(i)); - meta.add(metaMap); - } - - while (resultSet.next()) { - Map<String, Object> rowData = Maps.newLinkedHashMap(); - for (int i = 1; i <= metaData.getColumnCount(); i++) { - rowData.put(metaData.getColumnLabel(i), resultSet.getObject(i)); - } - data.add(rowData); - } - - result.put("meta", meta); - result.put("data", data); - - } catch (Exception e) { - throw new BusinessException("Engine execution SQL Error.", e); - - } finally { - if (resultSet != null) { - try { - resultSet.close(); - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - } - - if (statement != null) { - try { - statement.close(); - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - } - - if (connection != null) { - try { - connection.close(); - } catch (SQLException throwables) { - throwables.printStackTrace(); - } - } - } - - - return result; - } - - - /** - * 数据加载 ： MAP To List - * - * @param table 转换为table - * @param data 原始结果集合 - * @return - */ - private static Storage.DummyTable loadData(Storage.DummyTable table, List<Map<String, Object>> data) { - List<Storage.DummyColumn> columns = table.getColumns(); - - data.stream().forEach(item -> { - List list = Lists.newArrayList(); - for (Storage.DummyColumn column : columns) { - if (DataTypeMapping.BIGINT.equals(column.getType())) { - list.add(Long.valueOf(item.get(column.getName()).toString())); - } else { - list.add(item.get(column.getName()).toString()); - } - - } - - table.addRow(list.toArray()); - - }); - return table; - } - - /** - * 生成虚拟表 - * - * @param meta schema 表结构字段 - * @param tableName 表名 - * @return - */ - private static Storage.DummyTable buildDummyTable(List<Map<String, String>> meta, String tableName) { - Storage.DummyTable dummyTable = new Storage.DummyTable(tableName); - for (Map<String, String> column : meta) { - String fieldName = column.get("name"); - String fieldType = column.get("type"); - String dummyColumnType; - if (Splitter.on(",").trimResults().omitEmptyStrings().splitToList(intUnits).contains(fieldType)) { - dummyColumnType = DataTypeMapping.BIGINT; - } else if (Splitter.on(",").trimResults().omitEmptyStrings().splitToList(doubleUnits).contains(fieldType)) { - dummyColumnType = DataTypeMapping.DOUBLE; - } else { - dummyColumnType = DataTypeMapping.VARCHAR; - } - Storage.DummyColumn name = new Storage.DummyColumn(fieldName, dummyColumnType); - dummyTable.addColumn(name); - } - return dummyTable; - } - - private static String intUnits = "Int8,Int16,Int32,Int64,UInt8,UInt16,UInt32,UInt64,bigint"; - private static String doubleUnits = "Float32,Float64,float,double"; - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/DateFunction.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/DateFunction.java deleted file mode 100644 index ffa27b0..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/DateFunction.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.engine.component.calcite; - -import com.zdjizhi.utils.DateUtils; - -public class DateFunction { - - public String fromUnixTimeToDateTime24(Object i) { - Long aLong = Long.valueOf(String.valueOf(i)); - String s = DateUtils.convertTimestampToString(aLong, DateUtils.YYYY_MM_DD_HH24_MM_SS); - return s; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/InMemorySchema.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/InMemorySchema.java deleted file mode 100644 index c073304..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/InMemorySchema.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.mesalab.engine.component.calcite; - -import com.mesalab.engine.component.calcite.storage.Storage; -import org.apache.calcite.schema.Table; -import org.apache.calcite.schema.impl.AbstractSchema; - -import java.util.HashMap; -import java.util.Map; - -public class InMemorySchema extends AbstractSchema { - - private String dbName; - private Map<String, Object> operand; - - public InMemorySchema(String name, Map<String, Object> operand) { - this.operand = operand; - this.dbName = name; - } - - @Override - public Map<String, Table> getTableMap() { - - Map<String, Table> tables = new HashMap<String, Table>(); - - Storage.getTables().forEach(it -> { - tables.put(it.getName(), new InMemoryTable(it.getName(), it)); - }); - - return tables; - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/InMemorySchemaFactory.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/InMemorySchemaFactory.java deleted file mode 100644 index b19d88e..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/InMemorySchemaFactory.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.mesalab.engine.component.calcite; - -import org.apache.calcite.schema.Schema; -import org.apache.calcite.schema.SchemaFactory; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.impl.ScalarFunctionImpl; - -import java.util.Map; - -public class InMemorySchemaFactory implements SchemaFactory { - - @Override - public Schema create(SchemaPlus parentSchema, String name, Map<String, Object> operand) { - parentSchema.add("FROM_UNIXTIME", ScalarFunctionImpl.create(DateFunction.class, "fromUnixTimeToDateTime24")); - return new InMemorySchema(name, operand); - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/InMemoryTable.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/InMemoryTable.java deleted file mode 100644 index 6ff7463..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/InMemoryTable.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.mesalab.engine.component.calcite; - -import com.mesalab.engine.component.calcite.storage.Storage; -import org.apache.calcite.DataContext; -import org.apache.calcite.linq4j.AbstractEnumerable; -import org.apache.calcite.linq4j.Enumerable; -import org.apache.calcite.linq4j.Enumerator; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.schema.ScannableTable; -import org.apache.calcite.schema.impl.AbstractTable; -import org.apache.calcite.sql.type.SqlTypeUtil; - -public class InMemoryTable extends AbstractTable implements ScannableTable { - private String name; - private Storage.DummyTable _table; - private RelDataType dataType; - - InMemoryTable(String name) { - this.name = name; - } - - public InMemoryTable(String name, Storage.DummyTable it) { - this.name = name; - this._table = it; - } - - - @Override - public RelDataType getRowType(RelDataTypeFactory typeFactory) { - if (dataType == null) { - RelDataTypeFactory.Builder builder = typeFactory.builder(); - for (Storage.DummyColumn column : this._table.getColumns()) { - RelDataType sqlType = typeFactory.createJavaType(column.getJavaClass()); - sqlType = SqlTypeUtil.addCharsetAndCollation(sqlType, typeFactory); - builder.add(column.getName(), sqlType); - } - this.dataType = builder.build(); - } - return this.dataType; - } - - - @Override - public Enumerable<Object[]> scan(DataContext root) { - return new AbstractEnumerable<Object[]>() { - @Override - public Enumerator<Object[]> enumerator() { - return new Enumerator<Object[]>() { - private int cur = 0; - private int count = 0; - - @Override - public Object[] current() { - Object[] data = _table.getData(cur); - return data; - } - - @Override - public boolean moveNext() { - if (cur != 0 || count >= 1) { - ++cur; - } - if (count++ >= 1 && cur == 0) { - return false; - } - return cur < _table.getRowCount(); - } - - @Override - public void reset() { - } - - @Override - public void close() { - } - }; - } - }; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/storage/DataTypeMapping.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/storage/DataTypeMapping.java deleted file mode 100644 index cad66f0..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/storage/DataTypeMapping.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.mesalab.engine.component.calcite.storage; - -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Table; -import org.apache.calcite.sql.type.SqlTypeName; - -import java.math.BigDecimal; -import java.sql.Date; -import java.util.Set; - -/** - * 这里使用了GUAVA的table 作为存SQL和JAVA数据类型的数据结构 - * 这并不是一个好的设计，而是为了让大家更容易理解而做的设计 - */ -public class DataTypeMapping { - - public static Table<String, SqlTypeName, Class> TYPEMAPPING = HashBasedTable.create(); - public static final String CHAR = "char"; - public static final String VARCHAR = "varchar"; - public static final String BOOLEAN = "boolean"; - public static final String DATE = "date"; - public static final String INTEGER = "integer"; - public static final String INT = "int"; - public static final String TINYINT = "tinyint"; - public static final String SMALLINT = "smallint"; - public static final String BIGINT = "bigint"; - public static final String DECIMAL = "decimal"; - public static final String NUMERIC = "numeric"; - public static final String FLOAT = "float"; - public static final String REAL = "real"; - public static final String DOUBLE = "double"; - public static final String TIME = "time"; - public static final String TIMESTAMP = "timestamp"; - public static final String ANY = "any"; - - static { - TYPEMAPPING.put(DataTypeMapping.CHAR, SqlTypeName.CHAR, Character.class); - TYPEMAPPING.put(DataTypeMapping.VARCHAR, SqlTypeName.VARCHAR, String.class); - TYPEMAPPING.put(DataTypeMapping.BOOLEAN, SqlTypeName.BOOLEAN, Boolean.class); - TYPEMAPPING.put(DataTypeMapping.DATE, SqlTypeName.DATE, Date.class); - TYPEMAPPING.put(DataTypeMapping.INTEGER, SqlTypeName.INTEGER, Integer.class); - TYPEMAPPING.put(DataTypeMapping.INT, SqlTypeName.TIMESTAMP, String.class); - TYPEMAPPING.put(DataTypeMapping.TINYINT, SqlTypeName.TINYINT, Integer.class); - TYPEMAPPING.put(DataTypeMapping.SMALLINT, SqlTypeName.SMALLINT, Integer.class); - TYPEMAPPING.put(DataTypeMapping.BIGINT, SqlTypeName.BIGINT, Long.class); - TYPEMAPPING.put(DataTypeMapping.DECIMAL, SqlTypeName.DECIMAL, BigDecimal.class); - TYPEMAPPING.put(DataTypeMapping.NUMERIC, SqlTypeName.DECIMAL, Long.class); - TYPEMAPPING.put(DataTypeMapping.FLOAT, SqlTypeName.FLOAT, Float.class); - TYPEMAPPING.put(DataTypeMapping.REAL, SqlTypeName.REAL, Double.class); - TYPEMAPPING.put(DataTypeMapping.DOUBLE, SqlTypeName.DOUBLE, Double.class); - TYPEMAPPING.put(DataTypeMapping.TIME, SqlTypeName.TIME, Date.class); - TYPEMAPPING.put(DataTypeMapping.TIMESTAMP, SqlTypeName.TIMESTAMP, Long.class); - TYPEMAPPING.put(DataTypeMapping.ANY, SqlTypeName.ANY, String.class); - } - - /** - * 根据名字获取，对应的java类型 - */ - public static Class getJavaClassByName(String name) { - Set<Table.Cell<String, SqlTypeName, Class>> table = TYPEMAPPING.cellSet(); - for (Table.Cell<String, SqlTypeName, Class> it : table) { - if (it.getRowKey().equals(name)) { - return it.getValue(); - } - } - return null; - } - - public static SqlTypeName getSqlTypeByName(String name) { - for (Table.Cell<String, SqlTypeName, Class> it : TYPEMAPPING.cellSet()) { - if (it.getRowKey().equals(name)) { - return it.getColumnKey(); - } - } - return null; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/storage/Storage.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/storage/Storage.java deleted file mode 100644 index 79f8b9f..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/calcite/storage/Storage.java +++ /dev/null @@ -1,100 +0,0 @@ -package com.mesalab.engine.component.calcite.storage; - -import org.apache.calcite.sql.type.SqlTypeName; - -import java.util.*; - -/** - * 用于模拟数据库结构及数据 - */ -public class Storage { - - public static Hashtable<String, DummyTable> _bag = new Hashtable<>(); - - public static Collection<DummyTable> getTables() { - return _bag.values(); - } - - public static DummyTable getTable(String tableName) { - return _bag.get(tableName); - } - - public static class DummyTable { - private String name; - private List<DummyColumn> columns; - private List<List<Object>> datas = new ArrayList<>(); - - public DummyTable(String name) { - this.name = name; - } - - public String getName() { - return this.name; - } - - public List<DummyColumn> getColumns() { - return columns; - } - - public DummyTable addColumn(DummyColumn dc) { - if (this.columns == null) { - this.columns = new ArrayList<>(); - } - this.columns.add(dc); - return this; - } - - public void setColumns(List<DummyColumn> columns) { - this.columns = columns; - } - - public Object[] getData(int index) { - return this.datas.get(index).toArray(); - } - - public int getRowCount() { - return this.datas.size(); - } - - public void addRow(Object... objects) { - this.datas.add(Arrays.asList(objects)); - } - - - } - - public static class DummyColumn { - private String name; - private String type; - - public DummyColumn(String name, String type) { - this.name = name; - this.type = type; - } - - public String getName() { - return name; - } - - public String getType() { - return type; - } - - public void setName(String name) { - this.name = name; - } - - public void setType(String type) { - this.type = type; - } - - public Class getJavaClass() { - return DataTypeMapping.getJavaClassByName(this.type); - } - - public SqlTypeName getSqlTypeName() { - return DataTypeMapping.getSqlTypeByName(this.type); - } - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/ArangoConfig.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/ArangoConfig.java deleted file mode 100644 index 3756f11..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/ArangoConfig.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.mesalab.engine.component.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @description: for http - * @author: zhq - * @create: 2020-07-07 - **/ -@Data -@Component -public class ArangoConfig { - - @Value("${arango.query.url}") - private String queryurl; - - @Value("${arango.maxrows}") - private String maxrows; - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/ClickHouseConfig.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/ClickHouseConfig.java deleted file mode 100644 index 83429f0..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/ClickHouseConfig.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.mesalab.engine.component.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @Date: 2020-08-18 17:57 - * @Author : liuyongqiang - * @ClassName : ClickHouseConfig - * @Description : ClickHouseConfig - */ -@Data -@Component -public class ClickHouseConfig { - - @Value("${clickhouse.query.url}") - private String queryUrl; - - @Value("${clickhouse.dbname}") - private String dbName; - - @Value("${clickhouse.real.time.username}") - private String realTimeUserName; - - @Value("${clickhouse.real.time.password}") - private String realTimePassword; - - @Value("${clickhouse.long.term.username}") - private String longTermUserName; - - @Value("${clickhouse.long.term.password}") - private String longTermPassword; - - private int enableHttpCompression = 1; - - private String systemDBName = "system"; - - private long socketTimeout; -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/DruidConfig.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/DruidConfig.java deleted file mode 100644 index 63fa087..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/DruidConfig.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.mesalab.engine.component.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @Date: 2020-08-18 17:57 - * @Author : liuyongqiang - * @ClassName : DruidConfig - * @Description : DruidConfig - */ -@Data -@Component -public class DruidConfig { - - @Value("${druid.query.url}") - private String queryUrl; - @Value("${druid.dbname}") - private String dbname; - @Value("${druid.sqlTimeZone}") - private String sqlTimeZone; - - private String skipEmptyBuckets = "true"; - private long timeout = 600000; - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/EngineConfig.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/EngineConfig.java deleted file mode 100644 index f615aa6..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/EngineConfig.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.mesalab.engine.component.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import java.io.Serializable; - -@Data -@Component -public class EngineConfig implements Serializable { - - @Value("${engine.maxCacheNum}") - private int maxCacheNum; - @Value("${engine.defaultResultNum}") - private int defaultResultNum; - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/HbaseConfig.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/HbaseConfig.java deleted file mode 100644 index b3bf735..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/HbaseConfig.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.mesalab.engine.component.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @Date: 2020-08-18 17:35 - * @Author : liuyongqiang - * @ClassName : HbaseConfig - * @Description : HbaseConfig - */ -@Data -@Component -public class HbaseConfig { - @Value("${hbase.url}") - private String url; - @Value("${hbase.dbname}") - private String dbName; - @Value("${hbase.tableName}") - private String tableName; - @Value("${hbase.columnFamily}") - private String columnFamily; - @Value("${hbase.columnName}") - private String columnName; -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/HttpConfig.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/HttpConfig.java deleted file mode 100644 index 29d2ac7..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/HttpConfig.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.mesalab.engine.component.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class HttpConfig { - - @Value("${httpclient.connectTimeout}") - private Integer connectTimeout; - @Value("${httpclient.socketTimeout}") - private Integer socketTimeout; - @Value("${httpclient.maxConnection}") - private Integer maxConnection; - @Value("${httpclient.maxPerRoute}") - private Integer maxPerRoute; - - private String userAgent = "Mozilla/5.0(Windows NT 6.1;Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0"; - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/XxlJobConfig.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/XxlJobConfig.java deleted file mode 100644 index 5688d5f..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/config/XxlJobConfig.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.mesalab.engine.component.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -/** - * @Date: 2020-08-19 14:34 - * @Author : liuyongqiang - * @ClassName : JobAdminConfig - * @Description : JobAdminConfig - */ -@Data -@Component -public class XxlJobConfig { - - @Value("${xxl.job.admin.url}") - private String url; - - @Value("${xxl.job.admin.username}") - private String userName; - - @Value("${xxl.job.admin.password}") - private String password; -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/configuration/HttpConfiguration.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/configuration/HttpConfiguration.java deleted file mode 100644 index 29b10a6..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/configuration/HttpConfiguration.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.mesalab.engine.component.configuration; - -import com.mesalab.engine.component.config.HttpConfig; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.*; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.message.BasicHeaderElementIterator; -import org.apache.http.protocol.HTTP; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import javax.net.ssl.SSLException; -import java.io.InterruptedIOException; -import java.net.UnknownHostException; -import java.util.concurrent.TimeUnit; - -@Slf4j -@Configuration -public class HttpConfiguration { - - @Autowired - HttpConfig httpConfig; - - @Bean - @ConditionalOnMissingBean(RequestConfig.class) - public RequestConfig config() { - RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(httpConfig.getConnectTimeout()) - .setSocketTimeout(httpConfig.getSocketTimeout()) - .build(); - return requestConfig; - - } - - @Bean - public PoolingHttpClientConnectionManager poolingHttpClientConnectionManager() { - PoolingHttpClientConnectionManager poolHttpcConnManager = new PoolingHttpClientConnectionManager(60, TimeUnit.SECONDS); - // 最大连接数 - poolHttpcConnManager.setMaxTotal(httpConfig.getMaxConnection()); - // 路由基数 - poolHttpcConnManager.setDefaultMaxPerRoute(httpConfig.getMaxPerRoute()); - return poolHttpcConnManager; - } - - @Bean("connectionKeepAliveStrategy") - public ConnectionKeepAliveStrategy connectionKeepAliveStrategy() { - return (httpResponse, httpContext) -> { - HeaderElementIterator it = new BasicHeaderElementIterator( - httpResponse.headerIterator(HTTP.CONN_KEEP_ALIVE)); - while (it.hasNext()) { - HeaderElement he = it.nextElement(); - String param = he.getName(); - String value = he.getValue(); - if (value != null && param.equalsIgnoreCase("timeout")) { - try { - return Long.parseLong(value) * 1000; - } catch (NumberFormatException ignore) { - } - } - } - return 30 * 1000; - }; - } - - - @Bean - public HttpRequestRetryHandler httpRequestRetryHandler() { - // 请求重试 - final int retryTime = 3; - return (exception, executionCount, context) -> { - // Do not retry if over max retry count,如果重试次数超过了retryTime,则不再重试请求 - if (executionCount >= retryTime) { - return false; - } - // 服务端断掉客户端的连接异常 - if (exception instanceof NoHttpResponseException) { - return false; - } - // time out - if (exception instanceof InterruptedIOException) { - return false; - } - // Unknown host - if (exception instanceof UnknownHostException) { - return false; - } - // Connection refused - if (exception instanceof ConnectTimeoutException) { - return false; - } - // SSL handshake exception - if (exception instanceof SSLException) { - return false; - } - HttpClientContext clientContext = HttpClientContext.adapt(context); - HttpRequest request = clientContext.getRequest(); - return !(request instanceof HttpEntityEnclosingRequest); - }; - - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/configuration/ProjectAuthorProperties.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/configuration/ProjectAuthorProperties.java deleted file mode 100644 index 7b1ad67..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/configuration/ProjectAuthorProperties.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.mesalab.engine.component.configuration; - -import lombok.Data; - -/** - * 项目作者联系方式 - * - * @author dazzlzy - * @date 2018/5/26 - */ -@Data -public class ProjectAuthorProperties { - - private String name; - - private String url; - - private String email; - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/configuration/ProjectProperties.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/configuration/ProjectProperties.java deleted file mode 100644 index 8ab3250..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/configuration/ProjectProperties.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.mesalab.engine.component.configuration; - -import com.mesalab.common.enums.EnvironmentEnum; -import com.mesalab.common.enums.EnvironmentGroupEnum; -import lombok.Data; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Repository; - -import java.util.ArrayList; -import java.util.List; - -/** - * 项目配置 - * - * @author dazzlzy - * @date 2018/5/26 - */ -@Data -@Repository -@ConfigurationProperties("project") -public class ProjectProperties { - - /** - * 工程名 - */ - private String name; - - /** - * 版本 - */ - private String version; - - /** - * 工程描述 - */ - private String description; - - /** - * 项目组织标识 - */ - private String groupId; - - /** - * 项目标识 - */ - private String artifactId; - - /** - * 项目根目录 - */ - private String basedir; - - /** - * 核心项目包 - */ - private String corePackage; - - /** - * 业务项目包 - */ - private String servicePackage; - - /** - * 当前环境值 - */ - private String[] env; - - /** - * 项目作者 - */ - private ProjectAuthorProperties author; - - /** - * 注入的spring环境上下文 - */ - private final Environment environment; - - @Autowired - public ProjectProperties(Environment environment) { - this.environment = environment; - this.env = environment.getActiveProfiles(); - } - - /** - * 是否是生产环境 - * 从运行环境中读取最后一个是否为生产环境 - * - * @return boolean 是否为生产环境 - */ - public boolean isProduct() { - List<String> runtimeEnvs = new ArrayList<>(); - for (String s : this.env) { - if (EnvironmentGroupEnum.isRuntime(s)) { - runtimeEnvs.add(s); - } - } - if (runtimeEnvs.size() == 0) { - return false; - } - //最后一个运行环境, 如果spring.profiles.active=dev, prod, mysql 则运行环境为dev, prod, 最后一个运行环境为prod，是生产环境 - String env = runtimeEnvs.get(runtimeEnvs.size() - 1); - return EnvironmentEnum.PROD.getName().equals(env); - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/AbstractDataSourceDialect.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/AbstractDataSourceDialect.java deleted file mode 100644 index b4e694b..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/AbstractDataSourceDialect.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.mesalab.engine.component.dialect; - -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.util.BaseResultUtil; -import com.zdjizhi.utils.StringUtil; -import lombok.Data; - -@Data -public abstract class AbstractDataSourceDialect implements Dialect { - - public ApiQueryBean param; - public BaseResult baseResult; - - public void initialize(ApiQueryBean param) { - this.param = param; - if (StringUtil.isNotEmpty(param.getDbQuerySource())) { - param.setSql(param.getDbQuerySource().getSqlBody()); - } else { - param.setSql(param.getQuery()); - } - baseResult = BaseResultUtil.success(); - } - - /** - * SQL查询语句转换为方言所认知的查询语句。 - * - * @param sql - * @return - */ - abstract String convertQuery(String sql); - - - /** - * 基于RBO(rule-base optimizer)获取最优化的SQL - * - * @return - */ - abstract String findBestQuery(); - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/AbstractEngineDialect.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/AbstractEngineDialect.java deleted file mode 100644 index 7b94b49..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/AbstractEngineDialect.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.mesalab.engine.component.dialect; - -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.common.dto.results.BaseResult; -import com.zdjizhi.utils.StringUtil; -import lombok.Data; - -@Data -public abstract class AbstractEngineDialect implements Dialect { - public ApiQueryBean param; - public BaseResult baseResult; - - abstract void executeUDF(); - - public void initialize(ApiQueryBean param, BaseResult baseResult) { - this.param = param; - if (StringUtil.isNotEmpty(param.getEngineQuerySource())) { - param.setQuery(param.getEngineQuerySource().getSqlBody()); - } - this.baseResult = baseResult; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/ClickHouseDialect.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/ClickHouseDialect.java deleted file mode 100644 index 8cf17d6..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/ClickHouseDialect.java +++ /dev/null @@ -1,1098 +0,0 @@ -package com.mesalab.engine.component.dialect; - - -import com.google.common.collect.Maps; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.*; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.common.util.SqlHelper; -import com.mesalab.engine.component.ReportCache; -import com.mesalab.engine.component.SpringContext; -import com.mesalab.engine.component.SqlFunction; -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.engine.component.bean.SqlQueryBean; -import com.mesalab.engine.component.config.ClickHouseConfig; -import com.mesalab.engine.component.config.EngineConfig; -import com.mesalab.engine.service.HttpClientService; -import com.mesalab.engine.service.MetadataService; -import com.zdjizhi.utils.Encodes; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.expression.Alias; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.Function; -import net.sf.jsqlparser.expression.Parenthesis; -import net.sf.jsqlparser.expression.operators.conditional.AndExpression; -import net.sf.jsqlparser.expression.operators.conditional.OrExpression; -import net.sf.jsqlparser.expression.operators.relational.*; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.schema.Table; -import net.sf.jsqlparser.statement.Statement; -import net.sf.jsqlparser.statement.select.*; -import org.apache.avro.Schema; -import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; - -import java.nio.charset.Charset; -import java.util.List; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -@Slf4j -@SuppressWarnings("unchecked") -public class ClickHouseDialect extends AbstractDataSourceDialect { - - Pattern pGlobalIn = Pattern.compile("\\s+in\\s*\\(", Pattern.CASE_INSENSITIVE); - Pattern pUniq = Pattern.compile("count\\s*\\(\\s*distinct\\s*\\((.*?)\\)", Pattern.CASE_INSENSITIVE); - Pattern pUniqCombined = Pattern.compile("count\\s*\\(\\s*distinct\\s*\\((.*?)\\)", Pattern.CASE_INSENSITIVE); - Pattern pWhere = Pattern.compile("where\\s*", Pattern.CASE_INSENSITIVE); - Pattern pCount = Pattern.compile("\\bcount\\s*\\(", Pattern.CASE_INSENSITIVE); - Pattern pCount1 = Pattern.compile("count\\(\\s*1\\s*\\)", Pattern.CASE_INSENSITIVE); - - private MetadataService metadataService = (MetadataService) SpringContext.getBean("metadataService"); - private EngineConfig engineConfig = (EngineConfig) SpringContext.getBean("engineConfig"); - private ClickHouseConfig clickHouseConfig = (ClickHouseConfig) SpringContext.getBean("clickHouseConfig"); - private HttpClientService httpClientService = (HttpClientService) SpringContext.getBean("httpClientService"); - - public ClickHouseDialect(ApiQueryBean param) { - initialize(param); - } - - @Override - public boolean supportsLimit() { - return true; - } - - - @Override - public String convertQuery(String sql) { - return new Converter(sql). - generateStandard().generateDateFunction().generateSqlEncode().build(); - } - - @Override - public String findBestQuery() { - return new Optimizer().generateQueryLimit(). - generateSQL().generateUniq().generateSubquery().build(); - } - - - class Converter { - String sql = null; - - public Converter(String sql) { - this.sql = sql; - } - - /** - * 兼容标准SQL语言（支持CK19.14.6.12 新特性）： - * 1.子查询必须加表别名 - * 2.不支持 count(1) 需转为count(*) - * - * @return - */ - public Converter generateStandard() { - sql = generateCount(sql); - SelectBody body = generateSubqueryAlias(sql); - if (StringUtil.isNotEmpty(body)) { - sql = body.toString(); - } - return this; - } - - public Converter generateDateFunction() { - sql = SqlFunction.generateDateFunction(sql, DBTypeEnum.CLICKHOUSE.getValue()); - return this; - } - - /** - * http GET sql 里特殊符号需要做encode编码 - * - * @return - */ - public Converter generateSqlEncode() { - sql = Encodes.urlEncode(sql); - return this; - } - - public String build() { - return sql; - } - - /** - * 集群不支持count(1) 需转成count(*) - * - * @param sql - * @return - */ - private String generateCount(String sql) { - Matcher m = pCount1.matcher(sql); - StringBuffer sb = new StringBuffer(); - - while (m.find()) { - m.appendReplacement(sb, "count(*)"); - } - m.appendTail(sb); - return sb.toString(); - } - - /** - * clickhouse 集群模式下针对GLOBAL IN subquery 默认需要增加别名 - * - * @param sql - * @return - */ - private SelectBody generateSubqueryAlias(String sql) { - - SelectBody resultSelectBody = null; - try { - Statement statement = CCJSqlParserUtil.parse(sql); - - if (statement instanceof Select) { - - Select selectStatement = (Select) statement; - - SelectBody body = selectStatement.getSelectBody(); - - if (body instanceof PlainSelect) { - - PlainSelect plainSelect = (PlainSelect) body; - - //子查询处理 - Expression where = plainSelect.getWhere(); - if (where != null) { - setInSubqueryAlias(where); - } - - resultSelectBody = plainSelect; - - } else if (body instanceof SetOperationList) {//连接查询 - SetOperationList setOperationList = (SetOperationList) body; - List<SelectBody> selectsList = setOperationList.getSelects(); - for (int i = 0; i < selectsList.size(); i++) { - if (selectsList.get(i) instanceof PlainSelect) { - SelectBody selectBody = generateSubqueryAlias(selectsList.get(i).toString()); - setOperationList.getSelects().set(i, selectBody); - } - - } - - resultSelectBody = setOperationList; - - } - } - - } catch (JSQLParserException e) { - log.error("sqlParser error: ", e); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "参数校验：SQL 解析错误", e); - - } - - return resultSelectBody; - - } - - /** - * 子查询别名处理 - * - * @param where - */ - private void setInSubqueryAlias(Expression where) { - if (where instanceof InExpression) { - InExpression whereIn = (InExpression) where; - ItemsList rightItemsList = whereIn.getRightItemsList(); - if (rightItemsList instanceof SubSelect) { - SubSelect sub = (SubSelect) rightItemsList; - SelectBody selectBody = sub.getSelectBody(); - PlainSelect plainSelect = (PlainSelect) selectBody; - FromItem fromItem = plainSelect.getFromItem(); - if (fromItem instanceof Table) { - Table table = (Table) fromItem; - if (StringUtil.isEmpty(table.getAlias())) { - Alias alias = new Alias(table.getName()); - plainSelect.getFromItem().setAlias(alias); - } - } else if (fromItem instanceof SubSelect) { - SubSelect subSelect = (SubSelect) fromItem; - generateSubqueryAlias(subSelect.getSelectBody().toString()); - } - if (StringUtil.isNotEmpty(plainSelect.getWhere())) { - setInSubqueryAlias(plainSelect.getWhere()); - } - } - } else if (where instanceof AndExpression) { - AndExpression whereAnd = (AndExpression) where; - Expression leftExpression = whereAnd.getLeftExpression(); - setInSubqueryAlias(leftExpression); - Expression rightExpression = whereAnd.getRightExpression(); - setInSubqueryAlias(rightExpression); - } else if (where instanceof OrExpression) { - OrExpression whereOr = (OrExpression) where; - Expression leftExpression = whereOr.getLeftExpression(); - setInSubqueryAlias(leftExpression); - Expression rightExpression = whereOr.getRightExpression(); - setInSubqueryAlias(rightExpression); - } - } - } - - - class Optimizer { - private Boolean subIndexValidFlag = true; - String sql = null; - - /** - * 对SQL最内层的结果数据集进行限制 - * 参见查询网关系统参数：engine.maxCacheNum - * - * @return - */ - public Optimizer generateQueryLimit() { - setDefaultResultRows(StringUtil.isEmpty(param.getEngineQuerySource()) ? param.getDbQuerySource() : param.getEngineQuerySource(), engineConfig.getDefaultResultNum()); - setMaxCacheResultRows(param.getDbQuerySource(), engineConfig.getMaxCacheNum()); - return this; - } - - - private void setDefaultResultRows(SqlQueryBean sqlQuerySource, int defaultResultRows) { - if (StringUtil.isBlank(sqlQuerySource.getLimit())) { - sqlQuerySource.setSqlBody(sqlQuerySource.getSqlBody() + " limit " + defaultResultRows); - sqlQuerySource.setLimit(String.valueOf(defaultResultRows)); - } - - } - - /** - * 设置最大缓存数据集 - * - * @param sqlQuerySource - * @return - */ - private void setMaxCacheResultRows(SqlQueryBean sqlQuerySource, int maxCacheResultRows) { - SqlQueryBean innerSqlQuerySource = sqlQuerySource; - while (StringUtil.isNotEmpty(innerSqlQuerySource.getSubSelect())) { - innerSqlQuerySource = innerSqlQuerySource.getSubSqlQuerySources().get(0); - } - - if (!innerSqlQuerySource.isEnableLimit()) { - return; - } - - String orginInnerSql = innerSqlQuerySource.getSqlBody(); - String limitInnerSql = innerSqlQuerySource.getSqlBody(); - - if (StringUtil.isNotBlank(innerSqlQuerySource.getLimit())) { - - String[] limits = innerSqlQuerySource.getLimit().split(","); - - if (Integer.valueOf(limits[limits.length - 1]) > maxCacheResultRows) { - limits[limits.length - 1] = String.valueOf(maxCacheResultRows); - - if (limits.length == 2) { - limitInnerSql = SqlHelper.getLimitString(limitInnerSql, Integer.valueOf(limits[0]), Integer.valueOf(limits[1])); - } else { - limitInnerSql = SqlHelper.getLimitString(limitInnerSql, 0, Integer.valueOf(limits[0])); - } - - innerSqlQuerySource.setLimit(String.join(",", limits).toString()); - innerSqlQuerySource.setSqlBody(limitInnerSql); - - } - - } else { - limitInnerSql = innerSqlQuerySource.getSqlBody() + " limit " + maxCacheResultRows; - innerSqlQuerySource.setLimit(String.valueOf(maxCacheResultRows)); - innerSqlQuerySource.setSqlBody(limitInnerSql); - - } - - if (StringUtil.isNotEmpty(sqlQuerySource.getSubSqlQuerySources())) { - sqlQuerySource.setSqlBody(sqlQuerySource.getSqlBody().replace(orginInnerSql, limitInnerSql)); - } - - } - - public Optimizer generateSQL() { - SqlQueryBean sqlQuerySource = param.getDbQuerySource(); - sql = sqlQuerySource.getSqlBody(); - return this; - } - - /** - * count(distinct())转为uniq函数 - * - * @return - */ - public Optimizer generateUniq() { - Matcher m = pUniq.matcher(sql); - StringBuffer sb = new StringBuffer(); - while (m.find()) { - if (m.groupCount() == 1) { - String replaceValue = m.group(1); - m.appendReplacement(sb, "uniq(" + replaceValue); - - } - } - m.appendTail(sb); - sql = sb.toString(); - return this; - } - - /** - * IN 转换为 集群 Global IN 模式 - * - * @return - */ - public Optimizer generateGlobalIn() { - Matcher m = pGlobalIn.matcher(sql); - StringBuffer sb = new StringBuffer(); - while (m.find()) { - m.appendReplacement(sb, " global in ("); - } - - m.appendTail(sb); - sql = sb.toString(); - return this; - } - - /** - * count(distinct())转为uniqCombined函数 - * - * @return - */ - private Optimizer generateUniqCombined() { - - Matcher m = pUniqCombined.matcher(sql); - - StringBuffer sb = new StringBuffer(); - - while (m.find()) { - - if (m.groupCount() == 1) { - String replaceValue = m.group(1); - m.appendReplacement(sb, "uniqCombined(" + replaceValue); - - } - } - m.appendTail(sb); - - sql = sb.toString(); - return this; - } - - /** - * 事件索引子查询，目的提高日志查询检索性能: - * 方式1.基于索引表 - * 方式2.基于分区键 - * 限制： - * 1. 仅适用一级普通查询，带where （无group by 统计）的查询 - * 2. 不对count 统计进行生成 - * - * @return - */ - public Optimizer generateSubquery() { - - if (StringUtil.isBlank(param.getDbQuerySource().getExpr())) { - return this; - } - - if (StringUtil.isNotEmpty(param.getDbQuerySource().getGroupByElement())) { - return this; - } - - if (pCount.matcher(param.getDbQuerySource().getSelectItems()).find()) { - return this; - } - - String indexTable = getAvaliableIndexTable(); - if (StringUtil.isNotBlank(indexTable)) { - sql = getSubqueryByIndexTable(indexTable); - } else { - sql = getSubqueryByPartitionKey(sql); - } - return this; - } - - - /** - * 设置索引表子查询参数: 索引表有效性、索引表表名 - * - * @return - */ - private String getAvaliableIndexTable() { - String indexTable = ""; - String str = metadataService.getValueByKeyInSchemaDoc(param.getTableName(), "index_table"); - String[] indexTables = StringUtil.split(str, ","); - if (StringUtil.isEmpty(indexTables)) { - return StringUtil.EMPTY; - } - for (String tempIndexTable : indexTables) { - - String indexKey = metadataService.getValueByKeyInSchemaDoc(tempIndexTable, "index_key"); - //判断一: where 是否包含索引键 - if (!param.getDbQuerySource().getExpr().contains(indexKey)) { - continue; - } - - BaseResult indexTableData = metadataService.getSchemaInfo("fields", tempIndexTable); - Schema indexTableSchema = new Schema.Parser().parse(JsonMapper.toJsonString(indexTableData.getData())); - //判断二: order by 元素是否都在索引表中 - if (!isValidOfOrderElementsInSchema(indexTableSchema)) { - continue; - } - - BaseResult masterTableData = metadataService.getSchemaInfo("fields", param.getTableName()); - Schema masterTableSchema = new Schema.Parser().parse(JsonMapper.toJsonString(masterTableData.getData())); - //判断三: where 条件列是否都在索引表中 - setValidityByWhereExpressionsInSchema(param.getDbQuerySource().getWhereExpression(), masterTableSchema, indexTableSchema); - if (getSubIndexValidFlag() && isExist(indexTableSchema.getFullName())) { - indexTable = indexTableSchema.getFullName(); - break; - } - } - return indexTable; - } - - /** - * 基于索引表方式添加子查询 - * - * @param indexTable - * @return - */ - private String getSubqueryByIndexTable(String indexTable) { - - String primaryKey = metadataService.getValueByKeyInSchemaDoc(param.getTableName(), "primary_key"); - - StringBuffer subQuery = getSubquerySql(primaryKey, indexTable); - StringBuffer sb = new StringBuffer(); - sb.append("select ").append(param.getDbQuerySource().getSelectItems()).append(" from ").append(param.getTableName()).append(" where ").append(primaryKey).append(" in ( ").append(subQuery).append(")"); - if (StringUtil.isNotBlank(param.getDbQuerySource().getOrderBy())) { - sb.append(" order by ").append(param.getDbQuerySource().getOrderBy()); - } - if (StringUtil.isNotBlank(param.getDbQuerySource().getLimit())) { - sb.append(" limit ").append(param.getDbQuerySource().getLimit()); - } - return sb.toString(); - } - - /** - * 基于分区键方式 - * - * @param sql - * @return - */ - private String getSubqueryByPartitionKey(String sql) { - String partitionKey = getPartitionKey(param.getTableName()); - if (StringUtil.isBlank(partitionKey)) { - log.warn("Get Partition Key is Null.Focus in tableName {} schema", param.getTableName()); - return sql; - } else { - StringBuffer subQuery = getSubquerySql(partitionKey, - metadataService.getDBNameByTableName(param.getTableName()) + "." + param.getTableName()); - Matcher m = pWhere.matcher(sql); - StringBuffer sb = new StringBuffer(); - while (m.find()) { - m.appendReplacement(sb, "where " + partitionKey + " in ( " + subQuery + ") and "); - } - m.appendTail(sb); - return sb.toString(); - } - - } - - /** - * 获取子sql - * - * @param columnName - * @param tableName - * @return - */ - private StringBuffer getSubquerySql(String columnName, String tableName) { - StringBuffer subQuery = new StringBuffer(); - - subQuery.append("select " + columnName + " from ").append(tableName) - .append(" where ").append(param.getDbQuerySource().getExpr()); - - //order by中元素存在别名时做替换处理 - List<OrderByElement> listOrderElement = param.getDbQuerySource().getListOrderElement(); - if (StringUtil.isNotEmpty(listOrderElement)) { - String orderBy = param.getDbQuerySource().getOrderBy(); - for (OrderByElement order : listOrderElement) { - String orderField = order.getExpression().toString(); - if (orderField.startsWith("\"") && orderField.endsWith("\"")) { - orderField = orderField.substring(1, orderField.length() - 1); - } - String originalField = param.getDbQuerySource().getAliasFields().get(orderField); - if (StringUtil.isNotBlank(originalField)) { - orderBy = orderBy.replaceAll("(\"" + orderField + "\")|(\\b" + orderField + "\\b)", originalField); - } - - } - subQuery.append(" order by ").append(orderBy); - } - - - if (StringUtil.isNotBlank(param.getDbQuerySource().getLimit())) { - String[] limits = param.getDbQuerySource().getLimit().split(","); - if (limits.length == 2) { - int rows = Integer.valueOf(limits[0].trim()) + Integer.valueOf(limits[1].trim()); - subQuery.append(" limit ").append(rows); - } else { - subQuery.append(" limit ").append(param.getDbQuerySource().getLimit()); - } - - } else { - subQuery.append(" limit 0, " + engineConfig.getDefaultResultNum()); - } - return subQuery; - } - - /** - * 获取当前CK查询中的分区键，用于子查询优化操作; - * 若分区键被其它列当别名使用，返回空 - * - * @return - */ - private String getPartitionKey(String tableName) { - String partitionKey = metadataService.getPartitionKey(tableName); - - if (StringUtil.isNotBlank(partitionKey)) { - String tempValue = param.getDbQuerySource().getAliasFields().get(partitionKey); - if (StringUtil.isNotBlank(tempValue)) { - if (tempValue.contains(partitionKey)) { - partitionKey = tempValue; - } else { - partitionKey = ""; - } - } - - } - - return partitionKey; - - } - - /** - * 判断table是否在DB中 - * - * @param tableName - * @return - */ - private boolean isExist(String tableName) { - Map<String, String> result = executeHttpGet(String.format("select 1 from %s limit 1", tableName)); - if (result.get("status") == null || Integer.parseInt(result.get("status")) != HttpStatus.SC_OK) { - log.info("DB中不存在该表: {}", tableName); - return false; - } - return true; - } - - /** - * 判断order by 元素是否属于schema字段中 - * - * @param schema - * @return - */ - private boolean isValidOfOrderElementsInSchema(Schema schema) { - List<OrderByElement> listOrderElement = param.getDbQuerySource().getListOrderElement(); - if (StringUtil.isEmpty(listOrderElement)) { - return true; - } else { - Map<String, String> aliasFields = param.getDbQuerySource().getAliasFields(); - for (OrderByElement order : listOrderElement) { - String orderField = String.valueOf(order.getExpression()); - if (orderField.startsWith("\"") && orderField.endsWith("\"")) { - orderField = orderField.substring(1, orderField.length() - 1); - } - String originalField = StringUtil.isBlank(aliasFields.get(orderField)) ? orderField : aliasFields.get(orderField); - Expression expression = parserException(originalField); - if (expression instanceof Function) { - if (!getValidityByFunctionParamsInSchema(schema, (Function) expression)) { - return false; - } - } else { - if (!isFieldOfSchema(schema, originalField)) { - return false; - } - } - } - return true; - } - } - - /** - * 判断schema中是否存在field列 - * - * @param schema - * @param fieldName - * @return - */ - private boolean isFieldOfSchema(Schema schema, String fieldName) { - Schema.Field field = schema.getField(fieldName); - if (StringUtil.isEmpty(field)) { - return false; - } - return true; - } - - /** - * 判断where条件是否可以进行在索引表中的查询 - * - * @param where - * @param masterTableSchema 主表 - * @param indexTableSchema 索引表 - */ - private void setValidityByWhereExpressionsInSchema(Expression where, Schema masterTableSchema, Schema indexTableSchema) { - - if (StringUtil.isEmpty(masterTableSchema) || StringUtil.isEmpty(indexTableSchema)) { - setSubIndexValidFlag(false); - return; - } - - if (where instanceof InExpression) { - InExpression whereIn = (InExpression) where; - ItemsList rightItemsList = whereIn.getRightItemsList(); - if (rightItemsList instanceof SubSelect) { - setSubIndexValidFlag(false); - return; - } - String left; - if (!StringUtil.isEmpty(whereIn.getLeftExpression())) { - left = String.valueOf(whereIn.getLeftExpression()); - } else { - left = String.valueOf(whereIn.getLeftItemsList()); - } - if (!isFieldOfSchema(masterTableSchema, left) || !isFieldOfSchema(indexTableSchema, left)) { - setSubIndexValidFlag(false); - return; - } - } else if (where instanceof AndExpression) { - AndExpression whereAnd = (AndExpression) where; - Expression leftExpression = whereAnd.getLeftExpression(); - Expression rightExpression = whereAnd.getRightExpression(); - setValidityByWhereExpressionsInSchema(leftExpression, masterTableSchema, indexTableSchema); - setValidityByWhereExpressionsInSchema(rightExpression, masterTableSchema, indexTableSchema); - } else if (where instanceof OrExpression) { - OrExpression whereOr = (OrExpression) where; - Expression leftExpression = whereOr.getLeftExpression(); - Expression rightExpression = whereOr.getRightExpression(); - setValidityByWhereExpressionsInSchema(leftExpression, masterTableSchema, indexTableSchema); - setValidityByWhereExpressionsInSchema(rightExpression, masterTableSchema, indexTableSchema); - } else if (where instanceof ComparisonOperator) { - ComparisonOperator whereComparisonOperator = (ComparisonOperator) where; - setValidityByLRExpressionInSchema(masterTableSchema, indexTableSchema, whereComparisonOperator.getLeftExpression(), whereComparisonOperator.getRightExpression()); - } else if (where instanceof LikeExpression) { - LikeExpression whereLike = (LikeExpression) where; - setValidityByLRExpressionInSchema(masterTableSchema, indexTableSchema, whereLike.getLeftExpression(), whereLike.getRightExpression()); - } else if (where instanceof Function) { - Function function = (Function) where; - boolean bool = getValidityByFunctionParamsInSchema(indexTableSchema, function); - if (!bool) { - setSubIndexValidFlag(false); - } - } else if (where instanceof Parenthesis) { - Parenthesis whereExpressionParenthesis = (Parenthesis) where; - Expression expression = whereExpressionParenthesis.getExpression(); - setValidityByWhereExpressionsInSchema(expression, masterTableSchema, indexTableSchema); - } else { - setSubIndexValidFlag(false); - return; - } - } - - /** - * 根据函数参数判断索引表查询是否有效: 只支持0或1(判断是否在schema中)个参数, 其它均设为无效 - * - * @param schema - * @param function - */ - private boolean getValidityByFunctionParamsInSchema(Schema schema, Function function) { - ExpressionList parameters = function.getParameters(); - if (StringUtil.isEmpty(parameters)) { - return true; - } else { - List<Expression> expressions = parameters.getExpressions(); - if (expressions.size() != 1) { - return false; - } - if (!isFieldOfSchema(schema, String.valueOf(expressions.get(0)))) { - return false; - } - return true; - } - } - - /** - * 根据左右表达式判断索引表查询是否有效 - * - * @param masterTableSchema - * @param indexTableSchema - * @param leftExpression - * @param rightExpression - */ - private void setValidityByLRExpressionInSchema(Schema masterTableSchema, Schema indexTableSchema, Expression leftExpression, Expression rightExpression) { - String leftStr = String.valueOf(leftExpression); - String rightStr = String.valueOf(rightExpression); - if (leftExpression instanceof Function && rightExpression instanceof Function) { - setValidityByFunAndFunInSchema(masterTableSchema, indexTableSchema, (Function) leftExpression, (Function) rightExpression); - } else if (!(leftExpression instanceof Function) && rightExpression instanceof Function) { - setValidityByFunctionAndStrInSchema(masterTableSchema, indexTableSchema, (Function) rightExpression, leftStr); - } else if (leftExpression instanceof Function && !(rightExpression instanceof Function)) { - setValidityByFunctionAndStrInSchema(masterTableSchema, indexTableSchema, (Function) leftExpression, rightStr); - } else { - setValidityBuStrAndStrInSchema(masterTableSchema, indexTableSchema, leftStr, rightStr); - } - } - - /** - * 操作符左右两边都是非函数类型(转String) - * - * @param masterTableSchema - * @param indexTableSchema - * @param leftStr - * @param rightStr - */ - private void setValidityBuStrAndStrInSchema(Schema masterTableSchema, Schema indexTableSchema, String leftStr, String rightStr) { - if (isFieldOfSchema(masterTableSchema, leftStr) && !isFieldOfSchema(indexTableSchema, leftStr)) { - setSubIndexValidFlag(false); - return; - } - if (isFieldOfSchema(masterTableSchema, rightStr) && !isFieldOfSchema(indexTableSchema, rightStr)) { - setSubIndexValidFlag(false); - return; - } - if (!isFieldOfSchema(masterTableSchema, leftStr) && !isFieldOfSchema(masterTableSchema, rightStr)) { - setSubIndexValidFlag(false); - return; - } - } - - /** - * 操作符左右两边都是函数 - * - * @param masterTableSchema - * @param indexTableSchema - * @param leftExpression - * @param rightExpression - */ - private void setValidityByFunAndFunInSchema(Schema masterTableSchema, Schema indexTableSchema, Function leftExpression, Function rightExpression) { - boolean validLeftInMaster = getValidityByFunctionParamsInSchema(masterTableSchema, leftExpression); - boolean validLeftInIndex = getValidityByFunctionParamsInSchema(indexTableSchema, leftExpression); - if (validLeftInMaster && !validLeftInIndex) { - setSubIndexValidFlag(false); - return; - } - boolean validRightInMaster = getValidityByFunctionParamsInSchema(masterTableSchema, rightExpression); - boolean validRightInIndex = getValidityByFunctionParamsInSchema(indexTableSchema, rightExpression); - if (validRightInMaster && !validRightInIndex) { - setSubIndexValidFlag(false); - return; - } - if (!validLeftInMaster && !validRightInMaster) { - setSubIndexValidFlag(false); - return; - } - } - - /** - * 操作符左右一边是函数类型一边是非函数(转String) - * - * @param masterTableSchema - * @param indexTableSchema - * @param function - * @param str - */ - private void setValidityByFunctionAndStrInSchema(Schema masterTableSchema, Schema indexTableSchema, Function function, String str) { - boolean validFunInMaster = getValidityByFunctionParamsInSchema(masterTableSchema, function); - boolean validFunInIndex = getValidityByFunctionParamsInSchema(indexTableSchema, function); - if (validFunInMaster && !validFunInIndex) { - setSubIndexValidFlag(false); - return; - } - if (isFieldOfSchema(masterTableSchema, str) && !isFieldOfSchema(indexTableSchema, str)) { - setSubIndexValidFlag(false); - return; - } - if (!isFieldOfSchema(masterTableSchema, str) && !validFunInMaster) { - setSubIndexValidFlag(false); - return; - } - } - - private void setSubIndexValidFlag(Boolean subIndexValidFlag) { - this.subIndexValidFlag = subIndexValidFlag; - } - - private Boolean getSubIndexValidFlag() { - return subIndexValidFlag; - } - - public String build() { - return sql; - } - - - } - - private Expression parserException(String originalField) { - try { - return CCJSqlParserUtil.parseExpression(originalField); - } catch (JSQLParserException e) { - throw new BusinessException("parseExpression error: " + originalField); - } - } - - - @Override - public Dialect executeQuery() { - generateBaseResult(convertQuery(findBestQuery())); - return this; - } - - @Override - public Dialect executeAdministrativeQuery() { - generateBaseResult(param.getQuery()); - return this; - } - - - @Override - public Dialect executeSyntaxCheck() { - String sql = convertQuery(getSyntaxCheckSql()); - log.info("option=syntax-check: original sql: {}, after transformation(return message) info: {}", param.getQuery(), sql); - - Map<String, String> results = executeHttpGet(sql); - if (results.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), Encodes.urlDecode(sql), - JsonMapper.fromJsonString(results.get("result"), Map.class), null, null, QueryFormatEnum.JSON.getValue()); - } else { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), results.get("message"), - results.get("result"), null, null, param.getFormat()); - } - return this; - } - - private void generateBaseResult(String sql) { - - Map<String, String> results = executeHttpGet(sql); - if (results.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), "ok", - JsonMapper.fromJsonString(results.get("result"), Map.class), null, null, QueryFormatEnum.JSON.getValue()); - } else { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), results.get("message"), - results.get("result"), null, null, param.getFormat()); - } - } - - - @Override - public BaseResult build() { - - Object dataObject = baseResult.getData(); - if (baseResult.isSuccess()) { - if (StringUtil.isBlank(baseResult.getMessage())) { - baseResult.setMessage("ok"); - } - Map<String, Object> resultMap = (Map<String, Object>) dataObject; - Map<String, Object> statisticMap = (Map<String, Object>) resultMap.get("statistics"); - Map<String, Object> statistics = Maps.newLinkedHashMap(); - statistics.put("elapsed", Math.round(Double.parseDouble(String.valueOf(statisticMap.get("elapsed"))) * 1000)); - statistics.put("rows_read", statisticMap.get("rows_read")); - baseResult.setData(resultMap.get("data")); - baseResult.setMeta(resultMap.get("meta")); - baseResult.setStatistics(statistics); - } else if (baseResult.getStatus().equals(HttpStatus.SC_NOT_FOUND)) { - baseResult = BaseResultUtil.failure(HttpStatus.SC_NOT_FOUND, - ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), StringUtil.isEmpty(dataObject) ? "System Runtime Exception, Please contact Galaxy Data Platform Project manager" : String.valueOf(dataObject)); - } else if (baseResult.getStatus().equals(HttpStatus.SC_INTERNAL_SERVER_ERROR)) { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), - StringUtil.isEmpty(dataObject) ? "System Runtime Exception, Please contact Galaxy Data Platform Project manager" : String.valueOf(dataObject)); - } else { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), - StringUtil.isEmpty(dataObject) ? "System Runtime Exception, Please contact Galaxy Data Platform Project manager " : String.valueOf(dataObject)); - } - return baseResult; - } - - - /** - * 获取语法检测sql - * - * @return - */ - private String getSyntaxCheckSql() { - String sql = param.getDbQuerySource().getSqlBody(); - try { - sql = String.valueOf(parserAndUpdateSql(sql)); - } catch (Exception e) { - log.error("syntax-check sql error {}, execute original sql: {}, error is: {} ", - sql, sql = param.getDbQuerySource().getSqlBody(), e.getMessage() == null ? e.getCause() : e.getMessage()); - } - return sql; - } - - /** - * 做数据集限制: 将最内层sql的表名, 替换成(select * from tableName limit XXX) as alias - * 涉及from 和 where中存在子查询 - * - * @param sql - * @return - * @throws JSQLParserException - */ - private SelectBody parserAndUpdateSql(String sql) throws JSQLParserException { - Statement parse = CCJSqlParserUtil.parse(sql); - if (parse instanceof Select) { - Select select = (Select) parse; - SelectBody selectBody = select.getSelectBody(); - if (selectBody instanceof PlainSelect) { - PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); - FromItem fromItem = plainSelect.getFromItem(); - if (fromItem instanceof SubSelect) { - SubSelect subSelect = (SubSelect) plainSelect.getFromItem(); - subSelect.setSelectBody(parserAndUpdateSql(subSelect.getSelectBody().toString())); - } else if (fromItem instanceof Table) { - Table table = (Table) fromItem; - plainSelect.setFromItem(generateDataSetSql(table.getName(), table.getAlias() != null ? table.getAlias().getName() : table.getName())); - } - parseWhereAndUpdate(plainSelect.getWhere()); - return plainSelect; - } else if (selectBody instanceof SetOperationList) { - SetOperationList setOperationList = (SetOperationList) selectBody; - List<SelectBody> selects = setOperationList.getSelects(); - for (int i = 0; i < selects.size(); i++) { - selects.set(i, parserAndUpdateSql(selects.get(i).toString())); - } - return setOperationList; - } else { - throw new BusinessException("Only support selectBody operation: PlainSelect or SetOperationList"); - } - } - throw new BusinessException("Only support statement operation as select"); - } - - /** - * 将where中子查询语句表名更新为数据集形式,类同from中表的替换 - * - * @param whereExpression - * @throws JSQLParserException - */ - private void parseWhereAndUpdate(Expression whereExpression) throws JSQLParserException { - if (whereExpression instanceof ComparisonOperator) { - ComparisonOperator whereOperator = (ComparisonOperator) whereExpression; - Expression left = whereOperator.getLeftExpression(); - setSelectBody(left); - Expression right = whereOperator.getRightExpression(); - setSelectBody(right); - } else if (whereExpression instanceof InExpression) { - InExpression inExpression = (InExpression) whereExpression; - Expression leftExpression = inExpression.getLeftExpression(); - setSelectBody(leftExpression); - ItemsList rightItemsList = inExpression.getRightItemsList(); - if (rightItemsList instanceof SubSelect) { - SubSelect subSelect = (SubSelect) rightItemsList; - setSelectBody(subSelect); - } - } else if (whereExpression instanceof Between) { - Between whereBetween = (Between) whereExpression; - Expression betweenExpressionStart = whereBetween.getBetweenExpressionStart(); - setSelectBody(betweenExpressionStart); - Expression betweenExpressionEnd = whereBetween.getBetweenExpressionEnd(); - setSelectBody(betweenExpressionEnd); - } else if (whereExpression instanceof AndExpression) { - AndExpression whereAnd = (AndExpression) whereExpression; - parseWhereAndUpdate(whereAnd.getLeftExpression()); - parseWhereAndUpdate(whereAnd.getRightExpression()); - } else if (whereExpression instanceof OrExpression) { - OrExpression whereOr = (OrExpression) whereExpression; - parseWhereAndUpdate(whereOr.getLeftExpression()); - parseWhereAndUpdate(whereOr.getRightExpression()); - } else if (whereExpression instanceof Parenthesis) { - Parenthesis whereExpressionParenthesis = (Parenthesis) whereExpression; - Expression expression = whereExpressionParenthesis.getExpression(); - parseWhereAndUpdate(expression); - } - } - - private void setSelectBody(Expression expression) throws JSQLParserException { - if (expression instanceof SubSelect) { - SubSelect subSelect = (SubSelect) expression; - SelectBody selectBody = subSelect.getSelectBody(); - subSelect.setSelectBody(parserAndUpdateSql(selectBody.toString())); - } - } - - /** - * 生成数据集: 以FromItem对象形式的数据集表 sql,用来替换table - * - * @param tableName - * @param alias - * @return - * @throws JSQLParserException - */ - private FromItem generateDataSetSql(String tableName, String alias) throws JSQLParserException { - String sql = String.format("SELECT 1 FROM (SELECT * FROM %s LIMIT %s ) as %s ", - tableName, 10, alias); - Statement parse = CCJSqlParserUtil.parse(sql); - Select select = (Select) parse; - PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); - return plainSelect.getFromItem(); - } - - - /** - * ClickHouse Http Query Interface - * - * @param sql SQL语句 - * @return - */ - private Map<String, String> executeHttpGet(String sql) { - StringBuilder urlBuilder = new StringBuilder(clickHouseConfig.getQueryUrl()).append("/?"); - - String queryParameter = buildHttpQueryParameter(sql); - - log.info("DB Engine is :{}, Execute Http Query is: {}", DBTypeEnum.CLICKHOUSE.getValue(), urlBuilder.toString() + - Encodes.urlDecode(queryParameter)); - - List<NameValuePair> values = URLEncodedUtils.parse(queryParameter, Charset.forName("UTF-8")); - - return httpClientService.httpGet(urlBuilder.toString() + URLEncodedUtils.format(values, "utf-8")); - } - - private String buildHttpQueryParameter(String sql) { - - StringBuilder queryParameterBuilder = new StringBuilder(1024); - - String dbName = metadataService.getDBNameByTableName(param.getTableName()); - - if (dbName.equalsIgnoreCase(clickHouseConfig.getSystemDBName())) { - queryParameterBuilder.append("user=") - .append(clickHouseConfig.getLongTermUserName()).append("&") - .append("password=").append(clickHouseConfig.getLongTermPassword()).append("&") - .append("database=").append(dbName).append("&"); - - } else if (QueryOptionEnum.LONG_TERM.getValue().equalsIgnoreCase(param.getOption())) { - - queryParameterBuilder.append("user=") - .append(clickHouseConfig.getLongTermUserName()).append("&") - .append("password=").append(clickHouseConfig.getLongTermPassword()).append("&") - .append("database=").append(dbName).append("&") - .append("query_id=").append(ReportCache.getQueryId(param.getResultId(), param.getQuery())).append("&"); - } else { - queryParameterBuilder.append("user=") - .append(clickHouseConfig.getRealTimeUserName()).append("&") - .append("password=").append(clickHouseConfig.getRealTimePassword()).append("&") - .append("database=").append(dbName).append("&"); - } - - - queryParameterBuilder.append("query=").append(sql) - .append(" FORMAT ").append(QueryFormatEnum.JSON.getValue().toUpperCase()).append(" ;"); - return queryParameterBuilder.toString(); - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/Dialect.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/Dialect.java deleted file mode 100644 index e254d51..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/Dialect.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.mesalab.engine.component.dialect; - -import com.mesalab.common.dto.results.BaseResult; - -public interface Dialect { - - /** - * 数据库本身是否支持分页当前的分页查询方式 - * 如果数据库不支持的话，则不进行数据库分页 - * - * @return true：支持当前的分页查询方式 - */ - boolean supportsLimit(); - - /** - * 执行SQL语法检查 - * - * @return - */ - Dialect executeSyntaxCheck(); - - /** - * 执行查询操作 - * - * @return - */ - Dialect executeQuery(); - - - /** - * 执行数据库系统管理操作，包含 Show/Describe/Kill等 - * - * @return - */ - Dialect executeAdministrativeQuery(); - - /** - * 返回结果数据进行封装 - * - * @return - */ - BaseResult build(); - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/DruidDialect.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/DruidDialect.java deleted file mode 100644 index cad27fa..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/DruidDialect.java +++ /dev/null @@ -1,394 +0,0 @@ -package com.mesalab.engine.component.dialect; - -import com.google.common.base.Stopwatch; -import com.google.common.collect.Maps; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.DataTypeUtil; -import com.mesalab.engine.component.bean.SqlQueryBean; -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.common.enums.QueryFormatEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.common.util.SqlHelper; -import com.mesalab.engine.component.SpringContext; -import com.mesalab.engine.component.SqlFunction; -import com.mesalab.engine.component.config.DruidConfig; -import com.mesalab.engine.component.config.EngineConfig; -import com.mesalab.engine.service.HttpClientService; -import com.zdjizhi.utils.Encodes; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.schema.Table; -import net.sf.jsqlparser.statement.Statement; -import net.sf.jsqlparser.statement.select.*; -import org.apache.http.HttpStatus; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -@Slf4j -@SuppressWarnings("unchecked") -public class DruidDialect extends AbstractDataSourceDialect { - - private HttpClientService httpClientService = (HttpClientService) SpringContext.getBean("httpClientService"); - private DruidConfig druidConfig = (DruidConfig) SpringContext.getBean("druidConfig"); - private EngineConfig engineConfig = (EngineConfig) SpringContext.getBean("engineConfig"); - private int pageOffset; - private Stopwatch watch = Stopwatch.createUnstarted(); - - public DruidDialect(ApiQueryBean param) { - initialize(param); - } - - @Override - public boolean supportsLimit() { - return true; - } - - - @Override - public String convertQuery(String sql) { - return new Converter(sql).generateStandard().generateDateFunction().build(); - - } - - @Override - public String findBestQuery() { - - return new Optimizer().generateQueryLimit().generateSQL().build(); - - } - - @Override - public Dialect executeQuery() { - - watch.start(); - generateBaseResult(convertQuery(findBestQuery())); - return this; - - } - - @Override - public Dialect executeAdministrativeQuery() { - generateBaseResult(param.getQuery()); - return this; - } - - private void generateBaseResult(String sql) { - - Map<String, String> results = executeHttpPost(sql); - - if (results.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), null, - (List<Object>) JsonMapper.fromJsonString(results.get("result"), Object.class), null, null, QueryFormatEnum.JSON.getValue()); - } else { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), results.get("message"), - results.get("result"), null, null, param.getFormat()); - } - } - - /** - * Druid Http Query Interface - * - * @param sql SQL语句 - * @return - */ - private Map<String, String> executeHttpPost(String sql) { - StringBuilder urlBuilder = new StringBuilder(druidConfig.getQueryUrl()); - DruidQueryParam druidQueryParam = new DruidQueryParam(); - druidQueryParam.setQuery(sql); - druidQueryParam.getContext().put("sqlTimeZone", druidConfig.getSqlTimeZone()); - druidQueryParam.getContext().put("skipEmptyBuckets", druidConfig.getSkipEmptyBuckets()); - druidQueryParam.setResultFormat("object"); - log.info("DB Engine is :{}, Execute Query is: {}", DBTypeEnum.DRUID.getValue(), JsonMapper.toJsonString(druidQueryParam)); - return httpClientService.httpPost(urlBuilder.toString(), JsonMapper.toJsonString(druidQueryParam)); - } - - - @Override - public Dialect executeSyntaxCheck() { - String query = convertQuery(getSyntaxCheckSql()); - Map<String, String> results = executeHttpPost(query); - - if (results.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")),ResultCodeEnum.EXECUTE_SUCCESS.getCode(), query, - (List<Object>) JsonMapper.fromJsonString(results.get("result"), Object.class), null, null, param.getFormat()); - } else { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), results.get("message"), - results.get("result"), null, null, param.getFormat()); - } - return this; - } - - - @Override - public BaseResult build() { - - Object dataObject = baseResult.getData(); - - if (baseResult.isSuccess()) { - if (StringUtil.isBlank(baseResult.getMessage())) { - baseResult.setMessage("ok"); - } - List<Object> allResults = (List<Object>) dataObject; - if (pageOffset > 0 && StringUtil.isNotEmpty(allResults)) { - if (pageOffset >= allResults.size()) { - allResults = new ArrayList<>(); - } else { - allResults = allResults.subList(pageOffset, allResults.size()); - } - } - Map<String, Object> statistics = Maps.newLinkedHashMap(); - statistics.put("elapsed", watch.elapsed(TimeUnit.MILLISECONDS)); - statistics.put("rows_read", allResults.size()); - baseResult.setData(allResults); - baseResult.setMeta(getMeta(allResults)); - baseResult.setStatistics(statistics); - } else if (baseResult.getStatus().equals(HttpStatus.SC_NOT_FOUND)) { - baseResult = BaseResultUtil.failure(HttpStatus.SC_NOT_FOUND, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), - StringUtil.isEmpty(dataObject) ? "System Runtime Exception,Please contact Galaxy Data Platform Project manager. " : String.valueOf(dataObject)); - } else if (baseResult.getStatus().equals(HttpStatus.SC_INTERNAL_SERVER_ERROR)) { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), - StringUtil.isEmpty(dataObject) ? "System Runtime Exception,Please contact Galaxy Data Platform Project manager. " : String.valueOf(dataObject)); - } else { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), - StringUtil.isEmpty(dataObject) ? "System Runtime Exception,Please contact Galaxy Data Platform Project manager. " : String.valueOf(dataObject)); - } - return baseResult; - } - - - /** - * 获取语法检测sql - * - * @return - */ - private String getSyntaxCheckSql() { - String sql = param.getDbQuerySource().getSqlBody(); - try { - sql = String.valueOf(parserAndUpdateSql(sql)); - } catch (Exception e) { - log.error("syntax-check sql error {}, execute original sql: {}, error is: {} ", - sql, sql = param.getDbQuerySource().getSqlBody(), e.getMessage() == null ? e.getCause() : e.getMessage()); - } - return sql; - } - - /** - * 做时间范围条件过滤: 去除原有最内层条件, 重新添加时间范围 - * - * @param sql - * @return - * @throws JSQLParserException - */ - private SelectBody parserAndUpdateSql(String sql) throws JSQLParserException { - Statement parse = CCJSqlParserUtil.parse(sql); - if (parse instanceof Select) { - Select select = (Select) parse; - SelectBody selectBody = select.getSelectBody(); - if (selectBody instanceof PlainSelect) { - PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); - FromItem fromItem = plainSelect.getFromItem(); - if (fromItem instanceof SubSelect) { - SubSelect subSelect = (SubSelect) plainSelect.getFromItem(); - subSelect.setSelectBody(parserAndUpdateSql(subSelect.getSelectBody().toString())); - } else if (fromItem instanceof Table) { - plainSelect.setWhere(generateDataSetSql()); - return plainSelect; - } - return plainSelect; - } else if (selectBody instanceof SetOperationList) { - SetOperationList setOperationList = (SetOperationList) selectBody; - List<SelectBody> selects = setOperationList.getSelects(); - for (int i = 0; i < selects.size(); i++) { - selects.set(i, parserAndUpdateSql(selects.get(i).toString())); - } - return setOperationList; - } else { - throw new BusinessException("Only support selectBody operation: PlainSelect or SetOperationList"); - } - } - throw new BusinessException("Only support statement operation as select"); - } - - - /** - * 生成where条件: 以Expression对象形式 - * - * @return - * @throws JSQLParserException - */ - private Expression generateDataSetSql() throws JSQLParserException { - String sql = String.format("SELECT * FROM tableName WHERE %s >= CURRENT_TIMESTAMP - INTERVAL '%s' HOUR ", param.getDbQuerySource().getPartitionKey(), 1); - Statement parse = CCJSqlParserUtil.parse(sql); - Select select = (Select) parse; - PlainSelect plainSelect = (PlainSelect) select.getSelectBody(); - return plainSelect.getWhere(); - } - - - - private List getMeta(List<Object> results) { - List meta = new ArrayList<Map<String, String>>(); - if (results.size() > 0) { - Map<String, Object> map = (Map) results.get(0); - for (String key : map.keySet()) { - Object value = map.get(key); - Map<String, String> temp = new HashMap<>(); - temp.put("name", key); - if (value instanceof Integer) { - temp.put("type", DataTypeUtil.BIGINT); - } else if (value instanceof Double) { - temp.put("type", DataTypeUtil.DOUBLE); - } else { - temp.put("type", DataTypeUtil.VARCHAR); - } - meta.add(temp); - } - } - return meta; - } - - class Converter { - String sql; - - public Converter(String sql) { - this.sql = sql; - } - - /** - * SQL转换为DRUID语法格式 - * 1. 不支持limit offset,rows 转为 limit rows - * - * @return - */ - public Converter generateStandard() { - SqlQueryBean dbQuerySource = param.getDbQuerySource(); - if (StringUtil.isNotBlank(dbQuerySource.getLimit()) && - dbQuerySource.getLimit().split(",").length >= 2) { - - String[] limits = dbQuerySource.getLimit().trim().split(","); - int offset = Integer.valueOf(limits[0].trim()); - int rowNum = Integer.valueOf(limits[1].trim()); - pageOffset = offset; - sql = SqlHelper.getLimitString(dbQuerySource.getSqlBody(), 0, (offset + rowNum)); - } - return this; - } - - - public Converter generateDateFunction() { - sql = SqlFunction.generateDateFunction(sql, DBTypeEnum.DRUID.getValue()); - return this; - } - - - public String build() { - return sql; - } - } - - - class Optimizer { - String sql = null; - - /** - * 对SQL最内层的结果数据集进行限制 - * 参见查询网关系统参数：engine.maxCacheNum - * - * @return - */ - public Optimizer generateQueryLimit() { - setDefaultResultRows(StringUtil.isEmpty(param.getEngineQuerySource()) ? param.getDbQuerySource() : param.getEngineQuerySource(), engineConfig.getDefaultResultNum()); - setMaxCacheResultRows(param.getDbQuerySource(), engineConfig.getMaxCacheNum()); - return this; - } - - - private void setDefaultResultRows(SqlQueryBean sqlQuerySource, int defaultResultRows) { - if (StringUtil.isBlank(sqlQuerySource.getLimit())) { - sqlQuerySource.setSqlBody(sqlQuerySource.getSqlBody() + " limit " + defaultResultRows); - sqlQuerySource.setLimit(String.valueOf(defaultResultRows)); - } - - } - - /** - * 设置最大缓存数据集 - * - * @param sqlQuerySource - * @return - */ - private void setMaxCacheResultRows(SqlQueryBean sqlQuerySource, int maxCacheResultRows) { - SqlQueryBean innerSqlQuerySource = sqlQuerySource; - while (StringUtil.isNotEmpty(innerSqlQuerySource.getSubSelect())) { - innerSqlQuerySource = innerSqlQuerySource.getSubSqlQuerySources().get(0); - } - - if (!innerSqlQuerySource.isEnableLimit()) { - return; - } - - String orginInnerSql = innerSqlQuerySource.getSqlBody(); - - String limitInnerSql = innerSqlQuerySource.getSqlBody(); - if (StringUtil.isNotBlank(innerSqlQuerySource.getLimit())) { - - String[] limits = innerSqlQuerySource.getLimit().split(","); - - if (Integer.valueOf(limits[limits.length - 1]) > maxCacheResultRows) { - limits[limits.length - 1] = String.valueOf(maxCacheResultRows); - - if (limits.length == 2) { - limitInnerSql = SqlHelper.getLimitString(limitInnerSql, Integer.valueOf(limits[0]), Integer.valueOf(limits[1])); - } else { - limitInnerSql = SqlHelper.getLimitString(limitInnerSql, 0, Integer.valueOf(limits[0])); - } - - innerSqlQuerySource.setLimit(String.join(",", limits).toString()); - innerSqlQuerySource.setSqlBody(limitInnerSql); - - } - - } else { - limitInnerSql = innerSqlQuerySource.getSqlBody() + " limit " + maxCacheResultRows; - innerSqlQuerySource.setLimit(String.valueOf(maxCacheResultRows)); - innerSqlQuerySource.setSqlBody(limitInnerSql); - - } - - if (StringUtil.isNotEmpty(sqlQuerySource.getSubSqlQuerySources())) { - sqlQuerySource.setSqlBody(sqlQuerySource.getSqlBody().replace(orginInnerSql, limitInnerSql)); - } - - } - - public Optimizer generateSQL() { - SqlQueryBean sqlQuerySource = param.getDbQuerySource(); - sql = sqlQuerySource.getSqlBody(); - return this; - } - - public String build() { - return sql; - } - } - - @Data - class DruidQueryParam { - private String query; - private Map<String, String> context = Maps.newHashMap(); - private String resultFormat; - - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/FederationDialect.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/FederationDialect.java deleted file mode 100644 index 648ca67..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/FederationDialect.java +++ /dev/null @@ -1,147 +0,0 @@ -package com.mesalab.engine.component.dialect; - -import com.google.common.base.Function; -import com.google.common.base.Joiner; -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.mesalab.engine.component.bean.SqlQueryBean; -import com.mesalab.engine.component.calcite.CalciteMemory; -import com.mesalab.engine.component.udf.UDF; -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.QueryFormatEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; - -import javax.annotation.Nullable; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -/** - * 联邦查询-基于Calcite支持多源联合查询 - */ -@Slf4j -@SuppressWarnings("unchecked") -public class FederationDialect extends AbstractEngineDialect { - private Stopwatch watch = Stopwatch.createUnstarted(); - - public FederationDialect(ApiQueryBean param, BaseResult result) { - initialize(param, result); - } - - @Override - public boolean supportsLimit() { - return false; - } - - @Override - public Dialect executeSyntaxCheck() { - return this; - } - - - @Override - public Dialect executeQuery() { - try { - watch.start(); - - if (baseResult.isSuccess()) { - - executeUDF(); - - if (StringUtil.isNotEmpty(param.getEngineQuerySource())) { - SqlQueryBean sqlQuerySource = param.getEngineQuerySource(); - Map resultMap = CalciteMemory.executeMemoryQuery(sqlQuerySource.getTableName(), - (List<Map<String, String>>) baseResult.getMeta(), - (List<Map<String, Object>>) baseResult.getData(), param.getQuery()); - baseResult.setData(resultMap.get("data")); - baseResult.setMeta(resultMap.get("meta")); - } - - log.info("calcite execute time :" + watch.elapsed().toMillis() / 1000 + "s"); - } - } catch (Exception ex) { - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), - ResultCodeEnum.ENGINE_STATISTICS_ERROR.getCode(), "Engine execute query memory is error.", ex.getCause()); - } - - return this; - } - - @Override - public Dialect executeAdministrativeQuery() { - return this; - } - - - @Override - public BaseResult build() { - - if (baseResult.isSuccess()) { - - long executeTime = watch.elapsed(TimeUnit.MILLISECONDS); - Map<String, Object> statisticMap = (Map<String, Object>) baseResult.getStatistics(); - statisticMap.put("elapsed", Long.parseLong(String.valueOf(statisticMap.get("elapsed"))) + executeTime); - if (StringUtil.isEmpty(baseResult.getData())) { - statisticMap.put("result_size", 0); - statisticMap.put("result_rows", 0); - } else { - statisticMap.put("result_size", JsonMapper.toJsonString(baseResult.getData()).getBytes().length); - statisticMap.put("result_rows", ((List) baseResult.getData()).size()); - } - - if (param.getFormat().equalsIgnoreCase(QueryFormatEnum.CSV.getValue())) { - convertJsonToCSV(); - } - baseResult.setFormatType(param.getFormat()); - } - log.debug("Execute build Time : {}", watch.elapsed(TimeUnit.MILLISECONDS) / 1000.0); - return baseResult; - } - - - @Override - public void executeUDF() { - - SqlQueryBean sqlQuerySource = param.getDbQuerySource(); - Set<UDF> udfSet = sqlQuerySource.getUdfSet(); - for (UDF udf : udfSet) { - List<Map<String, String>> targetResult = (List<Map<String, String>>) udf.execute(sqlQuerySource, - (List<Object>) baseResult.getData()); - baseResult.setData(targetResult); - } - - - } - - - private void convertJsonToCSV() { - - List<String> results = Lists.transform((List<Map<String, Object>>) baseResult.getData(), new Function<Map<String, Object>, String>() { - - @Override - public String apply(@Nullable Map<String, Object> input) { - return Joiner.on(",").useForNull("").join(input.values()); - } - }); - - List<Object> metas = Lists.transform((List<Map<String, Object>>) baseResult.getMeta(), new Function<Map<String, Object>, Object>() { - - @Override - public Object apply(@Nullable Map<String, Object> input) { - return input.get("name"); - } - }); - - baseResult.setData(Joiner.on("\n").useForNull("").join(results)); - baseResult.setMeta(Joiner.on(",").useForNull("").join(metas)); - } - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/HbaseDialect.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/HbaseDialect.java deleted file mode 100644 index e0b5ff0..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/dialect/HbaseDialect.java +++ /dev/null @@ -1,212 +0,0 @@ -package com.mesalab.engine.component.dialect; - -import com.google.common.base.Stopwatch; -import com.google.common.collect.Maps; -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.QueryFormatEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.component.SpringContext; -import com.mesalab.engine.component.config.HbaseConfig; -import com.mesalab.engine.service.HttpClientService; -import com.mesalab.engine.service.SqlEngineService; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.http.HttpStatus; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -@Slf4j -@SuppressWarnings("unchecked") -public class HbaseDialect extends AbstractDataSourceDialect { - private static HbaseConfig hbaseConfig = (HbaseConfig) SpringContext.getBean("hbaseConfig"); - private static SqlEngineService sqlEngineService = (SqlEngineService) SpringContext.getBean("sqlEngineService"); - private HttpClientService httpClientService = (HttpClientService) SpringContext.getBean("httpClientService"); - private Stopwatch watch = Stopwatch.createUnstarted(); - - public HbaseDialect(ApiQueryBean param) { - initialize(param); - } - - @Override - public boolean supportsLimit() { - return false; - } - - @Override - public String convertQuery(String sql) { - return sql; - } - - @Override - public String findBestQuery() { - return param.getSql(); - } - - - @Override - public Dialect executeQuery() { - watch.start(); - StringBuilder urlBuilder = new StringBuilder("http://") - .append(hbaseConfig.getUrl()).append("/"); - - String queryId = StringUtil.isNotBlank(param.getResultId()) ? param.getResultId() : param.getReportId(); - StringBuilder queryparamBuilder = new StringBuilder(hbaseConfig.getDbName()) - .append(":").append(hbaseConfig.getTableName()).append("/") - .append(getQueryId(queryId, param.getQuery())).append("/") - .append(hbaseConfig.getColumnFamily()).append(":").append(hbaseConfig.getColumnName()); - - Map<String, String> headers = Maps.newHashMap(); - headers.put("Accept", "application/json"); - Map<String, String> results = httpClientService.httpGet(urlBuilder.toString() + queryparamBuilder.toString(), headers); - - if (results.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.EXECUTE_SUCCESS.getCode(), null, - JsonMapper.fromJsonString(decryptHbaseResult(results.get("result")), Map.class), null, null, QueryFormatEnum.JSON.getValue()); - getAndParserFormatSql(); - } else { - baseResult = BaseResultUtil.generate(Integer.valueOf(results.get("status")), ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), null, - results.get("result"), null, null, param.getFormat()); - } - return this; - } - - @Override - public Dialect executeAdministrativeQuery() { - return this; - } - - private Map<String, String> getFormatSql() { - StringBuilder urlBuilder = new StringBuilder("http://") - .append(hbaseConfig.getUrl()).append("/"); - String queryId = StringUtil.isNotBlank(param.getResultId()) ? param.getResultId() : param.getReportId(); - StringBuilder queryparamBuilder = new StringBuilder(hbaseConfig.getDbName()) - .append(":").append(hbaseConfig.getTableName()).append("/") - .append(getQueryId(queryId, param.getQuery())).append("/") - .append("detail:format_sql"); - log.info("Report Query：{}, queryId Id: {}, sql:{}", urlBuilder.toString() + queryparamBuilder.toString(), queryId, param.getQuery()); - - Map<String, String> headers = Maps.newHashMap(); - - headers.put("Accept", "application/json"); - - Map<String, String> formatSql = httpClientService.httpGet(urlBuilder.toString() + queryparamBuilder.toString(), headers); - if (formatSql.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - formatSql.put("result", decryptHbaseResult(formatSql.get("result"))); - } - - - return formatSql; - } - - - @Override - public Dialect executeSyntaxCheck() { - return this; - } - - - @Override - public BaseResult build() { - Object dataObject = baseResult.getData(); - - if (baseResult.isSuccess()) { - Map<String, Object> maps = (Map<String, Object>) dataObject; - Map<String, Object> statisticMap = (Map<String, Object>) maps.get("statistics"); - - Map<String, Object> statistics = Maps.newLinkedHashMap(); - Double executeTime = watch.elapsed(TimeUnit.MILLISECONDS) / 1000.0; - statistics.put("elapsed", String.format("%.2f", (double) statisticMap.get("elapsed") + executeTime)); - statistics.put("rows_read", statisticMap.get("rows_read")); - - int resultSize = StringUtil.isEmpty(maps.get("data")) ? 0 : JsonMapper.toJsonString(maps.get("data")).getBytes().length; - - statistics.put("result_size", resultSize); - baseResult.setData(maps.get("data")); - baseResult.setMeta(maps.get("meta")); - baseResult.setStatistics(statistics); - - - } else if (baseResult.getStatus().equals(HttpStatus.SC_NOT_FOUND)) { - baseResult = BaseResultUtil.failure(HttpStatus.SC_NOT_FOUND, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), - StringUtil.isEmpty(dataObject) ? "System Runtime Exception,Please contact Galaxy Data Platform Project manager" : String.valueOf(dataObject)); - } else if (baseResult.getStatus().equals(HttpStatus.SC_INTERNAL_SERVER_ERROR)) { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), - StringUtil.isEmpty(dataObject) ? "System Runtime Exception,Please contact Galaxy Data Platform Project manager" : String.valueOf(dataObject)); - } else { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, ResultCodeEnum.SQL_EXECUTION_ERROR.getCode(), - StringUtil.isEmpty(dataObject) ? "System Runtime Exception,Please contact Galaxy Data Platform Project manager" : String.valueOf(dataObject)); - } - return baseResult; - } - - /** - * 获取formatSql并解析 - */ - private void getAndParserFormatSql() { - if (baseResult.isSuccess()) { - Map<String, String> formatSql = getFormatSql(); - String sql = formatSql.get("result"); - log.info("从离线报表获取formatSql，解析并重制SQLQuerySource对象: " + sql); - sqlEngineService.convertQueryRecursive(param, sqlEngineService.parserSQLByAst(sql)); - param.setQuery(param.getDbQuerySource().getSqlBody()); - } - } - - - /** - * 解析json并将结果进行base64解密 - * - * @param hbaseResult - * @return - */ - private String decryptHbaseResult(String hbaseResult) { - - String result = ""; - - try { - Map<String, Object> maps = (Map<String, Object>) JsonMapper.fromJsonString(hbaseResult, Map.class); - - List<Object> rowList = (ArrayList) maps.get("Row"); - - if (StringUtil.isNotEmpty(rowList)) { - Map<String, Object> keyMap = (Map<String, Object>) rowList.get(0); - String encryptResult = ((Map<String, Object>) ((ArrayList) keyMap.get("Cell")).get(0)).get("$").toString(); - result = new String(Base64.getDecoder().decode(encryptResult), StandardCharsets.UTF_8); - - } - - - } catch (Exception e) { - - log.error("Parser HBase Result error: {}", e.getMessage()); - - } - - - return result; - - } - - - /** - * 通过SQL生成查询唯一kEY - * - * @param sql - * @param reportId - * @return - */ - private static String getQueryId(String reportId, String sql) { - return DigestUtils.md5Hex(reportId + sql.trim()); - } - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/filter/RequestParamFilter.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/filter/RequestParamFilter.java deleted file mode 100644 index 26f02c5..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/filter/RequestParamFilter.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.mesalab.engine.component.filter; - -import org.springframework.stereotype.Component; - -import javax.servlet.*; -import javax.servlet.annotation.WebFilter; -import javax.servlet.http.HttpServletRequest; -import java.io.IOException; - - -@Component -@WebFilter(urlPatterns = "/*") -public class RequestParamFilter implements Filter { - - - @Override - public void init(FilterConfig filterConfig) throws ServletException { - - } - - - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - filterChain.doFilter(new RequestParamWrapper((HttpServletRequest) servletRequest), servletResponse); - } - - @Override - public void destroy() { - - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/filter/RequestParamWrapper.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/filter/RequestParamWrapper.java deleted file mode 100644 index 779364a..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/filter/RequestParamWrapper.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.mesalab.engine.component.filter; - - -import com.google.common.collect.Lists; -import com.mesalab.common.enums.QueryParamEnum; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletRequestWrapper; -import java.nio.charset.Charset; -import java.util.List; - -public class RequestParamWrapper extends HttpServletRequestWrapper { - private String queryString = null; - - RequestParamWrapper(HttpServletRequest request) { - super(request); - queryString = request.getQueryString(); - - } - - - @Override - public String[] getParameterValues(String name) { - if (QueryParamEnum.QUERY.getValue().equals(name)) { - - queryString = queryString.replaceAll("\\+", "%2B"); - - List<NameValuePair> values = URLEncodedUtils.parse(queryString, Charset.forName("UTF-8")); - List<String> valueList = Lists.newArrayList(); - for (NameValuePair nameValuePair : values) { - if (nameValuePair.getName().equals(QueryParamEnum.QUERY.getValue())) { - valueList.add(nameValuePair.getValue()); - } - } - String[] valueString = new String[valueList.size()]; - valueList.toArray(valueString); - return valueString; - } - return super.getParameterValues(name); - } - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/BaseQueryProvider.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/BaseQueryProvider.java deleted file mode 100644 index 9973b75..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/BaseQueryProvider.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.mesalab.engine.component.strategy; - -import com.google.common.collect.ImmutableMap; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.HttpClientUtil; -import com.mesalab.common.util.SysConstant; -import com.mesalab.engine.component.config.ArangoConfig; -import com.mesalab.engine.service.AuthorizeService; -import com.zdjizhi.utils.JsonMapper; -import org.apache.http.Header; -import org.apache.http.message.BasicHeader; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpHeaders; -import org.springframework.stereotype.Component; - -import java.util.Map; - -/** - * @description: for base method - * @author: zhq - * @create: 2020-07-13 - **/ -@Component -public class BaseQueryProvider { - - @Autowired - ArangoConfig arangoConfig; - - @Autowired - AuthorizeService authorizeService; - - /*** - * @Author zhq - * @Description - * @Date 2020\8\12 0012 - * @Param [aql] - * @return java.lang.Object - **/ - public Object queryArango(String aql) throws BusinessException { - try { - Map queryMap = ImmutableMap.of("query", aql); - String jwtStr = SysConstant.ARANGO_JWT_PRE.concat(authorizeService.arangoJwtLogin()); - Header header = new BasicHeader(HttpHeaders.AUTHORIZATION, jwtStr); - String res = HttpClientUtil.httpPost(arangoConfig.getQueryurl(), JsonMapper.toJsonString(queryMap), header); - Map jsonMap = (Map) JsonMapper.fromJsonString(res, Map.class); - Object result = jsonMap.get("result"); - return result; - } catch (BusinessException e) { - throw e; - } - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/FqdnProviderImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/FqdnProviderImpl.java deleted file mode 100644 index a0801b7..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/FqdnProviderImpl.java +++ /dev/null @@ -1,201 +0,0 @@ -package com.mesalab.engine.component.strategy; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.CharMatcher; -import com.google.common.base.Joiner; -import com.google.common.base.Splitter; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.mesalab.common.enums.knowledge.RangeEnum; -import com.mesalab.common.knowledge.KnowDslObject; -import com.mesalab.common.knowledge.Match; -import com.mesalab.common.knowledge.Parameters; -import com.mesalab.common.knowledge.Range; -import com.mesalab.common.model.arango.IpLearningPath; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.StringUtil; -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.Validate; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.ObjectUtils; - -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * 根据fqdn查询ip的关系 - */ - -@Component("IP_LEARNING_VIEW") -@SuppressWarnings("unchecked") -public class FqdnProviderImpl implements QueryProvider { - - private static final String depthregex = "^[1-9]\\d*$"; //depth 取值范围大于0 - - @Value("${iplearning.uniqciprecent}") - private Integer uniqCipRecent;//统计最近时间 单位小时 - - @Value("${arango.maxrows}") - private String limit;//查询最大条数 - - @Autowired - BaseQueryProvider baseQueryProvider; - - @Override - public Object query(KnowDslObject.DSLQuery dslQeruy) { - Parameters parameters = dslQeruy.getParameters(); - //a.处理条件 b.需要的条件：fqdn_name 多个, depth, filter:last_found_time, dist_cip, http,tls,dns, limit - StringBuffer likeFilter = new StringBuffer(); - Object depth = 1;//数据库默认1度 - StringBuffer intervalsb = new StringBuffer(); - StringBuffer protocolsb = new StringBuffer(); - StringBuffer distCiptsb = new StringBuffer(); - if (!ObjectUtils.isEmpty(parameters)) { - limit = ObjectUtils.isEmpty(parameters.getLimit()) ? limit : parameters.getLimit(); - List<Match> matches = parameters.getMatch(); - //1.构建查询语句 match like - likeFilter.append(Match.like(matches, (m) -> m != null && "FQDN_NAME".equals(m.getFieldKey()) && !ObjectUtils.isEmpty(m.getFieldValues()))); - - //2.构建查询语句 intervals 时间 - List<String> intervals = parameters.getIntervals(); - if (!ObjectUtils.isEmpty(intervals)) { - String[] times = intervals.get(0).split("/"); - intervalsb.append(" and e.FIRST_FOUND_TIME >= ").append(DateUtils.convertStringToTimestamp(times[0], DateUtils.YYYY_MM_DD_HH24_MM_SS)).append(" and ").append("e.LAST_FOUND_TIME <").append(DateUtils.convertStringToTimestamp(times[1], DateUtils.YYYY_MM_DD_HH24_MM_SS)); - } - - //3.构建查询语句 range depth ,protocols, limit - List<Range> ranges = parameters.getRange(); - - if (!ObjectUtils.isEmpty(ranges)) { - - for (Range r : ranges) { - //查询深度 - if ("DEPTH".equals(r.getFieldKey())) { - Validate.notEmpty(r.getFieldValues(), "fieldValues of depth is null"); - Validate.matchesPattern(String.valueOf(r.getFieldValues().get(0)), depthregex, "fieldValues of depth is illegal, you should type in an integer greater than 0"); - depth = r.getFieldValues().stream().findFirst().get(); - } else if ("PROTOCOL".equals(r.getFieldKey())) { - List<String> protocols = Lists.newArrayList(); - for (Object protocol : r.getFieldValues()) { - if ("HTTP".equals(protocol)) { - protocols.add("e.HTTP_CNT_TOTAL > 0"); - } else if ("DNS".equals(protocol)) { - protocols.add("e.DNS_CNT_TOTAL > 0"); - } else if ("TLS".equals(protocol)) { - protocols.add("e.TLS_CNT_TOTAL > 0"); - } - } - if (protocols.size() > 0) { - protocolsb.append(Joiner.on("").skipNulls().join(" and (", Joiner.on(" or ").skipNulls().join(protocols), ")")); - } - } else if ("UNIQ_CIP".equals(r.getFieldKey())) { - distCiptsb.append(" and count(e.DIST_CIP) ").append(EnumUtils.getEnum(RangeEnum.class, StringUtil.upperCase(r.getType())).getSymbol()).append(r.getFieldValues().get(0)); - } - - } - } - } - - String fqdn2IpDepth = "1..1";//fqdn查询ip的深度 - depth = "0.." + (Integer.valueOf(String.valueOf(depth)) - Integer.valueOf(Splitter.on("..").splitToList(fqdn2IpDepth).get(1))); - StringBuffer aqlsb = new StringBuffer("let FQS = ( for doc in FQDN "); - aqlsb.append(ObjectUtils.isEmpty(likeFilter) ? "" : likeFilter); - aqlsb.append(" for fq2fq in "); - aqlsb.append(depth); - aqlsb.append(" outbound doc._id R_SAME_ORIGIN_FQDN2FQDN "); - aqlsb.append(" return distinct fq2fq ) "); - aqlsb.append(" for fq in FQS "); - aqlsb.append(" for v,e,p in ").append(fqdn2IpDepth).append(" outbound fq._id graph 'FQDN_IP_GRAPH' "); - aqlsb.append(" filter 1==1 "); - aqlsb.append(intervalsb); - aqlsb.append(distCiptsb); - aqlsb.append(protocolsb); - int pageRow = 0; - int offSet = 0; - int pageSize = 0; - List<String> limitList = Splitter.on(",").trimResults().omitEmptyStrings().splitToList(limit); - if(limitList.size()<2){ - pageSize= pageRow = Integer.valueOf(limitList.get(0)); - } else { - pageRow = Integer.valueOf(limitList.get(0)) + Integer.valueOf(limitList.get(1)); - offSet = Integer.valueOf(limitList.get(0)); - pageSize = Integer.valueOf(limitList.get(1)); - } - - List<IpLearningPath.IplearningEdges> ipEdgesList = getDistinctList(aqlsb, pageRow, offSet, pageSize); - - return parseResult(ipEdgesList); - - } - //循环查询 ，目的为了数量达到limit , 结果数据是否为limit , 1.当查询数量小于limit停止 2.当结果数量=limit停止 - private List<IpLearningPath.IplearningEdges> getDistinctList(StringBuffer aqlsb ,int pageRow,int offSet, int pageSize){ - pageRow = pageRow + offSet + pageSize;//每次多查一点点，去重 - StringBuffer sb = new StringBuffer(aqlsb); - sb.append(" limit ").append(pageRow); - sb.append(" return distinct p "); - - Object result = baseQueryProvider.queryArango(sb.toString()); - List<IpLearningPath> ipPathList = new ObjectMapper().convertValue(result, new TypeReference<List<IpLearningPath>>() { - }); - List<IpLearningPath.IplearningEdges> ipEdgesList = ipPathList.stream().flatMap(p -> p.getEdges().stream()).collect(Collectors.toList()); - List<IpLearningPath.IplearningEdges> edgesList = ipEdgesList.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(IpLearningPath.IplearningEdges::getTo))), ArrayList::new)); - if(ipEdgesList.size()<pageRow || edgesList.size()>= pageSize+offSet){ - if(offSet > edgesList.size()){ - return new ArrayList<>(); - } else { - if(offSet + pageSize > edgesList.size()){ - return edgesList.subList(offSet, edgesList.size()); - } - return edgesList.subList(offSet, offSet + pageSize); - } - } - return getDistinctList( aqlsb , pageRow, offSet, pageSize); - } - /** - * @return java.lang.Object - * @Author zhq - * @Description - * @Date 2020\7\15 0015 - * @Param [IpLearningPath] - **/ - private Object parseResult(List<IpLearningPath.IplearningEdges> edgesList) { - - List<Object> list = Lists.newArrayList(); - if(!ObjectUtils.isEmpty(edgesList)){ - for (IpLearningPath.IplearningEdges edges : edgesList) { - Map<String, Object> map = Maps.newHashMap(); - List<String> protocolsList = Lists.newArrayList(); - map.put("FQDN_NAME", CharMatcher.anyOf("FQDN/").trimLeadingFrom(edges.getFrom())); - map.put("IP", CharMatcher.anyOf("IP/").trimLeadingFrom(edges.getTo())); - map.put("FIRST_FOUND_TIME", DateUtils.convertTimestampToString(edges.getFirstFoundTime(), DateUtils.YYYY_MM_DD_HH24_MM_SS)); - map.put("LAST_FOUND_TIME", DateUtils.convertTimestampToString(edges.getLastFoundTime(), DateUtils.YYYY_MM_DD_HH24_MM_SS)); - //协议 按 TLS_CNT_TOTAL>0,SESSION_RECENT按 *_CNT_RECENT和，SESSION_TOTAL按 *_CNT_TOTAL和 - if (edges.getHttpCntTotal() > 0) { - protocolsList.add("HTTP"); - } - if (edges.getTlsCntTotal() > 0) { - protocolsList.add("TLS"); - } - if (edges.getDnsCntTotal() > 0) { - protocolsList.add("DNS"); - } - map.put("PROTOCOL", Joiner.on(",").join(protocolsList)); - map.put("SESSION_TOTAL", edges.getHttpCntTotal() + edges.getTlsCntTotal() + edges.getDnsCntTotal()); - map.put("SESSION_RECENT", Stream.of(edges.getHttpCntRecent(), edges.getTlsCntRecent(), edges.getDnsCntRecent()).flatMap(Collection::stream).mapToLong(x -> x).sum()); - - //UNIQ_CIP按DIST_CIP_TS.size()计算，UNIQ_CIP_RECENT按最近24小时计算 - map.put("UNIQ_CIP", edges.getDistCipTs().size()); - map.put("UNIQ_CIP_RECENT", edges.getDistCipTs().stream().filter(t -> t >= (DateUtils.getSomeHour(new Date(), Integer.valueOf(uniqCipRecent)).getTime() / 1000)).count()); - - list.add(map); - } - - } - return list; - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/IpProviderImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/IpProviderImpl.java deleted file mode 100644 index 9fedc02..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/IpProviderImpl.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.mesalab.engine.component.strategy; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.mesalab.common.knowledge.KnowDslObject; -import com.mesalab.common.knowledge.Parameters; -import com.mesalab.common.knowledge.Sort; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.StringUtil; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.ObjectUtils; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -/** - * @description: for ip search - * @author: zhq - * @create: 2020-07-17 - **/ -@Component("IP_VIEW") -@SuppressWarnings("unchecked") -public class IpProviderImpl implements QueryProvider { - - @Value("${arango.maxrows}") - private String limit;//查询最大条数 - - @Autowired - BaseQueryProvider baseQueryProvider; - - @Override - public Object query(KnowDslObject.DSLQuery dslQeruy) { - Parameters parameters = dslQeruy.getParameters(); - StringBuffer aqlsb = new StringBuffer(); - String sortAql = ""; - //需要条件 时间，limit - if(!ObjectUtils.isEmpty(parameters)){ - limit = ObjectUtils.isEmpty(parameters.getLimit()) ? limit : parameters.getLimit(); - List<Sort> sorts = parameters.getSort(); - - if (Objects.nonNull(sorts)) { - sortAql = sorts.stream().map(x -> { - if ("BYTES_TOTAL".equals(x.getFieldKey())) { - return "(doc.CLIENT_BYTES_SUM+doc.SERVER_BYTES_SUM) " + x.getType(); - } else if ("LAST_FOUND_TIME".equals(x.getFieldKey())) { - return "doc.LAST_FOUND_TIME " + x.getType(); - } - return null; - }).filter(x -> StringUtil.isNotBlank(x)).collect(Collectors.joining(",", " ", " ")); - } - } - aqlsb.append("for doc in IP "); - aqlsb.append(" filter doc.COMMON_LINK_INFO != '' "); - aqlsb.append(StringUtil.isBlank(sortAql) ? "" : " sort " + sortAql); - aqlsb.append(" limit ").append(limit); - aqlsb.append(" return distinct doc"); - //1.查询 - Object result = baseQueryProvider.queryArango(aqlsb.toString()); - //2.解析结果 - List<Map> resultList = new ObjectMapper().convertValue(result, new TypeReference<List<Map>>() { - }); - return parseResult(resultList); - - } - - private Object parseResult(List<Map> resultList) { - for (Map ip : resultList) { - ip.remove("_key"); - ip.remove("_id"); - ip.remove("_rev"); - ip.put("FIRST_FOUND_TIME", DateUtils.convertTimestampToString(Long.valueOf(String.valueOf(ip.get("FIRST_FOUND_TIME"))), DateUtils.YYYY_MM_DD_HH24_MM_SS)); - ip.put("LAST_FOUND_TIME", DateUtils.convertTimestampToString(Long.valueOf(String.valueOf(ip.get("LAST_FOUND_TIME"))), DateUtils.YYYY_MM_DD_HH24_MM_SS)); - ip.put("BYTES_TOTAL", Long.valueOf(String.valueOf(ip.get("CLIENT_BYTES_SUM"))) + Long.valueOf(String.valueOf(ip.get("SERVER_BYTES_SUM")))); - } - return resultList; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/QueryProvider.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/QueryProvider.java deleted file mode 100644 index 84dad2e..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/QueryProvider.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.engine.component.strategy; - -import com.mesalab.common.knowledge.KnowDslObject; - -/** - * @Description: 提供查询结果 - */ -public interface QueryProvider { - - Object query(KnowDslObject.DSLQuery dslQeruy); - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/SubscriberIdProviderImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/SubscriberIdProviderImpl.java deleted file mode 100644 index 4fb8b51..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/strategy/SubscriberIdProviderImpl.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.mesalab.engine.component.strategy; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.CharMatcher; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.mesalab.common.knowledge.KnowDslObject; -import com.mesalab.common.knowledge.Match; -import com.mesalab.common.knowledge.Parameters; -import com.zdjizhi.utils.DateUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.springframework.util.ObjectUtils; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * @description: - * @author: zhq - * @create: 2020-07-17 - **/ -@SuppressWarnings("unchecked") -@Component("SUBSCRIBER_ID_VIEW") -public class SubscriberIdProviderImpl implements QueryProvider { - - @Value("${arango.maxrows}") - private String limit;//查询最大条数 - - @Autowired - BaseQueryProvider baseQueryProvider; - - @Override - public Object query(KnowDslObject.DSLQuery dslQeruy) { - - //1.构建查询语句 match like - StringBuffer likeFilter = new StringBuffer(); - Parameters parameters = dslQeruy.getParameters(); - if(!ObjectUtils.isEmpty(parameters)){ - likeFilter.append(Match.like(parameters.getMatch(), (m) -> m != null && "SUBSCRIBER_ID".equals(m.getFieldKey()) && !ObjectUtils.isEmpty(m.getFieldValues()))); - limit = ObjectUtils.isEmpty(parameters.getLimit()) ? limit : parameters.getLimit(); - } - - StringBuffer aqlsb = new StringBuffer("for doc in SUBSCRIBER "); - aqlsb.append(ObjectUtils.isEmpty(likeFilter) ? "" : likeFilter); - aqlsb.append(" let maxTime = (let times = (for v,e,p in 1 outbound doc._id graph 'SUBSCRIBER_IP_GRAPH' return e.LAST_FOUND_TIME) return max(times)) "); - aqlsb.append(" for v,e,p in 1..1 outbound doc._id graph 'SUBSCRIBER_IP_GRAPH' "); - aqlsb.append(" filter e.LAST_FOUND_TIME == maxTime[0] "); - aqlsb.append(" limit ").append(limit); - aqlsb.append(" return distinct p "); - - //1.查询 - Object result = baseQueryProvider.queryArango(aqlsb.toString()); - //2.解析结果 - List<Map> resultList = new ObjectMapper().convertValue(result, new TypeReference<List<Map>>() { - }); - return parseResult(resultList); - - } - - private Object parseResult(List<Map> resultList) { - List<Object> list = Lists.newArrayList(); - for (Map<String, List<Map>> map : resultList) { - Map<String, Object> newMap = Maps.newHashMap(); - Map subIdVertice = map.get("vertices").get(0); - if (Objects.isNull(subIdVertice)) { - continue; - } - newMap.put("SUBSCRIBER_ID", subIdVertice.get("_key")); - newMap.put("FIRST_FOUND_TIME", DateUtils.convertTimestampToString(Long.valueOf(String.valueOf(subIdVertice.get("FIRST_FOUND_TIME"))), DateUtils.YYYY_MM_DD_HH24_MM_SS)); - newMap.put("LAST_FOUND_TIME", DateUtils.convertTimestampToString(Long.valueOf(String.valueOf(subIdVertice.get("LAST_FOUND_TIME"))), DateUtils.YYYY_MM_DD_HH24_MM_SS)); - Map ipEdges = map.get("edges").get(0); - if (Objects.isNull(ipEdges)||Objects.isNull(ipEdges.get("_to"))) { - newMap.put("IP", ""); - } else { - newMap.put("IP", CharMatcher.anyOf("IP/").trimLeadingFrom(String.valueOf(ipEdges.get("_to")))); - } - list.add(newMap); - } - return list; - } -}
\ No newline at end of file diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/IP_TO_CITY.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/IP_TO_CITY.java deleted file mode 100644 index 81efda8..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/IP_TO_CITY.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.mesalab.engine.component.udf; - -import com.google.common.collect.Lists; -import com.mesalab.engine.component.IPConverter; -import com.mesalab.engine.component.bean.SqlQueryBean; -import com.zdjizhi.utils.StringUtil; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.Map; - -@Slf4j -@Data -@AllArgsConstructor -@NoArgsConstructor -@SuppressWarnings("unchecked") -public class IP_TO_CITY implements UDF { - - private String name; - private String address; - - - public IP_TO_CITY(UDFElements udfElements) { - this.name = udfElements.getName(); - this.address = udfElements.getParams().get(0).toString(); - } - - @Override - public Object execute(SqlQueryBean sqlQuerySource, List<Object> results) { - List<Map<String, Object>> targetResult = Lists.newArrayList(); - String addressLabel = StringUtil.EMPTY; - try { - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name)) { - addressLabel = alias; - break; - } - } - - for (Object source : results) { - Map<String, Object> sourceMap = ((Map<String, Object>) source); - if (sourceMap.containsKey(addressLabel)) { - sourceMap.put(addressLabel, IPConverter.getCityDetail(sourceMap.get(addressLabel).toString())); - } - targetResult.add(sourceMap); - } - - } catch (Exception e) { - log.error("execute IP_TO_CITY ERROR: " + e); - } - - return targetResult; - } - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/IP_TO_COUNTRY.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/IP_TO_COUNTRY.java deleted file mode 100644 index b725269..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/IP_TO_COUNTRY.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.mesalab.engine.component.udf; - -import com.google.common.collect.Lists; -import com.mesalab.engine.component.IPConverter; -import com.mesalab.engine.component.bean.SqlQueryBean; -import com.zdjizhi.utils.StringUtil; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.Map; - -@Slf4j -@Data -@AllArgsConstructor -@NoArgsConstructor -@SuppressWarnings("unchecked") -public class IP_TO_COUNTRY implements UDF { - - private String name; - private String address; - - - public IP_TO_COUNTRY(UDFElements udfElements) { - this.name = udfElements.getName(); - this.address = udfElements.getParams().get(0).toString(); - } - - @Override - public Object execute(SqlQueryBean sqlQuerySource, List<Object> results) { - List<Map<String, Object>> targetResult = Lists.newArrayList(); - String addressLabel = StringUtil.EMPTY; - try { - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name)) { - addressLabel = alias; - break; - } - } - for (Object source : results) { - Map<String, Object> sourceMap = ((Map<String, Object>) source); - if (sourceMap.containsKey(addressLabel)) { - sourceMap.put(addressLabel, IPConverter.getCountry(sourceMap.get(addressLabel).toString())); - } - targetResult.add(sourceMap); - } - - } catch (Exception e) { - log.error("execute IP_TO_COUNTRY ERROR: " + e); - } - - return targetResult; - } - - -} -
\ No newline at end of file diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/IP_TO_GEO.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/IP_TO_GEO.java deleted file mode 100644 index 66921b6..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/IP_TO_GEO.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.mesalab.engine.component.udf; - -import com.google.common.collect.Lists; -import com.mesalab.engine.component.IPConverter; -import com.mesalab.engine.component.bean.SqlQueryBean; -import com.zdjizhi.utils.StringUtil; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; -import java.util.Map; - -@Slf4j -@Data -@AllArgsConstructor -@NoArgsConstructor -@SuppressWarnings("unchecked") -public class IP_TO_GEO implements UDF { - - private String name; - private String address; - - - public IP_TO_GEO(UDFElements udfElements) { - this.name = udfElements.getName(); - this.address = udfElements.getParams().get(0).toString(); - } - - @Override - public Object execute(SqlQueryBean sqlQuerySource, List<Object> results) { - List<Map<String, Object>> targetResult = Lists.newArrayList(); - String addressLabel = StringUtil.EMPTY; - try { - - for (String alias : sqlQuerySource.getAliasFields().keySet()) { - if (sqlQuerySource.getAliasFields().get(alias).contains(address) && - sqlQuerySource.getAliasFields().get(alias).contains(name)) { - addressLabel = alias; - break; - } - } - - - for (Object source : results) { - Map<String, Object> sourceMap = ((Map<String, Object>) source); - if (sourceMap.containsKey(addressLabel)) { - sourceMap.put(addressLabel, IPConverter.getGeo(sourceMap.get(addressLabel).toString())); - } - targetResult.add(sourceMap); - } - - } catch (Exception e) { - log.error("execute IP_TO_GEO ERROR: " + e); - } - - return targetResult; - } - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/TIME_FLOOR_WITH_FILL.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/TIME_FLOOR_WITH_FILL.java deleted file mode 100644 index 7e2a839..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/TIME_FLOOR_WITH_FILL.java +++ /dev/null @@ -1,275 +0,0 @@ -package com.mesalab.engine.component.udf; - -import com.google.common.base.Joiner; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.ConvertUtil; -import com.mesalab.engine.component.bean.SqlQueryBean; -import com.mesalab.engine.component.SpringContext; -import com.mesalab.engine.component.SqlFunction; -import com.mesalab.engine.component.config.EngineConfig; -import com.mesalab.engine.service.MetadataService; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.StringUtil; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.LongValue; -import net.sf.jsqlparser.expression.Parenthesis; -import net.sf.jsqlparser.expression.StringValue; -import net.sf.jsqlparser.expression.operators.conditional.AndExpression; -import net.sf.jsqlparser.expression.operators.conditional.OrExpression; -import net.sf.jsqlparser.expression.operators.relational.Between; -import net.sf.jsqlparser.expression.operators.relational.ComparisonOperator; -import org.joda.time.DateTime; - -import java.util.*; - -@Slf4j -@Data -@AllArgsConstructor -@NoArgsConstructor -@SuppressWarnings("unchecked") -public class TIME_FLOOR_WITH_FILL implements UDF { - - private MetadataService metadataService = (MetadataService) SpringContext.getBean("metadataService"); - private EngineConfig engineConfig = (EngineConfig) SpringContext.getBean("engineConfig"); - - private String name; - private String timestamp; - private String period; - private String fill; - - public TIME_FLOOR_WITH_FILL(UDFElements udfElements) { - this.name = udfElements.getName(); - - this.timestamp = udfElements.getParams().get(0).toString(); - this.period = ((StringValue) udfElements.getParams().get(1)).getValue(); - if (udfElements.getParams().size() >= 3) { - this.fill = ((StringValue) udfElements.getParams().get(2)).getValue(); - } - } - - - @Override - public Object execute(SqlQueryBean sqlQuerySource, List<Object> results) { - List<Map<String, Object>> targetResult = null; - String groupLabel = null; - String timeGranLabel = null; - - try { - Map<String, String> groupDimension = sqlQuerySource.getGroupDimension(); - if (results.isEmpty() || StringUtil.isBlank(fill) || StringUtil.isEmpty(groupDimension)) { - return results; - } - for (String groupKey : groupDimension.keySet()) { - if (groupDimension.get(groupKey).contains(name) && - groupDimension.get(groupKey).contains(timestamp)) { - timeGranLabel = groupKey; - } else { - groupLabel = Joiner.on(ConvertUtil.GROUP_SEPRATOR).skipNulls().join(groupKey, groupLabel); - } - - } - - WhereTimeRange whereTimeRange = getWhereTimeRange(sqlQuerySource); - Date benchmarkDate = getBenchmarkDate((Map<String, Object>) results.get(0), timeGranLabel); - - - List<Date> dateRangeList = Lists.newArrayList(); - - if (StringUtil.isNotBlank(whereTimeRange.getStart())) { - - if (StringUtil.isBlank(whereTimeRange.getEnd())) { - whereTimeRange.setEnd(DateUtils.getCurrentDate(DateUtils.YYYY_MM_DD_HH24_MM_SS)); - } - - List<Date> leftDateRange = DateUtils.getLeftDateRange(benchmarkDate, - DateUtils.convertStringToDate((whereTimeRange.getStart()), DateUtils.YYYY_MM_DD_HH24_MM_SS), period); - List<Date> rightDateRange = DateUtils.getRightDateRange(benchmarkDate, - DateUtils.convertStringToDate((whereTimeRange.getEnd()), DateUtils.YYYY_MM_DD_HH24_MM_SS), period); - - Collections.reverse(leftDateRange); - dateRangeList = Lists.newArrayList(Iterables.concat(leftDateRange, rightDateRange)); - - } else { - log.warn("SQL中时间条件不完整[startTime:{} - endTime:{}],补全功能无效!", whereTimeRange.getStart(), whereTimeRange.getEnd()); - return results; - } - - if (dateRangeList.size() >= 2) { - dateRangeList = dateRangeList.subList(1, dateRangeList.size() - 1); - } - if (dateRangeList.size() == 0) { - return results; - } - - int maxSeriesNum = engineConfig.getMaxCacheNum() / dateRangeList.size(); - if (maxSeriesNum == 0) { - log.warn("初始时间范围超出系统设置，无法进行补全[getMaxCacheNum:{} - dateRangeNum:{}],补全功能无效!", engineConfig.getMaxCacheNum(), dateRangeList.size()); - return results; - } - - - Map<String, Map<String, Map<String, Object>>> targetTimeseriesMap = ConvertUtil.convertResultListToTimeseries(results, - groupLabel, timeGranLabel); - - int actualSeriesNum = maxSeriesNum > targetTimeseriesMap.size() ? targetTimeseriesMap.size() : maxSeriesNum; - - if (actualSeriesNum > 0) { - - Map<String, Map<String, Map<String, Object>>> limitTimeseriesMap = Maps.newLinkedHashMap(); - int i = 1; - for (Map.Entry<String, Map<String, Map<String, Object>>> entry : targetTimeseriesMap.entrySet()) { - limitTimeseriesMap.put(entry.getKey(), entry.getValue()); - if (++i > actualSeriesNum) { - break; - } - - } - targetTimeseriesMap.clear(); - targetTimeseriesMap.putAll(limitTimeseriesMap); - } - - if (StringUtil.isNotEmpty(dateRangeList)) { - - for (String key : targetTimeseriesMap.keySet()) { - targetTimeseriesMap.put(key, ConvertUtil.completeTimeseries((TreeMap<String, Map<String, Object>>) targetTimeseriesMap.get(key), dateRangeList, fill.toString())); - } - } - - targetResult = ConvertUtil.convertTimeseriesToList(targetTimeseriesMap, groupLabel, timeGranLabel); - - } catch (Exception e) { - log.error("Parser function : TIME_FLOOR_WITH_FILL error : " + e); - } - - return targetResult; - } - - - /** - * 获取基准时间 - * - * @param firstElement - * @param timeGranLabel - * @return - */ - private Date getBenchmarkDate(Map<String, Object> firstElement, String timeGranLabel) { - Date benchmarkDate = null; - try { - Object benchmarkDateObject = firstElement.get(timeGranLabel); - - if (StringUtil.isNumeric(benchmarkDateObject.toString())) { - benchmarkDate = new DateTime(Long.valueOf(benchmarkDateObject.toString()) * 1000).toDate(); - } else { - benchmarkDate = DateUtils.convertStringToDate(benchmarkDateObject.toString(), DateUtils.YYYY_MM_DD_HH24_MM_SS); - } - } catch (Exception ex) { - log.error("Get BenchmarkDate Error ", ex); - - } - - return benchmarkDate; - } - - /** - * 获取where查询时间范围 - * - * @param sqlQuerySource - * @return - */ - private WhereTimeRange getWhereTimeRange(SqlQueryBean sqlQuerySource) { - String partitionKey = sqlQuerySource.getPartitionKey(); - Expression whereExpression = sqlQuerySource.getWhereExpression(); - String dbType = metadataService.getDBTypeByTableName(sqlQuerySource.getTableName()); - WhereTimeRange whereTimeRange = new WhereTimeRange(); - try { - parserWhereExpressionForWhereTimeRange(whereTimeRange, whereExpression, partitionKey, dbType); - } catch (Exception e) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), - ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "解析where表达式，获取数据的时间范围异常", e); - } - return whereTimeRange; - } - - /** - * 解析where表达式并设置数据的时间范围 - * - * @param whereTimeRange - * @param whereExpression - * @param partitionKey - * @param dbType - */ - private void parserWhereExpressionForWhereTimeRange(WhereTimeRange whereTimeRange, Expression whereExpression, String partitionKey, String dbType) { - if (StringUtil.isEmpty(whereExpression) || StringUtil.isBlank(partitionKey)) { - return; - } - if (whereExpression instanceof ComparisonOperator) { - ComparisonOperator whereOperator = (ComparisonOperator) whereExpression; - String operator = whereOperator.getStringExpression(); - Expression left = whereOperator.getLeftExpression(); - Expression right = whereOperator.getRightExpression(); - if (partitionKey.equals(left.toString())) { - String query; - if (right instanceof StringValue) { - query = ((StringValue) right).getValue(); - } else if (right instanceof LongValue) { - query = right.toString(); - } else { - query = SqlFunction.getQueryValue(right.toString(), dbType); - } - if (operator.contains("<")) { - whereTimeRange.setEnd(query); - } else if (operator.contains(">")) { - whereTimeRange.setStart(query); - } - } else if (partitionKey.equals(right.toString())) { - String query; - if (left instanceof StringValue) { - query = ((StringValue) right).getValue(); - } else if (left instanceof LongValue) { - query = left.toString(); - } else { - query = SqlFunction.getQueryValue(left.toString(), dbType); - } - if (operator.contains("<")) { - whereTimeRange.setStart(query); - } else if (operator.contains(">")) { - whereTimeRange.setEnd(query); - } - } - } else if (whereExpression instanceof Between) { - Between whereBetween = (Between) whereExpression; - Expression leftExpression = whereBetween.getLeftExpression(); - if (partitionKey.equals(leftExpression.toString())) { - whereTimeRange.setStart(whereBetween.getBetweenExpressionStart().toString()); - whereTimeRange.setEnd(whereBetween.getBetweenExpressionEnd().toString()); - } - } else if (whereExpression instanceof AndExpression) { - AndExpression whereAnd = (AndExpression) whereExpression; - parserWhereExpressionForWhereTimeRange(whereTimeRange, whereAnd.getLeftExpression(), partitionKey, dbType); - parserWhereExpressionForWhereTimeRange(whereTimeRange, whereAnd.getRightExpression(), partitionKey, dbType); - } else if (whereExpression instanceof OrExpression) { - OrExpression whereOr = (OrExpression) whereExpression; - parserWhereExpressionForWhereTimeRange(whereTimeRange, whereOr.getLeftExpression(), partitionKey, dbType); - parserWhereExpressionForWhereTimeRange(whereTimeRange, whereOr.getRightExpression(), partitionKey, dbType); - } else if (whereExpression instanceof Parenthesis) { - Parenthesis whereExpressionParenthesis = (Parenthesis) whereExpression; - Expression expression = whereExpressionParenthesis.getExpression(); - parserWhereExpressionForWhereTimeRange(whereTimeRange, expression, partitionKey, dbType); - } - } - - @Data - private class WhereTimeRange { - private String start; - private String end; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/UDF.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/UDF.java deleted file mode 100644 index 0317c30..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/UDF.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.mesalab.engine.component.udf; - -import com.mesalab.engine.component.bean.SqlQueryBean; - -import java.util.List; - -public interface UDF { - - Object execute(SqlQueryBean sqlQuerySource, List<Object> results); -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/UDFElements.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/UDFElements.java deleted file mode 100644 index 3a9e8f1..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/component/udf/UDFElements.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.mesalab.engine.component.udf; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; -import net.sf.jsqlparser.expression.Expression; - -import java.util.List; - -@Data -@AllArgsConstructor -@NoArgsConstructor -public class UDFElements { - private String name; - private List<Expression> params; -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/DslQueryController.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/DslQueryController.java deleted file mode 100644 index 09b7333..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/DslQueryController.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.mesalab.engine.controller; - -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.common.annotation.SysLog; -import com.mesalab.engine.component.DSLRouter; -import com.mesalab.engine.component.DSLValidate; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * @Date: 2020-08-17 10:55 - * @Author : liuyongqiang - * @ClassName : DslQueryController - * @Description : 基于DSL语言的单表数据查询 - */ -@Slf4j -@RestController -@RequestMapping("/v1/dsl/") -public class DslQueryController { - - @Autowired - protected DSLRouter dslRouter; - - @Autowired - protected DSLValidate dslValidate; - - @PostMapping("query") - @SysLog - public BaseResult query(@RequestBody DSLObject dslObject){ - BaseResult baseResult; - try { - dslValidate.executeValidate(dslObject); - baseResult = dslRouter.router(dslObject); - } catch (BusinessException e) { - return BaseResultUtil.failure( - e.getErrorStatus(), - e.getErrorCode(), - e.getErrorMessage()); - } - return baseResult; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/GlobalExceptionHandler.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/GlobalExceptionHandler.java deleted file mode 100644 index 568ebc2..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/GlobalExceptionHandler.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.mesalab.engine.controller; - -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.RestControllerAdvice; - -import javax.servlet.http.HttpServletResponse; - -@Slf4j -@RestControllerAdvice -public class GlobalExceptionHandler { - - - @ExceptionHandler(Exception.class) - public BaseResult handleException(Exception e) { - return BaseResultUtil.failure(e.getMessage()); - } - - - @ExceptionHandler({BusinessException.class}) - public BaseResult handleBusinessException(BusinessException e, HttpServletResponse response) { - response.setStatus(e.getErrorStatus()); - log.error("Execute Query Error: {}", e.getMessage() + ";" + (e.getCause() != null ? e.getCause().getMessage() : "")); - return BaseResultUtil.failure(e.getErrorStatus(), e.getErrorCode(), - (e.getMessage() != null ? e.getMessage() : e.getErrorMessage()) + " " + (e.getCause() != null ? e.getCause().getMessage() : "")); - } - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/KnowledgeController.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/KnowledgeController.java deleted file mode 100644 index 4fc5130..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/KnowledgeController.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.mesalab.engine.controller; - -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.QueryTypeEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.engine.service.KnowledgeService; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.EnumUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -import javax.servlet.http.HttpServletRequest; - -@Slf4j -@RestController -@RequestMapping("/knowledge") -public class KnowledgeController { - - @Autowired - KnowledgeService knowledgeService; - - @GetMapping("/info") - public String info() { - return "You Konow, For Graph Search!"; - } - - /*** - * @Description - * @Date 2020\7\16 - * @Param [request, dSLObject] - * @return com.mesalab.common.base.BaseResult - **/ - @PostMapping("/v1/") - public BaseResult knowledgeQuery(HttpServletRequest request, @RequestBody Object object) { - if (EnumUtils.isValidEnum(QueryTypeEnum.class,StringUtil.upperCase(request.getQueryString()))) { - log.info("服务推荐接口, 参数: queryString is {},params is {}", request.getQueryString(), object); - return knowledgeService.query(object); - } else { - log.error("服务推荐接口, 错误:queryString is {},params is {}", request.getQueryString(), object); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Not Support", null); - } - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/MetadataController.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/MetadataController.java deleted file mode 100644 index 9489d75..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/MetadataController.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.mesalab.engine.controller; - -import cn.hutool.core.util.StrUtil; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.common.annotation.SysLog; -import com.mesalab.engine.service.MetadataService; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.HttpStatus; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * @Date: 2020-08-17 15:57 - * @Author : liuyongqiang - * @ClassName : MetadataController - * @Description : 元数据相关接口 - */ -@Slf4j -@RestController -@RequestMapping(value = "/metadata") -public class MetadataController { - - @Autowired - MetadataService metadataService; - - @SysLog - @GetMapping("/schema/v1/{type}/{name}") - public BaseResult getSchema(@PathVariable(value = "type") String type, @PathVariable(value = "name") String name) { - if(StrUtil.isBlank(type)||StrUtil.isBlank(name)){ - return BaseResultUtil.failure(ResultStatusEnum.NOT_FOUND.getCode(),ResultStatusEnum.NOT_FOUND.getMessage()); - } - BaseResult baseResult; - try { - baseResult = metadataService.getSchemaInfo(type, name); - } catch (BusinessException e) { - return BaseResultUtil.failure(HttpStatus.SC_NOT_FOUND, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), e.getMessage()); - } - return baseResult; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/NetworkController.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/NetworkController.java deleted file mode 100644 index c0cef48..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/NetworkController.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.mesalab.engine.controller; - -import com.mesalab.common.dto.params.NetworkParam; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.engine.service.NetworkService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import javax.servlet.http.HttpServletRequest; - -/** - * @author wangwei - * @version 1.0 - * @date 2020/6/30 10:38 上午 - */ -@Slf4j -@RestController -@RequestMapping("/") -public class NetworkController { - - @Autowired - private NetworkService networkService; - private static final String PROTOCOL = "protocol"; - - @PostMapping("/traffic/v1/") - public BaseResult trafficDistribution(HttpServletRequest request, @Validated @RequestBody NetworkParam param) { - if (PROTOCOL.equalsIgnoreCase(request.getQueryString())) { - log.info("流量分布接口, 参数: queryString is {},params is {}", request.getQueryString(), param); - return networkService.getTrafficDistributedInfo(param); - } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Support protocol only", null); - } - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/SqlQueryController.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/SqlQueryController.java deleted file mode 100644 index 6ff20d1..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/SqlQueryController.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.mesalab.engine.controller; - -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.common.annotation.SysLog; -import com.mesalab.engine.service.SqlEngineService; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * @Date: 2020-08-18 09:34 - * @Author : liuyongqiang - * @ClassName : APIController - * @Description : 基于SQL语言的动态查询 - */ -@Slf4j -@RestController -@RequestMapping(value = "/") -public class SqlQueryController { - - @Autowired - SqlEngineService sqlEngineService; - - //基于SQL的公共查询服务 - @SysLog - @RequestMapping("/") - public BaseResult queryByGet(ApiQueryBean apiParam) { - if (StringUtil.isBlank(apiParam.getQuery())) { - return BaseResultUtil.success4Message("ok"); - } - return sqlEngineService.executeQuery(apiParam); - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/SystemController.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/SystemController.java deleted file mode 100644 index b41975e..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/SystemController.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.mesalab.engine.controller; - -import com.mesalab.common.annotation.SysLog; -import com.mesalab.common.model.StorageDeletion; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.service.SystemService; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; -import java.util.List; -import java.util.Map; - -@Slf4j -@RestController -@RequestMapping(value = "/sys") -public class SystemController { - - @Autowired - private SystemService systemService; - - - @RequestMapping("/storage/quota") - public BaseResult storageQuota() { - return systemService.getStorageQuota(); - } - - @SysLog - @RequestMapping("storage/daily-trend") - public BaseResult dailyTrendOfStorage(@RequestParam Map<String, Object> param) { - String searchStartTime = StringUtil.stripToEmpty((String)param.get("start_time")); - String searchEndTime = StringUtil.stripToEmpty((String)param.get("end_time")); - return systemService.dailyTrendOfStorage(searchStartTime, searchEndTime); - } - - @SysLog - @PostMapping(value = "storage/deletion") - public BaseResult storageDeletion(@RequestBody List<StorageDeletion> list) { - - if (StringUtil.isEmpty(list)) { - return BaseResultUtil.failure(ResultStatusEnum.FAIL.getCode(), "未获取到参数"); - } - - for (StorageDeletion info : list) { - if (StringUtil.isBlank(info.getLogType()) || StringUtil.isBlank(String.valueOf(info.getMaxDays()))) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "参数校验: logType 与 maxDays 需同时填写", null); - } - if ("ALL".equalsIgnoreCase(info.getLogType()) && list.size() != 1) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "参数校验: logType存在ALL时, 不能与其它类型共存", null); - } - if (!(info.getMaxDays() >= 0 && info.getMaxDays() <= 2000)) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "参数校验: logType非All: maxDays 必须是数值且[0-2000]", null); - } - } - return systemService.deleteStorage(list); - } - - @SysLog - @GetMapping(value = "storage/deletion") - public BaseResult storageDeletionStatus(StorageDeletion storageDeletionInfo) { - return systemService.getDeleteStorageStatus(storageDeletionInfo.getLogType()); - } - - @SysLog - @DeleteMapping(value = "engine/tasks/{queryId}") - public BaseResult killQuery(@PathVariable String queryId) { - log.info("停止查询, 参数: queryId is {}", queryId); - if (StringUtil.isBlank(queryId)) { - return BaseResultUtil.success4Message("ok"); - } - return systemService.deleteQueryTask(queryId); - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/TestCaseController.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/TestCaseController.java deleted file mode 100644 index 88b5633..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/TestCaseController.java +++ /dev/null @@ -1,123 +0,0 @@ -package com.mesalab.engine.controller; - -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.component.configuration.ProjectProperties; -import com.mesalab.engine.service.TestCaseService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Arrays; - -/** - * 测试案例控制层 - * - * @author darnell - * @date 2019/5/13 - */ -@Slf4j -@RestController -@RequestMapping(value = "test") -public class TestCaseController { - - ProjectProperties projectProperties; - - Environment environment; - - @Autowired - TestCaseService testCaseService; - - @Autowired - public TestCaseController(ProjectProperties projectProperties, Environment environment) { - this.projectProperties = projectProperties; - this.environment = environment; - } - - /** - * 测试 - * 使用@GetMapping相当于使用@RequestMapping(method={RequestMethod.GET}) - * - * @return BaseResult - */ - @GetMapping(value = "projectProperties") - public BaseResult projectProperties() { - //此处本意是将projectProperties返回至前端，但是projectProperties对象是由spring注入而来，其中包含过多的动态代理数据， - //使用lombok的@Data注解处理BaseResult时，数据过大，返回报错，因此只返回success - return BaseResultUtil.success(); - } - - /** - * 获取项目环境值，获取的是Environment对象中的activeProfiles，String[] - * - * @return 返回当前项目的环境值 - */ - @GetMapping(value = "activeProfiles") - public BaseResult activeProfiles() { - String[] activeProfiles = environment.getActiveProfiles(); - log.info("Active Profiles: {}", Arrays.toString(activeProfiles)); - return BaseResultUtil.success(activeProfiles); - } - - /** - * 运行环境，将Environment.activeProfiles注入到projectProfiles中 - * - * @return 返回当前项目的运行环境 - */ - @GetMapping(value = "env") - public BaseResult env() { - String[] env = projectProperties.getEnv(); - log.info("Project env: {}", Arrays.toString(env)); - return BaseResultUtil.success(env); - } - - /** - * 是否是生产环境 - * - * @return 返回当前项目的运行环境 - */ - @GetMapping(value = "isProduct") - public BaseResult isProduct() { - boolean isProduct = projectProperties.isProduct(); - String msg = "Current Environment is" + (isProduct ? "" : " not") + " product"; - log.info(msg); - return BaseResultUtil.success(msg); - } - - /** - * 检查是否授权 - * - * @return 检查授权 - */ - @GetMapping(value = "checkAuthc") - public BaseResult checkAuthc() { - return BaseResultUtil.success(); - } - - - /** - * 运行测试用例 - * - * @return - */ - @GetMapping(value = "runSql") - public BaseResult runSql() { - BaseResult result = testCaseService.runSql(); - return result; - } - - /** - * 校验schema - * - * @return - */ - @GetMapping(value = "runSchema") - public BaseResult runSchema() { - BaseResult result = testCaseService.runSchema(); - return result; - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/ArangoEngineService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/ArangoEngineService.java deleted file mode 100644 index 4dc29f9..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/ArangoEngineService.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.engine.service; - -/** - * @Date: 2020-07-20 10:01 - * @Author : liuyongqiang - * @ClassName : ArangoEngineService - * @Description : ArangoDB数据查询引擎接口 - */ -public interface ArangoEngineService extends CommonEngineService { - - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/AuthorizeService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/AuthorizeService.java deleted file mode 100644 index d293b1b..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/AuthorizeService.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.mesalab.engine.service; - -import com.mesalab.common.exception.BusinessException; - -/** - * @Date: 2020-07-19 14:22 - * @Author : liuyongqiang - * @ClassName : AuthorizeService - * @Description : JWT统一鉴权服务 - */ - -public interface AuthorizeService { - - /** - * @Description: ArangoDBJwt登录 - * @Author: liuyongqiang - * @Date: 2020/7/20 10:18 - * @return: java.lang.String - **/ - String arangoJwtLogin() throws BusinessException; -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/ClickHouseEngineService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/ClickHouseEngineService.java deleted file mode 100644 index d82cca0..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/ClickHouseEngineService.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.mesalab.engine.service; - -/** - * @Date: 2020-07-20 10:00 - * @Author : liuyongqiang - * @ClassName : ClickHouseEngineService - * @Description : ClickHouse数据查询引擎接口 - */ -public interface ClickHouseEngineService extends CommonEngineService { -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/CommonEngineService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/CommonEngineService.java deleted file mode 100644 index 215ba4d..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/CommonEngineService.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.mesalab.engine.service; - -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.exception.BusinessException; - -/** - * @Date: 2020-07-20 18:39 - * @Author : liuyongqiang - * @ClassName : CommonEngineService - * @Description : 公共查询接口定义 - */ -public interface CommonEngineService { - - - /** - * @Description: 单表数据查询 - * @Author: liuyongqiang - * @Date: 2020/7/21 9:41 - * @param dslObject: DSL查询对象 - * @return: com.mesalab.common.base.BaseResult - **/ - BaseResult dslQuery(DSLObject dslObject)throws BusinessException; - - - /** - * @Description: 查询数据源下的所有表 - * @Author: liuyongqiang - * @Date: 2020/7/23 15:16 - * @param dslObject: - * @return: com.mesalab.common.base.BaseResult - **/ - BaseResult tablesQuery(DSLObject dslObject)throws BusinessException; - - - /** - * @Description: 查询指定表中的所有字段 - * @Author: liuyongqiang - * @Date: 2020/7/23 11:08 - * @param dslObject: - * @return: com.mesalab.common.base.BaseResult - **/ - BaseResult schemaQuery(DSLObject dslObject)throws BusinessException; - - -} - diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/DruidEngineService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/DruidEngineService.java deleted file mode 100644 index d4ce603..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/DruidEngineService.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.mesalab.engine.service; - -/** - * @Date: 2020-07-20 10:02 - * @Author : liuyongqiang - * @ClassName : DruidEngineService - * @Description : Druid数据查询引擎接口 - */ -public interface DruidEngineService extends CommonEngineService { - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/EngineExecuteService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/EngineExecuteService.java deleted file mode 100644 index a6e899d..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/EngineExecuteService.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.mesalab.engine.service; - -import com.mesalab.common.exception.BusinessException; - -/** - * @Date: 2020-07-22 16:11 - * @Author : liuyongqiang - * @ClassName : EngineExecuteService - * @Description : 引擎执行查询接口 - */ -public interface EngineExecuteService { - - /** - * @Description: 执行Druid查询语句 - * @Author: liuyongqiang - * @Date: 2020/7/22 16:16 - * @param sql: - * @return: java.lang.String - **/ - String executeDruidSql (String sql) throws BusinessException; - - /** - * @Description: 执行ClickHouse查询语句 - * @Author: liuyongqiang - * @Date: 2020/7/22 16:16 - * @param sql: - * @return: java.lang.String - **/ - String executeClickHouseSql (String sql) throws BusinessException; - - /** - * @Description: 执行ArangoDB查询语句 - * @Author: liuyongqiang - * @Date: 2020/7/22 16:17 - * @param aql: - * @return: java.lang.String - **/ - String executeArangoDBSql (String aql) throws BusinessException; - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/HttpClientManager.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/HttpClientManager.java deleted file mode 100644 index 9cedf97..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/HttpClientManager.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.mesalab.engine.service; - -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.conn.ConnectionKeepAliveStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.FactoryBean; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Service("httpClientManager") -public class HttpClientManager implements FactoryBean<CloseableHttpClient>, InitializingBean, DisposableBean { - - private CloseableHttpClient client; - - @Autowired - private ConnectionKeepAliveStrategy connectionKeepAliveStrategy; - - @Autowired - private HttpRequestRetryHandler httpRequestRetryHandler; - - @Autowired - private PoolingHttpClientConnectionManager poolHttpcConnManager; - - @Autowired - private RequestConfig config; - - - @Override - public void destroy() throws Exception { - if(null != this.client){ - this.client.close(); - } - } - - @Override - public CloseableHttpClient getObject() throws Exception { - - return this.client; - } - - @Override - public Class<?> getObjectType() { - return (this.client == null ? CloseableHttpClient.class : this.client.getClass()); - } - - // 构建的实例为单例 - @Override - public boolean isSingleton() { - return true; - } - - @Override - public void afterPropertiesSet() throws Exception { - this.client = HttpClients.custom().setConnectionManager(poolHttpcConnManager) - .setRetryHandler(httpRequestRetryHandler) - .setKeepAliveStrategy(connectionKeepAliveStrategy) - .setDefaultRequestConfig(config) - .build(); - } - -} - - diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/HttpClientService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/HttpClientService.java deleted file mode 100644 index 73568f6..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/HttpClientService.java +++ /dev/null @@ -1,198 +0,0 @@ -package com.mesalab.engine.service; - -import com.google.common.collect.Maps; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.HttpStatus; -import org.apache.http.ParseException; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -@Slf4j -@Service -public class HttpClientService { - - @Resource(name = "httpClientManager") - private CloseableHttpClient httpClient; - - - public Map<String, String> httpGet(String url) { - - Map<String, String> resultMap = Maps.newHashMap(); - // 创建GET请求对象 - CloseableHttpResponse response = null; - try { - HttpGet httpGet = new HttpGet(url); - // 执行请求 - response = httpClient.execute(httpGet); - // 获取响应实体 - HttpEntity entity = response.getEntity(); - // 获取响应信息 - resultMap.put("status", String.valueOf(response.getStatusLine().getStatusCode())); - resultMap.put("result", EntityUtils.toString(entity, "UTF-8")); - } catch (ClientProtocolException e) { - log.error("协议错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_SERVICE_UNAVAILABLE)); - resultMap.put("message", e.getMessage()); - } catch (ParseException e) { - log.error("解析错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_SERVICE_UNAVAILABLE)); - resultMap.put("message", e.getMessage()); - } catch (IOException e) { - log.error("IO错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_BAD_GATEWAY)); - resultMap.put("message", e.getMessage()); - } catch (Exception e) { - log.error("其它错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_INTERNAL_SERVER_ERROR)); - resultMap.put("message", e.getMessage()); - } finally { - if (null != response) { - try { - EntityUtils.consumeQuietly(response.getEntity()); - response.close(); - } catch (IOException e) { - log.error("释放链接错误: {}", e.getMessage()); - } - } - } - return resultMap; - } - - - public Map<String, String> httpGet(String url, Map<String, String> headers) { - - Map<String, String> resultMap = Maps.newHashMap(); - // 创建GET请求对象 - CloseableHttpResponse response = null; - try { - HttpGet httpGet = new HttpGet(url); - for (String key : headers.keySet()) { - httpGet.setHeader(key, headers.get(key)); - } - // 执行请求 - response = httpClient.execute(httpGet); - // 获取响应实体 - HttpEntity entity = response.getEntity(); - // 获取响应信息 - resultMap.put("status", String.valueOf(response.getStatusLine().getStatusCode())); - resultMap.put("result", EntityUtils.toString(entity, "UTF-8")); - } catch (ClientProtocolException e) { - log.error("协议错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_SERVICE_UNAVAILABLE)); - resultMap.put("message", e.getMessage()); - } catch (ParseException e) { - log.error("解析错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_SERVICE_UNAVAILABLE)); - resultMap.put("message", e.getMessage()); - } catch (IOException e) { - log.error("IO错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_BAD_GATEWAY)); - resultMap.put("message", e.getMessage()); - } catch (Exception e) { - log.error("其它错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_INTERNAL_SERVER_ERROR)); - resultMap.put("message", e.getMessage()); - } finally { - if (null != response) { - try { - EntityUtils.consumeQuietly(response.getEntity()); - response.close(); - } catch (IOException e) { - log.error("释放链接错误: {}", e.getMessage()); - } - } - } - return resultMap; - } - - - public Map<String, String> httpPost(String url, String jsonString) { - - Map<String, String> resultMap = Maps.newHashMap(); - // 创建GET请求对象 - CloseableHttpResponse response = null; - try { - HttpPost httpPost = new HttpPost(url); - httpPost.setHeader("Content-Type", "application/json"); - httpPost.setEntity(new ByteArrayEntity(jsonString.getBytes("utf-8"))); - response = httpClient.execute(httpPost); - // 获取响应实体 - HttpEntity entity = response.getEntity(); - // 获取响应信息 - resultMap.put("status", String.valueOf(response.getStatusLine().getStatusCode())); - resultMap.put("result", EntityUtils.toString(entity, "UTF-8")); - } catch (ClientProtocolException e) { - log.error("协议错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_SERVICE_UNAVAILABLE)); - resultMap.put("message", e.getMessage()); - } catch (ParseException e) { - log.error("解析错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_SERVICE_UNAVAILABLE)); - resultMap.put("message", e.getMessage()); - } catch (IOException e) { - log.error("IO错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_BAD_GATEWAY)); - resultMap.put("message", e.getMessage()); - } catch (Exception e) { - log.error("其它错误: {}", e.getMessage()); - resultMap.put("status", String.valueOf(HttpStatus.SC_INTERNAL_SERVER_ERROR)); - resultMap.put("message", e.getMessage()); - } finally { - if (null != response) { - try { - EntityUtils.consumeQuietly(response.getEntity()); - response.close(); - } catch (IOException e) { - log.error("释放链接错误: {}", e.getMessage()); - } - } - } - return resultMap; - } - - public Map<String, String> getHttpPostResponseHeads(String url, Map<String, String> headers) { - - CloseableHttpResponse response = null; - HashMap<String, String> map = Maps.newHashMap(); - try { - HttpPost httpPost = new HttpPost(url); - for (Object k : headers.keySet()) { - httpPost.setHeader(k.toString(), headers.get(k).toString()); - } - response = httpClient.execute(httpPost); - Header[] Headers = response.getAllHeaders(); - for (Header h : Headers) { - map.put(h.getName().toUpperCase(), h.getValue()); - } - } catch (ClientProtocolException e) { - log.error("协议错误: {}", e.getMessage()); - } catch (ParseException e) { - log.error("解析错误: {}", e.getMessage()); - } catch (IOException e) { - log.error("IO错误: {}", e.getMessage()); - } finally { - if (null != response) { - try { - EntityUtils.consumeQuietly(response.getEntity()); - response.close(); - } catch (IOException e) { - log.error("释放链接错误: {}", e.getMessage()); - } - } - } - return map; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/KnowledgeService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/KnowledgeService.java deleted file mode 100644 index 2b61894..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/KnowledgeService.java +++ /dev/null @@ -1,162 +0,0 @@ -package com.mesalab.engine.service; - - -import com.fasterxml.jackson.databind.JsonNode; -import com.github.fge.jackson.JsonLoader; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.core.report.ProcessingMessage; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchema; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import com.google.common.base.Splitter; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.knowledge.DataSourceEnum; -import com.mesalab.common.enums.knowledge.MatchEnum; -import com.mesalab.common.enums.knowledge.RangeEnum; -import com.mesalab.common.knowledge.*; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.component.strategy.QueryProvider; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.Validate; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.util.ObjectUtils; - -import java.io.IOException; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static java.util.Objects.*; - -/** - * @description: for IP learning - * @author: zhq - * @create: 2020-07-06 - **/ -@Slf4j -@Service -public class KnowledgeService { - - @Value("${arango.maxrows}") - private String maxrows;//查询最大条数 - - @Autowired - Map<String, QueryProvider> provider = new ConcurrentHashMap<>(); - - public BaseResult query(Object object) { - //1.结构校验 - validateJson(JsonMapper.toJsonString(object)); - KnowDslObject dSLObject = (KnowDslObject) JsonMapper.fromJsonString(JsonMapper.toJsonString(object), KnowDslObject.class); - validate(dSLObject); - KnowDslObject.DSLQuery dslQeruy = dSLObject.getQuery(); - //2.查询 - Object result = provider.get(dslQeruy.getDataSource()).query(dslQeruy); - return BaseResultUtil.success(result); - - } - - /** - * @return void - * @Author zhq - * @Description - * @Date 2020\7\21 0021 - * @Param [dslQuery] - **/ - private void validate(KnowDslObject dSLObject) { - KnowDslObject.DSLQuery dslQuery = dSLObject.getQuery(); - requireNonNull(dslQuery, "query is null"); - Parameters parameters = dslQuery.getParameters(); - - requireNonNull(dslQuery.getDataSource(), "dataSource is null"); - requireNonNull(EnumUtils.getEnum(DataSourceEnum.class, StringUtil.upperCase(dslQuery.getDataSource())), "dataSource no match"); - - if (isNull(parameters)) { - return; - } - - List<Match> matchs = parameters.getMatch(); - List<Range> ranges = parameters.getRange(); - List<String> intervals = parameters.getIntervals(); - List<Sort> sorts = parameters.getSort(); - String limit = parameters.getLimit(); - if (!ObjectUtils.isEmpty(matchs)) { - matchs.forEach(m -> { - Validate.isTrue(EnumUtils.isValidEnum(MatchEnum.class, StringUtil.upperCase(m.getType())), "match type is illegal"); - //正则匹配方式 禁止 $xxx* 形式的value - if (MatchEnum.REGEX.getType().equals(m.getType())) { - m.getFieldValues().forEach(mv -> { - Validate.isTrue(!(String.valueOf(mv).startsWith("$") && String.valueOf(mv).endsWith("*")), "error in matching sign. begin with * and do not end with * for suffix matching, end with * and do not begin with $ or * for prefix matching, begin with $ and do not end with * for exactly matching, begin with * and end with * or no wildcard for substring matching"); - }); - } - }); - } - if (!ObjectUtils.isEmpty(ranges)) { - ranges.forEach(r -> { - Validate.isTrue(EnumUtils.isValidEnum(RangeEnum.class, StringUtil.upperCase(r.getType())), "range type is illegal"); - }); - } - if (!ObjectUtils.isEmpty(sorts)) { - sorts.forEach(s -> { - Validate.isTrue("asc".equalsIgnoreCase(s.getType()) || "desc".equalsIgnoreCase(s.getType()), "sort type is illegal"); - }); - } - if (!ObjectUtils.isEmpty(intervals)) { - List<String> times = Splitter.on("/").splitToList(intervals.get(0)); - Validate.notEmpty(times, "intervals should be [startTime/endTime],such as 2020-01-01 00:00:00/2020-02-02 00:00:00"); - Validate.isTrue(times.size() == 2, "intervals should be [startTime/endTime],such as 2020-01-01 00:00:00/2020-02-02 00:00:00"); - Validate.notBlank(times.get(0), "start time is null"); - Validate.notBlank(times.get(1), "end time is null"); - Date start = DateUtils.convertStringToDate(times.get(0), DateUtils.YYYY_MM_DD_HH24_MM_SS); - Date end = DateUtils.convertStringToDate(times.get(1), DateUtils.YYYY_MM_DD_HH24_MM_SS); - Validate.isTrue(start.before(end), "the end time must be after the start time"); - } - if (nonNull(limit)) { - List<String> limits = Splitter.on(",").trimResults().omitEmptyStrings().splitToList(limit); - Validate.notEmpty(limits, "limit is illegal"); - Validate.isTrue(!(limit.contains(",") && limits.size() != 2), "limit is illegal"); - for (String l : limits) { - Validate.isTrue(l.length() <= maxrows.length() && Integer.valueOf(maxrows) >= Integer.valueOf(l) && Integer.valueOf(l) >= 0, "limit is greater than 0 and less than " + maxrows); - } - } - } - - private void validateJson(String jsonString) { - JsonSchemaFactory factory = JsonSchemaFactory.byDefault(); - //通过jsonschemaFactory获取jsonnode对象 - try { - JsonNode schemaNode = JsonLoader.fromResource("/validate-dsl.json"); - //通过jsonstr字符串获取对应的jsonnode对象 - JsonNode dataNode = JsonLoader.fromString(jsonString); - JsonSchema jsonSchema = factory.getJsonSchema(schemaNode); - //使用json-schema-validator中的jsonschema对象的validate方法对数据进行校验 - //获取处理的报告信息 - ProcessingReport processingReport = jsonSchema.validate(dataNode); - //获取完整的报告信息 - Iterator<ProcessingMessage> iterator = processingReport.iterator(); - StringBuffer sb = new StringBuffer(); - while (iterator.hasNext()) { - ProcessingMessage next = iterator.next(); - JsonNode jsonNode = next.asJson(); - sb.append("pointer on "); - sb.append(jsonNode.get("instance").get("pointer")); - sb.append(", "); - sb.append(next.getMessage()); - sb.append(". "); - } - //判断校验是否成功，如果为true成功 - Validate.isTrue(processingReport.isSuccess(), sb.toString()); - } catch (ProcessingException e) { - log.error(e.getMessage()); - } catch (IOException e) { - log.error(e.getMessage()); - } - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/MetadataService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/MetadataService.java deleted file mode 100644 index 517ded9..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/MetadataService.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.mesalab.engine.service; - -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.exception.BusinessException; -import org.apache.avro.Schema; - -import java.util.Map; - -/** - * @Date: 2020-08-18 14:14 - * @Author : liuyongqiang - * @ClassName : MetadataService - * @Description : 元数据管理服务 - */ -public interface MetadataService { - /** - * 查询schema信息 - * - * @param type - * @param name - * @return - */ - BaseResult getSchemaInfo(String type, String name) throws BusinessException; - - /** - * 根据表名获取partitionKey - * - * @param tableName - * @return - */ - String getPartitionKey(String tableName); - - /** - * Schema Doc中: 通过key获取value - * - * @param tableName - * @param key - * @return - */ - String getValueByKeyInSchemaDoc(String tableName, String key); - - /** - * 通过数据源，获取所属数据库类型 - * - * @param tableName - * @return - */ - String getDBTypeByTableName(String tableName); - - - /** - * 通过数据源，获取所对应数据库名称 - * - * @param tableName - * @return - */ - String getDBNameByTableName(String tableName); - - - /** - * 获取所有已注册schema - * - * @return - */ - Map<String, Schema> getAllSchema(); -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/NetworkService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/NetworkService.java deleted file mode 100644 index eacff41..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/NetworkService.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.mesalab.engine.service; - -import com.mesalab.common.dto.params.NetworkParam; -import com.mesalab.common.dto.results.BaseResult; - -/** - * @author wangwei - * @version 1.0 - * @date 2020/6/30 4:20 下午 - */ -public interface NetworkService { - - BaseResult getTrafficDistributedInfo(NetworkParam param); -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/SqlEngineService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/SqlEngineService.java deleted file mode 100644 index f455cac..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/SqlEngineService.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.mesalab.engine.service; - -import com.mesalab.engine.component.bean.SqlQueryBean; -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.common.dto.results.BaseResult; - -/** - * @Date: 2020-08-18 09:54 - * @Author : liuyongqiang - * @ClassName : SqlEngineServiceImpl - * @Description : 基于SQL的动态查询服务 - */ -public interface SqlEngineService { - - /** - * @Description: 执行查询 - * @Author: liuyongqiang - * @Date: 2020/8/18 9:54 - * @param apiParam: - * @return: com.mesalab.common.dto.results.BaseResult - **/ - BaseResult executeQuery(ApiQueryBean apiParam); - - /** - * @Description: 递归解析SQLQuerySource - * @param param: - * @param sqlQuerySource: - * @return: void - **/ - void convertQueryRecursive(ApiQueryBean param, SqlQueryBean sqlQuerySource); - - /** - * @Description: 解析SQL - * @param sql: - * @return: com.mesalab.common.beans.SqlQueryBean - **/ - SqlQueryBean parserSQLByAst(String sql); -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/SystemService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/SystemService.java deleted file mode 100644 index 570ad68..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/SystemService.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.mesalab.engine.service; - -import com.mesalab.common.model.StorageDeletion; -import com.mesalab.common.dto.results.BaseResult; - -import java.util.List; - -public interface SystemService { - - /** - * 系统存储配额 - * - * @return - */ - BaseResult getStorageQuota(); - - /** - * 获取每天日志存储变化 - * - * @param searchStartTime - * @param searchEndTime - * @return - */ - BaseResult dailyTrendOfStorage(String searchStartTime, String searchEndTime); - - /** - * 数据配额设置 - * - * @param list - * @return - */ - BaseResult deleteStorage(List<StorageDeletion> list); - - /** - * 数据配额设置状态 - * - * @param logType - * @return - */ - BaseResult getDeleteStorageStatus(String logType); - - /** - * 终止查询任务 - * - * @param queryId - * @return - */ - BaseResult deleteQueryTask(String queryId); -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/TestCaseService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/TestCaseService.java deleted file mode 100644 index 81a5cba..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/TestCaseService.java +++ /dev/null @@ -1,617 +0,0 @@ -package com.mesalab.engine.service; - -import com.google.common.collect.Maps; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.component.SchemaCache; -import com.mesalab.engine.component.config.ClickHouseConfig; -import com.mesalab.engine.component.config.DruidConfig; -import com.mesalab.engine.service.HttpClientService; -import com.mesalab.engine.service.impl.SqlEngineServiceImpl; -import com.zdjizhi.utils.Encodes; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.avro.JsonProperties; -import org.apache.avro.Schema; -import org.apache.avro.data.Json; -import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Service; - -import java.io.File; -import java.net.URLDecoder; -import java.net.URLEncoder; -import java.nio.charset.Charset; -import java.text.SimpleDateFormat; -import java.util.*; - -@Slf4j -@Service -@SuppressWarnings("unchecked") -public class TestCaseService { - - @Autowired - HttpClientService httpClientService; - @Autowired - ClickHouseConfig clickHouseConfig; - @Autowired - SqlEngineServiceImpl sqlEngineService; - @Autowired - DruidConfig druidConfig; - @Autowired - SchemaCache schemaCache; - - List errorDate; - - /** - * 运行测试用例 - * - * @return - */ - public BaseResult runSql() { - long start = System.currentTimeMillis(); - List list = ckSql(); - List listDruid = druidSql(); - list.addAll(listDruid); - Map<String, String> resultTemp; - BaseResult result = null; - List<Object> data = new ArrayList<>(); - int countSuccess = 0; - SimpleDateFormat format1 = new SimpleDateFormat("yyyy-MM-dd HH:00:00"); - SimpleDateFormat format2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Date date = new Date(System.currentTimeMillis()); - for (int i = 0; i < list.size(); i++) { - String sql = String.valueOf(list.get(i)); - try { - sql = sql.replaceAll("\\$start_time", "'" + format1.format(date) + "'"); - sql = sql.replaceAll("\\$end_time", "'" + format2.format(date) + "'"); - sql = URLEncoder.encode(sql, "utf-8").replaceAll("\\+", "%20"); - } catch (Exception e) { - log.error("测试用例sql编码错误", e); - } - resultTemp = httpClientService.httpGet("http://127.0.0.1:9999/?query=" + sql); - if (resultTemp.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - resultTemp.remove("result"); - countSuccess++; - } else { - try { - resultTemp.put("query", URLDecoder.decode(sql, "UTF-8")); - } catch (Exception e) { - log.error("sql查询结果处理异常", e); - } - } - data.add(resultTemp); - } - - Collections.sort(data, (o1, o2) -> { - Map<String, Object> oneMap = (Map<String, Object>) o1; - Map<String, Object> twoMap = (Map<String, Object>) o2; - return Integer.valueOf(twoMap.get("status").toString()).compareTo(Integer.valueOf(oneMap.get("status").toString())); - }); - long cost = System.currentTimeMillis() - start; - HashMap<String, Object> statistics = new HashMap<>(); - statistics.put("elapsed", String.format("%.2f", (double) cost / 1000)); - statistics.put("total", list.size()); - statistics.put("success", countSuccess); - String message = countSuccess == list.size() ? "ok" : "error"; - result = BaseResultUtil.success(message, data, statistics); - return result; - } - - /** - * 运行schema校验 - * - * @return - */ - public BaseResult runSchema() { - schemaCache.removeAll(); - long start = System.currentTimeMillis(); - errorDate = new ArrayList(); - checkCKSchema("/schema/clickhouse"); - checkDruidSchema("/schema/druid"); - checkSchema("/schema/clickhouse"); - checkSchema("/schema/druid"); - long cost = System.currentTimeMillis() - start; - HashMap<String, Object> statistics = new HashMap<>(); - statistics.put("elapsed", String.format("%.2f", (double) cost / 1000)); - String message = errorDate.size() == 0 ? "ok" : "error"; - return BaseResultUtil.success(message, errorDate, statistics); - } - - - public List ckSql() { - List<String> list = new ArrayList<>(); - list.add("select toDateTime(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, ssl_sni, ssl_version from security_event_log where common_recv_time >= toDateTime($start_time) and common_recv_time< toDateTime($end_time) and common_client_ip like '49.7%' order by common_recv_time desc limit 0,20"); - list.add("select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, ssl_sni, ssl_version from security_event_log where common_recv_time >= UNIX_TIMESTAMP($start_time) and common_recv_time< UNIX_TIMESTAMP($end_time) and common_client_ip like '49.7%' order by common_recv_time desc limit 0,20"); - list.add("select toDateTime(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, http_host,http_url,http_user_agent from proxy_event_log where common_recv_time >= toDateTime($start_time) and common_recv_time< toDateTime($end_time) and http_host like '%joy.cn%' order by common_recv_time desc limit 0,20"); - list.add("select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, http_host,http_url,http_user_agent from proxy_event_log where common_recv_time >= UNIX_TIMESTAMP($start_time) and common_recv_time< UNIX_TIMESTAMP($end_time) and http_host like '%joy.cn%' order by common_recv_time desc limit 0,20"); - list.add("select toDateTime(common_recv_time) as common_recv_time, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, radius_framed_ip, radius_account from radius_record_log where common_recv_time >= toDateTime($start_time) and common_recv_time< toDateTime($start_time) and radius_account='T1yRd' order by common_recv_time desc limit 0,20"); - list.add("select FROM_UNIXTIME(common_recv_time) as common_recv_time,common_subscriber_id, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, http_host,http_url,http_user_agent from proxy_event_log where common_recv_time >= UNIX_TIMESTAMP($start_time) and common_recv_time< UNIX_TIMESTAMP($end_time) and http_host like '%joy.cn%' order by common_recv_time desc limit 0,20"); - list.add("select toDateTime(common_recv_time) as common_recv_time, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, radius_framed_ip, radius_account from radius_record_log where common_recv_time >= toDateTime($start_time) and common_recv_time< toDateTime($start_time) and radius_account='T1yRd' order by common_recv_time desc limit 0,20"); - list.add("select FROM_UNIXTIME(common_recv_time) as common_recv_time, common_address_type, common_l4_protocol, common_client_ip, common_server_ip, common_server_port, radius_framed_ip, radius_account from radius_record_log where common_recv_time >= UNIX_TIMESTAMP($start_time) and common_recv_time< UNIX_TIMESTAMP($end_time) and radius_account='T1yRd' order by common_recv_time desc limit 0,20"); - - list.add("select count(1) from connection_record_log"); - list.add("select count(*) from connection_record_log"); - list.add("select * from " + clickHouseConfig.getDbName() + ".connection_record_log limit 1"); - - //Radius 用户分析 - //1. radius账户申请客户端IP变化 - list.add("select framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries \n" + - "from radius_onff_log where event_timestamp >=toDateTime($start_time) and event_timestamp <toDateTime($end_time) and account='admin' group by framed_ip"); - list.add("select framed_ip, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries \n" + - "from radius_onff_log where event_timestamp >=$start_time and event_timestamp < $end_time and account='admin' group by framed_ip"); - //2. 用户IP承载用户变化 - list.add("select account, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries \n" + - "from radius_onff_log where event_timestamp >= $start_time and event_timestamp < $end_time and framed_ip='127.0.0.1' group by account"); - list.add("select account, arraySlice(groupUniqArray(concat(toString(event_timestamp),':', if(acct_status_type=1,'start','stop'))),1,100000) as timeseries \n" + - "from radius_onff_log where event_timestamp >= $start_time and event_timestamp < $end_time and framed_ip='127.0.0.1' group by account"); - //自定义报告API - //一. 预置Internal Hosts 报告 - list.add("select common_client_ip, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime($start_time))-604800 and common_recv_time< toStartOfDay(toDateTime($end_time)) group by common_client_ip order by sessions desc limit 0,100"); - //二. 预置External Hosts 报告 - list.add("select common_server_ip, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime($start_time))-604800 and common_recv_time< toStartOfDay(toDateTime($start_time)) group by common_server_ip order by sessions desc limit 0,100"); - //三. 预置Domains报告 - list.add("select http_domain AS domain,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time >= toStartOfDay(toDateTime($start_time))-86400 AND common_recv_time < toStartOfDay(toDateTime($start_time)) and notEmpty(domain) GROUP BY domain ORDER BY bytes DESC LIMIT 100"); - list.add("select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_domain, uniq (common_client_ip) as nums from connection_record_log where common_recv_time >= toStartOfDay(toDateTime($start_time))-86400 AND common_recv_time < toStartOfDay(toDateTime($start_time)) and http_domain in (select http_domain from connection_record_log where common_recv_time >= toStartOfDay(toDateTime($start_time))-86400 AND common_recv_time < toStartOfDay(toDateTime($start_time)) and notEmpty(http_domain) group by http_domain order by SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) desc limit 10 ) group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_domain order by stat_time asc limit 500\n"); - list.add("SELECT http_host as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time>= toStartOfDay(toDateTime($start_time))-604800 and common_recv_time< toStartOfDay(toDateTime($start_time)) and notEmpty(http_host) GROUP BY host ORDER BY bytes DESC " + - "union all " + - "SELECT 'totals' as host, SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes, SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes, SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes from connection_record_log where common_recv_time>= toStartOfDay(toDateTime($start_time))-604800 and common_recv_time< toStartOfDay(toDateTime($start_time)) and notEmpty(http_host)"); - //四. 预置HTTP/HTTPS URLS报告 - list.add("SELECT http_url AS url,count(*) AS sessions FROM proxy_event_log WHERE common_recv_time >= toStartOfDay(toDateTime($start_time))-86400 AND common_recv_time < toStartOfDay(toDateTime($start_time)) and notEmpty(http_url) GROUP BY url ORDER BY sessions DESC LIMIT 100"); - list.add("select toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300) as stat_time, http_url, count(distinct(common_client_ip)) as nums from proxy_event_log where common_recv_time >= toStartOfDay(toDateTime($start_time))-86400 AND common_recv_time < toStartOfDay(toDateTime($start_time)) and http_url IN (select http_url from proxy_event_log where common_recv_time >= toStartOfDay(toDateTime($start_time))-86400 AND common_recv_time < toStartOfDay(toDateTime($start_time)) and notEmpty(http_url) group by http_url order by count(*) desc limit 10 )group by toDateTime(intDiv(toUInt32(toDateTime(toDateTime(common_recv_time))), 300)*300), http_url order by stat_time asc limit 500"); - list.add("select common_subscriber_id as user, count(*) as sessions from connection_record_log where common_recv_time>= toStartOfDay(toDateTime($start_time))-604800 and common_recv_time< toStartOfDay(toDateTime($start_time)) and notEmpty(user) group by common_subscriber_id order by sessions desc limit 0,100"); - list.add("SELECT common_subscriber_id as user,SUM(coalesce(common_c2s_byte_num, 0)) AS sent_bytes,SUM(coalesce(common_s2c_byte_num, 0)) AS received_bytes,SUM(coalesce(common_c2s_byte_num, 0)+coalesce(common_s2c_byte_num, 0)) AS bytes FROM connection_record_log WHERE common_recv_time>= toStartOfDay(toDateTime($start_time))-604800 and common_recv_time< toStartOfDay(toDateTime($start_time)) and notEmpty(user) GROUP BY user ORDER BY bytes DESC LIMIT 100"); - - //RADIUS账户总计 - list.add("select count(distinct(framed_ip)) as active_ip_num , sum(acct_session_time) as online_duration from (select any(framed_ip) as framed_ip ,max(acct_session_time) as acct_session_time from radius_onff_log where account='000jS' and event_timestamp >= $start_time and event_timestamp < $end_time group by acct_session_id)"); - //RADIUS账户IP详情 - list.add("select distinct(framed_ip) as framed_ip from radius_onff_log where account='000iS' and event_timestamp >= $start_time and event_timestamp < $end_time"); - //RADIUS账户访问详情 - list.add("select max(if(acct_status_type=1,event_timestamp,0)) as start_time,max(if(acct_status_type=2,event_timestamp,0)) as end_time, any(framed_ip) as ip,max(acct_session_time) as online_duration from radius_onff_log where account='000jS' and event_timestamp >= $start_time and event_timestamp < $end_time group by acct_session_id order by start_time desc limit 200"); - //目标资源分析 - list.add("SELECT domain, groupUniqArrayMerge(ip_list) as ip_list, groupUniqArrayMerge(cdn_list) as cdn_list, groupUniqArrayMerge(protocol_type_list) as protocol_type_list, groupUniqArrayMerge(port_list) as port_list FROM security_website_domain_info where stat_time >= $start_time and stat_time < $end_time and policy_id=0 group by domain"); - //某IP的关联域名关系 - list.add("SELECT ip, FROM_UNIXTIME(max(stat_time)) as last_time, FROM_UNIXTIME(min(stat_time)) as first_time, groupUniqArrayMerge(domain_list) as domain_list, groupUniqArrayMerge(port_list) as port_list FROM security_ip_info where stat_time >= $start_time and stat_time < $end_time and ip='192.168.50.70' group by ip"); - //自定义函数测试 - list.add("SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= '2020-04-05 00:00:00' and __time < '2020-05-05 00:00:00' and policy_id=0 group by policy_id"); - list.add("SELECT ip, IP_TO_CITY(ip) as location, IP_TO_GEO(ip) as geo from (SELECT policy_id, arrayJoin(groupUniqArrayMerge(ip_list)) as ip FROM proxy_ip_info where stat_time >= $start_time and stat_time < $end_time and policy_id=0 group by policy_id)"); - list.add("select TIME_FLOOR_WITH_FILL(common_recv_time,'PT5M','previous') as stat_time from connection_record_log where common_recv_time > $start_time and common_recv_time < $end_time group by stat_time"); - - return list; - } - - public List druidSql() { - List<String> list = new ArrayList<>(); - //安全策略命中统计 - //1. 某策略命中计数 - list.add("select policy_id, sum(hits) as hits from security_event_hits_log where __time >$start_time and __time <$end_time and policy_id=40 group by policy_id"); - list.add("select policy_id, sum(hits) as hits from security_event_hits_log where __time >$start_time and __time <$end_time and policy_id in (9,10,88,45) group by policy_id"); - //2. 某策略命中计数趋势 - list.add("select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from security_event_hits_log where __time >= TIMESTAMP $start_time and __time < TIMESTAMP $end_time and policy_id=10 group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 100"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, sum(hits) as hits from security_event_hits_log where __time >= $start_time and __time < $end_time and policy_id=10 group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') limit 100"); - //3. 某策略命中时间（首次和最近一次) - list.add("select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from security_event_hits_log where policy_id in (100,101 ,105, 102) group by policy_id"); - list.add("select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from security_event_hits_log where policy_id in (100,101 ,105, 102) group by policy_id"); - //4. TopN 命中策略 - list.add("select policy_id, action, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by policy_id, action order by hits desc limit 200"); - list.add("select policy_id, action, sum(hits) as hits from security_event_hits_log where __time >=$start_time and __time <$end_time group by policy_id, action order by hits desc limit 200"); - //二、 代理策略命中统计 - //1. 某策略命中计数 - list.add("select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=$start_time and __time <$end_time and policy_id=100 group by policy_id"); - list.add("select policy_id, sum(hits) as hits from proxy_event_hits_log where __time >=$start_time and __time <$end_time and policy_id=100 group by policy_id"); - //2. 某策略命中趋势 - list.add("select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP $start_time and __time <TIMESTAMP $end_time and policy_id=100 group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') limit 101"); - list.add("select FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) as start_time, sum(hits) as hits from proxy_event_hits_log where __time >= $start_time and __time < $end_time and policy_id=100 group by FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300) limit 101"); - //3.某策略命中时间（首次和最近一次 - list.add("select policy_id,TIME_FORMAT(min(__time) ,'yyyy-MM-dd HH:mm:ss') as first_used, TIME_FORMAT(max(__time) ,'yyyy-MM-dd HH:mm:ss') as last_used from proxy_event_hits_log where policy_id in (100,101,102,105) group by policy_id"); - list.add("select policy_id, DATE_FORMAT(min(__time) ,'%Y-%m-%d %H:%i:%s') as first_used, DATE_FORMAT(max(__time) ,'%Y-%m-%d %H:%i:%s') as last_used from proxy_event_hits_log where policy_id in (100,101,102,105) group by policy_id"); - //4. TopN 命中策略 - list.add("select policy_id, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by policy_id, sub_action order by hits desc limit 200"); - list.add("select policy_id, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=$start_time and __time <$end_time group by policy_id, sub_action order by hits desc limit 200"); - //5. Proxy 操纵动作命中计数 - list.add("select sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= TIMESTAMP $start_time and __time < TIMESTAMP $end_time group by sub_action"); - list.add("select sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= $start_time and __time < $end_time group by sub_action"); - //6. Proxy 操纵动作命中趋势 - list.add("select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') , sub_action limit 100\n"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, sub_action as action, sum(hits) as hits from proxy_event_hits_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s'), sub_action limit 100\n"); - //7. Proxy Pinning TIMESTAMP 计数 - list.add("select sum(hits) as hits, 'not_pinning_num' as type from security_event_hits_log where __time >= $start_time and __time < $end_time and pinningst=0 union all select sum(hits) as hits, 'pinning_num' as type from security_event_hits_log where __time >= $start_time and __time < $end_time and pinningst=1 union all select sum(hits) as hits, 'maybe_pinning_num' as type from security_event_hits_log where __time >=TIMESTAMP $start_time and __time < TIMESTAMP $end_time and pinningst=2"); - list.add("select sum(hits) as hits, 'not_pinning_num' as type from security_event_hits_log where __time >= $start_time and __time < $end_time and pinningst=0 union all select sum(hits) as hits, 'pinning_num' as type from security_event_hits_log where __time >= $start_time and __time < $end_time and pinningst=1 union all select sum(hits) as hits, 'maybe_pinning_num' as type from security_event_hits_log where __time >= $start_time and __time < $end_time and pinningst=2"); - //8.Proxy Pinning计数趋势 - list.add("select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, 'not_pinning_num' as type, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time and pinningst=0 group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, 'pinning_num' as type, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time and pinningst=1 group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as start_time, 'maybe_pinning_num' as type, sum(hits) as hits from security_event_hits_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time and pinningst=2 group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, 'not_pinning_num' as type, sum(hits) as hits from security_event_hits_log where __time >= $start_time and __time < $end_time and pinningst=0 group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, 'pinning_num' as type, sum(hits) as hits from security_event_hits_log where __time >= $start_time and __time < $end_time and pinningst=1 group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as start_time, 'maybe_pinning_num' as type, sum(hits) as hits from security_event_hits_log where __time >= $start_time and __time < $end_time and pinningst=2 group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')"); - //三、 Traffics-带宽统计 - //1. Traffic IN/OUT 计数 - //Bytes - list.add("select sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time"); - list.add("select sum(total_in_bytes) as traffic_in_bytes, sum(total_out_bytes) as traffic_out_bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time"); - //Packets - list.add("select sum(total_in_packets) as traffic_in_packets, sum(total_out_packets) as traffic_out_packets from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time"); - list.add("select sum(total_in_packets) as traffic_in_packets, sum(total_out_packets) as traffic_out_packets from traffic_metrics_log where __time >= $start_time and __time < $end_time"); - //Sessions - list.add("select sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time"); - list.add("select sum(new_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time"); - //2. Traffic IN/OUT 带宽趋势 - //Bytes - list.add("select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_bytes' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_in_bytes' as type, sum(total_in_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_out_bytes' as type, sum(total_out_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s')"); - //Packets - list.add("select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s')"); - //Packets - list.add("select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_in_packets' as type, sum(total_in_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'traffic_out_packets' as type, sum(total_out_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s')"); - //Sessions - list.add("select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s')"); - //四、 Traffics-计数统计 - //1. 新建与活跃链接计数 - list.add("select sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time"); - list.add("select sum(new_conn_num) as new_conn_num, sum(established_conn_num) as established_conn_num from traffic_metrics_log where __time >= $start_time and __time < $end_time"); - //2.新建与活跃链接计数趋势 - list.add("select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time < TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'established_conn_num' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= TIMESTAMP $start_time and __time < TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT30S'),'yyyy-MM-dd HH:mm:ss')"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'new_conn_num' as type, sum(new_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, 'established_conn_num' as type, sum(established_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s')"); - //3. 安全策略动作计数 - //Bytes - list.add("select sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time < TIMESTAMP $end_time"); - list.add("select sum(default_in_bytes+default_out_bytes) as default_bytes, sum(allow_in_bytes+allow_out_bytes) as allow_bytes, sum(deny_in_bytes+deny_out_bytes) as deny_bytes, sum(monitor_in_bytes+monitor_out_bytes) as monitor_bytes, sum(intercept_in_bytes+intercept_out_bytes) as intercept_bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time"); - //Packets - list.add("select sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time\n"); - list.add("select sum(default_in_packets+default_out_packets) as default_packets, sum(allow_in_packets+allow_in_packets) as allow_packets, sum(deny_in_packets+deny_out_packets) as deny_packets, sum(monitor_in_packets+monitor_out_packets) as monitor_packets, sum(intercept_in_packets+intercept_out_packets) as intercept_packets from traffic_metrics_log where __time >= $start_time and __time < $end_time\n"); - //Sessions - list.add("select sum(default_conn_num) as default_sessions, sum(allow_conn_num) as allow_sessions, sum(deny_conn_num) as deny_sessions, sum(monitor_conn_num) as monitor_sessions, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time"); - list.add("select sum(default_conn_num) as default_sessions, sum(allow_conn_num) as allow_sessions, sum(deny_conn_num) as deny_sessions, sum(monitor_conn_num) as monitor_sessions, sum(intercept_conn_num) as intercept_sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time"); - //4. 安全策略动作趋势 - //Bytes - list.add("select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_bytes' as type, sum(default_in_bytes+default_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_bytes' as type, sum(allow_in_bytes+allow_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_bytes' as type, sum(deny_in_bytes+deny_out_bytes) as bytes from traffic_metrics_log where __time >= TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') \n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_bytes' as type, sum(monitor_in_bytes+monitor_out_bytes) as bytes from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_bytes' as type, sum(intercept_in_bytes+intercept_out_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'default_bytes' as type, sum(default_in_bytes+default_out_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'allow_bytes' as type, sum(allow_in_bytes+allow_out_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'deny_bytes' as type, sum(deny_in_bytes+deny_out_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') \n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'monitor_bytes' as type, sum(monitor_in_bytes+monitor_out_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'intercept_bytes' as type, sum(intercept_in_bytes+intercept_out_bytes) as bytes from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')"); - //packets - list.add("select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_packets' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_packets' as type, sum(allow_in_packets+allow_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_packets' as type, sum(deny_in_packets+deny_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') \n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_packets' as type, sum(monitor_in_packets+monitor_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_packets' as type, sum(intercept_in_packets+intercept_out_packets) as packets from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'default_packets' as type, sum(default_in_packets+default_out_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'allow_packets' as type, sum(allow_in_packets+allow_out_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'deny_packets' as type, sum(deny_in_packets+deny_out_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') \n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'monitor_packets' as type, sum(monitor_in_packets+monitor_out_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'intercept_packets' as type, sum(intercept_in_packets+intercept_out_packets) as packets from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')"); - //sessions - list.add("select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'default_conn_num' as type, sum(default_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'allow_conn_num' as type, sum(allow_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'deny_conn_num' as type, sum(deny_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') \n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'monitor_conn_num' as type, sum(monitor_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')\n" + - "union all\n" + - "select TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss') as stat_time, 'intercept_conn_num' as type, sum(intercept_conn_num) as sessions from traffic_metrics_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by TIME_FORMAT(time_floor(__time,'PT5M'),'yyyy-MM-dd HH:mm:ss')"); - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'default_conn_num' as type, sum(default_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'allow_conn_num' as type, sum(allow_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'deny_conn_num' as type, sum(deny_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') \n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'monitor_conn_num' as type, sum(monitor_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')\n" + - "union all\n" + - "select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s') as stat_time, 'intercept_conn_num' as type, sum(intercept_conn_num) as sessions from traffic_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/300)*300),'%Y-%m-%d %H:%i:%s')"); - //五. Traffics-TOPN统计 - //1. TopN 源IP - list.add("select source as client_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time and order_by=source group by source order by source desc limit 100"); - list.add("select source as client_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_internal_host_log where __time >= $start_time and __time < $end_time and order_by=source group by source order by source desc limit 100"); - //2. TopN 目的IP - list.add("select destination as server_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_external_host_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time and order_by=destination group by destination order by destination desc limit 100"); - list.add("select destination as server_ip, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_external_host_log where __time >= $start_time and __time < $end_time and order_by=destination group by destination order by destination desc limit 100"); - //3. TopN 域名 - list.add("select domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time and order_by=domain group by domain order by domain desc limit 200"); - list.add("select domain, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_website_domain_log where __time >= $start_time and __time < $end_time and order_by=domain group by domain order by domain desc limit 200"); - //4.TopN 用户 - list.add("select subscriber_id, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_user_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time and order_by=subscriber_id group by subscriber_id order by subscriber_id desc limit 200"); - list.add("select subscriber_id, sum(session_num) as sessions, sum(c2s_byte_num) as sent_bytes, sum(s2c_byte_num) as received_bytes, sum(c2s_byte_num + s2c_byte_num) as bytes, sum(c2s_pkt_num) as sent_packets ,sum(s2c_pkt_num) as received_packets, sum(c2s_pkt_num+s2c_pkt_num) as packets from top_user_log where __time >= $start_time and __time < $end_time and order_by=subscriber_id group by subscriber_id order by subscriber_id desc limit 200"); - //六、命中URL统计 - list.add("select url,sum(session_num) as hits from top_urls_log where __time >=TIMESTAMP $start_time and __time <TIMESTAMP $end_time group by url order by hits desc limit 100"); - list.add("select url,sum(session_num) as hits from top_urls_log where __time >= $start_time and __time < $end_time group by url order by hits desc limit 100"); - - //国家客户端数量 - list.add("SELECT country, APPROX_COUNT_DISTINCT_DS_HLL(ip_object) as num FROM source_country_ip_num_log group by country order by num desc limit 200"); - //域名推荐 - list.add("SELECT domain, protocol_type, ip_num, session_num FROM top_domain_recommend_daily_log where __time = now()"); - //推荐URL - list.add("SELECT url, content_length, ip_num, session_num FROM top_website_urls_daily_log where __time = now()"); - //劫持客户端数量 - list.add("SELECT policy_id, APPROX_COUNT_DISTINCT_DS_HLL(isp) as num FROM proxy_event_hits_log where __time >= $start_time and __time < $end_time and policy_id=0 group by policy_id"); - //劫持客户端IP地域分布 - list.add("select ip, IP_TO_CITY(ip) as location, IP_TO_GEO(ip) as geo from (SELECT policy_id, arrayJoin(groupUniqArrayMerge(ip_list)) as ip FROM proxy_ip_info where stat_time >= $start_time and stat_time < $end_time and policy_id=0 group by policy_id)"); - //注入流量统计 - list.add("select DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') as stat_time, sum(bytes) as bytes from injection_metrics_log where __time >= $start_time and __time < $end_time group by DATE_FORMAT(FROM_UNIXTIME(FLOOR(UNIX_TIMESTAMP(__time)/30)*30),'%Y-%m-%d %H:%i:%s') limit 100"); - - return list; - } - - - /** - * 查询clickhouse数据库 - * - * @param query - * @return - */ - private List queryCKDynamic(String query) { - StringBuilder urlBuilder = new StringBuilder(clickHouseConfig.getQueryUrl()).append("/?"); - StringBuilder queryparamBuilder = new StringBuilder("user=") - .append(clickHouseConfig.getRealTimeUserName()).append("&") - .append("password=").append(clickHouseConfig.getRealTimePassword()).append("&") - .append("database=") - .append(clickHouseConfig.getDbName()).append("&") - .append("query=").append(query) - .append(" FORMAT JSON;"); - - log.info("Query Session and Events Query : {}", urlBuilder.toString() + Encodes.urlDecode(queryparamBuilder.toString())); - List<NameValuePair> values = URLEncodedUtils.parse(queryparamBuilder.toString(), Charset.forName("UTF-8")); - String s = urlBuilder.toString() + URLEncodedUtils.format(values, "utf-8"); - Map<String, String> resultMap = httpClientService.httpGet(s); - List result = new ArrayList<>(); - if (resultMap.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - Map<String, Object> maps = (Map<String, Object>) JsonMapper.fromJsonString(resultMap.get("result"), Map.class); - List data = (List) maps.get("data"); - Iterator iterator = data.iterator(); - while (iterator.hasNext()) { - Map next = (Map) iterator.next(); - result.add(String.valueOf(next.get("name"))); - } - return result; - } - - return null; - } - - - /** - * 查询druid数据库 - * - * @param query - * @return - */ - private List queryDruidDynamic(String query) { - //headers - HttpHeaders requestHeaders = new HttpHeaders(); - requestHeaders.setContentType(MediaType.APPLICATION_JSON); - //body - Map<String, Object> params = Maps.newHashMap(); - params.put("query", query); - Map<String, String> resultMap = httpClientService.httpPost(druidConfig.getQueryUrl(), JsonMapper.toJsonString(params)); - if (resultMap.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - List<String> list = new ArrayList<>(); - String result = resultMap.get("result"); - List body = (List) Json.parseJson(result); - Iterator iterator = body.iterator(); - while (iterator.hasNext()) { - Map next = (Map) iterator.next(); - String name = String.valueOf(next.get("name")); - list.add(name); - } - return list; - } - return null; - } - - - private void checkDruidSchema(String path) { - Map<String, JsonProperties> druidSchemaInfo = getSchemaInfo(path); - Set<String> name = druidSchemaInfo.keySet(); - Iterator<String> iterator = name.iterator(); - while (iterator.hasNext()) {//遍历表,以Avro为基础 - String tableName = iterator.next(); - Map resultMap = (Map) com.zdjizhi.utils.JsonMapper.fromJsonString(String.valueOf(druidSchemaInfo.get(tableName)), Map.class); - if (resultMap == null) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "schema未注册此表: " + tableName, null); - } - //查数据库 - List resultSql = queryDruidDynamic("SELECT COLUMN_NAME as name FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '" + tableName + "'"); - - List resultAvro = new ArrayList(); - - //schema为基础 - List<Map> fields = (List<Map>) resultMap.get("fields");//遍历列 - Iterator<Map> fieldsAvro = fields.iterator(); - while (fieldsAvro.hasNext()) { - Map next = fieldsAvro.next(); - resultAvro.add(String.valueOf(next.get("name"))); - boolean b = resultSql.contains(next.get("name")); - if (!b) { - String str = String.format("%s.shcema中注册多:%s", tableName, next.get("name")); - errorDate.add(str); - } - } - - //Schema为基础 - Iterator iteratorSql = resultSql.iterator(); - while (iteratorSql.hasNext()) { - String next = String.valueOf(iteratorSql.next()); - boolean b = resultAvro.contains(next); - if (!b) { - String str = String.format("%s.avro中注册缺少:%s", tableName, next); - errorDate.add(str); - } - } - } - } - - - private void checkCKSchema(String path) { - Map<String, JsonProperties> ckSchemaInfo = getSchemaInfo(path); - Set<String> name = ckSchemaInfo.keySet(); - Iterator<String> iterator = name.iterator(); - while (iterator.hasNext()) {//遍历表,以Avro为基础 - String tableName = iterator.next(); - Map resultMap = (Map) com.zdjizhi.utils.JsonMapper.fromJsonString(String.valueOf(ckSchemaInfo.get(tableName)), Map.class); - if (resultMap == null) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "schema未注册此表: " + tableName, null); - } - //查数据库 - List resultSql = queryCKDynamic("describe " + tableName); - if (StringUtil.isEmpty(resultSql)) { - String str = String.format("shcema中注册多:%s.avsc", tableName); - errorDate.add(str); - continue; - } - - List resultAvro = new ArrayList(); - - //avro为基础 - List<Map> fields = (List<Map>) resultMap.get("fields");//遍历列 - Iterator<Map> fieldsAvro = fields.iterator(); - while (fieldsAvro.hasNext()) { - Map next = fieldsAvro.next(); - resultAvro.add(String.valueOf(next.get("name"))); - boolean b = resultSql.contains(next.get("name")); - if (!b) { - String str = String.format("%s.shcema中注册多:%s", tableName, next.get("name")); - errorDate.add(str); - } - } - - //db为基础 - Iterator iteratorSql = resultSql.iterator(); - while (iteratorSql.hasNext()) { - String next = String.valueOf(iteratorSql.next()); - boolean b = resultAvro.contains(next); - if (!b) { - String str = String.format("%s.avro中注册缺少:%s", tableName, next); - errorDate.add(str); - } - } - } - } - - - /** - * 获取avro文件 - * - * @param path - * @return - */ - private Map<String, JsonProperties> getSchemaInfo(String path) { - Map<String, JsonProperties> schema = new HashMap<>(); - String pathRoot = null; - try { - pathRoot = new File("").getCanonicalPath(); - File file = new File(pathRoot + path); - File[] files = file.listFiles(); - if (files != null) { - for (File f : files) { - if (!f.getName().toLowerCase().endsWith(".avsc")) { - continue; - } - try { - Schema ckSchema = new Schema.Parser().parse(new File(f.getPath())); - schema.put(ckSchema.getName(), ckSchema); - } catch (Exception ex) { - log.error("遍历" + path + "路径下文件失败: " + f.getName(), ex); - String str = String.format("%s读取文件失败:%s", path, f.getName()); - errorDate.add(str); - } - } - } - } catch (Exception e) { - log.error("获取 项目路径/{}失败", path); - String str = String.format("获取项目路径%s失败", path); - errorDate.add(str); - } - return schema; - } - - - /** - * 核对avro文件 - * - * @param path 路径 - */ - private void checkSchema(String path) { - Map<String, JsonProperties> ckeckSchemaInfo = getSchemaInfo(path); - Set<String> name = ckeckSchemaInfo.keySet(); - Iterator<String> iterator = name.iterator(); - while (iterator.hasNext()) {//遍历表 - String tableName = iterator.next(); - Map resultMap = (Map) com.zdjizhi.utils.JsonMapper.fromJsonString(String.valueOf(ckeckSchemaInfo.get(tableName)), Map.class); - if (resultMap == null) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "schema未注册此表: " + tableName, null); - } - List<Map> fields = (List<Map>) resultMap.get("fields");//遍历列 - Iterator<Map> fieldsIterator = fields.iterator(); - Map mapDoc = null; - while (fieldsIterator.hasNext()) { - Map next = fieldsIterator.next(); - try { - mapDoc = (Map) com.zdjizhi.utils.JsonMapper.fromJsonString(String.valueOf(next.get("doc")), Map.class); - } catch (Exception ex) { - log.error("该字段的 doc 为非JSON字符串，不需类型转换"); - String str = String.format("%s.avro中doc非json格式字段%s", tableName, next.get("name")); - errorDate.add(str); - } - if (mapDoc != null) { - next.put("doc", mapDoc); - } - if (StringUtil.isEmpty(mapDoc) && !StringUtil.isEmpty(next.get("doc"))) { - log.error("该字段的 doc 为非JSON字符串，不需类型转换"); - String str = String.format("%s.avro中doc非json格式字段%s", tableName, next.get("name")); - errorDate.add(str); - } - } - } - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/ArangoEngineServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/ArangoEngineServiceImpl.java deleted file mode 100644 index c17508d..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/ArangoEngineServiceImpl.java +++ /dev/null @@ -1,129 +0,0 @@ -package com.mesalab.engine.service.impl; - -import com.google.gson.Gson; -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.dto.results.ArangoCollectionResult; -import com.mesalab.common.dto.results.ArangoCursorResult; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.common.util.HttpClientUtil; -import com.mesalab.common.util.SqlKeywords; -import com.mesalab.common.util.SysConstant; -import com.mesalab.engine.component.ArangoParser; -import com.mesalab.engine.service.ArangoEngineService; -import com.mesalab.engine.service.AuthorizeService; -import com.mesalab.engine.service.EngineExecuteService; -import com.zdjizhi.utils.DateUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.Header; -import org.apache.http.message.BasicHeader; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpHeaders; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @Date: 2020-07-20 10:03 - * @Author : liuyongqiang - * @ClassName : ArangoEngineServiceImpl - * @Description : ArangoDB数据查询引擎实现 - */ -@Slf4j -@SuppressWarnings("unchecked") -@Service("arangoDBEngineService") -public class ArangoEngineServiceImpl implements ArangoEngineService { - - @Value("${arango.collection}") - String arangoCollection; - - @Autowired - ArangoParser arangoDSLParser; - - @Autowired - EngineExecuteService executeService; - - @Autowired - AuthorizeService authorizeService; - - @Override - public BaseResult dslQuery(DSLObject dslObject) throws BusinessException { - String executeSql = arangoDSLParser.allQuery(dslObject); - String jsonResult = executeService.executeArangoDBSql(executeSql); - ArangoCursorResult results = new Gson().fromJson(jsonResult, ArangoCursorResult.class); - dataFormat(results); - return BaseResultUtil.success(results.getResult()); - } - - - - @Override - public BaseResult tablesQuery(DSLObject dslObject) throws BusinessException { - String jwtStr = SysConstant.ARANGO_JWT_PRE.concat(authorizeService.arangoJwtLogin()); - Header header = new BasicHeader(HttpHeaders.AUTHORIZATION,jwtStr); - String jsonResult = HttpClientUtil.httpGet(arangoCollection,header); - ArangoCollectionResult collectionResultDTO = new Gson().fromJson(jsonResult, ArangoCollectionResult.class); - List<Map<String,String>> resultList = new ArrayList<>(); - if(!CollectionUtils.isEmpty(collectionResultDTO.getResult())){ - for (ArangoCollectionResult.ResultBean resultBean:collectionResultDTO.getResult()){ - if(!resultBean.isSystem()){ - Map<String,String> mapResult = new HashMap<>(); - mapResult.put("id",resultBean.getId()); - mapResult.put("name",resultBean.getName()); - mapResult.put("status",String.valueOf(resultBean.getStatus())); - resultList.add(mapResult); - } - } - } - return BaseResultUtil.success(resultList); - } - - @Override - public BaseResult schemaQuery(DSLObject dslObject) throws BusinessException { - return null; - } - - /** - * @Description: 数据格式转换 - * @Author: liuyongqiang - * @Date: 2020/7/27 11:37 - * @param arangoResultDTO: - * @return: com.mesalab.gateway.dtodefine.results.ArangoCursorResult - **/ - private void dataFormat(ArangoCursorResult arangoResultDTO) { - List<Map<String,Object>> result = arangoResultDTO.getResult(); - if(CollectionUtils.isEmpty(result)) return; - for(Map<String,Object> map : result){ - for (String key : map.keySet()){ - //时间格式处理 - if(key.lastIndexOf(SqlKeywords.A_TIME)>0){ - String time = String.format(SysConstant.DOUBLE_FORMAT, map.get(key)); - map.put(key, DateUtils.convertTimestampToString(Long.valueOf(time),DateUtils.YYYY_MM_DD_HH24_MM_SS)); - } - //数值类型数据处理 - if(map.get(key) instanceof Double){ - map.put(key,((Double) map.get(key)).longValue()); - } - if(map.get(key) instanceof ArrayList){ - List<Object> values = (List<Object>) map.get(key); - List<Object> temp = new ArrayList<>(); - for(Object val:values){ - if(val instanceof Double){ - temp.add(((Double) val).longValue()); - }else{ - temp.add(val); - } - } - map.put(key,temp); - } - } - } - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/AuthorizeServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/AuthorizeServiceImpl.java deleted file mode 100644 index f94a12f..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/AuthorizeServiceImpl.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.mesalab.engine.service.impl; - -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.annotation.SysLog; -import com.mesalab.engine.service.AuthorizeService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.ResponseEntity; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; - -/** - * @Date: 2020-07-19 14:24 - * @Author : liuyongqiang - * @ClassName : AuthorizeServiceImpl - * @Description : JWT统一鉴权服务实现 - */ -@Slf4j -@Service("authorizeService") -public class AuthorizeServiceImpl implements AuthorizeService { - - @Autowired - RestTemplate restTemplate; - - @Value("${arango.login}") - String arangoLogin; - - /** - * @Description: ArangoDBJwt登录 - * @Author: liuyongqiang - * @Date: 2020/7/20 10:18 - * @return: java.lang.String - **/ - @SysLog - @Override - public String arangoJwtLogin() throws BusinessException { - ResponseEntity<String> responseEntity = restTemplate.postForEntity(arangoLogin,null,String.class); - return responseEntity.getBody(); - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/ClickHouseEngineServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/ClickHouseEngineServiceImpl.java deleted file mode 100644 index 1c0db2e..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/ClickHouseEngineServiceImpl.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.mesalab.engine.service.impl; - -import com.google.gson.Gson; -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.dto.results.ClickHouseResult; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.component.ClickHouseParser; -import com.mesalab.engine.service.ClickHouseEngineService; -import com.mesalab.engine.service.EngineExecuteService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.text.MessageFormat; - -/** - * @Date: 2020-07-20 10:04 - * @Author : liuyongqiang - * @ClassName : ClickHouseEngineServiceImpl - * @Description : ClickHouse数据查询引擎实现 - */ -@Slf4j -@SuppressWarnings("unchecked") -@Service("clickHouseEngineService") -public class ClickHouseEngineServiceImpl implements ClickHouseEngineService { - - @Autowired - ClickHouseParser clickHouseDSLParser; - - @Autowired - EngineExecuteService executeService; - - /** - * @Description: 单表数据查询 - * @Author: liuyongqiang - * @Date: 2020/7/21 9:41 - * @param dslObject: DSL查询对象 - * @return: com.mesalab.common.base.BaseResult - **/ - @Override - public BaseResult dslQuery(DSLObject dslObject) throws BusinessException { - String executeSql = clickHouseDSLParser.allQuery(dslObject); - String jsonResult = executeService.executeClickHouseSql(executeSql); - return assemble(jsonResult); - } - - /** - * @Description: 查询数据源下的所有表 - * @Author: liuyongqiang - * @Date: 2020/7/23 15:16 - * @param dslObject: - * @return: com.mesalab.common.base.BaseResult - **/ - @Override - public BaseResult tablesQuery(DSLObject dslObject) throws BusinessException { - String executeSql = ClickHouseParser.TABLES_QUERY; - String jsonResult = executeService.executeClickHouseSql(executeSql); - return assemble(jsonResult); - } - - /** - * @Description: 查询指定表中的所有字段 - * @Author: liuyongqiang - * @Date: 2020/7/23 11:08 - * @param dslObject: - * @return: com.mesalab.common.base.BaseResult - **/ - @Override - public BaseResult schemaQuery(DSLObject dslObject) throws BusinessException { - String dataSource = dslObject.getDataSource(); - String executeSql = MessageFormat.format(ClickHouseParser.SCHEAM_QUERY,dataSource); - return assemble(executeService.executeClickHouseSql(executeSql)); - } - - /** - * @Description: 组装返回结果 - * @Author: liuyongqiang - * @Date: 2020/7/23 16:52 - * @param jsonResult: - * @return: com.mesalab.common.base.BaseResult - **/ - private BaseResult assemble(String jsonResult){ - ClickHouseResult clickHouseResultDTO = new Gson().fromJson(jsonResult, ClickHouseResult.class); - BaseResult baseResult = BaseResultUtil.success(clickHouseResultDTO.getData()); - baseResult.setStatistics(clickHouseResultDTO.getStatistics()); - baseResult.setMeta(clickHouseResultDTO.getMeta()); - return baseResult; - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/DruidEngineServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/DruidEngineServiceImpl.java deleted file mode 100644 index f065a45..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/DruidEngineServiceImpl.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.mesalab.engine.service.impl; - -import com.google.gson.Gson; -import com.mesalab.common.model.DSLObject; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.component.DruidParser; -import com.mesalab.engine.service.DruidEngineService; -import com.mesalab.engine.service.EngineExecuteService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import java.text.MessageFormat; -import java.util.List; -import java.util.Map; - -/** - * @Date: 2020-07-20 10:05 - * @Author : liuyongqiang - * @ClassName : DruidEngineServiceImpl - * @Description : Druid数据查询引擎实现 - */ -@Service("druidEngineService") -@SuppressWarnings("unchecked") -public class DruidEngineServiceImpl implements DruidEngineService { - - @Autowired - DruidParser druidDSLParse; - - @Autowired - EngineExecuteService executeService; - - /** - * @Description: 单表数据查询 - * @Author: liuyongqiang - * @Date: 2020/7/21 9:41 - * @param dslObject: DSL查询对象 - * @return: com.mesalab.common.base.BaseResult - **/ - @Override - public BaseResult dslQuery(DSLObject dslObject) throws BusinessException { - String executeSql = druidDSLParse.allQuery(dslObject); - String jsonResult = executeService.executeDruidSql(executeSql); - List<Map<String,Object>> results = new Gson().fromJson(jsonResult,List.class); - dataFormat(results); - return BaseResultUtil.success(results); - } - - - - /** - * @Description: 查询数据源下的所有表 - * @Author: liuyongqiang - * @Date: 2020/7/23 15:16 - * @param dslObject: - * @return: com.mesalab.common.base.BaseResult - **/ - @Override - public BaseResult tablesQuery(DSLObject dslObject) throws BusinessException { - String executeSql = DruidParser.TABLES_QUERY; - String jsonResult = executeService.executeDruidSql(executeSql); - List<Map<String,Object>> results = new Gson().fromJson(jsonResult,List.class); - return BaseResultUtil.success(results); - } - - /** - * @Description: 查询指定表中的所有字段 - * @Author: liuyongqiang - * @Date: 2020/7/23 11:08 - * @param dslObject: - * @return: com.mesalab.common.base.BaseResult - **/ - @Override - public BaseResult schemaQuery(DSLObject dslObject) throws BusinessException { - String dataSource = dslObject.getDataSource(); - String executeSql = MessageFormat.format(DruidParser.SCHEAM_QUERY,dataSource); - String jsonResult = executeService.executeDruidSql(executeSql); - List<Map<String,Object>> results = new Gson().fromJson(jsonResult,List.class); - return BaseResultUtil.success(results); - } - - /** - * @Description: 数据格式化 - * @Author: liuyongqiang - * @Date: 2020/7/28 16:56 - * @param results: - * @return: java.util.List<java.util.Map<java.lang.String,java.lang.Object>> - **/ - private void dataFormat(List<Map<String, Object>> results) { - for (Map<String,Object> map : results){ - for (String key : map.keySet()){ - if(map.get(key) instanceof Double){ - map.put(key,((Double) map.get(key)).longValue()); - } - } - } - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/EngineExecuteServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/EngineExecuteServiceImpl.java deleted file mode 100644 index 31c877c..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/EngineExecuteServiceImpl.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.mesalab.engine.service.impl; - -import com.google.gson.Gson; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.HttpClientUtil; -import com.mesalab.common.annotation.SysLog; -import com.mesalab.engine.service.AuthorizeService; -import com.mesalab.engine.service.EngineExecuteService; -import org.apache.http.Header; -import org.apache.http.message.BasicHeader; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.stereotype.Service; -import java.util.HashMap; -import java.util.Map; - -/** - * @Date: 2020-07-22 16:15 - * @Author : liuyongqiang - * @ClassName : EngineExecuteServiceImpl - * @Description : 引擎执行查询实现 - */ -@Service("engineExecuteService") -public class EngineExecuteServiceImpl implements EngineExecuteService { - - final Header JSON_HEADER = new BasicHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); - - @Autowired - AuthorizeService authorizedService; - - @Value("${druid.query.url}") - String druidQueryUrl; - @Value("${arango.query.url}") - String arangoQueryUrl; - @Value("${clickhouse.query.url}") - String clickHouseQueryUrl; - @Value("${clickhouse.dbname}") - String clickHouseDbName; - @Value("${clickhouse.real.time.username}") - String ckRealTimeUsername; - @Value("${clickhouse.real.time.password}") - String ckRealTimePassword; - - @SysLog - @Override - public String executeDruidSql(String sql) throws BusinessException { - Map<String,String> queryParams = new HashMap<>(); - queryParams.put("query",sql); - String jsonParam = new Gson().toJson(queryParams); - return HttpClientUtil.httpPost(druidQueryUrl,jsonParam,JSON_HEADER); - } - - @SysLog - @Override - public String executeClickHouseSql(String sql) throws BusinessException { - String url = clickHouseQueryUrl - .concat("/?user=") - .concat(ckRealTimeUsername) - .concat("&password=") - .concat(ckRealTimePassword) - .concat("&database=") - .concat(clickHouseDbName) - .concat("&query=").concat(sql) - .concat(" FORMAT JSON"); - return HttpClientUtil.httpGet(url); - } - - @SysLog - @Override - public String executeArangoDBSql(String aql) throws BusinessException { - Map<String,String> queryParams = new HashMap<>(); - queryParams.put("query",aql); - String jwtStr = "bearer " + authorizedService.arangoJwtLogin(); - Header header = new BasicHeader(HttpHeaders.AUTHORIZATION,jwtStr); - String jsonParam = new Gson().toJson(queryParams); - return HttpClientUtil.httpPost(arangoQueryUrl,jsonParam,header); - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/MetadataServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/MetadataServiceImpl.java deleted file mode 100644 index 36517dd..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/MetadataServiceImpl.java +++ /dev/null @@ -1,223 +0,0 @@ -package com.mesalab.engine.service.impl; - -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.enums.SchemaTypeEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.component.SchemaCache; -import com.mesalab.engine.component.config.ClickHouseConfig; -import com.mesalab.engine.component.config.DruidConfig; -import com.mesalab.engine.service.MetadataService; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.avro.Schema; -import org.apache.http.HttpStatus; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.io.File; -import java.util.*; - -@Slf4j -@Service("metadataService") -@SuppressWarnings("unchecked") -public class MetadataServiceImpl implements MetadataService { - - @Autowired - SchemaCache schemaCache; - - private static final String METADATA_SCHEMA_KEY = "metadata:schema"; - - @Autowired - ClickHouseConfig clickHouseConfig; - - @Autowired - DruidConfig druidConfig; - - @Override - public BaseResult getSchemaInfo(String type, String name)throws BusinessException { - if (SchemaTypeEnum.TABLES.getValue().equals(type)) { - List<String> tables = getTablesName(name); - return encapsulationTablesResult(tables, name); - } else if (SchemaTypeEnum.FIELDS.getValue().equals(type)) { - return BaseResultUtil.success("ok", parseAndGetSchemaMap(name)); - } else { - return BaseResultUtil.failure(HttpStatus.SC_NOT_FOUND, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Not found, check url please"); - } - } - - private List<String> getTablesName(String dbName) { - List<String> tables = new ArrayList<>(); - Map<String, Schema> map = (Map<String, Schema>) schemaCache.get(METADATA_SCHEMA_KEY); - map.values().forEach(schema -> { - Optional.ofNullable(schema).ifPresent(o -> { - if (dbName.equalsIgnoreCase(schema.getNamespace())) { - tables.add(schema.getName()); - } - }); - }); - return tables; - } - - private Map parseAndGetSchemaMap(String tableName) { - Schema schema = getSchemaByName(tableName); - if (StringUtil.isEmpty(schema)) { - throw new BusinessException(HttpStatus.SC_NOT_FOUND, ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Unregister In Schema: " + tableName, null); - } - Map resultMap = (Map) JsonMapper.fromJsonString(String.valueOf(schema), Map.class); - Map schemaDoc = (Map) JsonMapper.fromJsonString(schema.getDoc(), Map.class); - if (StringUtil.isEmpty(schemaDoc) && StringUtil.isNotEmpty(schema.getDoc())) { - log.error("{} schema's doc isn't jsonString and won't convert: {}", tableName, schema.getDoc()); - } - if (StringUtil.isNotEmpty(schemaDoc)) { - resultMap.put("doc", schemaDoc); - } - List<Map> fields = (List<Map>) resultMap.get("fields"); - Map mapDoc; - for (Map next : fields) { - mapDoc = (Map) JsonMapper.fromJsonString(String.valueOf(next.get("doc")), Map.class); - if (StringUtil.isNotEmpty(mapDoc)) { - next.put("doc", mapDoc); - } - if (StringUtil.isEmpty(mapDoc) && !StringUtil.isEmpty(next.get("doc"))) { - log.error("{} field's doc isn't jsonString and won't convert: {}", next.get("name"), next.get("doc")); - } - } - return resultMap; - } - - private Schema getSchemaByName(String tableName) { - Map<String, Schema> schemas = (Map<String, Schema>) schemaCache.get(METADATA_SCHEMA_KEY); - return schemas.get(tableName); - } - - /** - * 封装库中元数据信息 - * - * @param tables - * @return - */ - private BaseResult encapsulationTablesResult(List tables, String database) { - Map<String, Object> date = new LinkedHashMap<>(); - date.put("type", "enum"); - date.put("name", database); - date.put("symbols", tables); - BaseResult<Map> baseResult = BaseResultUtil.success("ok", date); - return baseResult; - } - - - /** - * 读取fileDir目录下的文件(.avsc) - * - * @param fileDir - * @return - */ - private Map<String, Schema> getSchemasByDirs(File fileDir) { - Map<String, Schema> result = new HashMap<>(); - List<File> files = recurseDirs(fileDir, ".avsc"); - for (File item : files) { - try { - Schema schema = new Schema.Parser().parse(item); - result.put(schema.getName(), schema); - } catch (Exception e) { - log.error("File: {}, Schema Parser Error: {}", item.getPath(), e); - } - } - return result; - } - - - /** - * 遍历读取startDir文件夹目录下所有以regex结尾的文件 - * - * @param startDir - * @param regex - * @return - */ - private List<File> recurseDirs(File startDir, String regex) { - List<File> result = new ArrayList<>(); - Optional.ofNullable(startDir.listFiles()).ifPresent(o -> { - for (File item : o) { - if (item.isDirectory()) { - result.addAll(recurseDirs(item, regex)); - } else { - if (item.getName().endsWith(regex)) { - result.add(item); - } - } - } - }); - return result; - } - - @Override - public String getPartitionKey(String tableName) { - String partitionKey = StringUtil.EMPTY; - try { - Schema schema = getSchemaByName(tableName); - Map doc = (Map) JsonMapper.fromJsonString(schema.getDoc(), Map.class); - if (StringUtil.isNotEmpty(doc)) { - Object partition_key = doc.get("partition_key"); - partitionKey = StringUtil.isEmpty(partition_key) ? StringUtil.EMPTY : partition_key.toString(); - } - } catch (Exception e) { - log.error("Get Partition key Error ", e); - } - return partitionKey; - } - - @Override - public String getValueByKeyInSchemaDoc(String tableName, String key) { - Schema schema = getSchemaByName(tableName); - Map doc = (Map) JsonMapper.fromJsonString(schema.getDoc(), Map.class); - if (StringUtil.isNotEmpty(doc)) { - Object value = doc.get(key); - if (value instanceof Map) { - return JsonMapper.toJsonString(value); - } else { - return StringUtil.isEmpty(value) ? StringUtil.EMPTY : value.toString(); - } - } - return StringUtil.EMPTY; - } - - @Override - public String getDBTypeByTableName(String tableName) { - Schema schema = getSchemaByName(tableName); - if (StringUtil.isEmpty(schema)) { - return StringUtil.EMPTY; - } - if (clickHouseConfig.getDbName().equalsIgnoreCase(schema.getNamespace()) - || "system".equalsIgnoreCase(schema.getNamespace())) { - return DBTypeEnum.CLICKHOUSE.getValue(); - } - if (druidConfig.getDbname().equalsIgnoreCase(schema.getNamespace())) { - return DBTypeEnum.DRUID.getValue(); - } - return StringUtil.EMPTY; - } - - @Override - public String getDBNameByTableName(String tableName) { - Schema schema = getSchemaByName(tableName); - return StringUtil.isNotEmpty(schema) ? schema.getNamespace() : clickHouseConfig.getDbName(); - } - - - @Override - public Map<String, Schema> getAllSchema() { - try { - String pathRoot = new File("").getCanonicalPath(); - File fileDir = new File(pathRoot + "/schema"); - return getSchemasByDirs(fileDir); - } catch (Exception e) { - log.error("Get Schema Error {}", e.getMessage()); - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), ResultCodeEnum.UNKNOW_ERROR.getCode(), "Get Schema Error:", e); - } - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/NetworkServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/NetworkServiceImpl.java deleted file mode 100644 index a14c850..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/NetworkServiceImpl.java +++ /dev/null @@ -1,359 +0,0 @@ -package com.mesalab.engine.service.impl; - -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.mesalab.common.dto.params.NetworkParam; -import com.mesalab.common.dto.params.QueryParam; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.model.ProtocolTree; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.common.util.TreeUtil; -import com.mesalab.engine.service.HttpClientService; -import com.mesalab.engine.service.NetworkService; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.HttpStatus; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import javax.annotation.Nullable; -import java.io.UnsupportedEncodingException; -import java.net.InetAddress; -import java.net.URLEncoder; -import java.net.UnknownHostException; -import java.util.*; -import java.util.stream.Collectors; - -/** - * @author wangwei - * @version 1.0 - * @date 2020/6/30 4:24 下午 - */ -@Slf4j -@SuppressWarnings("unchecked") -@Service("networkService") -public class NetworkServiceImpl implements NetworkService { - - @Autowired - HttpClientService httpClientService; - @Value("${server.port}") - private int serverPort; - private static String hostAddress; - - private static final String PROTOCOL_NODE = "Protocols/"; - - static { - try { - hostAddress = InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e) { - log.error("get LocalHost error: ", e); - } - } - - @Override - public BaseResult getTrafficDistributedInfo(NetworkParam param) { - BaseResult baseResult; - String sql = null; - if (QueryType.PROTOCOL_TREE_SUMMARY.getValue().equalsIgnoreCase(param.getQuery().getQueryType())) { - sql = generateProtocolTreeSql(param.getQuery()); - baseResult = getResultProtocolTree(sql); - } else if (QueryType.PROTOCOL_DATA_RATE_SUMMARY.getValue().equalsIgnoreCase(param.getQuery().getQueryType())) { - String parameters = param.getQuery().getParameters(); - Optional.ofNullable(parameters).ifPresent(o -> { - String[] split = parameters.split(","); - param.getQuery().setParameters(split[0].replaceFirst(PROTOCOL_NODE, "")); - }); - sql = generateProtocolDataRateSql(param.getQuery()); - baseResult = getProtocolDataRateSummary(sql); - } else if (QueryType.NETWORK_OVERVIEW_SUMMARY.getValue().equalsIgnoreCase(param.getQuery().getQueryType())) { - sql = generateOverviewSummarySql(param.getQuery()); - Map<String, String> dataRateResult = executeQuery(sql); - String statSql = generateStatSql(param.getQuery()); - Map<String, String> statResult = executeQuery(statSql); - baseResult = buildNetworkOverviewResult(dataRateResult, statResult); - } else { - baseResult = BaseResultUtil.failure(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "No match queryType"); - } - if (!baseResult.isSuccess()) { - throw new BusinessException(baseResult.getStatus(), baseResult.getCode(), baseResult.getMessage(), null); - } - return baseResult; - } - - private BaseResult buildNetworkOverviewResult(Map<String, String> dataRateResult, Map<String, String> currentSessionResult) { - BaseResult baseResult; - if (String.valueOf(HttpStatus.SC_OK).equals(currentSessionResult.get("status")) - && String.valueOf(HttpStatus.SC_OK).equals(dataRateResult.get("status"))) { - Map statDataResult = (Map) JsonMapper.fromJsonString(currentSessionResult.get("result"), Map.class); - List<Map> statData = (List<Map>) statDataResult.get("data"); - Map summaryDataResult = (Map) JsonMapper.fromJsonString(dataRateResult.get("result"), Map.class); - List<Map> summaryData = (List<Map>) summaryDataResult.get("data"); - - Map<String, Object> data = new HashMap<>(16); - if (!statData.isEmpty()) { - data.putAll(statData.get(0)); - } - if (statData.size() >= 4) { - data.put("current_sessions", statData.get(1).get("total_bytes")); - data.put("totalBytesTcp", statData.get(2).get("total_bytes")); - data.put("totalPackageTcpUdp", statData.get(3).get("total_bytes")); - } - if (!summaryData.isEmpty()) { - data.putAll(summaryData.get(0)); - } - long uniqClientIp = Long.parseLong(String.valueOf(data.get("uniq_client_ip") == null ? 0 : data.get("uniq_client_ip"))); - long totalSessions = Long.parseLong(String.valueOf(data.get("total_sessions") == null ? 0 : data.get("total_sessions"))); - long totalBytesTcp = Long.parseLong(String.valueOf(data.get("totalBytesTcp") == null ? 0 : data.get("totalBytesTcp"))); - long totalPackageTcpUdp = Long.parseLong(String.valueOf(data.get("totalPackageTcpUdp") == null ? 0 : data.get("totalPackageTcpUdp"))); - long currentSessions = Long.parseLong(String.valueOf(data.get("current_sessions") == null ? 0 : data.get("current_sessions"))); - long dataRate = Long.parseLong(String.valueOf(data.get("data_rate") == null ? 0 : data.get("data_rate"))); - long totalBytes = Long.parseLong(String.valueOf(data.get("total_bytes") == null ? 0 : data.get("total_bytes"))); - long totalUncategorizedBytes = Long.parseLong(String.valueOf(data.get("total_uncategorized_bytes") == null ? 0 : data.get("total_uncategorized_bytes"))); - double totalUncategorizedPercent = totalBytes == 0 ? 0 : totalUncategorizedBytes * 1.0 / totalBytes; - long oneSidedConnections = Long.parseLong(String.valueOf(data.get("one_sided_connections") == null ? 0 : data.get("one_sided_connections"))); - double oneSidedPercent = totalBytes == 0 ? 0 : oneSidedConnections * 1.0 / totalSessions; - long sequenceGapLossBytes = Long.parseLong(String.valueOf(data.get("sequence_gap_loss_bytes") == null ? 0 : data.get("sequence_gap_loss_bytes"))); - double sequenceGapLossBytesPercent = totalBytesTcp == 0 ? 0 : sequenceGapLossBytes * 1.0 / totalBytesTcp; - long fragmentationPackets = Long.parseLong(String.valueOf(data.get("fragmentation_packets") == null ? 0 : data.get("fragmentation_packets"))); - double fragmentationPacketsPercent = totalPackageTcpUdp == 0 ? 0 : fragmentationPackets * 1.0 / totalPackageTcpUdp; - - List result = new ArrayList<>(); - Map resultMap = new LinkedHashMap<>(); - resultMap.put("uniq_client_ip", uniqClientIp); - resultMap.put("total_sessions", totalSessions); - resultMap.put("current_sessions", currentSessions); - resultMap.put("data_rate", dataRate); - resultMap.put("total_bytes", totalBytes); - resultMap.put("total_uncategorized_bytes", totalUncategorizedBytes); - resultMap.put("total_uncategorized_percent", Double.parseDouble(String.format("%.4f", totalUncategorizedPercent))); - resultMap.put("one_sided_connections", oneSidedConnections); - resultMap.put("one_sided_percent", Double.parseDouble(String.format("%.4f", oneSidedPercent))); - resultMap.put("sequence_gap_loss_bytes", sequenceGapLossBytes); - resultMap.put("sequence_gap_loss_percent", Double.parseDouble(String.format("%.4f", sequenceGapLossBytesPercent))); - resultMap.put("fragmentation_packets", fragmentationPackets); - resultMap.put("fragmentation_percent", Double.parseDouble(String.format("%.4f", fragmentationPacketsPercent))); - result.add(resultMap); - - Map statistics = (Map) summaryDataResult.get("statistics"); - Map statStatistics = (Map) statDataResult.get("statistics"); - for (Object key : statistics.keySet()) { - if ("result_rows".equals(key.toString())) { - continue; - } - statistics.put(key, Long.parseLong(statistics.get(key).toString()) + Long.parseLong(statStatistics.get(key).toString())); - } - - baseResult = BaseResultUtil.success("ok", result, statistics); - } else { - baseResult = BaseResultUtil.error("dataRate result: " + dataRateResult.toString() + ";\n currentSession result is: " + currentSessionResult); - } - return baseResult; - } - - private BaseResult getProtocolDataRateSummary(String sql) { - BaseResult baseResult; - Map<String, String> result = executeQuery(sql); - if (String.valueOf(HttpStatus.SC_OK).equals(result.get("status"))) { - Map resultMap = (Map) JsonMapper.fromJsonString(result.get("result"), Map.class); - List<Map> data = (List<Map>) resultMap.get("data"); - data.forEach(o -> { - String[] protocolIds = String.valueOf(o.get("type")).split("/"); - String protocolId = protocolIds[protocolIds.length - 1]; - o.put("type", protocolId); - }); - baseResult = BaseResultUtil.success("ok", data, (Map) resultMap.get("statistics")); - } else { - baseResult = BaseResultUtil.error(result.get("result")); - } - return baseResult; - } - - private BaseResult getResultProtocolTree(String sql) { - BaseResult baseResult; - Map<String, String> result = executeQuery(sql); - if (String.valueOf(HttpStatus.SC_OK).equals(result.get("status"))) { - Map resultMap = (Map) JsonMapper.fromJsonString(result.get("result"), Map.class); - List<ProtocolTree> listProtocol = getListProtocol(getProtocolTrees((List<Map>) resultMap.get("data"))); - baseResult = BaseResultUtil.success("ok", listProtocol, (Map) resultMap.get("statistics")); - } else { - baseResult = BaseResultUtil.error(result.get("result")); - } - return baseResult; - } - - private String generateStatSql(QueryParam queryParam) { - return String.format("(SELECT SUM(c2s_byte_num + s2c_byte_num) as total_bytes, SUM(sessions) as total_sessions, SUM(c2s_byte_num + s2c_byte_num) * 8000/(TIMESTAMP_TO_MILLIS(TIMESTAMP '%s')-TIMESTAMP_TO_MILLIS(TIMESTAMP '%s')) AS data_rate " + - "FROM traffic_protocol_stat_log " + - "WHERE __time >= TIMESTAMP '%s' AND __time < TIMESTAMP '%s' " + - "AND LENGTH(protocol_id) = LENGTH(REPLACE(protocol_id,'/','')) LIMIT 1) " + - "UNION ALL " + - "( SELECT SUM(sessions), 0, 0 " + - "FROM traffic_protocol_stat_log " + - "WHERE __time >= TIMESTAMP '%s' AND __time < TIMESTAMP '%s' " + - "AND LENGTH(protocol_id) = LENGTH(REPLACE(protocol_id,'/','')) GROUP BY __time ORDER BY __time DESC LIMIT 1 ) " + - "UNION ALL " + - "( SELECT SUM(c2s_byte_num + s2c_byte_num),0 ,0 FROM traffic_protocol_stat_log WHERE (__time >= TIMESTAMP '%s' AND __time < TIMESTAMP '%s') AND (protocol_id = 'ETHERNET/IPv4/TCP' OR protocol_id = 'ETHERNET/IPv6/TCP') LIMIT 1) " + - "UNION ALL " + - "( SELECT SUM(c2s_pkt_num + s2c_pkt_num),0 ,0 FROM traffic_protocol_stat_log WHERE (__time >= TIMESTAMP '%s' AND __time < TIMESTAMP '%s') AND (protocol_id = 'ETHERNET/IPv4/TCP' OR protocol_id = 'ETHERNET/IPv6/TCP' OR protocol_id = 'ETHERNET/IPv4/UDP' OR protocol_id = 'ETHERNET/IPv6/UDP') LIMIT 1) ", - queryParam.getIntervals().getEndTime(), queryParam.getIntervals().getStartTime(), queryParam.getIntervals().getStartTime(), queryParam.getIntervals().getEndTime(), - queryParam.getIntervals().getStartTime(), queryParam.getIntervals().getEndTime(), - queryParam.getIntervals().getStartTime(), queryParam.getIntervals().getEndTime(), - queryParam.getIntervals().getStartTime(), queryParam.getIntervals().getEndTime()); - } - - private String generateProtocolDataRateSql(QueryParam queryParam) { - return String.format("(" + - "SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes " + - "from %s " + - "where protocol_id = '%s' and __time >= TIMESTAMP '%s' and __time < TIMESTAMP '%s' " + - "group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id " + - "order by stat_time asc" + - ") " + - "union all " + - "(" + - "SELECT TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss') as stat_time, protocol_id as type, sum(c2s_byte_num + s2c_byte_num) as bytes " + - "from %s " + - "where protocol_id like CONCAT('%s','/%s') and __time >= TIMESTAMP '%s' and __time < TIMESTAMP '%s' and LENGTH(protocol_id) = LENGTH(REPLACE(protocol_id,'/','')) + 1 + %s " + - "group by TIME_FORMAT(MILLIS_TO_TIMESTAMP( 1000 * TIME_FLOOR_WITH_FILL(TIMESTAMP_TO_MILLIS(__time)/1000, '%s', 'zero')), 'yyyy-MM-dd HH:mm:ss'), protocol_id " + - "order by stat_time asc) ", - queryParam.getGranularity(), queryParam.getDataSource(), queryParam.getParameters(), queryParam.getIntervals().getStartTime(), queryParam.getIntervals().getEndTime(), queryParam.getGranularity(), - queryParam.getGranularity(), queryParam.getDataSource(), queryParam.getParameters(), '%', queryParam.getIntervals().getStartTime(), queryParam.getIntervals().getEndTime(), - queryParam.getParameters().length() - queryParam.getParameters().replaceAll("/", "").length(), queryParam.getGranularity()); - - } - - private String generateOverviewSummarySql(QueryParam queryParam) { - return String.format("SELECT APPROX_COUNT_DISTINCT_DS_HLL(ip_object) AS uniq_client_ip, " + - "SUM(one_sided_connections) AS one_sided_connections, " + - "SUM(uncategorized_bytes) AS total_uncategorized_bytes, " + - "SUM(fragmentation_packets) AS fragmentation_packets, " + - "SUM(sequence_gap_loss) AS sequence_gap_loss_bytes " + - "FROM %s " + - "WHERE __time >= TIMESTAMP '%s' " + - "AND __time < TIMESTAMP '%s' LIMIT 1 ", - queryParam.getDataSource(), queryParam.getIntervals().getStartTime(), queryParam.getIntervals().getEndTime()); - } - - private String generateProtocolTreeSql(QueryParam queryParam) { - return String.format("SELECT protocol_id, SUM(sessions) as sessions,SUM(c2s_byte_num) as c2s_byte_num, SUM(c2s_pkt_num) as c2s_pkt_num, SUM(s2c_byte_num) as s2c_byte_num, SUM(s2c_pkt_num) as s2c_pkt_num " + - "FROM %s " + - "WHERE __time >= TIMESTAMP '%s' AND __time < TIMESTAMP '%s' " + - "GROUP BY protocol_id ", - queryParam.getDataSource(), queryParam.getIntervals().getStartTime(), queryParam.getIntervals().getEndTime()); - } - - private List<ProtocolTree> getProtocolTrees(List<Map> data) { - Iterable<ProtocolTree> resultConcat = Iterables.concat(new ArrayList<>()); - for (Map datum : data) { - String s = JsonMapper.toJsonString(datum); - List<ProtocolTree> nodes = parserRawData(s); - resultConcat = Iterables.concat(resultConcat, nodes); - } - List<ProtocolTree> listNodes = Lists.newArrayList(resultConcat); - ProtocolTree root = new ProtocolTree("Protocols", "Protocols", null); - List<ProtocolTree> roots = listNodes.stream().filter(o -> StringUtil.isBlank(o.getParentId())).collect(Collectors.toList()); - roots.forEach(item -> { - root.setSentBytes(root.getSentBytes() + item.getSentBytes()); - root.setReceivedBytes(root.getReceivedBytes() + item.getReceivedBytes()); - }); - - listNodes.forEach(item -> { - item.setId(PROTOCOL_NODE + item.getId()); - }); - - listNodes.add(root); - return listNodes; - } - - private Map<String, String> executeQuery(String sql) { - try { - sql = URLEncoder.encode(sql, "utf-8").replaceAll("\\+", "%20"); - } catch (UnsupportedEncodingException e) { - hostAddress = "127.0.0.1"; - log.error("sql Encode error: ", e); - } - String url = "http://" + hostAddress + ":" + serverPort + "/?query="; - return httpClientService.httpGet(url + sql); - } - - - private List<ProtocolTree> parserRawData(String jsonString) { - List<ProtocolTree> lists = Lists.newArrayList(); - Map<String, Object> results = (Map<String, Object>) JsonMapper.fromJsonString(jsonString, Map.class); - long sessions = Long.parseLong(results.get("sessions").toString()); - long sentBytes = Long.parseLong(results.get("c2s_byte_num").toString()); - long receivedBytes = Long.parseLong(results.get("s2c_byte_num").toString()); - long sentPackets = Long.parseLong(results.get("c2s_pkt_num").toString()); - long receivedPackets = Long.parseLong(results.get("s2c_pkt_num").toString()); - String protocolId = results.get("protocol_id").toString(); - List<String> protocolList = Lists.newArrayList(protocolId.split("/")); - ProtocolTree protocolTree = new ProtocolTree(protocolId, protocolList.size() <= 0 ? null : protocolList.get(protocolList.size() - 1), null); - protocolTree.setSentBytes(sentBytes); - protocolTree.setReceivedBytes(receivedBytes); - protocolTree.getMetrics().put("sentPackets", sentPackets); - protocolTree.getMetrics().put("receivedPackets", receivedPackets); - protocolTree.getMetrics().put("sessions", sessions); - lists.add(protocolTree); - return lists; - } - - private List<ProtocolTree> getListProtocol(List<ProtocolTree> listNodes) { - Map<String, List<ProtocolTree>> resultMap = listNodes.stream().collect(Collectors.groupingBy(ProtocolTree::getId)); - Map<String, Object> mergeResultMap = - Maps.transformValues(resultMap, new com.google.common.base.Function<List<ProtocolTree>, Object>() { - @Nullable - @Override - public Object apply(@Nullable List<ProtocolTree> input) { - if (StringUtil.isEmpty(input)) { - return null; - } - if (input.size() == 1) { - return input.get(0); - } - ProtocolTree firstProtocolTree = input.get(0); - ProtocolTree mergeProtocolTree = new ProtocolTree(firstProtocolTree.getId(), firstProtocolTree.getName(), - firstProtocolTree.getParentId(), 0, 0); - mergeProtocolTree.setMetrics(Maps.newLinkedHashMap()); - - for (ProtocolTree protocolTree : input) { - mergeProtocolTree.setSentBytes(mergeProtocolTree.getSentBytes() + protocolTree.getSentBytes()); - mergeProtocolTree.setReceivedBytes(mergeProtocolTree.getReceivedBytes() + protocolTree.getReceivedBytes()); - for (String key : protocolTree.getMetrics().keySet()) { - if (StringUtil.isEmpty(mergeProtocolTree.getMetrics().get(key))) { - mergeProtocolTree.getMetrics().put(key, protocolTree.getMetrics().get(key)); - } else { - long sumValue = Long.parseLong(mergeProtocolTree.getMetrics().get(key).toString()) - + Long.parseLong(protocolTree.getMetrics().get(key).toString()); - mergeProtocolTree.getMetrics().put(key, sumValue); - } - } - - } - - return mergeProtocolTree; - } - }); - List<ProtocolTree> mergeNodes = new ArrayList(mergeResultMap.values()); - return TreeUtil.mergeTree(mergeNodes, ProtocolTree::getId, ProtocolTree::getParentId, ProtocolTree::setChildrens); - } - - enum QueryType { - - PROTOCOL_TREE_SUMMARY("protocolTreeSummary"), PROTOCOL_DATA_RATE_SUMMARY("protocolDataRateSummary"), NETWORK_OVERVIEW_SUMMARY("networkOverviewSummary"); - private final String value; - - QueryType(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - } -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/SqlEngineServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/SqlEngineServiceImpl.java deleted file mode 100644 index b0abed9..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/SqlEngineServiceImpl.java +++ /dev/null @@ -1,661 +0,0 @@ -package com.mesalab.engine.service.impl; - -import com.google.common.base.CaseFormat; -import com.google.common.base.Joiner; -import com.mesalab.common.annotation.SysLog; -import com.mesalab.engine.component.bean.SqlQueryBean; -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.*; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.model.SchemaBase; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.common.util.SqlHelper; -import com.mesalab.engine.component.ReportCache; -import com.mesalab.engine.component.SqlFunction; -import com.mesalab.engine.component.udf.UDF; -import com.mesalab.engine.component.udf.UDFElements; -import com.mesalab.engine.component.config.ClickHouseConfig; -import com.mesalab.engine.component.config.DruidConfig; -import com.mesalab.engine.component.dialect.Dialect; -import com.mesalab.engine.component.dialect.FederationDialect; -import com.mesalab.engine.service.MetadataService; -import com.mesalab.engine.service.SqlEngineService; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import net.sf.jsqlparser.expression.BinaryExpression; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.Function; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.DescribeStatement; -import net.sf.jsqlparser.statement.ShowStatement; -import net.sf.jsqlparser.statement.Statement; -import net.sf.jsqlparser.statement.select.*; -import net.sf.jsqlparser.util.TablesNamesFinder; -import org.apache.http.HttpStatus; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.lang.reflect.Constructor; -import java.util.*; - -/** - * @Date: 2020-08-18 09:54 - * @Author : liuyongqiang - * @ClassName : SqlEngineServiceImpl - * @Description : 基于SQL的动态查询服务 - */ -@Slf4j -@SuppressWarnings("unchecked") -@Service("sqlQueryEngineService") -public class SqlEngineServiceImpl implements SqlEngineService { - - - @Autowired - MetadataService metadataService; - - @Autowired - ClickHouseConfig clickHouseConfig; - - @Autowired - DruidConfig druidConfig; - - - public static final int MAX_PARSER_LEVEL = 5; - - public static final String UNVALID_LIMIT_DESC = "NO LIMIT"; - - @SysLog - @Override - public BaseResult executeQuery(ApiQueryBean apiParam) { - if (StringUtil.isBlank(apiParam.getQuery())) { - return BaseResultUtil.success4Message("ok"); - } - Statement statement; - try { - statement = CCJSqlParserUtil.parse(apiParam.getQuery()); - } catch (Exception e) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error:", e.getCause()); - } - - BaseResult baseResult; - if (statement instanceof Select) { - baseResult = executeSelectStatement(apiParam); - } else if (statement instanceof DescribeStatement) { - DescribeStatement describeStatement = (DescribeStatement) statement; - baseResult = executeDescStatement(apiParam, describeStatement); - } else if (statement instanceof ShowStatement) { - ShowStatement showStatement = (ShowStatement) statement; - baseResult = executeShowStatement(apiParam, showStatement); - } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), - "SQL Syntax Error: Only support statement operations for select, show and describe.", null); - } - return baseResult; - } - - private BaseResult executeSelectStatement(ApiQueryBean param) { - - BaseResult baseResult = null; - if (QueryOptionEnum.LONG_TERM.getValue().equalsIgnoreCase(param.getOption())) { - baseResult = ReportCache.get(param); - if (baseResult.isSuccess()) { - return baseResult; - } else { - log.warn("Long-Term Results not exist, execute real-time query: resultID-{}, sql-{}", - param.getResultId(), StringUtil.strip(param.getQuery())); - } - } - - Dialect dbDialect = parserSQL(param); - if (param.getOption().equalsIgnoreCase(QueryOptionEnum.SYNTAX_CHECK.getValue())) { - baseResult = dbDialect.executeSyntaxCheck().build(); - } else { - baseResult = dbDialect.executeQuery().build(); - } - baseResult = new FederationDialect(param, baseResult).executeQuery().build(); - - if (!baseResult.isSuccess()) { - throw new BusinessException(baseResult.getStatus(), baseResult.getCode(), baseResult.getMessage(), null); - } - - return baseResult; - } - - /** - * 执行show语句 - * - * @param param - * @param showStatement - * @return - */ - private BaseResult executeShowStatement(ApiQueryBean param, ShowStatement showStatement) { - - if (showStatement != null && !"tables".equalsIgnoreCase(showStatement.getName())) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "SQL Syntax Error：不支持Show tables 之外的操作", null); - } - if (clickHouseConfig.getDbName().equalsIgnoreCase(param.getSchema())) { - param.setDbType(DBTypeEnum.CLICKHOUSE.getValue()); - param.setDialectDBType(DBTypeEnum.CLICKHOUSE.getValue()); - } else if (druidConfig.getDbname().equalsIgnoreCase(param.getSchema())) { - param.setDbType(DBTypeEnum.DRUID.getValue()); - param.setDialectDBType(DBTypeEnum.DRUID.getValue()); - param.setQuery("SELECT TABLE_NAME AS name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'TABLE'"); - } else if (StringUtil.isBlank(param.getSchema())) { - BaseResult result = getAllTables(); - return result; - } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "schema name error", null); - } - param.setFormat(QueryFormatEnum.JSON.getValue()); - - Dialect dialect = getDialect(param); - BaseResult result = dialect.executeAdministrativeQuery().build(); - if (result.getStatus() == null || result.getStatus() != HttpStatus.SC_OK) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), result.getMessage(), null); - } - List tables = (ArrayList) result.getData(); - List<String> list = new ArrayList<>(); - Iterator iterator = tables.iterator(); - while (iterator.hasNext()) { - Map<String, String> next = (Map) iterator.next(); - list.add(next.get("name")); - } - return tablesResultEncapsulation(list, param.getSchema()); - } - - /** - * 获取所有表信息 - * - * @return - */ - private BaseResult getAllTables() { - //查ck - ApiQueryBean temp = new ApiQueryBean(); - temp.setQuery("show tables"); - temp.setSchema(clickHouseConfig.getDbName()); - BaseResult ck = new BaseResult(); - try { - ck = executeShowStatement(temp, null); - } catch (Exception e) { - log.info("tables: 查询clickhouse异常", e); - } - - //查druid - BaseResult druid = new BaseResult(); - temp = new ApiQueryBean(); - temp.setSchema(druidConfig.getDbname()); - try { - druid = executeShowStatement(temp, null); - } catch (Exception e) { - log.info("tables: 查询druid 异常", e); - } - //合并结果 - Map<String, Object> date = new LinkedHashMap<>(); - List symbols = new ArrayList<>(); - String dbName = ""; - date.put("type", "enum"); - if (ck.getStatus() != null && ck.getStatus() == HttpStatus.SC_OK) { - Map ckDate = (Map) ck.getData(); - dbName += (String) ckDate.get("name"); - symbols.addAll((List) ckDate.get("symbols")); - } - if (druid.getStatus() != null && druid.getStatus() == HttpStatus.SC_OK) { - Map druidDate = (Map) druid.getData(); - if (!"".equals(dbName)) { - dbName += ","; - } - dbName += (String) druidDate.get("name"); - symbols.addAll((List) druidDate.get("symbols")); - } - date.put("name", dbName); - date.put("symbols", symbols); - if (ck.getStatus() == HttpStatus.SC_OK) { - ck.setData(date); - return ck; - } else if (druid.getStatus() == HttpStatus.SC_OK) { - druid.setData(date); - return druid; - } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "查询所有tables异常", null); - } - } - - private Dialect parserSQL(ApiQueryBean apiParam) { - apiParam.setTableName(validateAndGetTableName(apiParam)); - return parserAndGetDialect(apiParam); - } - - /** - * 执行describe语句 - * - * @param param - * @param statement - * @return - */ - private BaseResult executeDescStatement(ApiQueryBean param, DescribeStatement statement) { - String name = statement == null ? param.getTableName() : statement.getTable().getName(); - param.setTableName(name); - - - ApiQueryBean temp = new ApiQueryBean(); - temp.setSchema(clickHouseConfig.getDbName()); - temp.setTableName(name); - temp.setQuery("show tables"); - - - BaseResult ckResult = executeShowStatement(temp, null); - Map data = (Map) ckResult.getData(); - List symbols = (List) data.get("symbols"); - if (symbols.contains(param.getTableName())) { - //表在ck中 - param.setDbType(DBTypeEnum.CLICKHOUSE.getValue()); - param.setDialectDBType(DBTypeEnum.CLICKHOUSE.getValue()); - } else {//走druid去查 - param.setDbType(DBTypeEnum.DRUID.getValue()); - param.setDialectDBType(DBTypeEnum.DRUID.getValue()); - param.setQuery("SELECT COLUMN_NAME as name, DATA_TYPE as type FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '" + name + "'"); - } - param.setFormat(QueryFormatEnum.JSON.getValue()); - - SqlQueryBean dbQuerySource = new SqlQueryBean(); - dbQuerySource.setSqlBody(param.getQuery()); - param.setDbQuerySource(dbQuerySource); - - Dialect dialect = getDialect(param); - BaseResult result = dialect.executeAdministrativeQuery().build(); - if (result.getStatus() == null || result.getStatus() != HttpStatus.SC_OK) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "describe table error, please check tableName", null); - } - BaseResult baseResult = dataEncapsulationOfSchema(result, param); - return baseResult; - } - - /** - * 表描述信息结果封装 - * - * @param result - * @param param - * @return - */ - public BaseResult dataEncapsulationOfSchema(BaseResult result, ApiQueryBean param) { - SchemaBase schema = new SchemaBase(); - schema.setName(param.getTableName()); - List<Map> fields = new ArrayList<>(); - HashMap<String, Object> map = new HashMap<>(); - List<Map> list = (List<Map>) result.getData(); - Iterator iterator = list.iterator(); - while (iterator.hasNext()) { - Map<String, String> next = (Map) iterator.next(); - map = new HashMap<>(); - map.put("name", next.get("name")); - map.put("type", next.get("type")); - fields.add(map); - } - schema.setFields(fields); - result.setData(schema); - return result; - } - - /** - * 对请求参数进行验证 - * - * @param param query:查询语句，统一小写 - * format：输出格式，默认为JSON - * option：查询选项，默认为real-time - */ - private String validateAndGetTableName(ApiQueryBean param) { - - if (StringUtil.isBlank(param.getQuery())) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Param Syntax Error：The SQL must be exists.", null); - } - - if (StringUtil.isNotBlank(param.getOption())) { - if (!param.getOption().equalsIgnoreCase(QueryOptionEnum.REAL_TIME.getValue()) && - !param.getOption().equalsIgnoreCase(QueryOptionEnum.LONG_TERM.getValue()) && - !param.getOption().equalsIgnoreCase(QueryOptionEnum.SYNTAX_CHECK.getValue())) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Param Syntax Error：The option must be real-time or long-term or syntax-check. ", null); - } - if (param.getOption().equalsIgnoreCase(QueryOptionEnum.LONG_TERM.getValue())) { - if (StringUtil.isBlank(param.getReportId()) && StringUtil.isBlank(param.getResultId())) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Param Syntax Error：The option=long-term,must be reportId or resultId.", null); - } - } - } else { - param.setOption(QueryOptionEnum.REAL_TIME.getValue()); - } - - - if (StringUtil.isNotBlank(param.getFormat())) { - if (!param.getFormat().equalsIgnoreCase(QueryFormatEnum.JSON.getValue()) - && !param.getFormat().equalsIgnoreCase(QueryFormatEnum.CSV.getValue())) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.PARAM_SYNTAX_ERROR.getCode(), "Param Syntax Error: The format must be json or csv.", null); - } - } else { - param.setFormat(QueryFormatEnum.JSON.getValue()); - } - - return getTableName(param.getQuery()); - } - - private String getTableName(String sql) { - - String tableName = ""; - try { - Statement statement = CCJSqlParserUtil.parse(sql); - if (statement instanceof Select) { - Select selectStatement = (Select) statement; - TablesNamesFinder tablesNamesFinder = new TablesNamesFinder(); - List<String> tableList = tablesNamesFinder.getTableList(selectStatement); - if (StringUtil.isNotEmpty(tableList)) { - tableName = tableList.get(0); - int i = tableName.lastIndexOf("."); - tableName = i < 0 ? tableName : tableName.substring(i + 1); - } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error: Get Table Name Error."); - } - - - } else { - log.warn("Not support DML Parser"); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error, Not support DML Parser."); - } - - } catch (Exception e) { - - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "SQL Syntax Error: ", ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), e.getCause()); - } - - return tableName; - } - - private Dialect parserAndGetDialect(ApiQueryBean param) { - Dialect dialect; - String dbType = metadataService.getDBTypeByTableName(param.getTableName().toLowerCase()); - if (DBTypeEnum.CLICKHOUSE.getValue().equalsIgnoreCase(dbType)) { - param.setDbType(DBTypeEnum.CLICKHOUSE.getValue()); - } else if (DBTypeEnum.DRUID.getValue().equalsIgnoreCase(dbType)) { - param.setDbType(DBTypeEnum.DRUID.getValue()); - } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), param.getTableName() + " 表未在平台注册，请联系数据平台负责人", null); - } - param.setDialectDBType(SqlHelper.getDialectDBType(param.getDbType())); - convertQueryRecursive(param, parserSQLByAst(param.getQuery())); - dialect = getDialect(param); - return dialect; - } - - private Dialect getDialect(ApiQueryBean apiParam) { - Dialect dialect; - try { - Class dialectClazz = Class.forName("com.mesalab.engine.component.dialect." + CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_CAMEL, apiParam.getDbType()) + "Dialect"); - Constructor constructor = dialectClazz.getConstructor(ApiQueryBean.class); - dialect = (Dialect) constructor.newInstance(apiParam); - } catch (Exception e) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "方言转换实例异常！", e); - } - return dialect; - } - - /** - * 递归解析SQLQuerySource ，拆分为数据库执行SQL与引擎执行的SQL source - * - * @param sqlQuerySource SQL解析对象 - */ - @Override - public void convertQueryRecursive(ApiQueryBean param, SqlQueryBean sqlQuerySource) { - SqlQueryBean engineQuerySource = null; - SqlQueryBean dbQuerySource = null; - - SqlQueryBean indexUpSqlQuerySource = null; - SqlQueryBean indexSqlQuerySource = sqlQuerySource; - for (int i = 0; i < MAX_PARSER_LEVEL; i++) { - if (indexSqlQuerySource.getUdfSet().size() > 0) { - dbQuerySource = indexSqlQuerySource; - break; - } - if (StringUtil.isEmpty(indexSqlQuerySource.getSubSelect())) { - dbQuerySource = sqlQuerySource; - indexUpSqlQuerySource = null; - break; - } else { - indexUpSqlQuerySource = indexSqlQuerySource; - indexSqlQuerySource = indexSqlQuerySource.getSubSqlQuerySources().get(0); - if (indexSqlQuerySource.getUdfSet().size() > 0) { - dbQuerySource = indexSqlQuerySource; - break; - } - } - } - - if (StringUtil.isNotEmpty(indexUpSqlQuerySource)) { - engineQuerySource = new SqlQueryBean(); - String tableName = StringUtil.isEmpty(indexUpSqlQuerySource.getSubSelect().getAlias()) ? indexUpSqlQuerySource.getTableName() : indexUpSqlQuerySource.getSubSelect().getAlias().getName(); - String replace = sqlQuerySource.getSqlBody().replace(indexUpSqlQuerySource.getSubSelect().toString(), tableName); - engineQuerySource.setSqlBody(replace); - engineQuerySource.setTableName(tableName); - engineQuerySource.setLimit(sqlQuerySource.getLimit()); - } - - param.setDbQuerySource(dbQuerySource); - param.setEngineQuerySource(engineQuerySource); - } - - /** - * 解析SQL - * - * @param sql - * @return - */ - @Override - public SqlQueryBean parserSQLByAst(String sql) { - - SqlQueryBean sqlQuerySource = new SqlQueryBean(); - try { - - Statement statement = CCJSqlParserUtil.parse(sql); - if (statement instanceof Select) { - Select selectStatement = (Select) statement; - sqlQuerySource.setSqlBody(String.valueOf(selectStatement.getSelectBody())); - TablesNamesFinder tablesNamesFinder = new TablesNamesFinder(); - List<String> tableList = tablesNamesFinder.getTableList(selectStatement); - - if (StringUtil.isNotEmpty(tableList)) { - String tableName = tableList.get(0); - int i = tableName.lastIndexOf("."); - tableName = i < 0 ? tableName : tableName.substring(i + 1); - sqlQuerySource.setTableName(tableName); - sqlQuerySource.setPartitionKey(metadataService.getPartitionKey(tableName)); - } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error: Can't find table name."); - } - - - SelectBody body = selectStatement.getSelectBody(); - - - if (body instanceof PlainSelect) { //单条查询 - - PlainSelect select = (PlainSelect) body; - - FromItem fromItem = select.getFromItem(); - if (fromItem != null) { - sqlQuerySource.setFromItem(fromItem.toString()); - } - - Expression where = select.getWhere(); - - if (where != null) { - sqlQuerySource.setExpr(where.toString()); - sqlQuerySource.setWhereExpression(where); - } - - List<SelectItem> selectItemsList = select.getSelectItems(); - - if (StringUtil.isNotEmpty(selectItemsList)) { - String selectItemsString = Joiner.on(",").join(selectItemsList); - sqlQuerySource.setSelectItems(selectItemsString); - - } - - for (SelectItem item : selectItemsList) { - if (item instanceof SelectExpressionItem) { - SelectExpressionItem expressionItem = ((SelectExpressionItem) item); - if (StringUtil.isNotEmpty(expressionItem.getAlias())) { - String aliasName = expressionItem.getAlias().getName(); - if (aliasName.startsWith("\"") && aliasName.endsWith("\"")) { - aliasName = aliasName.substring(1, aliasName.length() - 1); - } - sqlQuerySource.getAliasFields().put(aliasName, expressionItem.getExpression().toString()); - } else { - sqlQuerySource.getAliasFields().put(expressionItem.getExpression().toString(), expressionItem.getExpression().toString()); - } - addUDFSet(sqlQuerySource.getUdfSet(), expressionItem.getExpression());//伪代码：后期需要在SQLQuerySource where对象中提取 - - } - } - - GroupByElement groupBy; - if (StringUtil.isNotEmpty(groupBy = select.getGroupBy())) { - sqlQuerySource.setGroupByElement(groupBy); - } - - List<OrderByElement> orderByList = select.getOrderByElements(); - if (StringUtil.isNotEmpty(orderByList)) { - String orderByString = Joiner.on(",").join(orderByList); - sqlQuerySource.setOrderBy(orderByString); - sqlQuerySource.setListOrderElement(orderByList); - } - - - FromItem subItem = select.getFromItem(); - - if (subItem instanceof SubSelect) { - SubSelect subSelect = (SubSelect) subItem; - log.debug("存在子查询为: {}", subSelect); - sqlQuerySource.setSubSelect(subSelect); - sqlQuerySource.getSubSqlQuerySources().add(0, parserSQLByAst(String.valueOf(subSelect.getSelectBody()))); - } - - - Limit limit = select.getLimit(); - - - if (limit != null) { - - if (StringUtil.isNotEmpty(limit.getOffset())) { - sqlQuerySource.setLimit(StringUtil.setDefaultIfEmpty(limit.getOffset(), 0) + "," + limit.getRowCount()); - } else { - sqlQuerySource.setLimit(String.valueOf(limit.getRowCount())); - } - } - - - } else if (body instanceof SetOperationList) { // 连接查询 - SetOperationList setOperationList = (SetOperationList) body; - List<SelectBody> selects = setOperationList.getSelects(); - //暂时只解析第一个结构，不接受不相同的where - if (StringUtil.isNotEmpty(selects)) { - SqlQueryBean parseSql = parserSQLByAst(selects.get(0).toString()); - sqlQuerySource.setUdfSet(parseSql.getUdfSet()); - sqlQuerySource.setWhereExpression(parseSql.getWhereExpression()); - sqlQuerySource.setGroupByElement(parseSql.getGroupByElement()); - sqlQuerySource.setAliasFields(parseSql.getAliasFields()); - } - sqlQuerySource.setLimit(UNVALID_LIMIT_DESC); - } else { //其它暂不支持 - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error：Only support statement operation as select."); - } - - - } else { - log.warn("Not support DML Parser"); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error: Only support statement operation as select."); - } - - - } catch (Exception e) { - log.error("sqlParser error: ", e); - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "SQL Syntax Error:", e); - - } - - return sqlQuerySource; - } - - /** - * 自定义函数封装 - * - * @param udfSet - * @param expr - */ - private void addUDFSet(Set<UDF> udfSet, Expression expr) { - parserExpressionForFun(udfSet, expr); - } - - private void parserExpressionForFun(Set<UDF> udfSet, Expression expr) { - if (expr instanceof Function) { - Function fun = (Function) expr; - if (SqlFunction.functions.keySet().contains(fun.getName().toUpperCase())) { - List<Expression> expressions = fun.getParameters().getExpressions(); - UDF udf = getUDF(fun, expressions); - udfSet.add(udf); - } - if (fun.getParameters() == null) { - return; - } - for (Expression expression : fun.getParameters().getExpressions()) { - parserExpressionForFun(udfSet, expression); - } - } else if (expr instanceof BinaryExpression) { - BinaryExpression binary = (BinaryExpression) expr; - Expression leftExpression = binary.getLeftExpression(); - parserExpressionForFun(udfSet, leftExpression); - Expression rightExpression = binary.getRightExpression(); - parserExpressionForFun(udfSet, rightExpression); - } - } - - /** - * 获取自定义函数类 - * - * @param fun - * @param expressions - * @return - */ - private UDF getUDF(Function fun, List<Expression> expressions) { - UDF udf; - String funName = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_UNDERSCORE, fun.getName()); - try { - Class dialectClazz = Class.forName("com.mesalab.engine.component.udf." + CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_UNDERSCORE, fun.getName().toUpperCase())); - Constructor constructor = dialectClazz.getConstructor(UDFElements.class); - UDFElements udfElements = new UDFElements(funName, expressions); - udf = (UDF) constructor.newInstance(udfElements); - } catch (Exception e) { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), ResultCodeEnum.SQL_SYNTAX_ERROR.getCode(), "自定义函数转换实例异常！", e); - } - return udf; - } - - /** - * 库中包含表表名查询结果处理 - * - * @param tables - * @return - */ - private BaseResult tablesResultEncapsulation(List tables, String database) { - List<String> list = new ArrayList<>(); - Map<String, Object> date = new LinkedHashMap<>(); - date.put("type", "enum"); - date.put("name", database); - Iterator iterator = tables.iterator(); - while (iterator.hasNext()) { - list.add(String.valueOf(iterator.next())); - date.put("symbols", list); - } - BaseResult<Map> baseResult = BaseResultUtil.success("ok", null); - baseResult.setData(date); - return baseResult; - } - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/SystemServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/SystemServiceImpl.java deleted file mode 100644 index 7c11d58..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/SystemServiceImpl.java +++ /dev/null @@ -1,500 +0,0 @@ -package com.mesalab.engine.service.impl; - -import com.google.common.collect.Maps; -import com.mesalab.common.model.StorageDeletion; -import com.mesalab.common.model.XxlJobInfo; -import com.mesalab.engine.component.bean.ApiQueryBean; -import com.mesalab.common.dto.params.ExecutorParam; -import com.mesalab.common.dto.results.BaseResult; -import com.mesalab.common.enums.JobHandlerEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.common.util.BaseResultUtil; -import com.mesalab.engine.component.config.ClickHouseConfig; -import com.mesalab.engine.component.config.XxlJobConfig; -import com.mesalab.engine.service.HttpClientService; -import com.mesalab.engine.service.SqlEngineService; -import com.mesalab.engine.service.SystemService; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.Encodes; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cglib.beans.BeanMap; -import org.springframework.stereotype.Service; - -import java.lang.reflect.Field; -import java.nio.charset.Charset; -import java.util.*; - -@Slf4j -@Service("systemService") -@SuppressWarnings("unchecked") -public class SystemServiceImpl implements SystemService { - - - @Autowired - private SqlEngineService sqlEngineService; - @Autowired - private XxlJobConfig xxlJobConfig; - @Autowired - private HttpClientService httpClientService; - @Autowired - private ClickHouseConfig clickHouseConfig; - - private Map<String, String> headers = Maps.newHashMap(); - - @Override - public BaseResult getStorageQuota() { - //封装sql进行查询：Analytic Logs、Files、Traffic Logs - ApiQueryBean param = new ApiQueryBean(); - param.setQuery("(select log_type, used_size, max_size, TIME_FORMAT(MILLIS_TO_TIMESTAMP(1000 * last_storage),'YYYY-MM-dd') as first_storage from sys_storage_log where log_type = 'Report and Metrics' ORDER BY __time desc limit 1)\n" + - "UNION ALL\n" + - "(select log_type, used_size, max_size, TIME_FORMAT(MILLIS_TO_TIMESTAMP(1000 * last_storage),'YYYY-MM-dd') as first_storage from sys_storage_log where log_type = 'Files' ORDER BY __time desc limit 1)\n" + - "UNION ALL\n" + - "(select log_type, used_size, max_size,TIME_FORMAT(MILLIS_TO_TIMESTAMP(1000 * last_storage),'YYYY-MM-dd') as first_storage from sys_storage_log where log_type = 'Traffic Logs' ORDER BY __time desc limit 1)"); - BaseResult result = sqlEngineService.executeQuery(param); - return result; - } - - @Override - public BaseResult dailyTrendOfStorage(String searchStartTime, String searchEndTime) { - - Date currentDate = DateUtils.convertStringToDate(DateUtils.getCurrentDate(), DateUtils.YYYY_MM_DD); - - if (StringUtil.isBlank(searchStartTime) && StringUtil.isBlank(searchEndTime)) { - searchStartTime = DateUtils.getFormatDate(DateUtils.getSomeDate(currentDate, -7), DateUtils.YYYY_MM_DD_HH24_MM_SS); - searchEndTime = DateUtils.getFormatDate(currentDate, DateUtils.YYYY_MM_DD_HH24_MM_SS); - } - - if (StringUtil.isNotBlank(searchStartTime) && StringUtil.isNotBlank(searchEndTime)) { - ApiQueryBean param = new ApiQueryBean(); - String sql = String.format("select TIME_FORMAT(__time,'YYYY-MM-dd') as stat_time,log_type as type, sum(aggregate_size) as used_size from sys_storage_log where __time >= '%s' and __time < '%s' group by TIME_FORMAT(__time,'YYYY-MM-dd'), log_type", searchStartTime, searchEndTime); - param.setQuery(sql); - return sqlEngineService.executeQuery(param); - } else { - throw new BusinessException(ResultStatusEnum.FAIL.getCode(), "参数校验：start_time 与 end_time 需同时填写", null); - } - } - - @Override - public BaseResult deleteStorage(List<StorageDeletion> list) { - - if (!logTypeIsAvaliable(list)) { - return BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, "匹配失败, 请检查日志类型!"); - } - - preProcessOfLogType(list); - - if (jobIsBusy(list)) { - return BaseResultUtil.failure(HttpStatus.SC_LOCKED, "任务繁忙中, 请稍后尝试!"); - } - - BaseResult baseResult = null; - for (StorageDeletion info : list) { - baseResult = executeDeleteStorageJob(info); - } - return baseResult; - } - - @Override - public BaseResult getDeleteStorageStatus(String logType) { - setCookie(); - List list = new ArrayList<>(); - if (StringUtil.isBlank(logType)) { - - StorageDeletion trafficInfo = getStorageDeletionInfoByHandler(LogType.TRAFFIC_LOGS.getValue(), JobHandlerEnum.DELETE_TRAFFIC_DATA_JOB_HANDLER.getValue()); - list.add(trafficInfo); - - StorageDeletion reportInfo = getStorageDeletionInfoByHandler(LogType.REPORT_AND_METRICS.getValue(), JobHandlerEnum.DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue()); - list.add(reportInfo); - - StorageDeletion fileInfo = getStorageDeletionInfoByHandler(LogType.FILES.getValue(), JobHandlerEnum.DELETE_FILES_JOB_HANDLER.getValue()); - list.add(fileInfo); - } else if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { - StorageDeletion trafficInfo = getStorageDeletionInfoByHandler(logType, JobHandlerEnum.DELETE_TRAFFIC_DATA_JOB_HANDLER.getValue()); - list.add(trafficInfo); - } else if (LogType.REPORT_AND_METRICS.getValue().equalsIgnoreCase(logType)) { - StorageDeletion reportInfo = getStorageDeletionInfoByHandler(logType, JobHandlerEnum.DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue()); - list.add(reportInfo); - } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { - StorageDeletion reportInfo = getStorageDeletionInfoByHandler(logType, JobHandlerEnum.DELETE_FILES_JOB_HANDLER.getValue()); - list.add(reportInfo); - } else { - return BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, "没有匹配的日志类型: " + logType); - } - return BaseResultUtil.success("ok", list); - } - - @Override - public BaseResult deleteQueryTask(String queryId) { - String executeSql = "kill query where query_id = '" + queryId + "'"; - StringBuilder urlBuilder = new StringBuilder(clickHouseConfig.getQueryUrl()).append("/?"); - StringBuilder queryParamBuilder = new StringBuilder("user=") - .append(clickHouseConfig.getLongTermUserName()).append("&") - .append("password=").append(clickHouseConfig.getLongTermPassword()).append("&") - .append("database=").append(clickHouseConfig.getDbName()).append("&") - .append("query=").append(executeSql) - .append(" FORMAT ").append("JSON").append(" ;"); - - List<NameValuePair> values = URLEncodedUtils.parse(queryParamBuilder.toString(), Charset.forName("UTF-8")); - - Map<String, String> map = httpClientService.httpGet(urlBuilder.toString() + URLEncodedUtils.format(values, "utf-8")); - if (map.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - return BaseResultUtil.success4Message("ok"); - } else { - return BaseResultUtil.failure(Integer.parseInt(map.get("status")), map.get("result")); - } - } - - - private boolean logTypeIsAvaliable(List<StorageDeletion> list) { - - List<LogType> logTypes = Arrays.asList(LogType.values()); - List<String> logType = new ArrayList<>(); - for (LogType type : logTypes) { - logType.add(type.getValue().toUpperCase()); - } - for (StorageDeletion info : list) { - if (!logType.contains(info.getLogType().toUpperCase())) { - return false; - } - } - return true; - } - - private List<StorageDeletion> preProcessOfLogType(List<StorageDeletion> list) { - if (list.size() == 1 && LogType.ALL.getValue().equalsIgnoreCase(list.get(0).getLogType())) { - Integer maxDays = list.get(0).getMaxDays(); - - list.clear(); - list.add(new StorageDeletion(LogType.TRAFFIC_LOGS.getValue(), maxDays)); - list.add(new StorageDeletion(LogType.REPORT_AND_METRICS.getValue(), maxDays)); - list.add(new StorageDeletion(LogType.FILES.getValue(), maxDays)); - } - return list; - } - - private boolean jobIsBusy(List<StorageDeletion> list) { - setCookie(); - for (StorageDeletion info : list) { - - String handler = getDeletePartHandlerByLogType(info.getLogType()); - if (jobIsBusyByHandler(handler)) { - return true; - } - handler = getDeleteAllHandlerByLogType(info.getLogType()); - if (jobIsBusyByHandler(handler)) { - return true; - } - } - return false; - } - - private boolean jobIsBusyByHandler(String handler) { - Map dataByHandler = getDataByHandler(handler); - String id = String.valueOf(dataByHandler.get("id")); - BaseResult result = queryJobStatusByJobId(Integer.parseInt(id)); - if (result.getStatus().equals(HttpStatus.SC_LOCKED)) { - return true; - } else if (!result.getStatus().equals(HttpStatus.SC_OK)) { - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), "获取任务状态异常: " + result.getMessage(), null); - } - return false; - } - - - /** - * 执行日志删除任务: 调用调度任务任务 - * - * @param info - * @return - */ - private BaseResult executeDeleteStorageJob(StorageDeletion info) { - - BaseResult baseResult; - String jobHandler; - String logType = info.getLogType(); - ExecutorParam executorParam = new ExecutorParam(); - if (0 == info.getMaxDays()) { - jobHandler = getDeleteAllHandlerByLogType(logType); - Map dataByHandler = getDataByHandler(jobHandler); - - if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { - executorParam.setMaxdays(30); - } else if (LogType.REPORT_AND_METRICS.getValue().equalsIgnoreCase(logType)) { - executorParam.setMaxdays(365); - } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { - executorParam.setMaxdays(365); - } - XxlJobInfo xxlJobInfo = setXxlJobInfoParam(dataByHandler, executorParam); - baseResult = executeTriggerAndUpdate(xxlJobInfo); - - if (baseResult.getStatus().equals(HttpStatus.SC_OK)) { - String deletePartHandler = getDeletePartHandlerByLogType(info.getLogType()); - Map date = getDataByHandler(deletePartHandler); - XxlJobInfo xxlJobInfoOfUpdate = setXxlJobInfoParam(date, executorParam); - baseResult = executeManageJob("update", xxlJobInfoOfUpdate); - } - } else { - jobHandler = getDeletePartHandlerByLogType(info.getLogType()); - executorParam.setMaxdays(info.getMaxDays()); - Map dataByHandler = getDataByHandler(jobHandler); - XxlJobInfo xxlJobInfo = setXxlJobInfoParam(dataByHandler, executorParam); - baseResult = executeTriggerAndUpdate(xxlJobInfo); - } - if (!baseResult.getStatus().equals(HttpStatus.SC_OK)) { - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), "调度任务数据配额设置任务执行失败", null); - } - return baseResult; - } - - private String getDeleteAllHandlerByLogType(String logType) { - String jobHandler = StringUtil.EMPTY; - if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_ALL_TRAFFIC_DATA_JOB_HANDLER.getValue(); - } else if (LogType.REPORT_AND_METRICS.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_ALL_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue(); - } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_ALL_FILES_JOB_HANDLER.getValue(); - } - return jobHandler; - } - - private String getDeletePartHandlerByLogType(String logType) { - String jobHandler = StringUtil.EMPTY; - if (LogType.TRAFFIC_LOGS.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_TRAFFIC_DATA_JOB_HANDLER.getValue(); - } else if (LogType.REPORT_AND_METRICS.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_REPORT_AND_METRICS_DATA_JOB_HANDLER.getValue(); - } else if (LogType.FILES.getValue().equalsIgnoreCase(logType)) { - jobHandler = JobHandlerEnum.DELETE_FILES_JOB_HANDLER.getValue(); - } - return jobHandler; - } - - /** - * 通过handler获取数据配额设置状态 - * - * @param logType - * @param jobHandlerValue - * @return - */ - private StorageDeletion getStorageDeletionInfoByHandler(String logType, String jobHandlerValue) { - StorageDeletion storageDeletionInfo = new StorageDeletion(); - Map trafficDate = getDataByHandler(jobHandlerValue); - XxlJobInfo xxlJobInfo = mapToBean(trafficDate, XxlJobInfo.class); - Map executorParam = (Map) com.zdjizhi.utils.JsonMapper.fromJsonString(xxlJobInfo.getExecutorParam(), Map.class); - Object maxDays = executorParam.get("maxdays"); - storageDeletionInfo.setLogType(logType); - storageDeletionInfo.setMaxDays(Integer.parseInt(String.valueOf(maxDays))); - return storageDeletionInfo; - } - - /** - * 触发执行器并更新任务 - * - * @param xxlJobInfo - * @return - */ - private BaseResult executeTriggerAndUpdate(XxlJobInfo xxlJobInfo) { - BaseResult baseResult; - BaseResult resultExecute = executeManageJob("trigger", xxlJobInfo); - if (resultExecute.getStatus().equals(HttpStatus.SC_OK)) { - BaseResult resultUpdate = executeManageJob("update", xxlJobInfo); - if (resultUpdate.getStatus().equals(HttpStatus.SC_OK)) { - baseResult = BaseResultUtil.success("ok", null); - } else { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, "更新任务失败: " + xxlJobInfo.getExecutorHandler()); - } - } else { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, "执行任务失败: " + xxlJobInfo.getExecutorHandler()); - } - return baseResult; - } - - /** - * 查询任务状态 - * - * @param jobId - * @return - */ - private BaseResult queryJobStatusByJobId(int jobId) { - BaseResult baseResult = null; - StringBuilder url = new StringBuilder(xxlJobConfig.getUrl()). - append("/jobinfo/jobBeat?jobId="). - append(jobId); - Map<String, String> resultMap = httpClientService.httpGet(url.toString(), headers); - log.info("请求调度任务jobBeat接口" + url); - baseResult = resultEncapsulationOfJob(resultMap); - return baseResult; - } - - /** - * 执行调度任务结果封装 - * - * @param resultMap - * @return - */ - private BaseResult resultEncapsulationOfJob(Map<String, String> resultMap) { - BaseResult baseResult; - if (StringUtil.isEmpty(resultMap)) { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, "服务繁忙，请联系调度平台方！"); - } else { - if (resultMap.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - Map result = (Map) com.zdjizhi.utils.JsonMapper.fromJsonString(String.valueOf(resultMap.get("result")), Map.class); - if (result.get("code").equals(HttpStatus.SC_OK)) { - baseResult = BaseResultUtil.success("ok", null); - } else if (result.get("code").equals(HttpStatus.SC_LOCKED)) { - baseResult = BaseResultUtil.failure(HttpStatus.SC_LOCKED, "资源被锁定: " + result.get("msg")); - } else { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, String.valueOf(result.get("msg"))); - } - } else { - baseResult = BaseResultUtil.failure(HttpStatus.SC_INTERNAL_SERVER_ERROR, resultMap.get("msg")); - } - } - return baseResult; - } - - - /** - * 执行调度平台任务更新 - * - * @param executeType - * @param xxlJobInfo - * @return - */ - private BaseResult executeManageJob(String executeType, XxlJobInfo xxlJobInfo) { - BaseResult baseResult = null; - String params = getUrlParamsByMap(getObjectToMap(xxlJobInfo)); - String url = String.format("%s/jobinfo/%s/?%s", xxlJobConfig.getUrl(), executeType, params); - Map<String, String> resultMap = httpClientService.httpGet(url, headers); - log.info("请求调度任务" + executeType + "接口" + url); - baseResult = resultEncapsulationOfJob(resultMap); - return baseResult; - } - - - /** - * 获取调度平台Cookie - * - * @return - */ - public void setCookie() { - StringBuilder url = new StringBuilder(xxlJobConfig.getUrl()); - headers.put("Content-Type", "application/json"); - String urlParamsByMap = getUrlParamsByMap(getObjectToMap(xxlJobConfig)); - Map httpPostResponseHeads = httpClientService.getHttpPostResponseHeads(url + "/login?" + urlParamsByMap, headers); - String cookie = String.valueOf(httpPostResponseHeads.get("SET-COOKIE")); - headers.put("Cookie", cookie); - } - - /** - * 通过handler Value获取jobInfo数据 - * - * @param handlerValue - * @return - */ - private Map getDataByHandler(String handlerValue) { - StringBuilder url = new StringBuilder(xxlJobConfig.getUrl()). - append("/jobinfo/pageList?jobGroup=-1&triggerStatus=-1&executorHandler="). - append(handlerValue); - Map<String, String> resultPageList = httpClientService.httpGet(url.toString(), headers); - log.info("请求调度任务pageList接口 " + url); - if (StringUtil.isNotEmpty(resultPageList) && resultPageList.get("status").equals(String.valueOf(HttpStatus.SC_OK))) { - Map<String, Object> maps = (Map<String, Object>) JsonMapper.fromJsonString(resultPageList.get("result"), Map.class); - if (StringUtil.isEmpty(maps)) { - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), "获取或封装结果异常: " + resultPageList, null); - } - List<Map> data = (List) maps.get("data"); - if (data.size() >= 1) { - return data.get(0); - } else { - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), "调度任务无匹配executorHandler(" + handlerValue + ")的任务信息", null); - } - } - throw new BusinessException(ResultStatusEnum.SERVER_ERROR.getCode(), "获取调度任务异常: " + resultPageList, null); - } - - /** - * 更新jobInfo - * - * @param data - * @param executorParam - * @return - */ - private XxlJobInfo setXxlJobInfoParam(Map data, ExecutorParam executorParam) { - XxlJobInfo xxlJobInfo = mapToBean(data, XxlJobInfo.class); - xxlJobInfo.setExecutorParam(JsonMapper.toJsonString(executorParam)); - return xxlJobInfo; - } - - public static String getUrlParamsByMap(Map<String, Object> map) { - if (map == null) { - return ""; - } - StringBuffer sb = new StringBuffer(); - for (Map.Entry<String, Object> entry : map.entrySet()) { - sb.append(entry.getKey() + "=" + Encodes.urlEncode(String.valueOf(entry.getValue()))); - sb.append("&"); - } - String s = sb.toString(); - if (s.endsWith("&")) { - s = StringUtil.substringBeforeLast(s, "&"); - } - return s; - } - - public static <T> T mapToBean(Map<String, Object> map, Class<T> clazz) { - T bean = null; - try { - bean = clazz.newInstance(); - } catch (InstantiationException e) { - log.error("mapToBean Error:{}",e.getMessage()); - } catch (IllegalAccessException e) { - log.error("mapToBean Error:{}",e.getMessage()); - } - BeanMap beanMap = BeanMap.create(bean); - beanMap.putAll(map); - return bean; - } - - public static Map<String, Object> getObjectToMap(Object obj) { - Map<String, Object> map = new LinkedHashMap<String, Object>(); - Class<?> clazz = obj.getClass(); - for (Field field : clazz.getDeclaredFields()) { - field.setAccessible(true); - String fieldName = field.getName(); - Object value = null; - try { - value = field.get(obj); - } catch (IllegalAccessException e) { - log.error(e.getMessage()); - } - if (value == null) { - value = ""; - } - map.put(fieldName, value); - } - return map; - } - - - enum LogType { - TRAFFIC_LOGS("Traffic Logs"), REPORT_AND_METRICS("Report and Metrics"), FILES("Files"), ALL("ALL"); - private final String value; - - LogType(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - } -} diff --git a/galaxy-query-engine/src/main/resources/banner.txt b/galaxy-query-engine/src/main/resources/banner.txt deleted file mode 100644 index c94616e..0000000 --- a/galaxy-query-engine/src/main/resources/banner.txt +++ /dev/null @@ -1,17 +0,0 @@ -${AnsiColor.BRIGHT_GREEN} -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -${AnsiColor.BRIGHT_BLUE} - _ _ _ _ - | | | | | | (_) - __ _ __ _| | __ ___ ___ _ ______ __| | __ _| |_ __ _ ______ ___ _ __ __ _ _ _ __ ___ - / _` |/ _` | |/ _` \ \/ / | | |______/ _` |/ _` | __/ _` |______/ _ \ '_ \ / _` | | '_ \ / _ \ - | (_| | (_| | | (_| |> <| |_| | | (_| | (_| | || (_| | | __/ | | | (_| | | | | | __/ - \__, |\__,_|_|\__,_/_/\_\\__, | \__,_|\__,_|\__\__,_| \___|_| |_|\__, |_|_| |_|\___| - __/ | __/ | __/ | - |___/ |___/ |___/ -${AnsiColor.BRIGHT_BLUE} -Spring Boot Version: ${spring-boot.version} -${AnsiColor.BRIGHT_GREEN} -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -${AnsiColor.DEFAULT} - diff --git a/galaxy-query-engine/src/main/resources/validate-dsl.json b/galaxy-query-engine/src/main/resources/validate-dsl.json deleted file mode 100644 index fef7e25..0000000 --- a/galaxy-query-engine/src/main/resources/validate-dsl.json +++ /dev/null @@ -1,221 +0,0 @@ -{ - "title": "服务推荐", - "description": "查询", - "type": "object", - "properties": { - "clientId": { - "description": "唯一标识符", - "maximum": 2147483647, - "minimum": -2147483648, - "type": [ - "null", - "integer" - ] - }, - "query": { - "description": "查询条件", - "type": "object", - "required": [ - "dataSource" - ], - "properties": { - "queryType": { - "description": "查询类型", - "maxLength": 65535, - "type": "string" - }, - "dataSource": { - "description": "查询数据源", - "maxLength": 65535, - "type": "string" - }, - "parameters": { - "description": "查询参数", - "type": "object", - "properties": { - "match": { - "description": "模糊查询参数", - "type": "array", - "items": [ - { - "type": "object", - "required": [ - "type", - "fieldKey", - "fieldValues" - ], - "properties": { - "type": { - "description": "匹配符号", - "maxLength": 65535, - "type": "string" - }, - "fieldKey": { - "description": "查询字段", - "maxLength": 65535, - "type": "string" - }, - "fieldValues": { - "description": "查询参数", - "type": "array", - "items": { - "maxLength": 65535, - "type": "string" - } - } - } - } - ], - "additionalItems": { - "type": "object", - "required": [ - "type", - "fieldKey", - "fieldValues" - ], - "properties": { - "type": { - "description": "匹配符号", - "maxLength": 65535, - "type": "string" - }, - "fieldKey": { - "description": "查询字段", - "maxLength": 65535, - "type": "string" - }, - "fieldValues": { - "description": "查询参数", - "type": "array", - "items": { - "maxLength": 65535, - "type": "string" - } - } - } - } - }, - "range": { - "description": "范围查询参数", - "type": "array", - "items": { - "type": "object", - "properties": { - "type": { - "description": "匹配符号", - "maxLength": 65535, - "type": "string" - }, - "fieldKey": { - "description": "查询字段", - "maxLength": 65535, - "type": "string" - }, - "fieldValues": { - "description": "查询参数", - "type": "array", - "items": { - "maximum": 2147483647, - "minimum": -2147483648, - "maxLength": 65535, - "type": [ - "integer", - "string" - ] - } - } - } - }, - "additionalItems": { - "type": "object", - "properties": { - "type": { - "description": "匹配符号", - "maxLength": 65535, - "type": "string" - }, - "fieldKey": { - "description": "查询字段", - "maxLength": 65535, - "type": "string" - }, - "fieldValues": { - "description": "查询参数", - "type": "array", - "items": { - "maximum": 2147483647, - "minimum": -2147483648, - "maxLength": 65535, - "type": [ - "integer", - "string" - ] - } - } - } - } - }, - "intervals": { - "description": "查询时间", - "type": "array", - "items": { - "maxLength": 65535, - "type": "string" - }, - "additionalItems": { - "maxLength": 65535, - "type": "string" - } - }, - "sort": { - "description": "排序", - "type": "array", - "items": { - "type": "object", - "required": ["type","fieldKey"], - "properties": { - "type": { - "description": "匹配符号", - "maxLength": 65535, - "type": "string" - }, - "fieldKey": { - "description": "排序字段", - "maxLength": 65535, - "type": "string" - } - } - }, - "additionalItems": { - "type": "object", - "required": ["type","fieldKey"], - "properties": { - "type": { - "description": "匹配符号", - "maxLength": 65535, - "type": "string" - }, - "fieldKey": { - "description": "排序字段", - "maxLength": 65535, - "type": "string" - } - } - } - }, - "limit": { - "description": "查询条数", - "maxLength": 65535, - "maximum": 2147483647, - "minimum": 0, - "type": [ - "integer", - "string" - ] - } - } - } - } - } - } -}
\ No newline at end of file @@ -31,23 +31,27 @@ </distributionManagement> <modules> - <!--公共组件--> + <!--公共模块--> <module>galaxy-common</module> <!--查询网关--> <module>galaxy-gateway</module> - <!--查询引擎--> - <module>galaxy-query-engine</module> <!--认证中心--> <module>galaxy-auth-center</module> <!--监控服务--> <module>galaxy-admin-server</module> <!--业务API--> <module>galaxy-business-api</module> + <!--文件服务--> <module>galaxy-hos-service</module> + <!--报表服务--> <module>galaxy-report-service</module> + <!--定时任务后台管理--> <module>galaxy-job-admin</module> + <!--定时任务执行器--> <module>galaxy-job-executor</module> + <!--定时任务核心模块--> <module>galaxy-job-core</module> + <module>galaxy-query-engine</module> </modules> <properties> |