diff options
| author | liuyongqiang <[email protected]> | 2020-12-09 10:46:35 +0800 |
|---|---|---|
| committer | liuyongqiang <[email protected]> | 2020-12-09 10:46:35 +0800 |
| commit | 1c4c5f826211e9572202ce7b9085dad7cd5a61dc (patch) | |
| tree | 46c271930f7e51ae2e44d69f7fd4ce01462576e7 | |
| parent | 886eea35087d567abee452bc7e9a0f3dd606e764 (diff) | |
galaxy-report-service迁移完成
43 files changed, 5055 insertions, 1 deletions
diff --git a/galaxy-query-engine/config/logback-spring.xml b/galaxy-query-engine/config/logback-spring.xml index b4e4240..166e52e 100644 --- a/galaxy-query-engine/config/logback-spring.xml +++ b/galaxy-query-engine/config/logback-spring.xml @@ -38,6 +38,9 @@ <level>DEBUG</level> </filter> </appender> + + <logger name="org.apache.hadoop.util.Shell" level="ERROR"/> + <root level="info"> <appender-ref ref="ALL_FILE" /> <appender-ref ref="STDOUT" /> diff --git a/galaxy-report-service/config/application.yml b/galaxy-report-service/config/application.yml index 3d122a8..5837501 100644 --- a/galaxy-report-service/config/application.yml +++ b/galaxy-report-service/config/application.yml @@ -4,6 +4,36 @@ server: spring: application: name: galaxy-report-service + datasource: + name: druidDataSource + url: jdbc:mariadb://192.168.44.3:3306/tsg-bifang?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false + username: root + password: 111111 + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: org.mariadb.jdbc.Driver + druid: + aop-patterns: com.mesalab.report.controller.*,com.mesalab.report.service.*,com.mesalab.report.mapper.* + connection-properties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 + filters: stat,wall,slf4j #配置监控统计拦截的filters，去掉后监控界面SQL无法进行统计，’wall’用于防火墙 + initial-size: 2 #初始化连接数 + max-active: 30 #最大连接数 + max-wait: 600000 #获取连接最大超时时间 + min-evictable-idle-time-millis: 300000 # 一个连接在池中最小生存的时间，单位是毫秒 + min-idle: 1 #最小连接数 + stat-view-servlet: + enabled: true #是否开启Druid监控信息显示页面 + login-password: admin + login-username: admin + reset-enable: false + url-pattern: /druid/* + test-on-borrow: true #申请连接时执行validationQuery检测连接是否有效，做了这个配置会降低性能 + test-on-return: false + test-while-idle: true # 执行validationQuery检测连接是否有效 + time-between-eviction-runs-millis: 60000 #间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒 + validation-query: select 1 #验证连接是否可用,在数据库中执行一条sql + web-stat-filter: + enabled: true #是否开启WebStatFilter + exclusions: '*.js,*.gif,*.jpg,*.png,*.css,*.ico,/druid/*' #设置不统计哪些URL(用于排除一些不必要的url) cloud: consul: host: 192.168.44.12 @@ -24,4 +54,45 @@ management: show-details: always ## log file path config logging: - config: ./config/logback-spring.xml
\ No newline at end of file + config: ./config/logback-spring.xml +## http connect config +http: + connectTimeout: 10000 #创建连接的最长时间 + connectionRequestTimeout: 10000 #从连接池中获取到连接的最长时间 + defaultMaxPerRoute: 100 #并发数 + maxTotal: 300 #最大连接数 + socketTimeout: 21605000 #数据传输的最长时间 + socketTimeoutShort: 30000 + staleConnectionCheckEnabled: true #提交请求前测试连接是否可用 +## hbase config +hbase: + client_retries_number: 3 + connect_pool: 10 + rpc_timeout: 100000 + table: tsg:report_result + zookeeper_property_clientPort: 2181 + zookeeper_quorum: 192.168.44.12 + zookeeper_znode_parent: /hbase +## mybatis config +mybatis: + mapperLocations: classpath*:/mappers/*.xml + typeAliasesPackage: com.mesalab.report.bean +## scan config +scan: + result: + scheduled: + plan: 0/10 * * * * ? #更新进度条的时间10s +## ck config +ck: + gateway_ip: 192.168.44.12:9999 #查询网关ip +globle: + job_thread: 2 #同时间执行是线程数 +## zookeeper config +zookeeper: + connectString: 192.168.44.12:2181 + connectionTimeoutMs: 50000 + elapsedTimeMs: 10000 + nameSpace: reportservice + open: 1 #是否启用zookeeper 0启用(集群) 1禁用(单机) + retryCount: 5 + sessionTimeoutMs: 50000
\ No newline at end of file diff --git a/galaxy-report-service/config/logback-spring.xml b/galaxy-report-service/config/logback-spring.xml index a1de056..a56f65b 100644 --- a/galaxy-report-service/config/logback-spring.xml +++ b/galaxy-report-service/config/logback-spring.xml @@ -38,6 +38,9 @@ <level>DEBUG</level> </filter> </appender> + + <logger name="org.apache.hadoop.util.Shell" level="ERROR"/> + <root level="info"> <appender-ref ref="ALL_FILE" /> <appender-ref ref="STDOUT" /> diff --git a/galaxy-report-service/pom.xml b/galaxy-report-service/pom.xml index d90b294..26158d7 100644 --- a/galaxy-report-service/pom.xml +++ b/galaxy-report-service/pom.xml @@ -30,6 +30,101 @@ <groupId>org.springframework.cloud</groupId> <artifactId>spring-cloud-starter-consul-discovery</artifactId> </dependency> + <dependency> + <groupId>org.mybatis.spring.boot</groupId> + <artifactId>mybatis-spring-boot-starter</artifactId> + <version>${mybatis-spring-boot-starter.version}</version> + </dependency> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-configuration-processor</artifactId> + <optional>true</optional> + </dependency> + <dependency> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + <version>${org.mariadb.version}</version> + </dependency> + <dependency> + <groupId>com.alibaba</groupId> + <artifactId>druid-spring-boot-starter</artifactId> + <version>${starter.druid.version}</version> + </dependency> + <dependency> + <groupId>com.alibaba</groupId> + <artifactId>fastjson</artifactId> + <version>1.2.69</version> + </dependency> + <dependency> + <groupId>ru.yandex.clickhouse</groupId> + <artifactId>clickhouse-jdbc</artifactId> + <version>0.1.51</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>4.5.2</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <version>3.5</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-framework</artifactId> + <version>2.12.0</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-recipes</artifactId> + <version>2.12.0</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>3.4.9</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>${hbase.client.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>2.7.1</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> <build> diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/GalaxyReportService.java b/galaxy-report-service/src/main/java/com/mesalab/report/GalaxyReportService.java index accf9a7..1b7f336 100644 --- a/galaxy-report-service/src/main/java/com/mesalab/report/GalaxyReportService.java +++ b/galaxy-report-service/src/main/java/com/mesalab/report/GalaxyReportService.java @@ -1,9 +1,17 @@ package com.mesalab.report; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; +import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.web.servlet.ServletComponentScan; import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; +@EnableScheduling +@ServletComponentScan @EnableDiscoveryClient @SpringBootApplication public class GalaxyReportService { @@ -11,4 +19,9 @@ public class GalaxyReportService { public static void main(String[] args) { SpringApplication.run(GalaxyReportService.class); } + + @Bean + MeterRegistryCustomizer<MeterRegistry> configurer(@Value("${spring.application.name}") String applicationName){ + return registry -> registry.config().commonTags("application", applicationName); + } } diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/bean/DBTypeEnum.java b/galaxy-report-service/src/main/java/com/mesalab/report/bean/DBTypeEnum.java new file mode 100644 index 0000000..0505c81 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/bean/DBTypeEnum.java @@ -0,0 +1,17 @@ +package com.mesalab.report.bean; + + +public enum DBTypeEnum { + CLICKHOUSE("clickhouse"), + DRUID("druid"), + HBASE("hbase"),; + private String value; + + DBTypeEnum(String value) { + this.value = value; + } + + public String getValue() { + return value; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/bean/HttpResult.java b/galaxy-report-service/src/main/java/com/mesalab/report/bean/HttpResult.java new file mode 100644 index 0000000..24c293e --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/bean/HttpResult.java @@ -0,0 +1,35 @@ +package com.mesalab.report.bean; + +/** + * Created by wk1 on 2019/5/15. + */ +public class HttpResult { + + private int code; + private String body; + // private byte[] bodybyte; + + + + public HttpResult() { + + } + + + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getBody() { + return body; + } + + public void setBody(String body) { + this.body = body; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/bean/JobEntity.java b/galaxy-report-service/src/main/java/com/mesalab/report/bean/JobEntity.java new file mode 100644 index 0000000..8f044ff --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/bean/JobEntity.java @@ -0,0 +1,263 @@ +package com.mesalab.report.bean; + +/** + * Created by wk1 on 2019/5/20. + */ +public class JobEntity implements Cloneable { + + private Integer resultId; + + private String resultName; + + private Integer jobId; + + private String startTime; + + private String endTime; + + private String querySql; + + private Integer status; + + private Integer excuteTime; + + private Long excuteRow; + + private Integer excuteProcess; + + private String excuteDetail; + + private Integer isSendNotice; + + private String opTime; + + private String chartType; + + private String issuedTime; + + private String query_id; + + private String query_duration_ms; + + private int resultRows; + + + private String memory_usage; + + private String result; + + private int excute_status; + + private String formatSql; + + + private int isValid; + + private long beginTime; + + public Integer getResultId() { + return resultId; + } + + public void setResultId(Integer resultId) { + this.resultId = resultId; + } + + public String getResultName() { + return resultName; + } + + public void setResultName(String resultName) { + this.resultName = resultName; + } + + public Integer getJobId() { + return jobId; + } + + public void setJobId(Integer jobId) { + this.jobId = jobId; + } + + public String getStartTime() { + return startTime; + } + + public void setStartTime(String startTime) { + this.startTime = startTime; + } + + public String getEndTime() { + return endTime; + } + + public void setEndTime(String endTime) { + this.endTime = endTime; + } + + public String getQuerySql() { + return querySql; + } + + public void setQuerySql(String querySql) { + this.querySql = querySql; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Integer getExcuteTime() { + return excuteTime; + } + + public void setExcuteTime(Integer excuteTime) { + this.excuteTime = excuteTime; + } + + public Long getExcuteRow() { + return excuteRow; + } + + public void setExcuteRow(Long excuteRow) { + this.excuteRow = excuteRow; + } + + public Integer getExcuteProcess() { + return excuteProcess; + } + + public void setExcuteProcess(Integer excuteProcess) { + this.excuteProcess = excuteProcess; + } + + public String getExcuteDetail() { + return excuteDetail; + } + + public void setExcuteDetail(String excuteDetail) { + this.excuteDetail = excuteDetail; + } + + public Integer getIsSendNotice() { + return isSendNotice; + } + + public void setIsSendNotice(Integer isSendNotice) { + this.isSendNotice = isSendNotice; + } + + public String getOpTime() { + return opTime; + } + + public void setOpTime(String opTime) { + this.opTime = opTime; + } + + public String getChartType() { + return chartType; + } + + public void setChartType(String chartType) { + this.chartType = chartType; + } + + public String getIssuedTime() { + return issuedTime; + } + + public void setIssuedTime(String issuedTime) { + this.issuedTime = issuedTime; + } + + public String getQuery_id() { + return query_id; + } + + public void setQuery_id(String query_id) { + this.query_id = query_id; + } + + public String getQuery_duration_ms() { + return query_duration_ms; + } + + public void setQuery_duration_ms(String query_duration_ms) { + this.query_duration_ms = query_duration_ms; + } + + + public String getMemory_usage() { + return memory_usage; + } + + public void setMemory_usage(String memory_usage) { + this.memory_usage = memory_usage; + } + + public String getResult() { + return result; + } + + public void setResult(String result) { + this.result = result; + } + + public int getExcute_status() { + return excute_status; + } + + public void setExcute_status(int excute_status) { + this.excute_status = excute_status; + } + + + public int getResultRows() { + return resultRows; + } + + public void setResultRows(int resultRows) { + this.resultRows = resultRows; + } + + public String getFormatSql() { + return formatSql; + } + + public void setFormatSql(String formatSql) { + this.formatSql = formatSql; + } + + + public int getIsValid() { + return isValid; + } + + public void setIsValid(int isValid) { + this.isValid = isValid; + } + + public long getBeginTime() { + return beginTime; + } + + public void setBeginTime(long beginTime) { + this.beginTime = beginTime; + } + + @Override + public Object clone() { + JobEntity o = null; + try { + o = (JobEntity) super.clone(); + } catch (CloneNotSupportedException e) { + System.out.println(e.toString()); + } + return o; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/bean/MonitorEntity.java b/galaxy-report-service/src/main/java/com/mesalab/report/bean/MonitorEntity.java new file mode 100644 index 0000000..6750540 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/bean/MonitorEntity.java @@ -0,0 +1,72 @@ +package com.mesalab.report.bean; + +import java.util.Map; + +/** + * Created by wk1 on 2019/5/15. + */ +public class MonitorEntity { + + private Long queueJobNum; + private Long excuteingJobNum; + private Long todaySuccessJobNum; + private Long todayErrorJobNum; + + private Map<String,JobEntity> joblist ; + + private String status; + + + public MonitorEntity() { + + } + + public Long getQueueJobNum() { + return queueJobNum; + } + + public void setQueueJobNum(Long queueJobNum) { + this.queueJobNum = queueJobNum; + } + + public Long getExcuteingJobNum() { + return excuteingJobNum; + } + + public void setExcuteingJobNum(Long excuteingJobNum) { + this.excuteingJobNum = excuteingJobNum; + } + + public Long getTodaySuccessJobNum() { + return todaySuccessJobNum; + } + + public void setTodaySuccessJobNum(Long todaySuccessJobNum) { + this.todaySuccessJobNum = todaySuccessJobNum; + } + + public Long getTodayErrorJobNum() { + return todayErrorJobNum; + } + + public void setTodayErrorJobNum(Long todayErrorJobNum) { + this.todayErrorJobNum = todayErrorJobNum; + } + + public Map<String, JobEntity> getJoblist() { + return joblist; + } + + public void setJoblist(Map<String, JobEntity> joblist) { + this.joblist = joblist; + } + + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ClickhouseConfig.java b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ClickhouseConfig.java new file mode 100644 index 0000000..d2c36ee --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ClickhouseConfig.java @@ -0,0 +1,66 @@ +package com.mesalab.report.configuration; + +import com.mesalab.report.util.Logs; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + +/** + * Created by wk1 on 2019/5/17. + */ +@Component +@ConfigurationProperties(prefix = "ck") +public class ClickhouseConfig { + + private static String gateway_ip; + public static String getJobUrl(String sql, Integer reportId) { + String url = "http://" + gateway_ip.trim() + "/?option=long-term&reportId=" + reportId + "&query="; + String jobsql = ""; + try { + + sql = URLEncoder.encode(sql , "utf8").replaceAll("\\+", "%20"); + jobsql = url + sql; + + } catch (UnsupportedEncodingException e) { + Logs.error(e.toString()); + } + return jobsql; + } + + public static String getProcessUrl(String query_id) { + + String url = "http://" + gateway_ip.trim() + "/?query="; + String processsql = ""; + try { + String sql = URLEncoder.encode("select elapsed,total_rows_approx,read_rows from `system`.processes where query_id='" + query_id + "'", "utf8").replaceAll("\\+", "%20"); + processsql = url + sql; + } catch (UnsupportedEncodingException e) { + Logs.error(e.toString()); + } + return processsql; + } + + public static String getFinishUrl(String query_id) { + + String url = "http://" + gateway_ip.trim() + "/?query="; + String finishsql = ""; + try { + String sql = URLEncoder.encode("select CAST(type, 'Int8') as type,read_rows,query_duration_ms,query,exception,memory_usage,event_time,result_rows,result_bytes from `system`.query_log where type>1 and query_id='" + query_id + "' order by event_time desc limit 1", "utf8").replaceAll("\\+", "%20"); + finishsql = url + sql; + } catch (UnsupportedEncodingException e) { + Logs.error(e.toString()); + } + return finishsql; + } + + public static String getKillUrl(String query_id) { + + String url = "http://" + gateway_ip.trim() + "/sys/engine/tasks/"+query_id; + return url; + } + public void setGateway_ip(String gateway_ip) { + ClickhouseConfig.gateway_ip = gateway_ip; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ExecutorConfig.java b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ExecutorConfig.java new file mode 100644 index 0000000..8a8996c --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ExecutorConfig.java @@ -0,0 +1,38 @@ +package com.mesalab.report.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.scheduling.config.ScheduledTaskRegistrar; +import org.springframework.stereotype.Component; + +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * Created by wk on 2020/8/28. + */ +@Configuration +@EnableAsync +public class ExecutorConfig { + + @Bean + public Executor taskExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(2); + executor.setMaxPoolSize(20); + executor.initialize(); + return executor; + } + @Bean + public Executor resultExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(2); + executor.setMaxPoolSize(20); + executor.initialize(); + return executor; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/configuration/GlobelConfig.java b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/GlobelConfig.java new file mode 100644 index 0000000..201b054 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/GlobelConfig.java @@ -0,0 +1,25 @@ +package com.mesalab.report.configuration; + +import com.mesalab.report.bean.JobEntity; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Created by wk1 on 2019/5/28. + */ + +@Component +@ConfigurationProperties(prefix = "globle") +public class GlobelConfig { + + public static int job_thread; + public static Map<String, JobEntity> mapresult = new ConcurrentHashMap<>(); + public void setJob_thread(int job_thread) { + GlobelConfig.job_thread = job_thread; + } + public final static String zkuuid = UUID.randomUUID().toString().replaceAll("-",""); +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HbaseConfig.java b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HbaseConfig.java new file mode 100644 index 0000000..db936b6 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HbaseConfig.java @@ -0,0 +1,85 @@ +package com.mesalab.report.configuration; + +import com.mesalab.report.bean.JobEntity; +import com.mesalab.report.util.StringUtil; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Created by wk1 on 2019/5/27. + */ +@Component +@ConfigurationProperties(prefix = "hbase") +public class HbaseConfig { + + private static String columefamily; + private static String url; + private static String table; + private static String colume_job_id; + private static String colume_query_id; + private static String colume_query_duration_ms; + private static String colume_read_rows; + private static String colume_memory_usage; + private static String colume_exception; + private static String colume_sql; + + public static String getHbaseJson(String hbaseid, String responsebody, JobEntity job) { + + String hbasejson = ""; + if (job.getStatus() == 2) { + hbasejson = "{\"Row\":[{\"key\":\"" + StringUtil.getBase64(hbaseid) + "\", \"Cell\": [{\"column\":\"" + StringUtil.getBase64(columefamily.trim()) + "\",\"$\":\"" + StringUtil.getBase64(responsebody.trim()) + "\"},{\"column\":\"" + StringUtil.getBase64(colume_job_id.trim()) + "\", \"$\":\"" + StringUtil.getBase64(String.valueOf(job.getResultId())) + "\"},{\"column\":\"" + StringUtil.getBase64(colume_query_id.trim()) + "\", \"$\":\"" + StringUtil.getBase64(job.getQuery_id().trim()) + "\"},{\"column\":\"" + StringUtil.getBase64(colume_query_duration_ms.trim()) + "\", \"$\":\"" + StringUtil.getBase64(job.getQuery_duration_ms().trim()) + "\"},{\"column\":\"" + StringUtil.getBase64(colume_read_rows.trim()) + "\", \"$\":\"" + StringUtil.getBase64(String.valueOf(job.getResultRows())) + "\"},{\"column\":\"" + StringUtil.getBase64(colume_memory_usage.trim()) + "\", \"$\":\"" + StringUtil.getBase64(job.getMemory_usage().trim()) + "\"}]}]}"; + } else { + hbasejson = "{\"Row\":[{\"key\":\"" + StringUtil.getBase64(hbaseid) + "\", \"Cell\": [{\"column\":\"" + StringUtil.getBase64(columefamily.trim()) + "\",\"$\":\"" + StringUtil.getBase64(responsebody.trim()) + "\"},{\"column\":\"" + StringUtil.getBase64(colume_sql.trim()) + "\", \"$\":\"" + StringUtil.getBase64(String.valueOf(job.getQuerySql())) + "\"},{\"column\":\"" + StringUtil.getBase64(colume_exception.trim()) + "\", \"$\":\"" + StringUtil.getBase64(job.getExcuteDetail().trim()) + "\"}]}]}"; + + } + return hbasejson; + } + + public static String getHbasePostUrl() { + + String hbaseurl = "http://" + url + "/" + table + "/rssss"; + + return hbaseurl; + } + + public void setColumefamily(String columefamily) { + HbaseConfig.columefamily = columefamily; + } + + public void setUrl(String url) { + HbaseConfig.url = url; + } + + public void setTable(String table) { + HbaseConfig.table = table; + } + + public void setColume_query_id(String colume_query_id) { + HbaseConfig.colume_query_id = colume_query_id; + } + + public void setColume_query_duration_ms(String colume_query_duration_ms) { + HbaseConfig.colume_query_duration_ms = colume_query_duration_ms; + } + + public void setColume_read_rows(String colume_read_rows) { + HbaseConfig.colume_read_rows = colume_read_rows; + } + + public void setColume_memory_usage(String colume_memory_usage) { + HbaseConfig.colume_memory_usage = colume_memory_usage; + } + + public void setColume_job_id(String colume_job_id) { + HbaseConfig.colume_job_id = colume_job_id; + } + + + public void setColume_exception(String colume_exception) { + HbaseConfig.colume_exception = colume_exception; + } + + public void setColume_sql(String colume_sql) { + HbaseConfig.colume_sql = colume_sql; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HbaseFactory.java b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HbaseFactory.java new file mode 100644 index 0000000..6d8410b --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HbaseFactory.java @@ -0,0 +1,64 @@ +package com.mesalab.report.configuration; + +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Created by Administrator on 2020/3/10. + */ +@Configuration +@EnableConfigurationProperties(HbaseProperties.class) +public class HbaseFactory { + private final HbaseProperties hbproperties; + + private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + + @Autowired + public HbaseFactory(HbaseProperties hbproperties) { + this.hbproperties = hbproperties; + } + + @Bean(name = "hbaseConfiguration") + public org.apache.hadoop.conf.Configuration getConfiguration() { + + org.apache.hadoop.conf.Configuration conf = null; + if (conf == null) { + conf = HBaseConfiguration.create(); + conf.set("hbase.zookeeper.quorum", hbproperties.getZookeeper_quorum()); + conf.set("hbase.zookeeper.property.clientPort", hbproperties.getZookeeper_property_clientPort()); + conf.set("zookeeper.znode.parent", hbproperties.getZookeeper_znode_parent()); + conf.set("hbase.client.retries.number", hbproperties.getClient_retries_number()); + conf.set("hbase.rpc.timeout", hbproperties.getRpc_timeout()); + conf.set("hbase.client.keyvalue.maxsize", "1024000000"); + conf.set("zookeeper.recovery.retry", "3"); + conf.set("hbase.client.ipc.pool.size", hbproperties.getConnect_pool().toString()); + + + } + + return conf; + } + + @Bean(name = "hbaseConnection") + public Connection getConnection(@Qualifier("hbaseConfiguration") org.apache.hadoop.conf.Configuration Conf) { + Connection con = null; + try { + con = ConnectionFactory.createConnection(Conf); + + } catch (IOException e) { + logger.error(e.toString()); + } + return con; + + + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HbaseProperties.java b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HbaseProperties.java new file mode 100644 index 0000000..636d792 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HbaseProperties.java @@ -0,0 +1,95 @@ +package com.mesalab.report.configuration; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Created by Administrator on 2020/3/10. + */ + +@ConfigurationProperties(prefix = "hbase") +public class HbaseProperties { + + private String zookeeper_quorum; + + private String zookeeper_property_clientPort; + + private String zookeeper_znode_parent; + + private String client_pause; + + private String client_retries_number; + + private String rpc_timeout; + + private Integer connect_pool; + + private String table; + + + + public String getZookeeper_quorum() { + return zookeeper_quorum; + } + + public void setZookeeper_quorum(String zookeeper_quorum) { + this.zookeeper_quorum = zookeeper_quorum; + } + + public String getZookeeper_property_clientPort() { + return zookeeper_property_clientPort; + } + + public void setZookeeper_property_clientPort(String zookeeper_property_clientPort) { + this.zookeeper_property_clientPort = zookeeper_property_clientPort; + } + + public String getZookeeper_znode_parent() { + return zookeeper_znode_parent; + } + + public void setZookeeper_znode_parent(String zookeeper_znode_parent) { + this.zookeeper_znode_parent = zookeeper_znode_parent; + } + + public String getClient_pause() { + return client_pause; + } + + public void setClient_pause(String client_pause) { + this.client_pause = client_pause; + } + + public String getClient_retries_number() { + return client_retries_number; + } + + public void setClient_retries_number(String client_retries_number) { + this.client_retries_number = client_retries_number; + } + + public String getRpc_timeout() { + return rpc_timeout; + } + + public void setRpc_timeout(String rpc_timeout) { + this.rpc_timeout = rpc_timeout; + } + + + public Integer getConnect_pool() { + return connect_pool; + } + + public void setConnect_pool(Integer connect_pool) { + this.connect_pool = connect_pool; + } + + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HttpClientPool.java b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HttpClientPool.java new file mode 100644 index 0000000..f10a117 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/HttpClientPool.java @@ -0,0 +1,160 @@ +package com.mesalab.report.configuration; + +import com.mesalab.report.util.Logs; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +/** + * Created by wk1 on 2019/5/15. + */ +@Component +@ConfigurationProperties(prefix = "http") +public class HttpClientPool { + + private Integer maxTotal; + + private Integer defaultMaxPerRoute; + + private Integer connectTimeout; + + private Integer connectionRequestTimeout; + + public static Integer socketTimeout; + + private boolean staleConnectionCheckEnabled; + + private Integer socketTimeoutShort; + + + public void setMaxTotal(Integer maxTotal) { + this.maxTotal = maxTotal; + } + + public void setDefaultMaxPerRoute(Integer defaultMaxPerRoute) { + this.defaultMaxPerRoute = defaultMaxPerRoute; + } + + public void setConnectTimeout(Integer connectTimeout) { + this.connectTimeout = connectTimeout; + } + + public void setConnectionRequestTimeout(Integer connectionRequestTimeout) { + this.connectionRequestTimeout = connectionRequestTimeout; + } + + public void setSocketTimeout(Integer socketTimeout) { + this.socketTimeout = socketTimeout; + } + + public void setStaleConnectionCheckEnabled(boolean staleConnectionCheckEnabled) { + this.staleConnectionCheckEnabled = staleConnectionCheckEnabled; + } + + public void setSocketTimeoutShort(Integer socketTimeoutShort) { + this.socketTimeoutShort = socketTimeoutShort; + } + + /** + * 首先实例化一个连接池管理器，设置最大连接数、并发连接数 + * + * @return + */ + @Bean(name = "httpClientConnectionManager") + public PoolingHttpClientConnectionManager getHttpClientConnectionManager() { + PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager(); + //最大连接数 + httpClientConnectionManager.setMaxTotal(maxTotal); + //并发数 + httpClientConnectionManager.setDefaultMaxPerRoute(defaultMaxPerRoute); + return httpClientConnectionManager; + } + + /** + * 实例化连接池，设置连接池管理器。 + * 这里需要以参数形式注入上面实例化的连接池管理器 + * + * @param httpClientConnectionManager + * @return + */ + @Bean(name = "httpClientBuilder") + public HttpClientBuilder getHttpClientBuilder(@Qualifier("httpClientConnectionManager") PoolingHttpClientConnectionManager httpClientConnectionManager) { + + //HttpClientBuilder中的构造方法被protected修饰，所以这里不能直接使用new来实例化一个HttpClientBuilder，可以使用HttpClientBuilder提供的静态方法create()来获取HttpClientBuilder对象 + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + + httpClientBuilder.setConnectionManager(httpClientConnectionManager); + + return httpClientBuilder; + } + + /** + * 注入连接池，用于获取httpClient + * + * @param httpClientBuilder + * @return + */ + @Bean + public CloseableHttpClient getCloseableHttpClient(@Qualifier("httpClientBuilder") HttpClientBuilder httpClientBuilder) { + return httpClientBuilder.build(); + } + + /** + * Builder是RequestConfig的一个内部类 + * 通过RequestConfig的custom方法来获取到一个Builder对象 + * 设置builder的连接信息 + * 这里还可以设置proxy，cookieSpec等属性。有需要的话可以在此设置 + * + * @return + */ + @Bean(name = "builder") + public RequestConfig.Builder getBuilder() { + RequestConfig.Builder builder = RequestConfig.custom(); + return builder.setConnectTimeout(connectTimeout) + .setConnectionRequestTimeout(connectionRequestTimeout) + .setSocketTimeout(socketTimeout) + .setStaleConnectionCheckEnabled(staleConnectionCheckEnabled); + } + + + + @Bean(name = "builderShort") + public RequestConfig.Builder getBuilderShort() { + RequestConfig.Builder builder = RequestConfig.custom(); + return builder.setConnectTimeout(connectTimeout) + .setConnectionRequestTimeout(connectionRequestTimeout) + .setSocketTimeout(socketTimeoutShort) + .setStaleConnectionCheckEnabled(staleConnectionCheckEnabled); + } + /** + * 使用builder构建一个RequestConfig对象 + * + * @param builder + * @return + */ + @Bean(name = "RequestLongConfig") + public RequestConfig getRequestConfig(@Qualifier("builder") RequestConfig.Builder builder) { + Logs.debug("httpclient success"); + return builder.build(); + } + + /** + * @author LBQ + * @version 1.0.0 + * @ClassName HbasePool + * @Description Hbase连接池 + * @Date 2018年5月31日 下午10:22:06 + */ + @Bean(name = "RequestShortConfig") + public RequestConfig getRequestShortConfig(@Qualifier("builderShort") RequestConfig.Builder builder) { + Logs.debug("httpclientshort success"); + return builder.build(); + } + +} + diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ZkConfig.java b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ZkConfig.java new file mode 100644 index 0000000..e2be3c1 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ZkConfig.java @@ -0,0 +1,51 @@ +package com.mesalab.report.configuration; + +import com.mesalab.report.util.Logs; +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Created by wk1 on 2020/1/6. + */ +@Configuration +@EnableConfigurationProperties(ZkProperties.class) + +public class ZkConfig { + + private final ZkProperties zkproperties; + + @Autowired + public ZkConfig(ZkProperties zkproperties) { + this.zkproperties = zkproperties; + } + + @Bean(name = "curatorConnect") + public CuratorFramework CuratorConnect() { + + CuratorFramework client = null; + + + RetryPolicy retryPolicy = new ExponentialBackoffRetry(zkproperties.getElapsedTimeMs(), zkproperties.getRetryCount()); + + // 实例化Curator客户端 + client = CuratorFrameworkFactory.builder() // 使用工厂类来建造客户端的实例对象 + .connectString(zkproperties.getConnectString()) // 放入zookeeper服务器ip + .sessionTimeoutMs(zkproperties.getSessionTimeoutMs()).connectionTimeoutMs(zkproperties.getConnectionTimeoutMs()).retryPolicy(retryPolicy) // 设定会话时间以及重连策略 + .namespace(zkproperties.getNameSpace()).build(); // 设置命名空间以及开始建立连接 + if (zkproperties.getOpen() == 0) { + client.start(); + } else { + Logs.info("repoertservice start local"); + } + + return client; + } + + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ZkProperties.java b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ZkProperties.java new file mode 100644 index 0000000..14d1051 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/configuration/ZkProperties.java @@ -0,0 +1,81 @@ +package com.mesalab.report.configuration; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Created by wk1 on 2020/1/6. + */ +@ConfigurationProperties(prefix = "zookeeper") +public class ZkProperties { + + private int open; + + private int retryCount; + + private int elapsedTimeMs; + + private String connectString; + + private int sessionTimeoutMs; + + private int connectionTimeoutMs; + + private String nameSpace; + + + public int getOpen() { + return open; + } + + public void setOpen(int open) { + this.open = open; + } + + public int getRetryCount() { + return retryCount; + } + + public void setRetryCount(int retryCount) { + this.retryCount = retryCount; + } + + public int getElapsedTimeMs() { + return elapsedTimeMs; + } + + public void setElapsedTimeMs(int elapsedTimeMs) { + this.elapsedTimeMs = elapsedTimeMs; + } + + public String getConnectString() { + return connectString; + } + + public void setConnectString(String connectString) { + this.connectString = connectString; + } + + public int getSessionTimeoutMs() { + return sessionTimeoutMs; + } + + public void setSessionTimeoutMs(int sessionTimeoutMs) { + this.sessionTimeoutMs = sessionTimeoutMs; + } + + public int getConnectionTimeoutMs() { + return connectionTimeoutMs; + } + + public void setConnectionTimeoutMs(int connectionTimeoutMs) { + this.connectionTimeoutMs = connectionTimeoutMs; + } + + public String getNameSpace() { + return nameSpace; + } + + public void setNameSpace(String nameSpace) { + this.nameSpace = nameSpace; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/controller/MonitorController.java b/galaxy-report-service/src/main/java/com/mesalab/report/controller/MonitorController.java new file mode 100644 index 0000000..1908de6 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/controller/MonitorController.java @@ -0,0 +1,60 @@ +package com.mesalab.report.controller; + +import com.alibaba.fastjson.JSONArray; +import com.mesalab.report.bean.MonitorEntity; +import com.mesalab.report.configuration.GlobelConfig; +import com.mesalab.report.service.MysqlService; +import com.mesalab.report.service.ZkService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.*; + +import java.util.Map; + + +/** + * Created by wk on 2020/4/16. + */ +@RestController +@Component +public class MonitorController { + + private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + @Autowired + private MysqlService mysqlService; + @Autowired + private ZkService zs; + + + @GetMapping(value = "/monitor") + public String getReportStatus() { + + + String json=""; + try { + MonitorEntity me = new MonitorEntity(); + Map<String,Long> numMap = mysqlService.getJobCount(); + me.setQueueJobNum(numMap.get("queueNum")); + me.setExcuteingJobNum(numMap.get("excuteingNum")); + me.setTodaySuccessJobNum(numMap.get("todaySuccessNum")); + me.setTodayErrorJobNum(numMap.get("todayErrorNum")); + me.setJoblist(GlobelConfig.mapresult); + if(zs.isMaster()){ + me.setStatus("active"); + } + else{ + me.setStatus("standby"); + } + GlobelConfig.mapresult.size(); + Object obj = JSONArray.toJSON(me); + json = obj.toString(); + } catch (Exception e) { + logger.error(e.toString()); + json=e.toString(); + } + + return json; + } + + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/controller/ScheduledResultController.java b/galaxy-report-service/src/main/java/com/mesalab/report/controller/ScheduledResultController.java new file mode 100644 index 0000000..1f1d7d9 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/controller/ScheduledResultController.java @@ -0,0 +1,127 @@ +package com.mesalab.report.controller; + +import com.mesalab.report.bean.JobEntity; +import com.mesalab.report.configuration.GlobelConfig; +import com.mesalab.report.configuration.HttpClientPool; +import com.mesalab.report.service.ExcuteProcessService; +import com.mesalab.report.service.ExcuteService; +import com.mesalab.report.service.MysqlService; +import com.mesalab.report.service.ZkService; +import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * @author wk1 + * @date 2020/1/8 + */ +@Component +@EnableScheduling +public class ScheduledResultController { + + private static Logger logger = LoggerFactory.getLogger(ScheduledResultController.class); + + protected static ExecutorService pool = Executors.newFixedThreadPool(30); + + + @Autowired + private MysqlService ms; + @Autowired + private ExcuteService es; + @Autowired + private ExcuteProcessService eps; + @Autowired + private ZkService zs; + + + @Scheduled(cron = "${scan.result.scheduled.plan}") + public void getExcuteResult() { + try { + if (zs.isMaster()) { + logger.info("开始查看结果"); + //先查询数据库是否有异常状态任务，killquery + List<JobEntity> joblist = ms.getJobForExcute(); + for (JobEntity jobEntity : joblist) { + String sql = jobEntity.getQuerySql().trim(); + sql = sql.replace("$exe_time", "toDateTime('" + jobEntity.getIssuedTime().trim() + "')"); + sql = sql.replace("$start_time", "toDateTime('" + jobEntity.getStartTime().trim() + "')"); + sql = sql.replace("$end_time", "toDateTime('" + jobEntity.getEndTime().trim() + "')"); + String queryid = DigestUtils.md5Hex(jobEntity.getResultId() + sql); + jobEntity.setQuery_id(queryid); + + if (jobEntity.getIsValid() == 0) { + eps.killQuery(jobEntity); + GlobelConfig.mapresult.get(jobEntity.getQuery_id()).setIsValid(0); + } else if (!GlobelConfig.mapresult.containsKey(jobEntity.getQuery_id())) { + eps.reSet(jobEntity); + } + if (GlobelConfig.mapresult.containsKey(jobEntity.getQuery_id())) { + if (jobEntity.getIsValid() == 0) { + eps.killQuery(jobEntity); + GlobelConfig.mapresult.get(jobEntity.getQuery_id()).setIsValid(0); + } + } else { + eps.reSet(jobEntity); + } + + } + //遍历内存中的任务对状态1的更新进度，其他更新数据库的状态 + for (Map.Entry<String, JobEntity> entry : GlobelConfig.mapresult.entrySet()) { + logger.info("key = " + entry.getKey() + ", value = " + entry.getValue().getStatus()); + long currentTime = System.currentTimeMillis(); + long excutetime = currentTime - entry.getValue().getBeginTime(); + logger.info("excute time=" + excutetime + "ttl_time=" + HttpClientPool.socketTimeout); + if (entry.getValue().getStatus() == 1 && excutetime > HttpClientPool.socketTimeout + 1) { + entry.getValue().setStatus(2); + entry.getValue().setExcute_status(500001); + eps.killQuery(entry.getValue()); + eps.updateResultMessage(entry.getValue()); + } else { + if (entry.getValue().getStatus() == 1) { + eps.updateProcessMessage(entry.getValue()); + } else { + eps.updateResultMessage(entry.getValue()); + } + } + } + int rows = GlobelConfig.job_thread - GlobelConfig.mapresult.size(); + if (rows > 0) { + List<JobEntity> jobs = ms.getJobTask(rows); + for (JobEntity job : jobs) { + logger.info("开始执行任务"); + long begintime = System.currentTimeMillis(); + job.setBeginTime(begintime); + pool.execute(new Runnable() { + @Override + public void run() { + es.excuteCkTask(job); + } + }); + } + } else { + logger.info("无待执行任务"); + } + } else { + if (GlobelConfig.mapresult.size() > 0) { + for (Map.Entry<String, JobEntity> entry : GlobelConfig.mapresult.entrySet()) { + logger.info("key = " + entry.getKey() + ", value = " + entry.getValue().getStatus()); + eps.killQuery(entry.getValue()); + } + GlobelConfig.mapresult.clear(); + } + } + } catch (Exception e) { + logger.error(e.toString()); + } + } + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/mapper/ReportResultMapper.java b/galaxy-report-service/src/main/java/com/mesalab/report/mapper/ReportResultMapper.java new file mode 100644 index 0000000..0440f26 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/mapper/ReportResultMapper.java @@ -0,0 +1,23 @@ +package com.mesalab.report.mapper; + +import com.mesalab.report.bean.JobEntity; +import org.apache.ibatis.annotations.Mapper; + +import java.util.List; +import java.util.Map; + +@Mapper +public interface ReportResultMapper { + + + List<JobEntity> getJobForExcute(); + + List<JobEntity> getJobTask(Map<String, Object> map); + + int updateProcesses(JobEntity job); + + int updateStatue(JobEntity job); + + Map<String,Long> getJobCount(Map<String, Object> map); + +}
\ No newline at end of file diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/ClickhouseService.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/ClickhouseService.java new file mode 100644 index 0000000..a9ff289 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/ClickhouseService.java @@ -0,0 +1,28 @@ +package com.mesalab.report.service; + +import com.mesalab.report.bean.HttpResult; + +import java.util.Map; + +/** + * Created by wk1 on 2020/1/2. + */ +public interface ClickhouseService { + + String doGet(String url) throws Exception; + + + String doGet(String url, Map<String, Object> map) throws Exception; + + + HttpResult sendQueryForGet(String url) throws Exception; + + + HttpResult QuerySystemForGet(String url) throws Exception; + + + HttpResult QuerySystemForDelete(String url) throws Exception; + + + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/ExcuteProcessService.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/ExcuteProcessService.java new file mode 100644 index 0000000..ac94f23 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/ExcuteProcessService.java @@ -0,0 +1,17 @@ +package com.mesalab.report.service; + +import com.mesalab.report.bean.JobEntity; + +import java.util.List; + +/** + * Created by wk1 on 2020/1/8. + */ +public interface ExcuteProcessService { + + void updateResultMessage(JobEntity job); + void reSet(JobEntity jobEntity); + void updateProcessMessage(JobEntity job); + void killQuery(JobEntity jobEntity); + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/ExcuteService.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/ExcuteService.java new file mode 100644 index 0000000..9ffdb46 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/ExcuteService.java @@ -0,0 +1,12 @@ +package com.mesalab.report.service; + +import com.mesalab.report.bean.JobEntity; + +/** + * Created by wk1 on 2020/1/8. + */ +public interface ExcuteService { + + void excuteCkTask(JobEntity jl); + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/FormatService.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/FormatService.java new file mode 100644 index 0000000..172caff --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/FormatService.java @@ -0,0 +1,11 @@ +package com.mesalab.report.service; + +/** + * Created by wk1 on 2020/4/15. + */ +public interface FormatService { + + + String doFormat(String url) throws Exception; + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/HbaseService.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/HbaseService.java new file mode 100644 index 0000000..d12729f --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/HbaseService.java @@ -0,0 +1,17 @@ +package com.mesalab.report.service; + +import com.mesalab.report.bean.HttpResult; +import com.mesalab.report.bean.JobEntity; +import org.apache.hadoop.yarn.webapp.hamlet.HamletSpec; + +import java.util.Map; + +/** + * Created by wk1 on 2020/1/2. + */ +public interface HbaseService { + + + Boolean put(JobEntity jobEntity) throws Exception; + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/MonitorService.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/MonitorService.java new file mode 100644 index 0000000..7481c2b --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/MonitorService.java @@ -0,0 +1,14 @@ +package com.mesalab.report.service; + + +import org.springframework.stereotype.Service; + +/** + * Created by wk1 on 2020/1/6. + */ +public interface MonitorService { + + + void addSuccess(); + void addFail(); +}
\ No newline at end of file diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/MysqlService.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/MysqlService.java new file mode 100644 index 0000000..dc49989 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/MysqlService.java @@ -0,0 +1,25 @@ +package com.mesalab.report.service; + + +import com.mesalab.report.bean.JobEntity; + +import java.util.List; +import java.util.Map; + + +public interface MysqlService { + + + List<JobEntity> getJobForExcute() throws Exception; + + + List<JobEntity> getJobTask(int Rows) throws Exception; + + + int updateProcesses(JobEntity job); + + int updateStatue(JobEntity job); + + Map<String,Long> getJobCount() throws Exception; + + } diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/ZkService.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/ZkService.java new file mode 100644 index 0000000..e33c971 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/ZkService.java @@ -0,0 +1,16 @@ +package com.mesalab.report.service; + + +import org.springframework.stereotype.Service; + +import java.net.InetAddress; + +/** + * Created by wk1 on 2020/1/6. + */ +@Service +public interface ZkService { + + + boolean isMaster(); +}
\ No newline at end of file diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ClickhouseServiceImpl.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ClickhouseServiceImpl.java new file mode 100644 index 0000000..90dadf9 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ClickhouseServiceImpl.java @@ -0,0 +1,217 @@ +package com.mesalab.report.service.impl; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.mesalab.report.bean.HttpResult; +import com.mesalab.report.service.ClickhouseService; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.params.CoreConnectionPNames; +import org.apache.http.util.EntityUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import java.util.Map; + +/** + * Created by wk1 on 2019/5/15. + */ + +@Service +public class ClickhouseServiceImpl implements ClickhouseService { + + @Autowired + private CloseableHttpClient httpClient; + + @Autowired + @Qualifier("RequestShortConfig") + private RequestConfig RequestshortConfig; + + @Autowired + @Qualifier("RequestLongConfig") + private RequestConfig RequestLongConfig; + private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + + /** + * 不带参数的get请求，如果状态码为200，则返回body，如果不为200，则返回null + * + * @param url + * @return + * @throws Exception + */ + @Override + public String doGet(String url) throws Exception { + // 声明 http get 请求 + + HttpGet httpGet = new HttpGet(url); + httpGet.setHeader("Accept", "application/json"); + // 装载配置信息 + httpGet.setConfig(RequestLongConfig); + + try(CloseableHttpResponse response = this.httpClient.execute(httpGet)){ + if (response.getStatusLine().getStatusCode() == 200) { + // 返回响应体的内容 + + return EntityUtils.toString(response.getEntity(), "UTF-8"); + } else { + throw new Exception(); + + } + } + catch (Exception e){ + + logger.error(e.toString()); + throw new Exception(); + + } + // 发起请求 + + + // 判断状态码是否为200 + + + } + + /** + * 带参数的get请求，如果状态码为200，则返回body，如果不为200，则返回null + * + * @param url + * @return + * @throws Exception + */ + @Override + public String doGet(String url, Map<String, Object> map) throws Exception { + URIBuilder uriBuilder = new URIBuilder(url); + + if (map != null) { + // 遍历map,拼接请求参数 + for (Map.Entry<String, Object> entry : map.entrySet()) { + uriBuilder.setParameter(entry.getKey(), entry.getValue().toString()); + } + } + + // 调用不带参数的get请求 + return this.doGet(uriBuilder.build().toString()); + + } + + /** + * 带参数的post请求 + * + * @param url + * @return + * @throws Exception + */ + @Override + public HttpResult sendQueryForGet(String url) throws Exception,OutOfMemoryError{ + // 声明httpPost请求 + HttpGet httpGet = new HttpGet(url); + + + // 加入配置信息 + httpGet.setConfig(RequestLongConfig); + + try(CloseableHttpResponse response = this.httpClient.execute(httpGet) ){ + HttpResult rs = new HttpResult(); + rs.setCode(response.getStatusLine().getStatusCode()); + rs.setBody(EntityUtils.toString(response.getEntity(), "UTF-8")); + return rs; + + }catch (Exception e){ + logger.error(e.toString()); + throw new Exception(); + } + + } + + + + + @Override + public HttpResult QuerySystemForGet(String url) throws Exception { + // 声明httpPost请求 + HttpGet httpGet = new HttpGet(url); + // 加入配置信息 + httpGet.setConfig(RequestshortConfig); + + // 发起请求 + try(CloseableHttpResponse response = this.httpClient.execute(httpGet) ){ + + HttpResult rs = new HttpResult(); + rs.setCode(response.getStatusLine().getStatusCode()); + rs.setBody(EntityUtils.toString(response.getEntity(), "UTF-8")); + + //Thread.sleep(120); + return rs; + + }catch (Exception e){ + logger.error(e.toString()); + throw new Exception(); + } + } + + + @Override + public HttpResult QuerySystemForDelete(String url) throws Exception { + // 声明httpPost请求 + HttpDelete HttpDelete = new HttpDelete(url); + // 加入配置信息 + HttpDelete.setConfig(RequestshortConfig); + + // 发起请求 + try(CloseableHttpResponse response = this.httpClient.execute(HttpDelete)) { + + HttpResult rs = new HttpResult(); + rs.setCode(response.getStatusLine().getStatusCode()); + rs.setBody(EntityUtils.toString(response.getEntity(), "UTF-8")); + + //Thread.sleep(120); + return rs; + } + catch (Exception e){ + logger.error(e.toString()); + throw new Exception(); + } + + } + + public CloseableHttpClient getHttpClient() { + return httpClient; + } + + public void setHttpClient(CloseableHttpClient httpClient) { + this.httpClient = httpClient; + } + + public RequestConfig getRequestshortConfig() { + return RequestshortConfig; + } + + public void setRequestshortConfig(RequestConfig requestshortConfig) { + RequestshortConfig = requestshortConfig; + } + + public RequestConfig getRequestLongConfig() { + return RequestLongConfig; + } + + public void setRequestLongConfig(RequestConfig requestLongConfig) { + RequestLongConfig = requestLongConfig; + } + + /** + * 不带参数post请求 + * + * @param url + * @return + * @throws Exception + */ + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ExcuteProcessServiceImpl.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ExcuteProcessServiceImpl.java new file mode 100644 index 0000000..113d874 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ExcuteProcessServiceImpl.java @@ -0,0 +1,214 @@ +package com.mesalab.report.service.impl; + +import com.alibaba.fastjson.JSON; +import com.mesalab.report.bean.HttpResult; +import com.mesalab.report.bean.JobEntity; +import com.mesalab.report.configuration.ClickhouseConfig; +import com.mesalab.report.configuration.GlobelConfig; +import com.mesalab.report.service.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; + +/** + * Created by wk1 on 2020/1/8. + */ +@Service + +public class ExcuteProcessServiceImpl implements ExcuteProcessService { + private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + + + @Autowired + private ClickhouseService cs; + @Autowired + private MysqlService ms; + @Autowired + private HbaseService hs; + @Autowired + private MonitorService mons; + + @Override + public void updateResultMessage(JobEntity je) { + + + try { + if (je.getIsValid() == 0) { + je.setStatus(9); + je.setExcuteDetail("CANCEL"); + } else { + switch (je.getExcute_status()) { + + case 200666: + Boolean isok = saveToHbase(je); + if (isok) { + je.setExcuteDetail("SUCCESS"); + je.setExcuteProcess(100); + je.setStatus(2); + logger.info("success save to hbase query_id="+je.getQuery_id()+" resultid =" + je.getResultId() + " excutesql=" + je.getQuerySql()); + } else { + je.setStatus(5); + je.setExcuteDetail("Write Data Error"); + mons.addFail(); + logger.error("save hbase error query_id="+je.getQuery_id()+" resultid =" + je.getResultId() + " excutesql=" + je.getQuerySql()); + } + break; + + case 400001: + je.setStatus(3); + je.setExcuteDetail("Param Syntax Error"); + logger.error("Param Syntax Error query_id="+je.getQuery_id()+" resultid =" + je.getResultId() + " excutesql=" + je.getQuerySql()); + break; + case 400010: + je.setStatus(3); + je.setExcuteDetail("SQL Syntax Error"); + logger.error("SQL Syntax Error query_id="+je.getQuery_id()+" resultid =" + je.getResultId() + " excutesql=" + je.getQuerySql()); + break; + case 500001: + je.setStatus(4); + je.setExcuteDetail("SQL Execution Error"); + logger.error("SQL Execution Error query_id="+je.getQuery_id()+" resultid =" + je.getResultId() + " excutesql=" + je.getQuerySql()); + break; + case 500010: + je.setStatus(6); + je.setExcuteDetail("Engine Statistics Error"); + logger.error("Engine Statistics Error query_id="+je.getQuery_id()+" resultid =" + je.getResultId() + " excutesql=" + je.getQuerySql()); + break; + case 555999: + je.setStatus(7); + je.setExcuteDetail("Unknow Error"); + logger.error("Unknow Error query_id="+je.getQuery_id()+" resultid =" + je.getResultId() + " excutesql=" + je.getQuerySql()); + break; + default: + je.setStatus(8); + je.setExcuteDetail("System Error"); + logger.error("System Error query_id="+je.getQuery_id()+" resultid =" + je.getResultId() + " excutesql=" + je.getQuerySql()); + } + + } + int number = 0; + int z = 3; + do { + number = ms.updateProcesses(je); + z--; + } + while (number != 1 && z >= 0); + } catch (Exception e) { + je.setStatus(10); + je.setExcuteDetail("Database Error"); + ms.updateProcesses(je); + logger.error("save database error resultid =" + je.getResultId()+" queryid=" + je.getResultId() + e.toString()); + + } finally { + saveToMonitor(je); + GlobelConfig.mapresult.remove(je.getQuery_id()); + } + + + } + + @Override + public void reSet(JobEntity jobEntity) { + String killurl = ClickhouseConfig.getKillUrl(jobEntity.getQuery_id()); + try { + cs.QuerySystemForDelete(killurl); + + } catch (Exception e) { + logger.error(e.toString()); + } finally { + jobEntity.setStatus(0); + jobEntity.setExcuteDetail("Re Execution"); + ms.updateProcesses(jobEntity); + } + } + + @Override + public void killQuery(JobEntity jobEntity) { + String killurl = ClickhouseConfig.getKillUrl(jobEntity.getQuery_id()); + try { + cs.QuerySystemForDelete(killurl); + + } catch (Exception e) { + logger.error(e.toString()); + } + } + /** + * 获取进度条信息 + */ + @Override + public void updateProcessMessage(JobEntity job) { + String queryProccess = ClickhouseConfig.getProcessUrl(job.getQuery_id()); + HttpResult hr = null; + try { + hr = cs.QuerySystemForGet(queryProccess); + String rs = hr.getBody().trim(); + Map data = JSON.parseObject(rs); + + if (!rs.isEmpty() && !rs.equals("")) { + List listdata = (List) data.get("data"); + if (null != listdata && listdata.size() > 0) { + Map map = JSON.parseObject(JSON.toJSONString(listdata.get(0))); + long total_rows_approx = Long.parseLong(map.get("total_rows_approx").toString()); + long read_rows = Long.parseLong(map.get("read_rows").toString()); + float elapsed = Float.parseFloat(map.get("elapsed").toString()) * 1000; + double persent = (read_rows * 1.00 / total_rows_approx); + int result = (int) (persent * 100); + if (result > 98) { + result = 98; + } + job.setExcuteTime((int) elapsed); + job.setExcuteRow(total_rows_approx); + job.setExcuteProcess(result); + if (job.getExcuteRow() != 0 || job.getExcuteTime() != 0) { + ms.updateProcesses(job); + } + } else { + logger.info("responsedata is null"); + } + } else { + logger.error("responsebody is null"); + } + } catch (Exception e) { + logger.error(e.toString()); + } + } + + /** + * 结果存入hbase + */ + public Boolean saveToHbase(JobEntity entity) { + int k = 3; + Boolean isok = false; + do { + try { + k--; + isok = hs.put(entity); + break; + } catch (Exception e) { + logger.error("写入hbase报错重试次数" + (3 - k) + e.toString()); + k--; + } + } + while (k >= 0); + return isok; + } + + /** + * promethus记录结果 + */ + public void saveToMonitor(JobEntity entity) { + + try { + if (entity.getStatus() == 2) { + + mons.addSuccess(); + } else { + mons.addFail(); + } + } catch (Exception e) { + logger.error("监控结果失败" + e.toString()); + } + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ExcuteserviceImpl.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ExcuteserviceImpl.java new file mode 100644 index 0000000..e845a32 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ExcuteserviceImpl.java @@ -0,0 +1,133 @@ +package com.mesalab.report.service.impl; + +import com.alibaba.fastjson.JSON; +import com.mesalab.report.bean.HttpResult; +import com.mesalab.report.bean.JobEntity; +import com.mesalab.report.configuration.ClickhouseConfig; +import com.mesalab.report.configuration.GlobelConfig; +import com.mesalab.report.service.ClickhouseService; +import com.mesalab.report.service.ExcuteService; +import com.mesalab.report.service.FormatService; +import com.mesalab.report.service.MysqlService; +import io.netty.channel.ConnectTimeoutException; +import org.apache.commons.codec.digest.DigestUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.net.SocketTimeoutException; +import java.util.Map; + +import static com.mesalab.report.configuration.ClickhouseConfig.getJobUrl; + +/** + * Created by wk1 on 2020/1/8. + */ +@Service +public class ExcuteserviceImpl implements ExcuteService { + private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + + + @Autowired + private ClickhouseService cs; + @Autowired + private MysqlService ms; + + @Autowired + private FormatService fs; + + @Override + public void excuteCkTask(JobEntity job) { + + + String sql = job.getQuerySql().trim(); + sql = sql.replace("$exe_time", "toDateTime('" + job.getIssuedTime().trim() + "')"); + sql = sql.replace("$start_time", "toDateTime('" + job.getStartTime().trim() + "')"); + sql = sql.replace("$end_time", "toDateTime('" + job.getEndTime().trim() + "')"); + job.setQuerySql(sql); + String queryid = DigestUtils.md5Hex(job.getResultId() + sql); + job.setQuery_id(queryid); + job.setQuerySql(sql); + job.setStatus(1); + job.setExcute_status(1); + job.setExcuteDetail("EXECUTING"); + job.setExcuteRow(0L); + job.setExcuteTime(0); + job.setExcuteProcess(0); + job.setResultRows(0); + GlobelConfig.mapresult.put(queryid, job); + ms.updateProcesses(job); + logger.info("execute queryid=" + queryid + " sql=" + sql + "mapresult size=" + GlobelConfig.mapresult.size()); + String joburl = getJobUrl(job.getQuerySql(), job.getResultId()); + String killurl = ClickhouseConfig.getKillUrl(job.getQuery_id()); + HttpResult hr = new HttpResult(); + int k = 3; + do { + try { + hr = cs.sendQueryForGet(joburl); + logger.info("httpcode" + hr.getCode()); + + if (hr != null && hr.getCode() != 200) { + + k = 0; + Map mapresult = JSON.parseObject(hr.getBody()); + job.setExcute_status(Integer.parseInt(mapresult.get("code").toString())); + logger.error("excute sql Error "); + + } else { + + k = 0; + Map mapresult = JSON.parseObject(hr.getBody()); + + Map rows = (Map) mapresult.get("statistics"); + job.setResultRows(Integer.parseInt(rows.get("result_rows").toString())); + job.setExcuteRow(Long.parseLong(rows.get("rows_read").toString())); + job.setResult(hr.getBody()); + job.setExcuteTime((int) Float.parseFloat(rows.get("elapsed").toString())); + job.setExcute_status(Integer.parseInt(mapresult.get("code").toString())); + logger.info("success resultid = "+job.getResultId()+" queryid=" + queryid + " sql=" + sql); + + } + } catch (SocketTimeoutException e) { + k--; + job.setExcuteDetail(e.toString()); + if (k == 0) { + job.setExcute_status(500001); + job.setExcuteDetail("SQL Execution Error excute query time out"); + logger.info("timeout resultid = "+job.getResultId()+" queryid=" + queryid + " sql=" + sql); + } else { + logger.info("Socket warn " + e.toString() + "retry time " + (3 - k)); + } + + + } catch (ConnectTimeoutException e) { + + job.setExcute_status(555999); + job.setExcuteDetail(e.toString()); + logger.error("Unknow Error" + e.toString()); + k = 0; + + } catch (OutOfMemoryError e) { + + job.setExcute_status(555999); + job.setExcuteDetail("result too large"); + logger.error("outofmemery Error" + e.toString()); + k = 0; + } catch (Exception e) { + job.setExcute_status(555999); + job.setExcuteDetail(e.toString()); + logger.error("Unknow Error" + e.toString()); + k = 0; + } + + try { + cs.QuerySystemForDelete(killurl); + } catch (Exception e) { + logger.error("Kill Query Error" + e.toString()); + } + } + while (k > 0); + job.setStatus(2); + + } + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/FormatServiceImpl.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/FormatServiceImpl.java new file mode 100644 index 0000000..d448a1c --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/FormatServiceImpl.java @@ -0,0 +1,53 @@ +package com.mesalab.report.service.impl; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.mesalab.report.service.FormatService; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.params.HttpParams; +import org.apache.http.util.EntityUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.util.Map; + +/** + * Created by wk1 on 2020/4/15. + */ +@Service +public class FormatServiceImpl implements FormatService { + + @Autowired + private CloseableHttpClient httpClient; + + @Autowired + @Qualifier("RequestShortConfig") + private RequestConfig config; + + @Override + public String doFormat(String url) throws Exception { + + HttpGet httpGet = new HttpGet(url); + httpGet.setHeader("Accept", "application/json"); + // 装载配置信息 + httpGet.setConfig(config); + CloseableHttpResponse response = this.httpClient.execute(httpGet); + if (response.getStatusLine().getStatusCode() == 200) { + // 返回响应体的内容 + + JSONObject jsonObject = JSON.parseObject(EntityUtils.toString(response.getEntity(), "UTF-8")); + String responseMessage = jsonObject.getString("message"); + return responseMessage.trim(); + } + else{ + throw new Exception("Formate sql error url = "+url); + } + + } + + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/HbaseServiceImpl.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/HbaseServiceImpl.java new file mode 100644 index 0000000..e4194a6 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/HbaseServiceImpl.java @@ -0,0 +1,58 @@ +package com.mesalab.report.service.impl; + +import com.mesalab.report.bean.HttpResult; +import com.mesalab.report.bean.JobEntity; +import com.mesalab.report.configuration.HbaseProperties; +import com.mesalab.report.service.HbaseService; +import com.mesalab.report.util.Logs; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.util.Bytes; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + + +/** + * Created by wk1 on 2019/5/15. + */ + +@Service +public class HbaseServiceImpl implements HbaseService { + + @Autowired + private Connection hbaseConnection; + + @Autowired + private HbaseProperties hbproperties; + + @Override + public Boolean put(JobEntity jobEntity) throws Exception { + + + Boolean status = false; + try(Table table =hbaseConnection.getTable(TableName.valueOf(hbproperties.getTable()))) { + Put put = new Put(Bytes.toBytes(jobEntity.getQuery_id())); + put.addColumn(Bytes.toBytes("response"), Bytes.toBytes("result"), Bytes.toBytes(jobEntity.getResult())); + put.addColumn(Bytes.toBytes("detail"), Bytes.toBytes("result_id"), Bytes.toBytes(jobEntity.getResultId())); + + put.addColumn(Bytes.toBytes("detail"), Bytes.toBytes("excute_sql"), Bytes.toBytes(jobEntity.getQuerySql())); + //put.addColumn(Bytes.toBytes("detail"), Bytes.toBytes("format_sql"), Bytes.toBytes(jobEntity.getFormatSql())); + + if(jobEntity.getExcuteRow()!=null){ + put.addColumn(Bytes.toBytes("detail"), Bytes.toBytes("read_rows"), Bytes.toBytes(jobEntity.getExcuteRow())); + } + if(jobEntity.getQuery_duration_ms()!=null){ + put.addColumn(Bytes.toBytes("detail"), Bytes.toBytes("query_duration_ms"), Bytes.toBytes(jobEntity.getQuery_duration_ms())); + } + if(jobEntity.getMemory_usage()!=null){ + put.addColumn(Bytes.toBytes("detail"), Bytes.toBytes("memory_usage"), Bytes.toBytes(jobEntity.getMemory_usage())); + } + table.put(put); + status =true; + }catch(Exception e) { + Logs.error(e.toString()); + } + return status; + } + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/MonitorServiceImpl.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/MonitorServiceImpl.java new file mode 100644 index 0000000..59fbfbe --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/MonitorServiceImpl.java @@ -0,0 +1,71 @@ +package com.mesalab.report.service.impl; + +import com.mesalab.report.configuration.GlobelConfig; +import com.mesalab.report.configuration.ZkProperties; +import com.mesalab.report.service.MonitorService; +import com.mesalab.report.service.ZkService; +import com.mesalab.report.util.Logs; +import io.micrometer.core.instrument.*; +import org.apache.commons.collections.map.HashedMap; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.ExistsBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.Stat; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.ObjectUtils; + +import javax.annotation.PostConstruct; +import java.net.InetAddress; +import java.util.List; +import java.util.Map; + +/** + * Created by wk1 on 2020/1/6. + */ +@Service +public class MonitorServiceImpl implements MonitorService { + + private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + + @Autowired + MeterRegistry registry; + + private Counter counter_report_success; + private Counter counter_report_fail; + + @PostConstruct + public void init(){ + + counter_report_success = registry.counter("report_success_count", "reportJob", "report_success"); + + counter_report_fail = registry.counter("report_fail_count", "reportJob", "report_error"); + } + @Override + public void addSuccess() { + + try{ + counter_report_success.increment(); + } catch (Exception e) { + logger.error(e.toString()); + } + } + @Override + public void addFail() { + + try{ + counter_report_fail.increment(); + } catch (Exception e) { + logger.error(e.toString()); + } + } + + + public MeterRegistry getRegistry() { + return registry; + } + + public void setRegistry(MeterRegistry registry) { + this.registry = registry; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/MysqlServiceImpl.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/MysqlServiceImpl.java new file mode 100644 index 0000000..101b015 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/MysqlServiceImpl.java @@ -0,0 +1,91 @@ +package com.mesalab.report.service.impl; + +import com.mesalab.report.bean.JobEntity; +import com.mesalab.report.mapper.ReportResultMapper; +import com.mesalab.report.service.MysqlService; +import com.mesalab.report.util.DateUtil; +import com.mesalab.report.util.Logs; +import com.mesalab.report.util.StringUtil; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by wk1 on 2020/1/2. + */ +@Service +public class MysqlServiceImpl implements MysqlService { + private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + + @Autowired + public ReportResultMapper rrm; + + @Override + public List<JobEntity> getJobForExcute() throws Exception{ + + return rrm.getJobForExcute(); + + } + + @Override + public List<JobEntity> getJobTask(int rows) throws Exception{ + + + String current_time = DateUtil.getDate(); + HashMap map = new HashMap(); + map.put("endtime", current_time); + map.put("rows", rows); + + return rrm.getJobTask(map); + + } + + @Override + public int updateProcesses(JobEntity job){ + + try { + Date currentTime = new Date(); + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String sDate = formatter.format(currentTime); + java.sql.Timestamp nowTime = java.sql.Timestamp.valueOf(sDate); + job.setOpTime(sDate); + return rrm.updateProcesses(job); + } + catch(Exception e){ + logger.error(e.toString()); + return 0; + } + } + + @Override + public int updateStatue(JobEntity job) { + try { + Date currentTime = new Date(); + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String sDate = formatter.format(currentTime); + java.sql.Timestamp nowTime = java.sql.Timestamp.valueOf(sDate); + job.setOpTime(sDate); + return rrm.updateStatue(job); + } + catch(Exception e){ + logger.error(e.toString()); + return 0; + } + } + + @Override + public Map<String,Long> getJobCount() throws Exception { + String current_time = DateUtil.getDate(); + String today_time = DateUtil.getCurrentDate(); + HashMap map = new HashMap(); + map.put("endtime", current_time); + map.put("optime", today_time); + return rrm.getJobCount(map); + } + +}
\ No newline at end of file diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ZkServiceImpl.java b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ZkServiceImpl.java new file mode 100644 index 0000000..785a30d --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/service/impl/ZkServiceImpl.java @@ -0,0 +1,71 @@ +package com.mesalab.report.service.impl; + +import com.mesalab.report.configuration.GlobelConfig; +import com.mesalab.report.configuration.ZkProperties; +import com.mesalab.report.service.ZkService; +import com.mesalab.report.util.Logs; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.ExistsBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.Stat; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.net.InetAddress; + +/** + * Created by wk1 on 2020/1/6. + */ +@Service +public class ZkServiceImpl implements ZkService { + @Autowired + private CuratorFramework curatorConnect; + @Autowired + private ZkProperties zp; + + + @Override + public boolean isMaster() { + + try { + if (zp.getOpen() == 0) { + boolean isZkCuratorStarted = curatorConnect.isStarted(); + String nodePath = "/masterip"; + System.out.println("当前客户端的状态：" + (isZkCuratorStarted ? "连接中..." : "已关闭...")); + ExistsBuilder eb = curatorConnect.checkExists(); + Stat statExist = curatorConnect.checkExists().forPath(nodePath); + if (statExist == null) { + byte[] data = GlobelConfig.zkuuid.getBytes(); // 节点数据 + + String result = curatorConnect.create().creatingParentsIfNeeded() // 创建父节点，也就是会递归创建 + .withMode(CreateMode.EPHEMERAL) // 节点类型 + .forPath(nodePath, data); + return true; + + } else { + System.out.println(nodePath + " 节点存在"); + Stat stat = new Stat(); + + byte[] nodeData = curatorConnect.getData().storingStatIn(stat).forPath(nodePath); + String masterid = new String(nodeData).trim(); + System.out.println("uuid="+ GlobelConfig.zkuuid+" 节点 " + nodePath + " 的数据为：" + new String(nodeData)); + + if (masterid.equals(GlobelConfig.zkuuid)) { + + return true; + + } else { + Logs.debug("不是主节点"); + return false; + } + } + } else { + return true; + } + } catch (Exception e) { + Logs.error(e.toString()); + return true; + } + } + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/util/DateUtil.java b/galaxy-report-service/src/main/java/com/mesalab/report/util/DateUtil.java new file mode 100644 index 0000000..96c6d63 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/util/DateUtil.java @@ -0,0 +1,574 @@ +package com.mesalab.report.util; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; + + +/** + * <p>时间/日期处理工具类</p> + * + * @author 中科智源育成信息有限公司 E-mail: [email protected] + * @version 1.0 创建时间：2010-11-8 下午04:48:57 + */ +public final class DateUtil { + + private static SimpleDateFormat simpleDateFormat = null; + /** + * 处理日期时，用到参数。格式24小时制yyyy-MM-dd HH:mm:ss + */ + private static String YYYY_MM_DD_HH24_MM_SS = "yyyy-MM-dd HH:mm:ss"; + /** + * 处理日期时，用到参数。格式12小时制yyyy-MM-dd hh:mm:ss + */ + private static String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd hh:mm:ss"; + /** + * 处理日期时，用到参数。格式为yyyy-MM-dd + */ + private static String YYYY_MM_DD = "yyyy-MM-dd"; + /** + * 处理日期时，用到参数。格式24小时制yyyy-MM-dd HH:mm + */ + private static String YYYY_MM_DD_HH_MM = "yyyy-MM-dd HH:mm"; + /** + * 处理日期时，用到参数。格式24小时制yyyyMMddHHmmss + */ + private static String YYYYMMDDHH24MMSS = "yyyyMMddHHmmss"; + /** + * 处理日期时，用到参数。格式为yyyyMMdd + */ + private static String YYYYMMDD = "yyyyMMdd"; + /** + * 处理日期时，用到参数。格式为HHmmss + */ + private static String HH24MMSS = "HHmmss"; + + /** + * <p>Description:抑制默认的构造器,避免实例化对象 </p> + */ + private DateUtil() { + + } + + + /** + * <p>获得当前日期，格式为默认yyyy-MM-dd.</p> + * <pre> + * DateUtil.getCurrentDate() = "2010-11-02" + * </pre> + * + * @return String 返回标准日期格式(yyyy-MM-dd) + * @see #getCurrentDate(String) + */ + public static String getCurrentDate() { + + simpleDateFormat = new SimpleDateFormat(YYYY_MM_DD); + return simpleDateFormat.format(new Date()); + } + + /** + * <p> + * 获得当前时间. + * 如果pattern为空,将抛出{@link NullPointerException} + * </p> + * <pre> + * DateUtil.getCurrentDate(TimeConstants.YYYY_MM_DD) = "2010-11-02" + * DateUtil.getCurrentDate("yyyy-MM-dd HH:mm:ss") = "2010-11-02 17:20:59" + * </pre> + * + * @param pattern 描述日期和时间格式. + * @return 当前日期 + */ + public static String getCurrentDate(String pattern) { + + if (StringUtil.isEmpty(pattern)) { + throw new NullPointerException("未规定时间格式."); + } + simpleDateFormat = new SimpleDateFormat(DateUtil.YYYY_MM_DD_HH_MM_SS); + return simpleDateFormat.format(new Date()); + + + } + + public static String getDate() { + + + simpleDateFormat = new SimpleDateFormat(DateUtil.YYYY_MM_DD_HH24_MM_SS); + return simpleDateFormat.format(new Date()); + + + } + + public static String getBeforeDate(int time) { + + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String dt = df.format(new Date()); + String[] s = dt.split(":"); + Integer i = Integer.valueOf(s[1]); + String ret = ""; + switch (time) { + case 300: + if (i < 5) { + ret = s[0] + ":00:00"; + } else if (i < 10) { + ret = s[0] + ":05:00"; + } else if (i < 15) { + ret = s[0] + ":10:00"; + } else if (i < 20) { + ret = s[0] + ":15:00"; + } else if (i < 25) { + ret = s[0] + ":20:00"; + } else if (i < 30) { + ret = s[0] + ":25:00"; + } else if (i < 35) { + ret = s[0] + ":30:00"; + } else if (i < 40) { + ret = s[0] + ":35:00"; + } else if (i < 45) { + ret = s[0] + ":40:00"; + } else if (i < 50) { + ret = s[0] + ":45:00"; + } else if (i < 55) { + ret = s[0] + ":50:00"; + } else if (i <= 59) { + ret = s[0] + ":55:00"; + } + break; + case 900: + if (i < 15) { + ret = s[0] + ":00:00"; + } else if (i < 30) { + ret = s[0] + ":15:00"; + } else if (i < 45) { + ret = s[0] + ":30:00"; + } else if (i < 60) { + ret = s[0] + ":45:00"; + } + break; + case 60: + /*Long l = new Date().getTime(); + Long w = l - 60000; + SimpleDateFormat dd = new SimpleDateFormat(DateUtil.YYYY_MM_DD_HH_MM); + ret = dd.format(w); + break;*/ + if (i < 1) { + ret = s[0] + ":00:00"; + + } else if (i >= 59) { + ret = s[0] + ":59:00"; + } else { + ret = s[0] + ":" + i + ":00"; + } + break; + + default: + simpleDateFormat = new SimpleDateFormat(DateUtil.YYYY_MM_DD_HH_MM); + ret = simpleDateFormat.format(new Date()); + break; + + } + + return ret; + + } + + + /** + * <p> 获得昨天时间.通过指定日期和时间格式,具体可参考本类提供的时间格式常量.</p> + * <pre> + * DateUtil.getDateOfYesterday(TimeConstants.YYYY_MM_DD_HH24_MM_SS) = "2010-11-04 16:59:13" + * </pre> + * + * @param pattern 描述日期和时间格式 + * @return String 返回日期的字符串表现形式值. + * @see #getFormatDate(Date, String) + */ + public static String getDateOfYesterday(String pattern) { + + Calendar calendar = Calendar.getInstance(); + calendar.add(calendar.DAY_OF_MONTH, -1); + Date time = calendar.getTime(); + return getFormatDate(time, pattern); + + } + + /** + * <p> + * 得到距离当天<code>offset</code>天数的某天日期，通过 <code>offset</code> 的值（+/-）来确定增减计算. + * </p> + * <pre> + * 当天日期： 2010-11-05 + * DateUtil.getSomeDate("yyyy-MM-dd", 2) = 2010-11-07 + * DateUtil.getSomeDate("yyyy-MM-dd", -2) = 2010-11-03 + * DateUtil.getSomeDate("yyyy-MM-dd", 0) = 2010-11-05 + * </pre> + * + * @param pattern 描述日期和时间格式 + * @param offset 描述增加\减少几日,为0时为当前日期. + * @return 返回日期格式的字符串 + * @see #getFormatDate(Date, String) + */ + + public static String getSomeDate(String pattern, int offset) { + + Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.DAY_OF_MONTH, offset); + Date time = calendar.getTime(); + + return getFormatDate(time, pattern); + + } + + /** + * <p>获得指定日期当月第一天</p> + * <pre> + * DateUtil.getFirstDayOfMonth(TimeConstants.YYYY_MM_DD_HH24_MM_SS, new Date()) = "2010-11-01 00:00:00" + * </pre> + * + * @param date 指定日期. + * @param pattern 描述日期和时间格式 + * @return 返回日期格式的字符串 + */ + public static String getFirstDayOfMonth(Date date, + String pattern) { + + if (StringUtil.isEmpty(pattern) || StringUtil.isEmpty(date)) { + throw new NullPointerException(); + } + + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + + Integer minDateNum = calendar.getActualMinimum(Calendar.DAY_OF_MONTH); + + calendar.set(Calendar.DAY_OF_MONTH, minDateNum); + calendar.set(Calendar.HOUR, calendar.getActualMinimum(Calendar.HOUR)); + calendar.set(Calendar.AM_PM, Calendar.AM); + calendar.set(Calendar.MINUTE, calendar + .getActualMinimum(Calendar.MINUTE)); + calendar.set(Calendar.SECOND, calendar + .getActualMinimum(Calendar.SECOND)); + + return getFormatDate(calendar.getTime(), pattern); + } + + /** + * <p>获得指定日期当月最后一天</p> + * <pre> + * DateUtil.getLastDayOfMonth(TimeConstants.YYYY_MM_DD_HH24_MM_SS, new Date()) = "2010-11-30 23:59:59" + * </pre> + * + * @param date 指定日期 + * @param pattern 描述日期和时间格式 + * @return 返回日期格式的字符串 + */ + public static String getLastDayOfMonth(Date date, + String pattern) { + + if (StringUtil.isEmpty(pattern) || StringUtil.isEmpty(date)) { + throw new NullPointerException(); + } + + Calendar c = Calendar.getInstance(); + c.setTime(date); + Integer maxDateNum = c.getActualMaximum(Calendar.DAY_OF_MONTH); + c.set(Calendar.DAY_OF_MONTH, maxDateNum); + c.set(Calendar.HOUR, c.getActualMaximum(Calendar.HOUR)); + c.set(Calendar.AM_PM, Calendar.PM); + c.set(Calendar.MINUTE, c.getActualMaximum(Calendar.MINUTE)); + c.set(Calendar.SECOND, c.getActualMaximum(Calendar.SECOND)); + return getFormatDate(c.getTime(), pattern); + } + + /** + * <p>获得指定日期的日.</p> + * + * @param date 日期值 + * @return 返回日期的日.如果{@link java.util.Date}为<code>null</code>,抛出异常. + * @throws NullPointerException + */ + public static String getDDFromDate(Date date) { + if (null == date) { + throw new NullPointerException("未存储日期对象"); + } + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + + return calendar.get(Calendar.DATE) + ""; + + } + + /** + * <p>获得指定日期的月.</p> + * + * @param date 指定日期 + * @return 返回日期的月.如果{@link java.util.Date}为<code>null</code>,抛出异常. + * @throws NullPointerException + */ + public static String getMMFromDate(Date date) { + + if (null == date) { + throw new NullPointerException(); + } + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + calendar.add(Calendar.MONTH, 1); + + return calendar.get(Calendar.MONTH) + ""; + + } + + + /** + * <p>获得指定日期的年.</p> + * + * @param date 指定日期 + * @return 返回日期的年.如果{@link java.util.Date}为<code>null</code>,抛出异常. + * @throws NullPointerException + */ + public static String getYYFromDate(Date date) { + + if (null == date) { + throw new NullPointerException(); + } + Calendar calendar = Calendar.getInstance(); + calendar.setTime(date); + + return calendar.get(Calendar.YEAR) + ""; + + } + + + /** + * <p>返回两个日期之间的所有天数,放入集合{@link List}中.</p> + * <pre> + * DateUtil.getDateList(DateUtil.convertStringToDate(DateUtil.DATE_PATTERN_YYYY_MM_DD, + * "2010-10-22"),new Date()) = list + * </pre> + * + * @param begin 开始时间 + * @param end 结束时间 + * @return 返回两个日期相隔时间集合 + * @throws NullPointerException + */ + public static List<Date> getDateList(Date begin, Date end) { + + if (null == begin || null == end) { + throw new NullPointerException(); + } + + List<Date> result = new ArrayList<Date>(); + Calendar calendar1 = Calendar.getInstance(); + Calendar calendar2 = Calendar.getInstance(); + calendar1.setTime(begin); + calendar2.setTime(end); + while (calendar1.before(calendar2)) { + result.add(calendar1.getTime()); + calendar1.add(calendar1.DAY_OF_MONTH, 1); + } + + return result; + + } + + /** + * <p>根据两个日期获得之间所有的工作日期返回一个日期数组 (工作日不包括星期六和星期日)</p> + * <pre> + * DateUtil.getWorkingdayList(DateUtil.convertStringToDate(TimeConstants.YYYY_MM_DD, + * "2010-10-15"),new Date()) = list + * </pre> + * + * @param begin 开始日期 + * @param end 结束日期 + * @return 返回两个日期相隔时间集合 + * @throws NullPointerException + */ + public static List<Date> getWorkingdayList(Date begin, + Date end) { + if (null == begin || null == end) { + throw new NullPointerException(); + } + List<Date> list = getDateList(begin, end); + List<Date> result = new ArrayList<Date>(); + Calendar calendar = Calendar.getInstance(); + + for (int i = 0; i < list.size(); i++) { + + calendar.setTime(list.get(i)); + if (calendar.get(Calendar.DAY_OF_WEEK) == 1 + || calendar.get(Calendar.DAY_OF_WEEK) == 7) { + continue; + } else { + result.add(list.get(i)); + } + + } + return result; + } + + /** + * <p>根据日期类型的数组获得数组内所有日期对应的星期X列表</p> + * + * @param list 日期类型的数组 + * @return 星期X列表 + */ + public static List<String> getListOfDayOfWeek(List<Date> list) { + + List<String> result = new ArrayList<String>(); + Calendar calendar = Calendar.getInstance(); + for (Date date : list) { + calendar.setTime(date); + switch (calendar.get(Calendar.DAY_OF_WEEK)) { + case 1: + result.add("星期天"); + break; + case 2: + result.add("星期一"); + break; + case 3: + result.add("星期二"); + break; + case 4: + result.add("星期三"); + break; + case 5: + result.add("星期四"); + break; + case 6: + result.add("星期五"); + break; + case 7: + result.add("星期六"); + break; + default: + break; + } + + } + return result; + } + + + /** + * <p>将日期,转换成10进制日期</p> + * <p> + * pattern需跟date格式相同,才可进行转换.转换后的长整型. + * </p> + * <pre> + * DateUtil.convertStringToTimestamp(TimeConstants.YYYY_MM_DD, "2010-10-11") = 1286726400 + * </pre> + * + * @param date 时间值 + * @param pattern 描述日期与格式 + * @return 返回10进制时间值字符串 + * @throws NullPointerException + * @see #convertTimestampToString(long, String) + */ + public static long convertStringToTimestamp(String date, + String pattern) { + + if (StringUtil.isEmpty(date) || StringUtil.isEmpty(pattern)) { + throw new NullPointerException("参数为NULL"); + } + + long time = 0l; + simpleDateFormat = new SimpleDateFormat(pattern); + try { + time = simpleDateFormat.parse(date).getTime() / 1000; + } catch (ParseException e) { + e.printStackTrace(); + } + return time; + + } + + + /** + * <p>将10进制,转换日期字符串类型.</p> + * <pre> + * DateUtil.convertTimestampToString(TimeConstants.YYYY_MM_DD, 1286726400l) = "2010-10-11 00:00:00" + * </pre> + * + * @param time10 时间戳值 + * @param pattern 描述日期与格式 + * @return 返回日期字符串 + * @throws NullPointerException + * @see #getFormatDate(Date, String) + * @see #convertStringToTimestamp(String, String) + */ + public static String convertTimestampToString( + long time10, String pattern) { + + if (StringUtil.isEmpty(pattern)) { + throw new NullPointerException("参数为NULL"); + } + Date date = new Date(); + date.setTime(time10 * 1000); + return getFormatDate(date, pattern); + } + + /** + * <p> + * 将日期转变为字符串的表现形式. + * </p> + * <p> + * 日期格式可以参考本类提供的日期常量,也可自己定义日期格式. + * 当<code>pattern</code> 或 <code>date</code> 为空时,抛出空指针异常. + * </p> + * <pre> + * DateUtil.getFormatDate("yyyyMMddHHmmss", new Date()) = "20101102174448" + * </pre> + * + * @param date 解析的日期值 + * @param pattern 描述日期与时间格式 + * @return 返回日期的字符串表现形式. + * @throws NullPointerException + */ + public static String getFormatDate(Date date, String pattern) { + + if (StringUtil.isEmpty(pattern) || StringUtil.isEmpty(date)) { + throw new NullPointerException("参数为NULL"); + } + simpleDateFormat = new SimpleDateFormat(pattern); + + return simpleDateFormat.format(date); + } + + + /** + * <p>通过字符串日期类型,转换为Date类型日期.</p> + * <p> + * pattern格式，需跟提供的日期字符串格式相统一，否则解析异常返回NULL值。 + * </P> + * <pre> + * DateUtil.convertStringToDate(TimeConstants.YYYY_MM_DD, "2010-09-11") = date类型日期 + * DateUtil.convertStringToDate(TimeConstants.YYYY_MM_DD_HH_MM, "2010-09-11") = NULL. + * </pre> + * + * @param date 字符串类型日期. + * @param pattern 描述日期与时间格式. + * @return 返回解析后的{@link java.util.Date }类型日期. + * @throws NullPointerException + */ + public static Date convertStringToDate(String date, + String pattern) { + + if (StringUtil.isEmpty(date) || StringUtil.isEmpty(pattern)) { + throw new NullPointerException(); + } + simpleDateFormat = new SimpleDateFormat(pattern); + Date parseDate = null; + + try { + parseDate = simpleDateFormat.parse(date); + } catch (ParseException e) { + e.printStackTrace(); + } + + return parseDate; + } + +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/util/Logs.java b/galaxy-report-service/src/main/java/com/mesalab/report/util/Logs.java new file mode 100644 index 0000000..a8f0695 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/util/Logs.java @@ -0,0 +1,30 @@ +package com.mesalab.report.util; + +import com.google.gson.Gson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Created by wk1 on 2019/5/23. + */ + + +public class Logs { + private static Logger logger = LoggerFactory.getLogger(Logs.class); + + //debug方法，其他error、warn类似定义 + public static void debug(Object obj) { + //如果更改日志出工工具，只需修改该行代码即可。如：可改为 System.out.println(obj); + logger.debug(new Gson().toJson(obj)); + } + + public static void error(Object obj) { + //如果更改日志出工工具，只需修改该行代码即可。如：可改为 System.out.println(obj); + logger.error(new Gson().toJson(obj)); + } + + public static void info(Object obj) { + //如果更改日志出工工具，只需修改该行代码即可。如：可改为 System.out.println(obj); + logger.info(new Gson().toJson(obj)); + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/util/SQLDateFunctionUtil.java b/galaxy-report-service/src/main/java/com/mesalab/report/util/SQLDateFunctionUtil.java new file mode 100644 index 0000000..e11f938 --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/util/SQLDateFunctionUtil.java @@ -0,0 +1,1012 @@ +package com.mesalab.report.util; + +import com.mesalab.report.bean.DBTypeEnum; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author GouGe + * @data 2019/8/1 10:13 + */ +public class SQLDateFunctionUtil { + public static Pattern pDateFormat = Pattern.compile("\\W(FROM_UNIXTIME|DATE_FORMAT|STR_TO_DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static Pattern pGroupByFormat = Pattern.compile("(SELECT\\s+|GROUP\\s*BY\\s+|,\\s*)(FROM_UNIXTIME|DATE_FORMAT|STR_TO_DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static Pattern pDateRelative = Pattern.compile("\\W(ADDDATE|DATE_ADD|DATE_SUB|SUBDATE)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static Pattern pIntervalNumUnit = Pattern.compile("INTERVAL(.*?)(YEAR|QUARTER|MONTH|WEEK|DAY|HOUR|MINUTE|SECOND)$", Pattern.CASE_INSENSITIVE); + public static Pattern pUnixTime = Pattern.compile("\\WUNIX_TIMESTAMP\\s*\\(", Pattern.CASE_INSENSITIVE); + public static Pattern pDate = Pattern.compile("\\W(DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static Pattern pDateCalculateNum = Pattern.compile("\\(\\s*(DATE)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static Pattern pUnitFuncGetNum = Pattern.compile("\\W(YEAR|QUARTER|MONTH|DAY|HOUR|MINUTE|SECOND|DAYOFYEAR|DAYOFMONTH|DAYOFWEEK)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static Pattern pMarkDate = Pattern.compile("\\W(MAKEDATE)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static Pattern pLastDateFunc = Pattern.compile("\\W(LAST_DAY)\\s*\\(", Pattern.CASE_INSENSITIVE); + public static Pattern pCKTimeStamp = Pattern.compile("\\s+TIMESTAMP\\s+([^'])", Pattern.CASE_INSENSITIVE); + public static Pattern pNow = Pattern.compile("NOW\\(\\s*\\)", Pattern.CASE_INSENSITIVE); + public static Pattern pStrDateTime = Pattern.compile("(<|>|=|between|and)\\s*('\\d{4}-\\d{2}-\\d{2}\\s*\\d{2}:\\d{2}:\\d{2}')", Pattern.CASE_INSENSITIVE); + + + /** + * 解析mysql函数替换为对应数据库类型函数 + * + * @param sql 要解析的sql + * @param dbType 数据库类型 + * @return + */ + public static String generateDateFunction(String sql, String dbType) { + sql = SQLDateFunctionUtil.parseMarkDate(sql, dbType); + sql = SQLDateFunctionUtil.parseDateCalculateNum(sql, dbType); + sql = SQLDateFunctionUtil.parseLastDayFunc(sql, dbType); + sql = SQLDateFunctionUtil.parseGroupByFormat(sql, dbType); + sql = SQLDateFunctionUtil.parseDateFormat(sql, dbType); + sql = SQLDateFunctionUtil.parseDateRelative(sql, dbType); + sql = SQLDateFunctionUtil.parseUnixTime(sql, dbType); + sql = SQLDateFunctionUtil.parseDate(sql, dbType); + sql = SQLDateFunctionUtil.parseUnitFuncGetNum(sql, dbType); + sql = SQLDateFunctionUtil.specialDisposeEnd(sql, dbType); + sql = SQLDateFunctionUtil.replaceMark(sql); + return sql; + } + + + /** + * 匹配范围：FROM_UNIXTIME(、DATE_FORMAT(、STR_TO_DATE( + * + * @param sql 需要解析的sql + * @param dbType + * @return + */ + public static String parseDateFormat(String sql, String dbType) { + int count = 0; + while (true) { + Matcher matcher = pDateFormat.matcher(sql); + if (!matcher.find() || count++ >= 40) { + return sql; + } + int start = matcher.start() + 1; + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = dateFormatHavaMark(sqlParse, bracketsMatch[0], false, dbType); + sql = sql.substring(0, start) + str; + } + } + + /** + * 匹配格式化 FROM_UNIXTIME(、DATE_FORMAT(、STR_TO_DATE( + * + * @param sql + * @param dbType + * @return + */ + public static String parseGroupByFormat(String sql, String dbType) { + int count = 0; + while (true) { + Matcher matcher = pGroupByFormat.matcher(sql); + if (!matcher.find() || count++ >= 30) { + return sql; + } + int start = matcher.start(); + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = dateFormatHavaMark(sqlParse, bracketsMatch[0], true, dbType); + sql = sql.substring(0, start) + str; + } + } + + /** + * 匹配 SUBDATE(、SUBDATE(、SUBDATE(、ADDDATE(、DATE_SUB(、DATE_ADD( + * + * @param sql + * @param dbType + * @return + */ + public static String parseDateRelative(String sql, String dbType) { + int count = 0; + while (true) { + Matcher matcher = pDateRelative.matcher(sql); + if (!matcher.find() || count++ >= 30) { + return sql; + } + int start = matcher.start() + 1; + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, null, false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = dateRelativeHaveMark(sqlParse, bracketsMatch[0], dbType); + sql = sql.substring(0, start) + str; + } + } + + /** + * 匹配 UNIX_TIMESTAMP( + * + * @param sql + * @param dbType + * @return + */ + public static String parseUnixTime(String sql, String dbType) { + int count = 0; + while (true) { + Matcher matcher = pUnixTime.matcher(sql); + if (!matcher.find() || count++ == 30) { + return sql; + } + int start = matcher.start() + 1; + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, null, false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = parseUnixTimeHavaMark(sqlParse, bracketsMatch[0], dbType); + sql = sql.substring(0, start) + str; + } + } + + /** + * 匹配 DATE( + * + * @param sql + * @param dbType + * @return + */ + public static String parseDate(String sql, String dbType) { + int count = 0; + while (true) { + Matcher matcher = pDate.matcher(sql); + if (!matcher.find() || count++ >= 30) { + return sql; + } + int start = matcher.start() + 1; + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = parseDateHavaMark(sqlParse, bracketsMatch[0], dbType); + sql = sql.substring(0, start) + str; + } + } + + /** + * 匹配 YEAR(、QUARTER(、MONTH(、DAY(、HOUR(、MINUTE(、SECOND、DAYOFYEAR(、DAYOFMONTH(、DAYOFWEEK( + * + * @param sql + * @param dbType + * @return + */ + public static String parseUnitFuncGetNum(String sql, String dbType) { + int count = 0; + while (true) { + Matcher matcher = pUnitFuncGetNum.matcher(sql); + if (!matcher.find() || count++ >= 30) { + return sql; + } + int start = matcher.start() + 1; + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = parseUnitFuncGetNumHaveMark(sqlParse, bracketsMatch[0], dbType); + sql = sql.substring(0, start) + str; + } + } + + /** + * 匹配 pMarkDate( + * + * @param sql + * @param dbType + * @return + */ + public static String parseMarkDate(String sql, String dbType) { + int count = 0; + while (true) { + Matcher matcher = pMarkDate.matcher(sql); + if (!matcher.find() || count++ >= 30) { + return sql; + } + int start = matcher.start() + 1; + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = parseMarkDateHaveMark(sqlParse, bracketsMatch[0], dbType); + sql = sql.substring(0, start) + str; + } + } + + /** + * 匹配 LAST_DAY( + * + * @param sql + * @param dbType + * @return + */ + public static String parseLastDayFunc(String sql, String dbType) { + int count = 0; + while (true) { + Matcher matcher = pLastDateFunc.matcher(sql); + if (!matcher.find() || count++ >= 30) { + return sql; + } + int start = matcher.start() + 1; + String sqlParse = sql.substring(start); + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String str = parseLastDayFuncHaveMark(sqlParse, bracketsMatch[0], dbType); + sql = sql.substring(0, start) + str; + } + } + + /** + * 匹配 涉及DATE()-、DATE()+ + * + * @param sql + * @param dbType + * @return + */ + public static String parseDateCalculateNum(String sql, String dbType) { + int count = 0; + String sqlTmp = ""; + while (true) { + Matcher matcher = pDateCalculateNum.matcher(sql); + if (!matcher.find() || count++ >= 30) { + sql = sql.replace("##", "("); + return sql; + } + String sqlParse = sql.substring(matcher.start()); + if (sqlTmp.equals(sqlParse)) { + sqlParse = "##" + sqlParse.substring(1); + sql = sql.substring(0, matcher.start()) + sqlParse; + continue; + } + sqlTmp = sqlParse; + int[] bracketsMatch = StringUtil.getBracketsMatch(sqlParse, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + String sqlParse2 = sql.substring(matcher.start() + 1); + int[] bracketsMatch2 = StringUtil.getBracketsMatch(sqlParse2, "(", false); + if (bracketsMatch2[0] >= 1) { + --bracketsMatch2[0]; + } + String str = parseDateCalculateNumHaveMark(sqlParse, bracketsMatch[0], bracketsMatch2[0], dbType); + sql = sql.substring(0, matcher.start()) + str; + } + } + + /** + * 替换 涉及DATE()-、DATE()+ + * + * @param sqlParse + * @param num 括号总个数 + * @param num2 date函数内括号数 + * @param dbType + * @return + */ + private static String parseDateCalculateNumHaveMark(String sqlParse, int num, int num2, String dbType) { + Pattern pDateCalculateNumParse = Pattern.compile("(DATE)\\s*\\((.*?(.*?\\).*?){" + num2 + "})\\)(.*?(.*?\\).*?){" + (num - num2 - 1) + "})\\)", Pattern.CASE_INSENSITIVE); + StringBuffer sb = new StringBuffer(); + Matcher mDateCalculateNumParse = pDateCalculateNumParse.matcher(sqlParse); + if (mDateCalculateNumParse.find() && (mDateCalculateNumParse.group(4).trim().startsWith("+") || mDateCalculateNumParse.group(4).trim().startsWith("-"))) { + String group0 = mDateCalculateNumParse.group(0); + String group2 = mDateCalculateNumParse.group(2); + String group4 = mDateCalculateNumParse.group(4); + if (StringUtil.getBracketsMatch(group0.substring(0, group0.length() - 1), "(", true)[0] == num) { + String[] split = group4.split(","); + String replaceValue = ""; + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (split.length == 1) { + replaceValue = "toStartOfDay#[toDateTime#[" + group2 + "]]+[" + group4 + "]*86400 )"; + } else if (split.length == 2) { + replaceValue = "toStartOfDay#[toDateTime#[" + group2 + "]]+[" + split[0] + "]*86400 ," + split[1] + ")"; + } + } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (split.length == 1) { + replaceValue = "TIME_SHIFT#[FLOOR#[ TIMESTAMP " + group2 + " to day],'P1D'," + group4 + "])"; + } else if (split.length == 2) { + replaceValue = "TIME_SHIFT#[FLOOR#[ TIMESTAMP " + group2 + "to day],'P1D'," + split[0] + "]," + split[1] + ")"; + } + } + mDateCalculateNumParse.appendReplacement(sb, replaceValue); + } + } + mDateCalculateNumParse.appendTail(sb); + return sb.toString(); + } + + + /** + * 替换 LAST_DAY(date) + * + * @param sqlParse + * @param num 包含括号个数 + * @param dbType + * @return + */ + public static String parseLastDayFuncHaveMark(String sqlParse, int num, String dbType) { + Pattern pLastDayParse = Pattern.compile("(LAST_DAY)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); + StringBuffer sb = new StringBuffer(); + Matcher mLastDayParse = pLastDayParse.matcher(sqlParse); + if (mLastDayParse.find()) { + String replaceValue = lastDayRealize(mLastDayParse.group(1), mLastDayParse.group(2), dbType); + mLastDayParse.appendReplacement(sb, replaceValue); + } + mLastDayParse.appendTail(sb); + return sb.toString(); + } + + /** + * 实现替换：LAST_DAY(date) + * + * @param unit + * @param param + * @param dbType + * @return + */ + private static String lastDayRealize(String unit, String param, String dbType) { + String replaceValue = null; + if ("LAST_DAY".equals(unit.toUpperCase())) { + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + replaceValue = "addDays#[addMonths#[toStartOfMonth#[toDateTime#[" + param + "]],1],-1]"; + } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + replaceValue = "TIME_SHIFT#[FLOOR#[TIME_SHIFT#[ TIMESTAMP " + param + ", 'P1M', 1] to month], 'P1D', -1]"; + } + } + return replaceValue; + } + + /** + * 替换：MAKEDATE(year,dayofyear) + * + * @param sqlParse + * @param num 函数参数包含括号的个数 + * @param dbType + * @return + */ + public static String parseMarkDateHaveMark(String sqlParse, int num, String dbType) { + Pattern pMarkDateParse = Pattern.compile("(MAKEDATE)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); + StringBuffer sb = new StringBuffer(); + Matcher mMarkDateParse = pMarkDateParse.matcher(sqlParse); + if (mMarkDateParse.find()) { + String replaceValue = markDateRealize(mMarkDateParse.group(1), mMarkDateParse.group(2), dbType); + mMarkDateParse.appendReplacement(sb, replaceValue); + } + mMarkDateParse.appendTail(sb); + return sb.toString(); + } + + /** + * 实现替换：MAKEDATE(year,dayofyear) + * + * @param func + * @param expr + * @param dbType + * @return + */ + private static String markDateRealize(String func, String expr, String dbType) { + String replaceValue = null; + List<String> params = diviParam(expr, ","); + if ("MAKEDATE".equals(func.toUpperCase()) && params.size() == 2) { + Pattern pYearFunNum = Pattern.compile("(YEAR)\\s*\\(", Pattern.CASE_INSENSITIVE); + Matcher matcher = pYearFunNum.matcher(expr); + if (matcher.find() && "YEAR".equals(matcher.group(1).toUpperCase())) { + String temp = expr.substring(matcher.start()); + int[] bracketsMatch = StringUtil.getBracketsMatch(temp, "(", false); + if (bracketsMatch[0] >= 1) { + --bracketsMatch[0]; + } + Pattern pYear = Pattern.compile("^(.*?)(YEAR)\\s*\\((.*?(.*?\\).*?){" + bracketsMatch[0] + "})\\)(.*?)$", Pattern.CASE_INSENSITIVE); + Matcher mYear = pYear.matcher(params.get(0)); + if (mYear.find()) { + String group1 = mYear.group(1); + String group5 = mYear.group(5); + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (group1 != null && !"".equals(group1.trim()) && group5 != null && !"".equals(group5.trim())) { + replaceValue = "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "]]," + group1 + " " + group5 + "]," + params.get(1) + " -1 ]]"; + } else if (group5 != null && !"".equals(group5.trim())) { + replaceValue = "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "]]," + group5 + "]," + params.get(1) + " -1 ]]"; + } else if (group1 != null && !"".equals(group1.trim())) { + replaceValue = "toDateTime#[addDays#[addYears#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "]]," + group1 + "0 ]," + params.get(1) + " -1 ]]"; + } else { + replaceValue = "toDateTime#[addDays#[toStartOfYear#[toDateTime#[" + mYear.group(3) + "]]," + params.get(1) + " -1 ]]"; + } + } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (group1 != null && !"".equals(group1.trim()) && group5 != null && !"".equals(group5.trim())) { + replaceValue = "TIME_SHIFT#[TIME_SHIFT#[FLOOR#[ TIMESTAMP " + mYear.group(3) + " to year],'P1Y'," + group1 + " " + group5 + " ],'P1D'," + params.get(1) + " -1 ]"; + } else if (group5 != null && !"".equals(group5.trim())) { + replaceValue = "TIME_SHIFT#[TIME_SHIFT#[FLOOR#[ TIMESTAMP " + mYear.group(3) + " to year],'P1Y'," + group5 + "],'P1D'," + params.get(1) + " -1 ]"; + } else if (group1 != null && !"".equals(group1.trim())) { + replaceValue = "TIME_SHIFT#[TIME_SHIFT#[FLOOR#[ TIMESTAMP " + mYear.group(3) + " to year],'P1Y'," + group1 + "0 ],'P1D'," + params.get(1) + " -1 ]"; + } else { + replaceValue = "TIME_SHIFT#[FLOOR#[ TIMESTAMP " + mYear.group(3) + " to year],'P1D'," + params.get(1) + "-1 ]"; + } + } + } + } + } + return replaceValue; + } + + /** + * 替换 YEAR(expr)、QUARTER(expr)、MONTH(expr)、DAY(expr)、HOUR(expr)、MINUTE(expr)、SECONDexpr)、DAYOFYEAR(expr)、DAYOFMONTH(expr)、DAYOFWEEK(expr) + * + * @param sqlParse + * @param num + * @param dbType + * @return + */ + private static String parseUnitFuncGetNumHaveMark(String sqlParse, int num, String dbType) { + Pattern pUnitFuncParse = Pattern.compile("(YEAR|QUARTER|MONTH|DAY|HOUR|MINUTE|SECOND|DAYOFYEAR|DAYOFMONTH|DAYOFWEEK)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); + StringBuffer sb = new StringBuffer(); + Matcher mUnitFuncParse = pUnitFuncParse.matcher(sqlParse); + if (mUnitFuncParse.find()) { + String replaceValue = unitFuncGetNumRealize(mUnitFuncParse.group(1), mUnitFuncParse.group(2), dbType); + mUnitFuncParse.appendReplacement(sb, replaceValue); + } + mUnitFuncParse.appendTail(sb); + return sb.toString(); + } + + /** + * 实现替换：YEAR(expr)、QUARTER(expr)、MONTH(expr)、DAY(expr)、HOUR(expr)、MINUTE(expr)、SECONDexpr)、DAYOFYEAR(expr)、DAYOFMONTH(expr)、DAYOFWEEK(expr) + * + * @param unit + * @param expr + * @param dbType + * @return + */ + private static String unitFuncGetNumRealize(String unit, String expr, String dbType) { + String replaceValue = null; + unit = unit.toUpperCase(); + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + switch (unit) { + case "YEAR": + replaceValue = "toYear#[toDateTime#[" + expr + "]]"; + break; + case "QUARTER": + replaceValue = "toQuarter#[toDateTime#[" + expr + "]]"; + break; + case "MONTH": + replaceValue = "toMonth#[toDateTime#[" + expr + "]]"; + break; + case "DAY": + replaceValue = "toDayOfMonth#[toDateTime#[" + expr + "]]"; + break; + case "HOUR": + replaceValue = "toHour#[toDateTime#[" + expr + "]]"; + break; + case "MINUTE": + replaceValue = "toMinute#[toDateTime#[" + expr + "]]"; + break; + case "SECOND": + replaceValue = "toSecond#[toDateTime#[" + expr + "]]"; + break; + case "DAYOFYEAR": + replaceValue = "toDayOfYear#[toDateTime#[" + expr + "]]"; + break; + case "DAYOFMONTH": + replaceValue = "toDayOfMonth#[toDateTime#[" + expr + "]]"; + break; + case "DAYOFWEEK": + replaceValue = "toDayOfWeek#[addDays#[toDateTime#[" + expr + "],1]]"; + break; + default: + replaceValue = unit + "#[" + expr + "]"; + } + } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + switch (unit) { + case "YEAR": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'YEAR' ]"; + break; + case "QUARTER": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'QUARTER' ]"; + break; + case "MONTH": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'MONTH' ]"; + break; + case "DAY": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'DAY' ]"; + break; + case "HOUR": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'HOUR' ]"; + break; + case "MINUTE": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'MINUTE' ]"; + break; + case "SECOND": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'SECOND' ]"; + break; + case "DAYOFYEAR": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'DOY' ]"; + break; + case "DAYOFMONTH": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'PT1M',0], 'DAY' ]"; + break; + case "DAYOFWEEK": + replaceValue = "TIME_EXTRACT#[TIME_SHIFT#[ TIMESTAMP " + expr + ",'P1D',1], 'DOW' ]"; + break; + default: + replaceValue = unit + "#[" + expr + "]"; + } + } + return replaceValue; + } + + /** + * 实现相对时间相关替换： + * + * @param params + * @param sign ADD相关为true,SUB相关为false; + * @param dbType + * @return + */ + private static String dateRelative(List<String> params, Boolean sign, String dbType) { + String replaceValue = ""; + if (params.size() == 2 && params.get(1).toUpperCase().contains("INTERVAL")) { + String param1 = params.get(0); + String param2 = params.get(1); + Matcher matcher = pIntervalNumUnit.matcher(param2); + if (matcher.find()) { + param2 = matcher.group(1); + if (!sign) { + param2 = "- #[" + param2 + "]"; + } + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + switch (matcher.group(2).toUpperCase()) { + case "SECOND": + replaceValue = "toDateTime#[" + param1 + "]" + param2; + break; + case "MINUTE": + replaceValue = "toDateTime#[" + param1 + "]" + param2 + "*60"; + break; + case "HOUR": + replaceValue = "toDateTime#[" + param1 + "]" + param2 + "*3600"; + break; + case "DAY": + replaceValue = "toDateTime#[" + param1 + "]" + param2 + "*86400"; + break; + case "WEEK": + replaceValue = "toDateTime#[" + param1 + "]" + param2 + "*604800"; + break; + case "MONTH": + replaceValue = "addMonths#[toDateTime#[" + param1 + "] ," + param2 + "]"; + break; + case "QUARTER": + replaceValue = "addQuarter#[toDateTime#[" + param1 + "]," + param2 + "]"; + break; + case "YEAR": + replaceValue = "addYears#[toDateTime#[" + param1 + "]," + param2 + "]"; + break; + default: + replaceValue = param1 + param2; + break; + } + } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + switch (matcher.group(2).toUpperCase()) { + case "SECOND": + replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'PT1S'," + param2 + "]"; + break; + case "MINUTE": + replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'PT1M'," + param2 + "]"; + break; + case "HOUR": + replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'PT1H'," + param2 + "]"; + break; + case "DAY": + replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1D'," + param2 + "]"; + break; + case "WEEK": + replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1W'," + param2 + "]"; + break; + case "MONTH": + replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1M'," + param2 + "]"; + break; + case "QUARTER": + replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1M'," + param2 + "]"; + break; + case "YEAR": + replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1Y'," + param2 + "]"; + break; + default: + replaceValue = param1 + param2; + break; + } + } + return replaceValue; + } + } else if (params.size() == 2) { + String param1 = params.get(0); + String param2 = params.get(1); + if (!sign) { + param2 = "- #[" + param2 + "]"; + } + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + replaceValue = "addDays#[toDateTime#[" + param1 + "]," + param2 + "]"; + } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + replaceValue = "TIME_SHIFT#[ TIMESTAMP " + param1 + ",'P1D'," + param2 + "]"; + } + } + return replaceValue; + } + + /** + * 实现替换：FROM_UNIXTIME(unix_timestamp)、FROM_UNIXTIME(date,format)、DATE_FORMAT(date,format)、STR_TO_DATE(date,format) + * + * @param func + * @param param + * @param dbType + * @return + */ + private static String dateFormatRealize(String func, String param, boolean bool, String dbType) { + List<String> params = diviParam(param, ","); + String param1 = params.get(0); + String replaceValue = null; + if ("FROM_UNIXTIME".equals(func.toUpperCase()) && params.size() == 1) { + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + replaceValue = "toDateTime#[" + param + "]"; + } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && !bool) { + replaceValue = "MILLIS_TO_TIMESTAMP#[ 1000 * [" + param + "]]"; + } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && bool) { + replaceValue = "TIME_FORMAT#[MILLIS_TO_TIMESTAMP#[ 1000 * [" + param + "]],'YYYY-MM-dd HH:mm:ss']"; + } + } else if (func.contains("FROM_UNIXTIME") && DBTypeEnum.DRUID.getValue().equals(dbType) && "%Y-%m-%d %H:%i:%s".equals(params.get(1).replaceAll("\\'|\\\"", "").trim())) { + if (!bool) { + replaceValue = "MILLIS_TO_TIMESTAMP#[ 1000*[" + param1 + "]]"; + } else if (bool) { + replaceValue = "TIME_FORMAT#[MILLIS_TO_TIMESTAMP#[ 1000 * [" + param + "]],'YYYY-MM-dd HH:mm:ss']"; + } + } else { + String param2 = params.get(1).replaceAll("\\'|\\\"", "").trim(); + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType) && bool) { + switch (param2) { + case "%Y-%m-%d %H:%i:%s": + replaceValue = "toDateTime#[" + param1 + "]"; + break; + case "%Y-%m-%d %H:%i:00": + replaceValue = "toStartOfMinute#[toDateTime#[" + param1 + "]]"; + break; + case "%Y-%m-%d %H:00:00": + replaceValue = "toStartOfHour#[toDateTime#[" + param1 + "]]"; + break; + case "%Y-%m-%d 00:00:00": + replaceValue = "toStartOfDay#[toDateTime#[" + param1 + "]]"; + break; + case "%Y-%m-01 00:00:00": + replaceValue = "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + "]]]"; + break; + case "%Y-01-01 00:00:00": + replaceValue = "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + "]]]"; + break; + case "%Y-%m-%d": + replaceValue = "formatDateTime#[toDateTime#[" + param1 + "], '%Y-%m-%d]"; + break; + case "%Y-%m-01": + replaceValue = "formatDateTime#[toDateTime#[" + param1 + "], '%Y-%m-01]"; + break; + case "%Y-01-01": + replaceValue = "formatDateTime#[toDateTime#[" + param1 + "], '%Y-01-01]"; + break; + default: + replaceValue = "toDateTime#[" + param1 + "]"; + } + } else if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType) && !bool) { + switch (param2) { + case "%Y-%m-%d %H:%i:%s": + replaceValue = "toDateTime#[" + param1 + "]"; + break; + case "%Y-%m-%d %H:%i:00": + replaceValue = "toStartOfMinute#[toDateTime#[" + param1 + "]]"; + break; + case "%Y-%m-%d %H:00:00": + replaceValue = "toStartOfHour#[toDateTime#[" + param1 + "]]"; + break; + case "%Y-%m-%d 00:00:00": + replaceValue = "toStartOfDay#[toDateTime#[" + param1 + "]]"; + break; + case "%Y-%m-01 00:00:00": + replaceValue = "toDateTime#[toStartOfMonth#[toDateTime#[" + param1 + "]]]"; + break; + case "%Y-01-01 00:00:00": + replaceValue = "toDateTime#[toStartOfYear#[toDateTime#[" + param1 + "]]]"; + break; + case "%Y-%m-%d": + replaceValue = "toStartOfDay#[toDateTime#[" + param1 + "]]"; + break; + case "%Y-%m-01": + replaceValue = "toStartOfMonth#[toDateTime#[" + param1 + "]]"; + break; + case "%Y-01-01": + replaceValue = "toStartOfYear#[toDateTime#[" + param1 + "]]"; + break; + default: + replaceValue = "toDateTime#[" + param1 + "]"; + } + } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && bool) { + switch (param2) { + case "%Y-%m-%d %H:%i:%s": + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:mm:ss']"; + break; + case "%Y-%m-%d %H:%i:00": + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:mm:00']"; + break; + case "%Y-%m-%d %H:00:00": + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:00:00']"; + break; + case "%Y-%m-%d 00:00:00": + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd 00:00:00']"; + break; + case "%Y-%m-01 00:00:00": + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-01 00:00:00']"; + break; + case "%Y-01-01 00:00:00": + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-01-01 00:00:00']"; + break; + case "%Y-%m-%d": + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd']"; + break; + case "%Y-%m-01": + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-01']"; + break; + case "%Y-01-01": + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-01-01']"; + break; + default: + replaceValue = "TIME_FORMAT#[ TIMESTAMP " + param1 + ",'YYYY-MM-dd HH:mm:ss']"; + } + } else if (DBTypeEnum.DRUID.getValue().equals(dbType) && !bool) { + switch (param2) { + case "%Y-%m-%d %H:%i:%s": + replaceValue = " TIMESTAMP " + param1; + break; + case "%Y-%m-%d %H:%i:00": + replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to minute]"; + break; + case "%Y-%m-%d %H:00:00": + replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to hour]"; + break; + case "%Y-%m-%d 00:00:00": + replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to day]"; + break; + case "%Y-%m-01 00:00:00": + replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to month]"; + break; + case "%Y-01-01 00:00:00": + replaceValue = "FLOOR#[ TIMESTAMP " + param1 + " to year]"; + break; + case "%Y-%m-%d": + replaceValue = "TIME_FLOOR#[ TIMESTAMP " + param1 + ",'P1D']"; + break; + case "%Y-%m-01": + replaceValue = "TIME_FLOOR#[ TIMESTAMP " + param1 + ",'P1M']"; + break; + case "%Y-01-01": + replaceValue = "TIME_FLOOR#[ TIMESTAMP " + param1 + ",'P1Y']"; + break; + default: + replaceValue = " TIMESTAMP " + param1; + break; + } + } + } + return replaceValue; + } + + /** + * 解析之后的sql处理 + * + * @param sql + * @param dbType + * @return + */ + public static String specialDisposeEnd(String sql, String dbType) { + if (sql == null) { + return sql; + } + StringBuffer sb = new StringBuffer(); + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + Matcher mStrDateTime = pStrDateTime.matcher(sql); + while (mStrDateTime.find()) { + String innerValue = mStrDateTime.group(1) + " toDateTime#[" + mStrDateTime.group(2) + "]"; + mStrDateTime.appendReplacement(sb, innerValue); + } + mStrDateTime.appendTail(sb); + sql = sb.toString(); + } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + + Matcher mNowFun = pNow.matcher(sql); + while (mNowFun.find()) { + String innerValue = "CURRENT_TIMESTAMP"; + mNowFun.appendReplacement(sb, innerValue); + } + mNowFun.appendTail(sb); + sql = sb.toString(); + + Matcher mStrDateTime = pStrDateTime.matcher(sql); + sb = new StringBuffer(); + while (mStrDateTime.find()) { + String innerValue = mStrDateTime.group(1) + " TIMESTAMP " + mStrDateTime.group(2); + mStrDateTime.appendReplacement(sb, innerValue); + } + mStrDateTime.appendTail(sb); + sql = sb.toString(); + + + Matcher matcher = pCKTimeStamp.matcher(sql); + sb = new StringBuffer(); + while (matcher.find()) { + if (matcher.groupCount() == 1) { + matcher.appendReplacement(sb, " " + matcher.group(1)); + } + } + matcher.appendTail(sb); + sql = sb.toString(); + } + return sql; + } + + /** + * 替换解析时候加入的[、]、# + * + * @param sql + * @return + */ + public static String replaceMark(String sql) { + sql = sql.replace("[", "("); + sql = sql.replace("]", ")"); + sql = sql.replace("#", ""); + return sql; + } + + /** + * 替换：FROM_UNIXTIME(unix_timestamp)、FROM_UNIXTIME(date,format)、DATE_FORMAT(date,format)、STR_TO_DATE(date,format) + * + * @param sqlParse + * @param num 包含括号的个数 + * @param bool 应用的分组中为true 条件范围中则为false + * @param dbType + * @return + */ + public static String dateFormatHavaMark(String sqlParse, int num, boolean bool, String dbType) { + Pattern pDateFormatMark = Pattern.compile("(FROM_UNIXTIME|DATE_FORMAT|STR_TO_DATE)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); + Matcher mDateFormatMark = pDateFormatMark.matcher(sqlParse); + StringBuffer sb = new StringBuffer(); + if (mDateFormatMark.find()) { + String group2 = mDateFormatMark.group(2); + String replaceValue = group2; + if (StringUtil.getBracketsMatch(group2, "(", false)[0] >= 0) { + replaceValue = dateFormatRealize(mDateFormatMark.group(1), group2, bool, dbType); + } + mDateFormatMark.appendReplacement(sb, replaceValue); + } + mDateFormatMark.appendTail(sb); + return sb.toString(); + } + + /** + * 替换SUBDATE(expr,days)、SUBDATE(expr,days)、SUBDATE(date,INTERVAL expr dbType)、ADDDATE(date,INTERVAL expr dbType)、DATE_SUB(date,INTERVAL expr dbType)、DATE_ADD(date,INTERVAL expr dbType) + * + * @param sqlParse + * @param num 包含的括号个数 + * @param dbType + * @return + */ + public static String dateRelativeHaveMark(String sqlParse, int num, String dbType) { + Pattern pDateRelativeParse = Pattern.compile("(ADDDATE|DATE_ADD|SUBDATE|DATE_SUB)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); + Matcher matcherParse = pDateRelativeParse.matcher(sqlParse); + String innerValue = ""; + StringBuffer sb = new StringBuffer(); + if (matcherParse.find()) { + Boolean sign; + if (matcherParse.group(1).toUpperCase().contains("ADD")) { + sign = true; + } else { + sign = false; + } + if (StringUtil.getBracketsMatch(matcherParse.group(2), null, false)[0] >= 0) { + List<String> params = diviParam(matcherParse.group(2), ","); + innerValue = dateRelative(params, sign, dbType); + } + matcherParse.appendReplacement(sb, innerValue); + } + matcherParse.appendTail(sb); + return sb.toString(); + } + + /** + * 实现替换：UNIX_TIMESTAMP()、UNIX_TIMESTAMP(expr) + * + * @param sqlParse + * @param num 包含的括号个数 + * @param dbType + * @return + */ + public static String parseUnixTimeHavaMark(String sqlParse, int num, String dbType) { + Pattern pUnixTimeParse = Pattern.compile("(UNIX_TIMESTAMP)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); + Matcher matcherParse = pUnixTimeParse.matcher(sqlParse); + StringBuffer sb = new StringBuffer(); + String innerValue = null; + if (matcherParse.find()) { + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { + innerValue = "toUnixTimestamp#[now#[]]"; + } else { + innerValue = "toUnixTimestamp#[ " + matcherParse.group(2) + "]"; + } + } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + if (matcherParse.group(2) == null || "".equals(matcherParse.group(2).trim())) { + innerValue = "0.001 * TIMESTAMP_TO_MILLIS[CURRENT_TIMESTAMP]"; + } else { + innerValue = "0.001 * TIMESTAMP_TO_MILLIS#[ TIMESTAMP " + matcherParse.group(2) + "]"; + } + } + matcherParse.appendReplacement(sb, innerValue); + } + matcherParse.appendTail(sb); + return sb.toString(); + } + + /** + * 实现替换：DATE(expr) + * + * @param sqlParse + * @param num 包含的括号个数 + * @param dbType + * @return + */ + public static String parseDateHavaMark(String sqlParse, int num, String dbType) { + Pattern pDateParse = Pattern.compile("(DATE)\\s*\\((.*?(.*?\\).*?){" + num + "})\\)", Pattern.CASE_INSENSITIVE); + Matcher matcherParse = pDateParse.matcher(sqlParse); + StringBuffer sb = new StringBuffer(); + String innerValue = null; + if (matcherParse.find()) { + if (DBTypeEnum.CLICKHOUSE.getValue().equals(dbType)) { + innerValue = "toStartOfDay#[toDateTime#[" + matcherParse.group(2) + "]]"; + } else if (DBTypeEnum.DRUID.getValue().equals(dbType)) { + innerValue = "TIME_FLOOR#[ TIMESTAMP " + matcherParse.group(2) + ",'P1D']"; + } + matcherParse.appendReplacement(sb, innerValue); + } + matcherParse.appendTail(sb); + return sb.toString(); + } + + /** + * 获取函数参数 + * + * @param str + * @param div 参数分隔符 + * @return + */ + public static List<String> diviParam(String str, String div) { + if (str == null) { + return null; + } + List<String> result = new ArrayList<>(); + String[] split = str.split(div); + String resultTemp = ""; + for (int i = 0; i < split.length; i++) { + resultTemp += split[i]; + if (StringUtil.getBracketsMatch(resultTemp, "(", true)[0] >= 0 + && StringUtil.getBracketsMatch(resultTemp, "[", true)[0] >= 0) { + result.add(resultTemp); + resultTemp = ""; + continue; + } + resultTemp += div; + } + return result; + } +} diff --git a/galaxy-report-service/src/main/java/com/mesalab/report/util/StringUtil.java b/galaxy-report-service/src/main/java/com/mesalab/report/util/StringUtil.java new file mode 100644 index 0000000..829c15c --- /dev/null +++ b/galaxy-report-service/src/main/java/com/mesalab/report/util/StringUtil.java @@ -0,0 +1,845 @@ +package com.mesalab.report.util; + + +import org.apache.commons.lang3.StringUtils; +import sun.misc.BASE64Encoder; + +import java.io.UnsupportedEncodingException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Map; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * <p>字符串处理工具类</p> + * + * @author 中电积至有限公司 + * @version 1.0 创建时间：2010-11-8 下午04:50:39 + */ + + +public final class StringUtil extends StringUtils { + + /** + * The empty String <code>""</code>. + */ + public static final String EMPTY = ""; + /** + * 空 <code>String</code> 数组. + */ + public static final String[] EMPTY_STRING_ARRAY = new String[0]; + public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + /** + * ISO8859_1 编码集 + */ + public static final String CODE_ISO8859_1 = "ISO8859_1"; + /** + * GB2312 编码集 + */ + public static final String CODE_GB2312 = "GB2312"; + /** + * GBK 编码集 + */ + public static final String CODE_GBK = "GBK"; + /** + * UTF-8 编码集 + */ + public static final String CODE_UTF_8 = "UTF-8"; + private static final String[] SIMPLIFIED_CASE = {"O", "一", "二", "三", "四", "五", + "六", "七", "八", "九", "十"}; + private static final String[] TRADITIONAL_CASE = {"零", "壹", "贰", "叁", "肆", "伍", + "陆", "柒", "捌", "玖", "拾"}; + private static final String PATTERN_EMPTY_FULL = "[\u4e00-\u9fa5]"; + private static final String PATTERN_EMPTY_CHARACTER = "\\s*|\t|\r|\n"; + private static int size = 0; + + + /** + * <p>Description:抑制默认的构造器,避免实例化对象 </p> + */ + private StringUtil() { + + } + + /** + * <p>判断一个对象是否为空</p> + * <p> + * <code>object</code>元素判断所有对象是否为空. + * 另外对{@link String}、{@link Collection}及其子类 、{@link Map}及其子类、数组 进行长度验证,如果长度为0,视为空对象. + * </p> + * <pre> + * String aa = " "; + * List list = new ArrayList() + * LinkedHashSet set = new LinkedHashSet(); + * StringUtil.isEmpty(aa) = true + * StringUtil.isEmpty(list) = true + * StringUtil.isEmpty(set) = true + * StringUtil.isEmpty("\t") = true + * </pre> + * + * @param object 对象元素 + * @return <code>true</code> 对象为<code>null</code>,<code>false</code> 对象不为<code>null</code>. + */ + public static boolean isEmpty(Object object) { + initSize(object); + return size == 0; + } + + + /** + * <p>判断一个对象不为空</p> + * <p> + * <code>object</code>元素判断不为空验证 + * 另外对{@link String}、{@link Collection}及其子类 、{@link Map}及其子类、数组 进行长度验证,如果长度为0,视为空对象. + * </p> + * <pre> + * String aa = " "; + * List list = new ArrayList() + * LinkedHashSet set = new LinkedHashSet(); + * StringUtil.isNotEmpty(aa) = false + * StringUtil.isEmpty(list) = false + * StringUtil.isEmpty(set) = false + * StringUtil.isEmpty("\t") = false + * </pre> + * + * @param object 对象元素 + * @return <code>true</code> 对象为<code>null</code>,<code>false</code> 对象不为<code>null</code>. + */ + public static boolean isNotEmpty(Object object) { + initSize(object); + return size != 0; + } + + + public static String bSubstring(String s, int length) throws Exception { + + byte[] bytes = s.getBytes("Unicode"); + int n = 0; // 表示当前的字节数 + int i = 2; // 要截取的字节数，从第3个字节开始 + for (; i < bytes.length && n < length; i++) { + // 奇数位置，如3、5、7等，为UCS2编码中两个字节的第二个字节 + if (i % 2 == 1) { + n++; // 在UCS2第二个字节时n加1 + } else { + // 当UCS2编码的第一个字节不等于0时，该UCS2字符为汉字，一个汉字算两个字节 + if (bytes[i] != 0) { + n++; + } + } + } + // 如果i为奇数时，处理成偶数 + if (i % 2 == 1) + + { + // 该UCS2字符是汉字时，去掉这个截一半的汉字 + if (bytes[i - 1] != 0) + i = i - 1; + // 该UCS2字符是字母或数字，则保留该字符 + else + i = i + 1; + } + + return new String(bytes, 0, i, "Unicode"); + } + + + /** + * 判断对象是否有数据存在? 不存在为0、存在不为0的值. + * + * @param object 对象值 + */ + private static void initSize(Object object) { + if (object == null) { + size = 0; + } else { + if (object instanceof String) { + size = isBlank((String) object) ? 0 : 1; + } else if (object instanceof Collection) { + size = ((Collection) object).size(); + } else if (object instanceof Map) { + size = ((Map) object).size(); + } else if (object instanceof Object[]) { + Object[] objectArray = (Object[]) object; + size = objectArray.length; + //其他数据类型 + } else { + size = 1; + } + + } + + } + + + /** + * <p>判断字符串是否为空. + * 为空条件:全角\半角\tab 等没有实际意义的字符. + * 具体参看{@link Character#isWhitespace(char)}对空格的定义. + * </p> + * <p> + * <pre> + * StringUtils.isBlank(" ") = true 为半角空格 + * StringUtils.isBlank(" ") = true 为全角空格 + * StringUtils.isBlank(" ") = true 为tab键 + * </pre> + * + * @param str 字符串 + * @return <code>true</code> 字符串为空 ,<code>false</code> 不为空 + */ + public static boolean isBlank(String str) { + int strLen; + if (str == null || (strLen = str.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return false; + } + } + return true; + } + + + /** + * <p>判断字符串不为空. + * 为空条件:全角\半角\tab 等没有实际意义的字符. + * 具体参看{@link Character#isWhitespace(char)}对空格的定义. + * </p> + * <pre> + * StringUtils.isBlank(" ") = false 为半角空格 + * StringUtils.isBlank(" ") = false 为全角空格 + * StringUtils.isBlank(" ") = false 为tab键 + * </pre> + * + * @param str 字符串 + * @return <code>true</code> 字符串为空 ,<code>false</code> 不为空 + */ + public static boolean isNotBlank(String str) { + return !isBlank(str); + } + + + /** + * <p> + * 获得文件扩展名 + * TXT,HTML,HTM,EML,DOC,DOCX,XLS,XLSX,PPT,PPTX,PDF + * </p> + * + * @param filePath 文件路径 + * @return <code>string</code> 扩展名 ,<code>null</code> 无扩展名 + */ + public static String getFileExtendName(String filePath) { + String extendName = null; + if (filePath.lastIndexOf(".") != -1) { + extendName = filePath.substring(filePath.lastIndexOf(".") + 1).trim().toUpperCase(); + } + return extendName; + } + + + /** + * <p>去掉字符串两边空格. + * 去掉空格规则: + * 半角\全角\tab等无实际意义的字符. + * 具体参看{@link Character#isWhitespace(char)}对空格的定义. + * </p> + * <pre> + * StringUtil.strip(" aa ").length() = 2 + * StringUtil.strip(" ").length() = 0 + * </pre> + * + * @param str 字符 + * @return <code>str</code> 两边去空后字符串,<code>""</code> 字符串为null时. + */ + public static String strip(String str) { + + if (isEmpty(str)) { + return EMPTY; + } + int start = 0; + int end = str.length(); + //去掉前边空格 + while ((start != end) && Character.isWhitespace(str.charAt(start))) { + start++; + } + //去掉后边空格 + while ((end != 0) && Character.isWhitespace(str.charAt(end - 1))) { + end--; + } + + return str.substring(start, end); + } + + /** + * <p>去掉字符串空格，如果为中文去掉所有空格，英文去两边空格。 + * 去掉空格规则: + * 半角\全角\tab等无实际意义的字符. + * <p> + * </p> + * <pre> + * StringUtil.strip(" a a ").length() = 2 + * StringUtil.strip(" ").length() = 0 + * </pre> + * + * @param str 字符 + * @return <code>str</code> 去空后字符串,<code>""<code> 字符串为null时. + */ + public static String stripAll(String str) { + String dest = ""; + Pattern p = Pattern.compile(PATTERN_EMPTY_FULL); + Matcher m = p.matcher(str); + if (m.find()) { + if (isNotBlank(str)) { + p = Pattern.compile(PATTERN_EMPTY_CHARACTER); + str = str.replace((char) 12288, ' '); + Matcher matcher = p.matcher(str); + dest = matcher.replaceAll(""); + } + } else { + dest = str.trim(); + } + + return dest; + } + + + /** + * <p>如果对象为空时,返回默认值.</p> + * + * @param object 要判断是否为空的对象 + * @param defaultValue 为空时设的默认值 + * @return 获取处理后的数据 + * @see #isEmpty(Object) + */ + public static Object setDefaultIfEmpty(Object object, Object defaultValue) { + + return isEmpty(object) ? defaultValue : object; + } + + /** + * <p>对字符串进行MD5加密.</p> + * <p> + * 一般作为密码的处理方式,首先通过MD5进行加密，然后将字符串进行Base64编码获得所需字符. + * </p> + * <pre> + * String str = "ceshi"; + * StringUtil.md5(str) = "zBfDDNERxyFfyPUfh5Dg4Q==" + * </pre> + * + * @param msg 要加密的字符串 + * @return 返回加密后的24位字符, 如果解析出现异常将返回<code>null</code>. + */ + public static String md5(String msg) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] b = md.digest(msg.getBytes()); + BASE64Encoder encoder = new BASE64Encoder(); + return encoder.encode(b); + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + return EMPTY; + } + } + + public static String getBase64(String msg) { + try { + + BASE64Encoder encoder = new BASE64Encoder(); + return encoder.encode(msg.getBytes()); + } catch (Exception e) { + e.printStackTrace(); + return EMPTY; + } + } + + /** + * <p>截取处理字符串,当字符串超过指定的截取长度时,用“......”补充</p> + * <pre> + * String str = "中华人民共和国"; + * StringUtil.getMoreString(str, 6) = "中华人民共和......" + * </pre> + * + * @param text 字符串数据 + * @param length 截取的长度值 + * @return 返回处理后字符 + */ + public static String getMoreString(String text, int length) { + StringBuilder textBuilder = new StringBuilder(); + + if (isEmpty(text)) { + return EMPTY; + } + int endIndex = Math.min(length, text.length()); + if (endIndex == length) { + text = text.substring(0, length); + textBuilder.append(text).append("......"); + } else { + textBuilder.append(text); + } + + + return textBuilder.toString(); + } + + + /** + * <p>通过给定url获取域名地址.</p> + * <p> + * 获得域名信息需去除 http/https/ftp 与 www 一些头信息,获取有效地址. + * </p> + * <pre> + * StringUtil.getDomain("http://www.baidu.com") = "baidu.com" + * </pre> + * + * @param webSiteUrl url 地址 + * @return 返回截取后的域名地址 + */ + public static String getDomain(String webSiteUrl) { + String url; + + if (isEmpty(webSiteUrl)) { + return EMPTY; + } + + if (webSiteUrl.indexOf("http://") >= 0) { + url = webSiteUrl.substring("http://".length(), webSiteUrl.length()); + } else if (webSiteUrl.indexOf("https://", 0) >= 0) { + url = webSiteUrl.substring("https://".length(), webSiteUrl.length()); + } else if (webSiteUrl.indexOf("ftp://", 0) >= 0) { + url = webSiteUrl.substring("ftp://".length(), webSiteUrl.length()); + } else { + url = webSiteUrl; + } + + if (url.indexOf("/", 0) >= 0) { + url = url.substring(0, url.indexOf("/", 1)); + } + if (url.indexOf("www.") == 0) { + url = url.substring(url.indexOf(".") + 1, url.length()); + } + if (url.indexOf("?") >= 0) { + url = url.substring(0, url.indexOf("?")); + } + + return url; + } + + /** + * <p>按照规则解析字符串得到一个数组.</p> + * <pre> + * StringUtil.Split(null, "*") = null + * StringUtil.Split("", *) = [] + * StringUtil.Split("a.b.c", ".") = ["a", "b", "c"] + * StringUtil.Split("a\tb\nc", null) = [] + * StringUtil.Split("a b c", " ") = ["a", "b", "c"] + * </pre> + * + * @param str 需要分隔的字符串 + * @param regex 分隔规则 + * @return 返回字符串数组, 如果分隔字符串为空返回<code>null</code>. + */ + public static String[] split(String str, String regex) { + + if (str == null) { + return null; + } + + if (str.length() == 0 || regex == null) { + return EMPTY_STRING_ARRAY; + } + + + return str.split(regex); + + } + + /** + * <p>判断字符集编码是否在java中支持.</p> + * + * @param name 所要求的字符集 + * @return <code>true</code> java 虚拟机下支持该字符集,<code>false</code> 不支持该字符集 + */ + public static boolean isSupported(String name) { + if (name == null) { + return false; + } + try { + new String(EMPTY_BYTE_ARRAY, name); + } catch (UnsupportedEncodingException e) { + return false; + } + return true; + } + + + /** + * <p>字符编码转换，需要提供字符串的源编码与目的编码格式.</p> + * <p> + * 字符串工具类中提供一些字符编码常量： + * {@link #CODE_GB2312}\{@link #CODE_GBK}\{@link #CODE_ISO8859_1}\{@link #CODE_UTF_8} + * </p> + * + * @param value 要编码的值 + * @param sourceCodingFormat 字符的原始编码格式,具体编码格式可看本类提供的编码样式. + * @param destCodingFormat 要转换字符的编码格式,具体编码格式可看本类提供的编码样式. + * @return 返回编码后的字符串. + * @throws UnsupportedEncodingException + */ + public static String getCodingConversionResult(String value, String sourceCodingFormat, + String destCodingFormat) throws UnsupportedEncodingException { + + if (!isSupported(sourceCodingFormat) || !isSupported(destCodingFormat)) { + throw new UnsupportedEncodingException("JVM 下不支持该字符集编码"); + } + + if (isEmpty(value)) { + return EMPTY; + } + + + return new String(value.getBytes(sourceCodingFormat), destCodingFormat); + } + + + /** + * <p>判断字符串是否是数字格式(包括小数形式).</p> + * <pre> + * String a1 = "12"; + * String a2 = "0.01"; + * String a3 = "0.0.1"; + * String a4 = "123a"; + * StringUtil.isNumeric(a1) = true + * StringUtil.isNumeric(a2) = true + * StringUtil.isNumeric(a3) = false + * StringUtil.isNumeric(a4) = false + * </pre> + * + * @param numberString 数字格式字符串 + * @return <code>true</code> 符合数字格式(包括小数),<code>false</code> 不符合数字格式. + */ + public static boolean isNumeric(String numberString) { + + if (isEmpty(numberString)) { + return false; + } + + if (numberString.startsWith(".") || numberString.endsWith(".")) { + return false; + } + + int length = numberString.split("\\.").length - 1; //判断小数点在字符串中出现的次数。 + + + if (length > 1) { //小数点大于1次，不符合数字规范 + + return false; + } + + + for (int i = 0; i < numberString.length(); i++) { + if (!Character.isDigit(numberString.charAt(i)) && !".".equals(String.valueOf(numberString.charAt(i)))) { + return false; + } + } + + return true; + } + + /** + * <p>压缩字符串</p> + * <p> + * 将带有换行\回车等字符替换为空格格式.具体替换有： + * \t,\n,\f,\r . + * </p> + * <pre> + * StringUtil.compressString("ceshi\too\f") = "ceshi oo " + * </pre> + * + * @param text 要替换的字符串,可能为<code>null</code> + */ + public static String compressString(String text) { + if (isBlank(text)) { + return text; + } + return text.replaceAll("[ \t\n\f\r ]+", " "); + } + + /** + * <p>将查找的字符串全部替换为设置的字符串</p> + * <p> + * 当该方法无操作时,返回<code>null</code>引用. + * </p> + * <pre> + * StringUtil.replace(null, "*","*") = null + * StringUtil.replace("str", null, "*") = "str" + * StringUtil.replace("ceshi", "c", "") = "eshi" + * StringUtil.replace("aba", "a", "z") = "zbz" + * </pre> + * + * @param text 需处理的字符串 + * @param searchString 查找的字符 + * @param replaceString 替换的字符 + * @return 返回替换后的字符串, 文本为空时返回null. + * @see #replace(String, String, String, int) + */ + public static String replace(String text, String searchString, String replaceString) { + + return replace(text, searchString, replaceString, -1); + } + + /** + * <p>将查找字符串替换为设置字符串,根据<code>count</code>决定替换几次. + * 当<code>text</code> 为空不进行任何操作.. + * </p> + * <pre> + * StringUtil.replace("abaa", "a", null, -1) = "abaa" + * StringUtil.replace("abaa", "a", "", -1) = "b" + * StringUtil.replace("abaa", "a", "z", 0) = "abaa" + * StringUtil.replace("abaa", "a", "z", 1) = "zbaa" + * </pre> + * + * @param text 需要处理的字符串 + * @param searchString 查找的字符 + * @param replaceString 替换字符 + * @param count 替换的总次数,<code>-1</code> 替换所有满足条件的字符,<code>0</code> 不进行替换. + * @return 返回替换后的字符串, 文本为空时返回null. + */ + public static String replace(String text, String searchString, String replaceString, int count) { + + if (isBlank(text) || isBlank(searchString) || replaceString == null || count == 0) { + return text; + } + + int start = 0; + int end = text.indexOf(searchString, start); + if (end == -1) { + return text; + } + int replLength = searchString.length(); + + int increase = replaceString.length() - replLength; + increase = (increase < 0 ? 0 : increase); + increase *= (count < 0 ? 16 : (count > 64 ? 64 : count)); + StringBuffer buf = new StringBuffer(text.length() + increase); + while (end != -1) { + buf.append(text.substring(start, end)).append(replaceString); + start = end + replLength; + if (--count == 0) { + break; + } + end = text.indexOf(searchString, start); + } + buf.append(text.substring(start)); + return buf.toString(); + + } + + + /** + * <p>将字符串数字转换为简体大写中文格式.</p> + * <pre> + * StringUtil.convertSimplifiedCase("325") = ”三二五" + * </pre> + * + * @param numberString 数字字符串 + * @return 返回简体大写后的数字 + */ + public static String convertSimplifiedCase(String numberString) { + + StringBuilder simplifiedBuilder = new StringBuilder(); + + if (isEmpty(numberString)) { + return null; + } + + + for (int i = 0; i < numberString.length(); i++) { + String tempNumberString = String.valueOf(numberString.charAt(i)); + if ("0123456789".indexOf(tempNumberString) >= 0) { + int number = Integer.parseInt(tempNumberString); + simplifiedBuilder.append(SIMPLIFIED_CASE[number]); + } else { + simplifiedBuilder.append(tempNumberString); + } + } + + return simplifiedBuilder.toString(); + } + + + /** + * <p>把字符串中的数字转换成繁体大写中文的格式.</p> + * <pre> + * StringUtil.convertTraditionalCase("325") = "叁贰伍" + * </pre> + * + * @param numberString 数字字符串 + * @return 返回繁体大写后的数字 + */ + public static String convertTraditionalCase(String numberString) { + + StringBuilder simplifiedBuilder = new StringBuilder(); + + if (isEmpty(numberString)) { + return null; + } + + + for (int i = 0; i < numberString.length(); i++) { + String tempNumberString = String.valueOf(numberString.charAt(i)); + if ("0123456789".indexOf(tempNumberString) >= 0) { + int number = Integer.parseInt(tempNumberString); + simplifiedBuilder.append(TRADITIONAL_CASE[number]); + } else { + simplifiedBuilder.append(tempNumberString); + } + } + + return simplifiedBuilder.toString(); + } + + /** + * <p>创建唯一标识字符串.</p> + * <pre> + * StringUtil.createUUID() = "00000DAF3CFC4E0B8DF2D5BEACB14D75" + * </pre> + * + * @return 返回标识符字符串 + */ + public static String createUUID() { + String uuid = UUID.randomUUID().toString(); + uuid = uuid.replace("-", ""); + return uuid.toUpperCase(); + } + + + /** + * <p>字符串过滤,负责将html的textarea属性获得的字符转换成html格式的字符集.</p> + * + * @param str 要解析的字符串 + * @return 是解析后的字符串 + */ + public static String htmlFilter(String str) { + StringBuffer stringbuffer = new StringBuffer(); + for (int i = 0; i < str.length(); i++) { + char c = str.charAt(i); + switch (c) { + + case 39: + stringbuffer.append("'"); + break; + + case 34: + stringbuffer.append("""); + break; + + case 60: + stringbuffer.append("<"); + break; + + case 62: + stringbuffer.append(">"); + break; + + case 38: + stringbuffer.append("&"); + break; + + case 32: + stringbuffer.append(" "); + break; + + case 10: + stringbuffer.append("<br>"); + break; + + case 8220: + stringbuffer.append("“"); + break; + + case 8221: + stringbuffer.append("”"); + break; + + default: + stringbuffer.append(c); + break; + } + } + + return stringbuffer.toString(); + } + + + /** + * 统一正则抽取字符方法 + * + * @param str 原始值 + * @param pattern 正则字符串 + * @param index 取符合规范下标属性 + * @return String 返回类型 + * @author （darnell） 2017年8月21日 下午4:25:31 + */ + public static String getFromRegEx(String str, String pattern, int index) { + String value = ""; + Pattern p = Pattern.compile(pattern); + Matcher matcher = p.matcher(str); + while (matcher.find()) { + value = matcher.group(index); + } + return value; + } + + + /** + * 获取第一个括号对应右括号的索引以及包含括号的个数,向后匹配 + * + * @param str + * @param bracketL 支持：( OR [ OR { + * @param bool 是否将传入的字符串全部匹配完成,若不需要则在匹配到第一个括号匹配完成时结束 + * @return int[0]:匹配括号的个数,int[1]:第一个括号对应右括号的索引;若无匹配括号返回{0,-1};若有其中一个不成对则返回{-1,index} + */ + public static int[] getBracketsMatch(String str, String bracketL, boolean bool) { + int[] result = {0, -1}; + if (str == null) { + return result; + } + String bracketR = ")"; + if (bracketL == null) { + bracketL = "("; + } else if (bracketL.equals("[")) { + bracketR = "]"; + } else if (bracketL.equals("{")) { + bracketR = "}"; + } + StringBuffer sb = new StringBuffer(str); + int countL = 0, countR = 0; + for (int a = 0; a < sb.length(); a++) { + if (sb.indexOf(bracketR) == a) { + ++countR; + sb.replace(a, a + 1, "$"); + } else if (sb.indexOf(bracketL) == a) { + ++countL; + sb.replace(a, a + 1, "$"); + } + if (countR > countL) { + result[0] = -1; + result[1] = -1; + return result; + } + result[0] = countL; + result[1] = a; + if (countL != 0 && (countL == countR) && !bool) { + return result; + } + } + if (countR != countL) { + result[0] = -1; + } + return result; + } + +} diff --git a/galaxy-report-service/src/main/resources/mappers/ReportResultMapper.xml b/galaxy-report-service/src/main/resources/mappers/ReportResultMapper.xml new file mode 100644 index 0000000..3bbb033 --- /dev/null +++ b/galaxy-report-service/src/main/resources/mappers/ReportResultMapper.xml @@ -0,0 +1,97 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" > +<mapper namespace="com.mesalab.report.mapper.ReportResultMapper" > + <resultMap type="com.mesalab.report.bean.JobEntity" id="BaseResultMap"> + <id property="resultId" column="result_id"/> + <result property="resultName" column="result_name"/> + <result property="startTime" column="start_time"/> + <result property="endTime" column="end_time"/> + <result property="querySql" column="query_sql"/> + <result property="status" column="status"/> + <result property="excuteTime" column="excute_time"/> + <result property="excuteRow" column="excute_row"/> + <result property="excuteProcess" column="excute_process"/> + <result property="excuteDetail" column="excute_detail"/> + <result property="opTime" column="op_time"/> + <result property="issuedTime" column="issued_time"/> + <result property="resultRows" column="result_rows"/> + <result property="isValid" column="is_valid"/> + + </resultMap> + <sql id="Base_Column_List" > + result_id, result_name, DATE_FORMAT(start_time,'%Y-%m-%d %H:%i:%s') as start_time,DATE_FORMAT(end_time,'%Y-%m-%d %H:%i:%s') as end_time, query_sql, status, excute_time, + excute_row, excute_process, excute_detail, is_send_notice,DATE_FORMAT(op_time,'%Y-%m-%d %H:%i:%s') as op_time,DATE_FORMAT(issued_time,'%Y-%m-%d %H:%i:%s') as issued_time,result_rows,is_valid + </sql> + <select id="getJobForExcute" resultMap="BaseResultMap" parameterType="java.lang.Integer" > + select + <include refid="Base_Column_List" /> + from report_result + where status = 1 order by end_time limit 10 + </select> + + <select id="getJobTask" resultMap="BaseResultMap" parameterType="hashmap"> + select + <include refid="Base_Column_List" /> + from report_result + where status = 0 and is_valid = 1 and end_time < #{endtime} order by end_time limit ${rows} + </select> + + <update id="updateProcesses" parameterType="com.mesalab.report.bean.JobEntity" > + update report_result + <set > + <if test="status != null" > + status = #{status,jdbcType=INTEGER}, + </if> + <if test="excuteTime != null" > + excute_time = #{excuteTime,jdbcType=INTEGER}, + </if> + <if test="excuteRow != null" > + excute_row = #{excuteRow,jdbcType=BIGINT}, + </if> + <if test="excuteProcess != null" > + excute_process = #{excuteProcess,jdbcType=INTEGER}, + </if> + <if test="excuteDetail != null" > + excute_detail = #{excuteDetail,jdbcType=VARCHAR}, + </if> + <if test="opTime != null" > + op_time = #{opTime,jdbcType=TIMESTAMP}, + </if> + <if test="resultRows != null" > + result_rows = #{resultRows,jdbcType=INTEGER}, + </if> + </set> + where result_id = #{resultId,jdbcType=INTEGER} + </update> + + + + + + <update id="updateStatue" parameterType="com.mesalab.report.bean.JobEntity" > + update report_result + <set > + <if test="status != null" > + status = #{status,jdbcType=INTEGER}, + </if> + <if test="excuteDetail != null" > + excute_detail = #{excuteDetail,jdbcType=VARCHAR}, + </if> + <if test="opTime != null" > + op_time = #{opTime,jdbcType=TIMESTAMP}, + </if> + <if test="resultRows != null" > + result_rows = #{resultRows,jdbcType=INTEGER}, + </if> + </set> + where result_id = #{resultId,jdbcType=INTEGER} + </update> + + + <select id="getJobCount" resultType="map" parameterType="hashmap"> + SELECT (SELECT COUNT(1) FROM report_result where status = 0 and is_valid = 1 and end_time < #{endtime}) as queueNum, + (SELECT COUNT(1) FROM report_result where status = 1 and is_valid = 1 ) as excuteingNum, + (SELECT COUNT(1) FROM report_result where status = 2 and op_time > #{optime} ) as todaySuccessNum, + (SELECT COUNT(1) FROM report_result where status > 2 and op_time > #{optime} ) as todayErrorNum + </select> +</mapper>
\ No newline at end of file @@ -68,6 +68,7 @@ <jsqlparser.version>3.0</jsqlparser.version> <jsqlparser.version>3.0</jsqlparser.version> <httpclient.version>4.5.6</httpclient.version> + <org.mariadb.version>1.7.2</org.mariadb.version> <commons.lang.version>2.6</commons.lang.version> <google.gson.version>2.8.6</google.gson.version> <docker.registry>192.168.40.153</docker.registry> |