diff options
| author | LAPTOP-CUUVN8AS\wk <[email protected]> | 2020-12-02 17:36:55 +0800 |
|---|---|---|
| committer | LAPTOP-CUUVN8AS\wk <[email protected]> | 2020-12-02 17:36:55 +0800 |
| commit | d8b40364006ddadeffdcb57606b84605affea5d8 (patch) | |
| tree | 12cc9908c52438347aea83d3a109e2b4c451c440 | |
| parent | 11425f1b9eea2bf47120c31fa1accdcb68335a42 (diff) | |
代码标准化kzformat
27 files changed, 420 insertions, 706 deletions
diff --git a/config/application.properties b/config/application.properties deleted file mode 100644 index 94a07e1..0000000 --- a/config/application.properties +++ /dev/null @@ -1,99 +0,0 @@ -#Hbasehttp�Ķ˿� -server.port=9093 -#���½�������ʱ��10s -scan.result.scheduled.plan=0/15 * * * * ? -#ͬʱ��ִ�����߳��� -globle.job_thread=2 -#Hbasehttp�Ķ˿� -#Hbase�ı���������ͨ������Ҫ���� -hbase.table=tsg:report_result -hbase.zookeeper_quorum=192.168.10.222,192.168.10.223 -hbase.zookeeper_property_clientPort=2181 -hbase.zookeeper_znode_parent=/hbase -hbase.client_retries_number=3 -hbase.rpc_timeout=100000 -hbase.connect_pool=10 -#��ѯ����ip -ck.gateway_ip=192.168.40.223:9999 -#mariadb��url -spring.datasource.url=jdbc:mariadb://192.168.40.204:3306/trepor?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false -#spring.datasource.url=jdbc:mysql://192.168.11.208:3306/ntc-api?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false -#mariadb���û��� -spring.datasource.username=root -#mariadb������ -spring.datasource.password=111111 - -#zk��Ⱥ��ip -zookeeper.connectString=192.168.44.34,192.168.44.35:2181 -#�Ƿ�����zookeeper 0����(��Ⱥ) 1����(����) -zookeeper.open=0 -zookeeper.retryCount=5 -zookeeper.elapsedTimeMs=10000 -zookeeper.sessionTimeoutMs=50000 -zookeeper.connectionTimeoutMs=50000 -zookeeper.nameSpace=reportservice - -#��������� -http.maxTotal=300 -#������ -http.defaultMaxPerRoute=100 -#�������ӵ��ʱ�� -http.connectTimeout=10000 -#�����ӳ��л�ȡ�����ӵ��ʱ�� -http.connectionRequestTimeout=10000 -#���ݴ�����ʱ�� -http.socketTimeout=21605000 -#�ύ����ǰ���������Ƿ���� -http.staleConnectionCheckEnabled=true -http.socketTimeoutShort=30000 - -#�������ò���Ҫ����ͨ�� -spring.datasource.name=druidDataSource -spring.datasource.type=com.alibaba.druid.pool.DruidDataSource -spring.datasource.driver-class-name=org.mariadb.jdbc.Driver - -#���ü��ͳ�����ص�filters��ȥ�����ؽ���SQL������ͳ�ƣ���wall�����ڷ���ǽ -spring.datasource.druid.filters=stat,wall,slf4j -#��������� -spring.datasource.druid.max-active=30 -#��С������ -spring.datasource.druid.min-idle=1 -#��ʼ�������� -spring.datasource.druid.initial-size=2 -#��ȡ�������ʱʱ�� -spring.datasource.druid.max-wait=600000 -#�����òŽ���һ�μ�⣬�����Ҫ�رյĿ������ӣ���λ�Ǻ��� -spring.datasource.druid.time-between-eviction-runs-millis=60000 -# һ�������ڳ�����С�����ʱ�䣬��λ�Ǻ��� -spring.datasource.druid.min-evictable-idle-time-millis=300000 -#��֤�����Ƿ����,�����ݿ���ִ��һ��sql -spring.datasource.druid.validation-query=select 1 -#��������Ϊtrue����Ӱ�����ܣ����ұ�֤��ȫ�ԡ��������ӵ�ʱ���⣬�������ʱ�����timeBetweenEvictionRunsMillis�� -# ִ��validationQuery��������Ƿ���Ч -spring.datasource.druid.test-while-idle=true -#��������ʱִ��validationQuery��������Ƿ���Ч������������ûή������ -spring.datasource.druid.test-on-borrow=true -spring.datasource.druid.test-on-return=false -spring.datasource.druid.connection-properties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 -#�Ƿ���WebStatFilter -spring.datasource.druid.web-stat-filter.enabled=true -#���ò�ͳ����ЩURL(�����ų�һЩ����Ҫ��url) -spring.datasource.druid.web-stat-filter.exclusions=*.js,*.gif,*.jpg,*.png,*.css,*.ico,/druid/* -#�Ƿ���Druid�����Ϣ��ʾҳ�� -spring.datasource.druid.stat-view-servlet.enabled=true -#�������������·�� -spring.datasource.druid.stat-view-servlet.url-pattern=/druid/* -#��ֹ�ֶ����ü������ -spring.datasource.druid.stat-view-servlet.reset-enable=false -#durid-uiҳ���˻����� -spring.datasource.druid.stat-view-servlet.login-username=admin -spring.datasource.druid.stat-view-servlet.login-password=admin -#Spring��أ����ڲ����ӿڵ��õļ��,��Ҫ����aop��ذ� -spring.datasource.druid.aop-patterns=com.mesa.reportservice.controller.*,com.mesa.reportservice.service.*,com.mesa.reportservice.mapper.* -mybatis.typeAliasesPackage=com.mesa.reportservice.bean -mybatis.mapperLocations=classpath*:/mappers/*.xml -spring.application.name=galaxy_reort_service -management.endpoints.web.exposure.include=* -management.metrics.tags.application=${spring.application.name} - -logging.config= ./config/log4j2-dev.xml diff --git a/config/application.yml b/config/application.yml new file mode 100644 index 0000000..18e2381 --- /dev/null +++ b/config/application.yml @@ -0,0 +1,124 @@ +#http的端口 +server: + port: 9093 +#更新进度条的时间10s +scan: + result: + scheduled: + plan: 0/15 * * * * ? +#同时间执行是线程数 +globle: + job_thread: 2 +#Hbasehttp的端口 +#Hbase的表名等配置通畅不需要更改 +hbase: + table: tsg:report_result + zookeeper_quorum: 192.168.40.194,192.168.40.193 + zookeeper_property_clientPort: 2181 + zookeeper_znode_parent: /hbase + client_retries_number: 3 + rpc_timeout: 100000 + connect_pool: 10 +#查询网关ip +ck: + gateway_ip: http://192.168.40.223:9999 + +#zk集群的ip +zookeeper: + connectString: 192.168.44.34,192.168.44.35:2181 +#是否启用zookeeper 0启用(集群) 1禁用(单机) + open: 0 + retryCount: 5 + elapsedTimeMs: 10000 + sessionTimeoutMs: 50000 + connectionTimeoutMs: 50000 + nameSpace: reportservice + +#最大连接数 +http: + maxTotal: 300 +#并发数 + defaultMaxPerRoute: 100 +#创建连接的最长时间 + connectTimeout: 10000 +#从连接池中获取到连接的最长时间 + connectionRequestTimeout: 10000 +#数据传输的最长时间 + socketTimeout: 21605000 +#提交请求前测试连接是否可用 + staleConnectionCheckEnabled: true + socketTimeoutShort: 30000 + +#mariadb的url +spring: + application: + name: galaxy_report_service + + datasource: + url: jdbc:mariadb://192.168.40.204:3306/trepor?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false +#spring.datasource.url=jdbc:mysql://192.168.11.208:3306/ntc-api?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false +#mariadb的用户名 + username: root +#mariadb的密码 + password: 111111 + +#以下配置不需要更改通常 + name: druidDataSource + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: org.mariadb.jdbc.Driver + +#配置监控统计拦截的filters,去掉后监控界面SQL无法进行统计,’wall’用于防火墙 + druid: + filters: stat,wall,slf4j + #最大连接数 + max-active: 30 + #最小连接数 + min-idle: 1 + #初始化连接数 + initial-size: 2 + #获取连接最大超时时间 + max-wait: 600000 + #间隔多久才进行一次检测,检测需要关闭的空闲连接,单位是毫秒 + time-between-eviction-runs-millis: 60000 + # 一个连接在池中最小生存的时间,单位是毫秒 + min-evictable-idle-time-millis: 300000 + #验证连接是否可用,在数据库中执行一条sql + validation-query: select 1 + #建议配置为true,不影响性能,并且保证安全性。申请连接的时候检测,如果空闲时间大于timeBetweenEvictionRunsMillis, + # 执行validationQuery检测连接是否有效 + test-while-idle: true + #申请连接时执行validationQuery检测连接是否有效,做了这个配置会降低性能 + test-on-borrow: true + test-on-return: false + connection-properties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 + #是否开启WebStatFilter + web-stat-filter: + enabled: true + #设置不统计哪些URL(用于排除一些不必要的url) + exclusions: "*.js,*.gif,*.jpg,*..ng,*.css,*.ico,/druid/*" + #是否开启Druid监控信息显示页面 + stat-view-servlet: + enabled: true + #甚至浏览器访问路径 + url-pattern: /druid/* + #禁止手动重置监控数据 + reset-enable: false + #durid-ui页面账户密码 + login-username: admin + login-password: admin + #Spring监控,对内部各接口调用的监控,需要导入aop相关包 + aop-patterns: com.mesa.reportservice.controller.*,com.mesa.reportservice.service.*,com.mesa.reportservice.mapper.* +mybatis: + typeAliasesPackage: com.mesa.reportservice.bean + mapperLocations: classpath*:/mappers/*.xml +management: + endpoints: + web: + exposure: + include: "*" + metrics: + tags: + application: galaxy_report_service + +logging: + config: ./config/log4j2-dev.xml
\ No newline at end of file diff --git a/src/main/java/com/mesa/reportservice/bean/MonitorEntity.java b/src/main/java/com/mesa/reportservice/bean/MonitorEntity.java index 54c7d84..a611071 100644 --- a/src/main/java/com/mesa/reportservice/bean/MonitorEntity.java +++ b/src/main/java/com/mesa/reportservice/bean/MonitorEntity.java @@ -14,6 +14,7 @@ public class MonitorEntity { private Map<String,JobEntity> joblist ; + private String status; public MonitorEntity() { @@ -59,4 +60,12 @@ public class MonitorEntity { public void setJoblist(Map<String, JobEntity> joblist) { this.joblist = joblist; } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } } diff --git a/src/main/java/com/mesa/reportservice/configuration/ClickhouseConfig.java b/src/main/java/com/mesa/reportservice/configuration/ClickhouseConfig.java index 983048c..1c25752 100644 --- a/src/main/java/com/mesa/reportservice/configuration/ClickhouseConfig.java +++ b/src/main/java/com/mesa/reportservice/configuration/ClickhouseConfig.java @@ -1,6 +1,6 @@ package com.mesa.reportservice.configuration; -import com.mesa.reportservice.util.Logs; +import com.mesa.reportservice.util.LogUtil; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @@ -16,7 +16,7 @@ public class ClickhouseConfig { private static String gateway_ip; public static String getJobUrl(String sql, Integer reportId) { - String url = "http://" + gateway_ip.trim() + "/?option=long-term&reportId=" + reportId + "&query="; + String url = gateway_ip.trim() + "/?option=long-term&reportId=" + reportId + "&query="; String jobsql = ""; try { @@ -24,40 +24,40 @@ public class ClickhouseConfig { jobsql = url + sql; } catch (UnsupportedEncodingException e) { - Logs.error(e.toString()); + LogUtil.error("ClickhouseConfig:27",e); } return jobsql; } public static String getProcessUrl(String query_id) { - String url = "http://" + gateway_ip.trim() + "/?query="; + String url = gateway_ip.trim() + "/?query="; String processsql = ""; try { String sql = URLEncoder.encode("select elapsed,total_rows_approx,read_rows from `system`.processes where query_id='" + query_id + "'", "utf8").replaceAll("\\+", "%20"); processsql = url + sql; } catch (UnsupportedEncodingException e) { - Logs.error(e.toString()); + LogUtil.error("ClickhouseConfig:40",e); } return processsql; } public static String getFinishUrl(String query_id) { - String url = "http://" + gateway_ip.trim() + "/?query="; + String url = gateway_ip.trim() + "/?query="; String finishsql = ""; try { String sql = URLEncoder.encode("select CAST(type, 'Int8') as type,read_rows,query_duration_ms,query,exception,memory_usage,event_time,result_rows,result_bytes from `system`.query_log where type>1 and query_id='" + query_id + "' order by event_time desc limit 1", "utf8").replaceAll("\\+", "%20"); finishsql = url + sql; } catch (UnsupportedEncodingException e) { - Logs.error(e.toString()); + LogUtil.error("ClickhouseConfig:53",e); } return finishsql; } public static String getKillUrl(String query_id) { - String url = "http://" + gateway_ip.trim() + "/sys/engine/tasks/"+query_id; + String url = gateway_ip.trim() + "/sys/engine/tasks/"+query_id; return url; } public void setGateway_ip(String gateway_ip) { diff --git a/src/main/java/com/mesa/reportservice/configuration/GlobelConfig.java b/src/main/java/com/mesa/reportservice/configuration/GlobelConfig.java index 643d198..e0b9c0d 100644 --- a/src/main/java/com/mesa/reportservice/configuration/GlobelConfig.java +++ b/src/main/java/com/mesa/reportservice/configuration/GlobelConfig.java @@ -1,10 +1,12 @@ package com.mesa.reportservice.configuration; import com.mesa.reportservice.bean.JobEntity; +import com.mesa.reportservice.util.StringUtil; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; /** @@ -20,4 +22,7 @@ public class GlobelConfig { public void setJob_thread(int job_thread) { GlobelConfig.job_thread = job_thread; } + + public final static String zkuuid = UUID.randomUUID().toString().replaceAll("-","");; + } diff --git a/src/main/java/com/mesa/reportservice/configuration/HbaseConfig.java b/src/main/java/com/mesa/reportservice/configuration/HbaseConfig.java index 2288bd4..5948257 100644 --- a/src/main/java/com/mesa/reportservice/configuration/HbaseConfig.java +++ b/src/main/java/com/mesa/reportservice/configuration/HbaseConfig.java @@ -35,12 +35,6 @@ public class HbaseConfig { return hbasejson; } - public static String getHbasePostUrl() { - - String hbaseurl = "http://" + url + "/" + table + "/rssss"; - - return hbaseurl; - } public void setColumefamily(String columefamily) { HbaseConfig.columefamily = columefamily; diff --git a/src/main/java/com/mesa/reportservice/configuration/HbaseFactory.java b/src/main/java/com/mesa/reportservice/configuration/HbaseFactory.java index 9e4ef1c..01fdaeb 100644 --- a/src/main/java/com/mesa/reportservice/configuration/HbaseFactory.java +++ b/src/main/java/com/mesa/reportservice/configuration/HbaseFactory.java @@ -1,5 +1,6 @@ package com.mesa.reportservice.configuration; +import com.mesa.reportservice.util.LogUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.*; import org.springframework.beans.factory.annotation.Autowired; @@ -9,8 +10,6 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** * Created by Administrator on 2020/3/10. @@ -55,7 +54,7 @@ public class HbaseFactory { con = ConnectionFactory.createConnection(Conf); } catch (IOException e) { - logger.error(e.toString()); + LogUtil.error("HbaseFactory:58",e); } return con; diff --git a/src/main/java/com/mesa/reportservice/configuration/HttpClientPool.java b/src/main/java/com/mesa/reportservice/configuration/HttpClientPool.java index 28d408b..9188db7 100644 --- a/src/main/java/com/mesa/reportservice/configuration/HttpClientPool.java +++ b/src/main/java/com/mesa/reportservice/configuration/HttpClientPool.java @@ -1,6 +1,5 @@ package com.mesa.reportservice.configuration; -import com.mesa.reportservice.util.Logs; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -139,7 +138,6 @@ public class HttpClientPool { */ @Bean(name = "RequestLongConfig") public RequestConfig getRequestConfig(@Qualifier("builder") RequestConfig.Builder builder) { - Logs.debug("httpclient success"); return builder.build(); } @@ -152,7 +150,6 @@ public class HttpClientPool { */ @Bean(name = "RequestShortConfig") public RequestConfig getRequestShortConfig(@Qualifier("builderShort") RequestConfig.Builder builder) { - Logs.debug("httpclientshort success"); return builder.build(); } diff --git a/src/main/java/com/mesa/reportservice/configuration/ZkConfig.java b/src/main/java/com/mesa/reportservice/configuration/ZkConfig.java index 4bedaf1..f8f5fc9 100644 --- a/src/main/java/com/mesa/reportservice/configuration/ZkConfig.java +++ b/src/main/java/com/mesa/reportservice/configuration/ZkConfig.java @@ -1,6 +1,6 @@ package com.mesa.reportservice.configuration; -import com.mesa.reportservice.util.Logs; +import com.mesa.reportservice.util.LogUtil; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -41,7 +41,7 @@ public class ZkConfig { if (zkproperties.getOpen() == 0) { client.start(); } else { - Logs.info("repoertservice start local"); + // LogUtil.info("","reportservice start local"); } return client; diff --git a/src/main/java/com/mesa/reportservice/controller/MonitorController.java b/src/main/java/com/mesa/reportservice/controller/MonitorController.java index 094e867..1035512 100644 --- a/src/main/java/com/mesa/reportservice/controller/MonitorController.java +++ b/src/main/java/com/mesa/reportservice/controller/MonitorController.java @@ -4,6 +4,10 @@ import com.alibaba.fastjson.JSONArray; import com.mesa.reportservice.bean.MonitorEntity; import com.mesa.reportservice.configuration.GlobelConfig; import com.mesa.reportservice.service.MysqlService; +import com.mesa.reportservice.service.ZkService; +import com.mesa.reportservice.util.LogUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.web.bind.annotation.*; @@ -17,11 +21,12 @@ import java.util.Map; @RestController @Component public class MonitorController { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); - private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); @Autowired private MysqlService mysqlService; - + @Autowired + private ZkService zs; @GetMapping(value = "/monitor") public String getReportStatus() { @@ -36,14 +41,19 @@ public class MonitorController { me.setTodaySuccessJobNum(numMap.get("todaySuccessNum")); me.setTodayErrorJobNum(numMap.get("todayErrorNum")); me.setJoblist(GlobelConfig.mapresult); + if(zs.isMaster()){ + me.setStatus("active"); + } + else{ + me.setStatus("standby"); + } GlobelConfig.mapresult.size(); Object obj = JSONArray.toJSON(me); json = obj.toString(); } catch (Exception e) { - logger.error(e.toString()); + LogUtil.error("MonitorController:47",e); json=e.toString(); } - return json; } diff --git a/src/main/java/com/mesa/reportservice/controller/ScheduledResultController.java b/src/main/java/com/mesa/reportservice/controller/ScheduledResultController.java index c193e15..0ace2a3 100644 --- a/src/main/java/com/mesa/reportservice/controller/ScheduledResultController.java +++ b/src/main/java/com/mesa/reportservice/controller/ScheduledResultController.java @@ -3,8 +3,14 @@ package com.mesa.reportservice.controller; import com.mesa.reportservice.bean.JobEntity; import com.mesa.reportservice.configuration.GlobelConfig; import com.mesa.reportservice.configuration.HttpClientPool; -import com.mesa.reportservice.service.*; -import org.apache.commons.codec.digest.DigestUtils; +import com.mesa.reportservice.service.ExcuteProcessService; +import com.mesa.reportservice.service.ExcuteService; +import com.mesa.reportservice.service.MysqlService; +import com.mesa.reportservice.service.ZkService; +import com.mesa.reportservice.util.LogUtil; +import com.mesa.reportservice.util.StringUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; @@ -23,8 +29,7 @@ import java.util.concurrent.Executors; @EnableScheduling public class ScheduledResultController { - private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); - + private final Logger logger = LoggerFactory.getLogger(this.getClass()); protected static ExecutorService pool = Executors.newFixedThreadPool(30); @@ -48,60 +53,74 @@ public class ScheduledResultController { for (JobEntity jobEntity : joblist) { String sql = jobEntity.getQuerySql().trim(); sql = sql.replace("$exe_time", "toDateTime('" + jobEntity.getIssuedTime().trim() + "')"); - //sql = sql.replace("$start_time", "toDateTime('" + jobEntity.getStartTime().trim() + "')"); - //sql = sql.replace("$end_time", "toDateTime('" + jobEntity.getEndTime().trim() + "')"); - String queryid = DigestUtils.md5Hex(jobEntity.getResultId() + sql); + sql = sql.replace("$start_time", "toDateTime('" + jobEntity.getStartTime().trim() + "')"); + sql = sql.replace("$end_time", "toDateTime('" + jobEntity.getEndTime().trim() + "')"); + String queryid = StringUtil.md5(jobEntity.getResultId() + sql); jobEntity.setQuery_id(queryid); - if (jobEntity.getIsValid() == 0) { + /* if (jobEntity.getIsValid() == 0) { eps.killQuery(jobEntity); GlobelConfig.mapresult.get(jobEntity.getQuery_id()).setIsValid(0); } else if (!GlobelConfig.mapresult.containsKey(jobEntity.getQuery_id())) { eps.killQuery(jobEntity); + }*/ + if (GlobelConfig.mapresult.containsKey(jobEntity.getQuery_id())) { + if (jobEntity.getIsValid() == 0) { + eps.killQuery(jobEntity); + GlobelConfig.mapresult.get(jobEntity.getQuery_id()).setIsValid(0); + } + } else { + eps.killQuery(jobEntity); } - } //遍历内存中的任务对状态1的更新进度,其他更新数据库的状态 for (Map.Entry<String, JobEntity> entry : GlobelConfig.mapresult.entrySet()) { logger.info("key = " + entry.getKey() + ", value = " + entry.getValue().getStatus()); long currentTime = System.currentTimeMillis(); - long excutetime =currentTime-entry.getValue().getBeginTime(); - logger.info("excute time="+excutetime+"ttl_time="+HttpClientPool.socketTimeout); - if (entry.getValue().getStatus()==1 && excutetime> HttpClientPool.socketTimeout+1){ + long excutetime = currentTime - entry.getValue().getBeginTime(); + logger.info("excute time=" + excutetime + "ttl_time=" + HttpClientPool.socketTimeout); + if (entry.getValue().getStatus() == 1 && excutetime > HttpClientPool.socketTimeout + 1) { entry.getValue().setStatus(2); entry.getValue().setExcute_status(500001); eps.killQuery(entry.getValue()); eps.updateResultMessage(entry.getValue()); - } - else{ + } else { if (entry.getValue().getStatus() == 1) { eps.updateProcessMessage(entry.getValue()); - } - else { + } else { eps.updateResultMessage(entry.getValue()); } } } - int rows = GlobelConfig.job_thread - GlobelConfig.mapresult.size(); - if (rows > 0) { - List<JobEntity> jobs = ms.getJobTask(rows); - for (JobEntity job : jobs) { - logger.info("开始执行任务"); - long begintime = System.currentTimeMillis(); - job.setBeginTime(begintime); - pool.execute(new Runnable() { - @Override - public void run() { - es.excuteCkTask(job); - } - }); - } - } else { - logger.info("无待执行任务"); + int rows = GlobelConfig.job_thread - GlobelConfig.mapresult.size(); + if (rows > 0) { + List<JobEntity> jobs = ms.getJobTask(rows); + for (JobEntity job : jobs) { + logger.info("开始执行任务"); + long begintime = System.currentTimeMillis(); + job.setBeginTime(begintime); + pool.execute(new Runnable() { + @Override + public void run() { + es.excuteCkTask(job); + } + }); } + } else { + logger.info("无待执行任务"); + } + } else { + if (GlobelConfig.mapresult.size() > 0) { + for (Map.Entry<String, JobEntity> entry : GlobelConfig.mapresult.entrySet()) { + logger.info("key = " + entry.getKey() + ", value = " + entry.getValue().getStatus()); + eps.killQuery(entry.getValue()); + } + GlobelConfig.mapresult.clear(); + } } } catch (Exception e) { - logger.error(e.toString()); + LogUtil.error("ScheduledResultController:106", e); + } } diff --git a/src/main/java/com/mesa/reportservice/service/impl/ClickhouseServiceImpl.java b/src/main/java/com/mesa/reportservice/service/impl/ClickhouseServiceImpl.java index 54cab5c..edb9ee4 100644 --- a/src/main/java/com/mesa/reportservice/service/impl/ClickhouseServiceImpl.java +++ b/src/main/java/com/mesa/reportservice/service/impl/ClickhouseServiceImpl.java @@ -1,27 +1,21 @@ package com.mesa.reportservice.service.impl; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.JSONObject; import com.mesa.reportservice.bean.HttpResult; import com.mesa.reportservice.service.ClickhouseService; +import com.mesa.reportservice.util.LogUtil; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; import org.apache.http.client.utils.URIBuilder; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.params.CoreConnectionPNames; import org.apache.http.util.EntityUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import javax.annotation.Resource; import java.util.Map; -import static com.mesa.reportservice.util.GZIPUtils.compress; -import static com.mesa.reportservice.util.GZIPUtils.uncompress; /** * Created by wk1 on 2019/5/15. @@ -40,7 +34,6 @@ public class ClickhouseServiceImpl implements ClickhouseService { @Autowired @Qualifier("RequestLongConfig") private RequestConfig RequestLongConfig; - private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); /** * 不带参数的get请求,如果状态码为200,则返回body,如果不为200,则返回null @@ -70,7 +63,7 @@ public class ClickhouseServiceImpl implements ClickhouseService { } catch (Exception e){ - logger.error(e.toString()); + LogUtil.error("ClickhouseServerImpl:77 error", e); throw new Exception(); } @@ -128,7 +121,7 @@ public class ClickhouseServiceImpl implements ClickhouseService { return rs; }catch (Exception e){ - logger.error(e.toString()); + LogUtil.error("ClickhouseServerImpl:136 error", e); throw new Exception(); } @@ -155,7 +148,7 @@ public class ClickhouseServiceImpl implements ClickhouseService { return rs; }catch (Exception e){ - logger.error(e.toString()); + LogUtil.error("ClickhouseServerImpl:164 error", e); throw new Exception(); } } @@ -179,7 +172,7 @@ public class ClickhouseServiceImpl implements ClickhouseService { return rs; } catch (Exception e){ - logger.error(e.toString()); + LogUtil.error("ClickhouseServerImpl:188 error", e); throw new Exception(); } diff --git a/src/main/java/com/mesa/reportservice/service/impl/ExcuteProcessServiceImpl.java b/src/main/java/com/mesa/reportservice/service/impl/ExcuteProcessServiceImpl.java index c79190a..96a4cf7 100644 --- a/src/main/java/com/mesa/reportservice/service/impl/ExcuteProcessServiceImpl.java +++ b/src/main/java/com/mesa/reportservice/service/impl/ExcuteProcessServiceImpl.java @@ -6,6 +6,9 @@ import com.mesa.reportservice.bean.JobEntity; import com.mesa.reportservice.configuration.ClickhouseConfig; import com.mesa.reportservice.configuration.GlobelConfig; import com.mesa.reportservice.service.*; +import com.mesa.reportservice.util.LogUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -18,7 +21,7 @@ import java.util.Map; @Service public class ExcuteProcessServiceImpl implements ExcuteProcessService { - private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + private final Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired @@ -99,7 +102,7 @@ public class ExcuteProcessServiceImpl implements ExcuteProcessService { je.setStatus(10); je.setExcuteDetail("Database Error"); ms.updateProcesses(je); - logger.error("save database error resultid =" + je.getResultId()+" queryid=" + je.getResultId() + e.toString()); + logger.error("save database error resultid =" + je.getResultId()+" queryid=" + je.getResultId() ); } finally { saveToMonitor(je); @@ -116,7 +119,7 @@ public class ExcuteProcessServiceImpl implements ExcuteProcessService { cs.QuerySystemForDelete(killurl); } catch (Exception e) { - logger.error(e.toString()); + LogUtil.error("ExcuteProcessServiceImpl:122 error",e); } finally { jobEntity.setStatus(0); jobEntity.setExcuteDetail("Re Execution"); @@ -162,7 +165,7 @@ public class ExcuteProcessServiceImpl implements ExcuteProcessService { logger.error("responsebody is null"); } } catch (Exception e) { - logger.error(e.toString()); + logger.error("ExcuteProcessServiceImpl:168",e); } } @@ -178,7 +181,7 @@ public class ExcuteProcessServiceImpl implements ExcuteProcessService { isok = hs.put(entity); break; } catch (Exception e) { - logger.error("写入hbase报错重试次数" + (3 - k) + e.toString()); + logger.error("写入hbase报错重试次数" + (3 - k) ); k--; } } @@ -199,7 +202,7 @@ public class ExcuteProcessServiceImpl implements ExcuteProcessService { mons.addFail(); } } catch (Exception e) { - logger.error("监控结果失败" + e.toString()); + logger.error("监控结果失败"); } } } diff --git a/src/main/java/com/mesa/reportservice/service/impl/ExcuteserviceImpl.java b/src/main/java/com/mesa/reportservice/service/impl/ExcuteserviceImpl.java index 7109c93..ea76110 100644 --- a/src/main/java/com/mesa/reportservice/service/impl/ExcuteserviceImpl.java +++ b/src/main/java/com/mesa/reportservice/service/impl/ExcuteserviceImpl.java @@ -9,8 +9,12 @@ import com.mesa.reportservice.service.ClickhouseService; import com.mesa.reportservice.service.ExcuteService; import com.mesa.reportservice.service.FormatService; import com.mesa.reportservice.service.MysqlService; +import com.mesa.reportservice.util.LogUtil; +import com.mesa.reportservice.util.StringUtil; import io.netty.channel.ConnectTimeoutException; import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -24,7 +28,7 @@ import static com.mesa.reportservice.configuration.ClickhouseConfig.getJobUrl; */ @Service public class ExcuteserviceImpl implements ExcuteService { - private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + private final Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired @@ -41,10 +45,10 @@ public class ExcuteserviceImpl implements ExcuteService { String sql = job.getQuerySql().trim(); sql = sql.replace("$exe_time", "toDateTime('" + job.getIssuedTime().trim() + "')"); - //sql = sql.replace("$start_time", "toDateTime('" + job.getStartTime().trim() + "')"); - //sql = sql.replace("$end_time", "toDateTime('" + job.getEndTime().trim() + "')"); + sql = sql.replace("$start_time", "toDateTime('" + job.getStartTime().trim() + "')"); + sql = sql.replace("$end_time", "toDateTime('" + job.getEndTime().trim() + "')"); job.setQuerySql(sql); - String queryid = DigestUtils.md5Hex(job.getResultId() + sql); + String queryid = StringUtil.md5(job.getResultId() + sql); job.setQuery_id(queryid); job.setQuerySql(sql); job.setStatus(1); @@ -67,7 +71,6 @@ public class ExcuteserviceImpl implements ExcuteService { logger.info("httpcode" + hr.getCode()); if (hr != null && hr.getCode() != 200) { - k = 0; Map mapresult = JSON.parseObject(hr.getBody()); job.setExcute_status(Integer.parseInt(mapresult.get("code").toString())); @@ -77,52 +80,54 @@ public class ExcuteserviceImpl implements ExcuteService { k = 0; Map mapresult = JSON.parseObject(hr.getBody()); - Map rows = (Map) mapresult.get("statistics"); job.setResultRows(Integer.parseInt(rows.get("result_rows").toString())); job.setExcuteRow(Long.parseLong(rows.get("rows_read").toString())); job.setResult(hr.getBody()); job.setExcuteTime((int) Float.parseFloat(rows.get("elapsed").toString())); job.setExcute_status(Integer.parseInt(mapresult.get("code").toString())); - logger.info("success resultid = "+job.getResultId()+" queryid=" + queryid + " sql=" + sql); + // String message = "success resultid = "+job.getResultId()+" queryid=" + queryid + " sql=" + sql; + // logger.info("success resultid = "+job.getResultId()); } } catch (SocketTimeoutException e) { k--; - job.setExcuteDetail(e.toString()); + String errorMessage = e.getMessage(); + job.setExcuteDetail(errorMessage); if (k == 0) { job.setExcute_status(500001); job.setExcuteDetail("SQL Execution Error excute query time out"); - logger.info("timeout resultid = "+job.getResultId()+" queryid=" + queryid + " sql=" + sql); + String message ="timeout resultid = "+job.getResultId()+" queryid=" + queryid + " sql=" + sql; + logger.error("ExcuteserviceImpl:104 timeout"); } else { - logger.info("Socket warn " + e.toString() + "retry time " + (3 - k)); + // logger.info("Socket warn retry time " + (3 - k)); } } catch (ConnectTimeoutException e) { job.setExcute_status(555999); - job.setExcuteDetail(e.toString()); - logger.error("Unknow Error" + e.toString()); + job.setExcuteDetail("ConnectTimeoutException "); + logger.error("ConnectTimeoutException Error "); k = 0; } catch (OutOfMemoryError e) { job.setExcute_status(555999); job.setExcuteDetail("result too large"); - logger.error("outofmemery Error" + e.toString()); + logger.error("outofmemery Error"); k = 0; } catch (Exception e) { job.setExcute_status(555999); - job.setExcuteDetail(e.toString()); - logger.error("Unknow Error" + e.toString()); + job.setExcuteDetail("unknow Error"); + LogUtil.error("Unknow Error {}" , e); k = 0; } try { cs.QuerySystemForDelete(killurl); } catch (Exception e) { - logger.error("Kill Query Error" + e.toString()); + LogUtil.error("ExcuteserviceImpl:131 Kill Query Error" , e); } } while (k > 0); diff --git a/src/main/java/com/mesa/reportservice/service/impl/HbaseServiceImpl.java b/src/main/java/com/mesa/reportservice/service/impl/HbaseServiceImpl.java index 4509736..579e8d1 100644 --- a/src/main/java/com/mesa/reportservice/service/impl/HbaseServiceImpl.java +++ b/src/main/java/com/mesa/reportservice/service/impl/HbaseServiceImpl.java @@ -1,28 +1,15 @@ package com.mesa.reportservice.service.impl; -import com.mesa.reportservice.bean.HttpResult; import com.mesa.reportservice.bean.JobEntity; import com.mesa.reportservice.configuration.HbaseProperties; import com.mesa.reportservice.service.HbaseService; -import com.mesa.reportservice.util.Logs; -import com.mesa.reportservice.util.ziputil; +import com.mesa.reportservice.util.LogUtil; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.util.EntityUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.io.IOException; -import java.util.Map; - /** * Created by wk1 on 2019/5/15. */ @@ -61,7 +48,7 @@ public class HbaseServiceImpl implements HbaseService { table.put(put); status =true; }catch(Exception e) { - Logs.error(e.toString()); + LogUtil.error("HbaseServiceImpl:52",e); } return status; } diff --git a/src/main/java/com/mesa/reportservice/service/impl/MonitorServiceImpl.java b/src/main/java/com/mesa/reportservice/service/impl/MonitorServiceImpl.java index 2ab4170..8b2cdd6 100644 --- a/src/main/java/com/mesa/reportservice/service/impl/MonitorServiceImpl.java +++ b/src/main/java/com/mesa/reportservice/service/impl/MonitorServiceImpl.java @@ -1,24 +1,14 @@ package com.mesa.reportservice.service.impl; -import com.mesa.reportservice.configuration.GlobelConfig; -import com.mesa.reportservice.configuration.ZkProperties; import com.mesa.reportservice.service.MonitorService; -import com.mesa.reportservice.service.ZkService; -import com.mesa.reportservice.util.Logs; +import com.mesa.reportservice.util.LogUtil; import io.micrometer.core.instrument.*; -import org.apache.commons.collections.map.HashedMap; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.api.ExistsBuilder; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.util.ObjectUtils; import javax.annotation.PostConstruct; -import java.net.InetAddress; -import java.util.List; -import java.util.Map; /** * Created by wk1 on 2020/1/6. @@ -26,7 +16,7 @@ import java.util.Map; @Service public class MonitorServiceImpl implements MonitorService { - private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + private final Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired MeterRegistry registry; @@ -47,7 +37,7 @@ public class MonitorServiceImpl implements MonitorService { try{ counter_report_success.increment(); } catch (Exception e) { - logger.error(e.toString()); + LogUtil.error("MonitorServiceImpl:40",e); } } @Override @@ -56,7 +46,7 @@ public class MonitorServiceImpl implements MonitorService { try{ counter_report_fail.increment(); } catch (Exception e) { - logger.error(e.toString()); + LogUtil.error("MonitorServiceImpl:40",e); } } diff --git a/src/main/java/com/mesa/reportservice/service/impl/MysqlServiceImpl.java b/src/main/java/com/mesa/reportservice/service/impl/MysqlServiceImpl.java index 534a90e..e4b237c 100644 --- a/src/main/java/com/mesa/reportservice/service/impl/MysqlServiceImpl.java +++ b/src/main/java/com/mesa/reportservice/service/impl/MysqlServiceImpl.java @@ -4,8 +4,9 @@ import com.mesa.reportservice.bean.JobEntity; import com.mesa.reportservice.mapper.ReportResultMapper; import com.mesa.reportservice.service.MysqlService; import com.mesa.reportservice.util.DateUtil; -import com.mesa.reportservice.util.Logs; -import com.mesa.reportservice.util.StringUtil; +import com.mesa.reportservice.util.LogUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -20,7 +21,7 @@ import java.util.Map; */ @Service public class MysqlServiceImpl implements MysqlService { - private final org.apache.log4j.Logger logger = org.apache.log4j.Logger.getLogger(this.getClass()); + private final Logger print = LoggerFactory.getLogger(this.getClass()); @Autowired public ReportResultMapper rrm; @@ -57,7 +58,7 @@ public class MysqlServiceImpl implements MysqlService { return rrm.updateProcesses(job); } catch(Exception e){ - logger.error(e.toString()); + LogUtil.error("MysplServiceImpl:61",e); return 0; } } @@ -73,7 +74,7 @@ public class MysqlServiceImpl implements MysqlService { return rrm.updateStatue(job); } catch(Exception e){ - logger.error(e.toString()); + LogUtil.error("MysplServiceImpl:78",e); return 0; } } diff --git a/src/main/java/com/mesa/reportservice/service/impl/ZkServiceImpl.java b/src/main/java/com/mesa/reportservice/service/impl/ZkServiceImpl.java index 23e970f..fb1f43f 100644 --- a/src/main/java/com/mesa/reportservice/service/impl/ZkServiceImpl.java +++ b/src/main/java/com/mesa/reportservice/service/impl/ZkServiceImpl.java @@ -1,8 +1,9 @@ package com.mesa.reportservice.service.impl; +import com.mesa.reportservice.configuration.GlobelConfig; import com.mesa.reportservice.configuration.ZkProperties; import com.mesa.reportservice.service.ZkService; -import com.mesa.reportservice.util.Logs; +import com.mesa.reportservice.util.LogUtil; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.ExistsBuilder; import org.apache.zookeeper.CreateMode; @@ -30,35 +31,36 @@ public class ZkServiceImpl implements ZkService { if (zp.getOpen() == 0) { boolean isZkCuratorStarted = curatorConnect.isStarted(); String nodePath = "/masterip"; - System.out.println("当前客户端的状态:" + (isZkCuratorStarted ? "连接中..." : "已关闭...")); + // System.out.println("当前客户端的状态:" + (isZkCuratorStarted ? "连接中..." : "已关闭...")); ExistsBuilder sss = curatorConnect.checkExists(); Stat statExist = curatorConnect.checkExists().forPath(nodePath); - InetAddress address = InetAddress.getLocalHost(); - String localip = address.getHostAddress().trim(); + //InetAddress address = InetAddress.getLocalHost(); + // String localip = address.getHostAddress().trim(); + //String zkuuid = GlobelConfig.zkuuid; + // System.out.print(GlobelConfig.zkuuid); if (statExist == null) { - System.out.println(address.getHostAddress()); - byte[] data = address.getHostAddress().getBytes(); // 节点数据 + byte[] data = GlobelConfig.zkuuid.getBytes(); // 节点数据 String result = curatorConnect.create().creatingParentsIfNeeded() // 创建父节点,也就是会递归创建 .withMode(CreateMode.EPHEMERAL) // 节点类型 .forPath(nodePath, data); - System.out.println(result + "节点,创建成功..."); return true; } else { - System.out.println(nodePath + " 节点存在"); Stat stat = new Stat(); byte[] nodeData = curatorConnect.getData().storingStatIn(stat).forPath(nodePath); String masterip = new String(nodeData).trim(); - System.out.println("节点 " + nodePath + " 的数据为:" + new String(nodeData)); - - if (masterip.equals(localip)) { + // System.out.println("节点 " + nodePath + " 的数据为:" + new String(nodeData)); + /// System.out.print("-------------"+GlobelConfig.zkuuid); + if (masterip.equals(GlobelConfig.zkuuid)) { + // System.out.print("-------------s是主节点"); return true; } else { - Logs.debug("不是主节点"); + // System.out.print("-------------不是主节点"); + return false; } } @@ -66,7 +68,7 @@ public class ZkServiceImpl implements ZkService { return true; } } catch (Exception e) { - Logs.error(e.toString()); + LogUtil.error("ZKServiceImpl:61",e); return true; } } diff --git a/src/main/java/com/mesa/reportservice/util/DateUtil.java b/src/main/java/com/mesa/reportservice/util/DateUtil.java index 05ee225..09571a8 100644 --- a/src/main/java/com/mesa/reportservice/util/DateUtil.java +++ b/src/main/java/com/mesa/reportservice/util/DateUtil.java @@ -460,30 +460,6 @@ public final class DateUtil { * <pre> * DateUtil.convertStringToTimestamp(TimeConstants.YYYY_MM_DD, "2010-10-11") = 1286726400 * </pre> - * - * @param date 时间值 - * @param pattern 描述日期与格式 - * @return 返回10进制时间值字符串 - * @throws NullPointerException - * @see #convertTimestampToString(long, String) - */ - public static long convertStringToTimestamp(String date, - String pattern) { - - if (StringUtil.isEmpty(date) || StringUtil.isEmpty(pattern)) { - throw new NullPointerException("参数为NULL"); - } - - long time = 0l; - simpleDateFormat = new SimpleDateFormat(pattern); - try { - time = simpleDateFormat.parse(date).getTime() / 1000; - } catch (ParseException e) { - e.printStackTrace(); - } - return time; - - } /** @@ -538,37 +514,4 @@ public final class DateUtil { } - /** - * <p>通过字符串日期类型,转换为Date类型日期.</p> - * <p> - * pattern格式,需跟提供的日期字符串格式相统一,否则解析异常返回NULL值。 - * </P> - * <pre> - * DateUtil.convertStringToDate(TimeConstants.YYYY_MM_DD, "2010-09-11") = date类型日期 - * DateUtil.convertStringToDate(TimeConstants.YYYY_MM_DD_HH_MM, "2010-09-11") = NULL. - * </pre> - * - * @param date 字符串类型日期. - * @param pattern 描述日期与时间格式. - * @return 返回解析后的{@link java.util.Date }类型日期. - * @throws NullPointerException - */ - public static Date convertStringToDate(String date, - String pattern) { - - if (StringUtil.isEmpty(date) || StringUtil.isEmpty(pattern)) { - throw new NullPointerException(); - } - simpleDateFormat = new SimpleDateFormat(pattern); - Date parseDate = null; - - try { - parseDate = simpleDateFormat.parse(date); - } catch (ParseException e) { - e.printStackTrace(); - } - - return parseDate; - } - } diff --git a/src/main/java/com/mesa/reportservice/util/GZIPUtils.java b/src/main/java/com/mesa/reportservice/util/GZIPUtils.java deleted file mode 100644 index 5185351..0000000 --- a/src/main/java/com/mesa/reportservice/util/GZIPUtils.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.mesa.reportservice.util; - -/** - * Created by wk1 on 2020/5/14. - */ -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -public class GZIPUtils { - public static final String GZIP_ENCODE_UTF_8 = "UTF-8"; - public static final String GZIP_ENCODE_ISO_8859_1 = "ISO-8859-1"; - - - public static byte[] compress(byte[] str) { - if (str == null || str.length == 0) { - return null; - } - ByteArrayOutputStream out = new ByteArrayOutputStream(); - GZIPOutputStream gzip; - try { - gzip = new GZIPOutputStream(out); - gzip.write(str); - gzip.close(); - } catch ( Exception e) { - e.printStackTrace(); - } - return out.toByteArray(); - } - - - - public static byte[] uncompress(byte[] bytes) { - if (bytes == null || bytes.length == 0) { - return null; - } - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ByteArrayInputStream in = new ByteArrayInputStream(bytes); - try { - GZIPInputStream ungzip = new GZIPInputStream(in); - byte[] buffer = new byte[256]; - int n; - while ((n = ungzip.read(buffer)) >= 0) { - out.write(buffer, 0, n); - } - } catch (Exception e) { - e.printStackTrace(); - } - return out.toByteArray(); - } - - public static String uncompressToString(byte[] bytes, String encoding) { - if (bytes == null || bytes.length == 0) { - return null; - } - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ByteArrayInputStream in = new ByteArrayInputStream(bytes); - try { - GZIPInputStream ungzip = new GZIPInputStream(in); - byte[] buffer = new byte[256]; - int n; - while ((n = ungzip.read(buffer)) >= 0) { - out.write(buffer, 0, n); - } - return out.toString(encoding); - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - public static String uncompressToString(byte[] bytes) { - return uncompressToString(bytes, GZIP_ENCODE_UTF_8); - } - - public static void main(String[] args) throws IOException { - String s = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - System.out.println("字符串长度:"+s.length()); - //System.out.println("压缩后::"+compress(s).length); - // System.out.println("解压后:"+uncompress(compress(s)).length); - // System.out.println("解压字符串后::"+uncompressToString(compress(s)).length()); - } -} diff --git a/src/main/java/com/mesa/reportservice/util/IdleConnectionEvictor.java b/src/main/java/com/mesa/reportservice/util/IdleConnectionEvictor.java deleted file mode 100644 index cca9f05..0000000 --- a/src/main/java/com/mesa/reportservice/util/IdleConnectionEvictor.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.mesa.reportservice.util; - -import org.apache.http.conn.HttpClientConnectionManager; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -/** - * Created by wk1 on 2019/5/15. - */ - - -@Component - -public class IdleConnectionEvictor extends Thread { - - @Autowired - private HttpClientConnectionManager connMgr; - - private volatile boolean shutdown; - - public IdleConnectionEvictor() { - super(); - super.start(); - } - - @Override - public void run() { - try { - while (!shutdown) { - synchronized (this) { - wait(50000); -// 关闭失效的连接 - connMgr.closeExpiredConnections(); - } - } - } catch (InterruptedException ex) { -// 结束 - } - } - - //关闭清理无效连接的线程 - public void shutdown() { - shutdown = true; - synchronized (this) { - notifyAll(); - } - } -} - - diff --git a/src/main/java/com/mesa/reportservice/util/Logs.java b/src/main/java/com/mesa/reportservice/util/LogUtil.java index 3c471da..b169881 100644 --- a/src/main/java/com/mesa/reportservice/util/Logs.java +++ b/src/main/java/com/mesa/reportservice/util/LogUtil.java @@ -4,24 +4,25 @@ package com.mesa.reportservice.util; * Created by wk1 on 2019/5/23. */ -import org.apache.log4j.Logger; +import org.slf4j.LoggerFactory; -public class Logs { - private static Logger logger = Logger.getLogger(Logs.class); +public class LogUtil { + private static org.slf4j.Logger print = LoggerFactory.getLogger(LogUtil.class); //debug方法,其他error、warn类似定义 - public static void debug(Object obj) { + public static void debug(String message,Object obj) { //如果更改日志出工工具,只需修改该行代码即可。如:可改为 System.out.println(obj); - logger.debug(obj); + print.debug(message+" {}", obj); } - public static void error(Object obj) { + public static void error(String message,Exception obj) { //如果更改日志出工工具,只需修改该行代码即可。如:可改为 System.out.println(obj); - logger.error(obj); + print.error(message+" {}", obj); + } - public static void info(Object obj) { + public static void info(String message,Object obj) { //如果更改日志出工工具,只需修改该行代码即可。如:可改为 System.out.println(obj); - logger.info(obj); + print.info(message+" {}", obj); } } diff --git a/src/main/java/com/mesa/reportservice/util/StringUtil.java b/src/main/java/com/mesa/reportservice/util/StringUtil.java index 56a5458..6e3c846 100644 --- a/src/main/java/com/mesa/reportservice/util/StringUtil.java +++ b/src/main/java/com/mesa/reportservice/util/StringUtil.java @@ -1,6 +1,7 @@ package com.mesa.reportservice.util; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import sun.misc.BASE64Encoder; @@ -334,11 +335,10 @@ public final class StringUtil extends StringUtils { */ public static String md5(String msg) { try { - MessageDigest md = MessageDigest.getInstance("MD5"); - byte[] b = md.digest(msg.getBytes()); - BASE64Encoder encoder = new BASE64Encoder(); - return encoder.encode(b); - } catch (NoSuchAlgorithmException e) { + String shaString=DigestUtils.sha256Hex(msg); + //String md5String = DigestUtils.md5Hex(msg); + return shaString; + } catch (Exception e) { e.printStackTrace(); return EMPTY; } @@ -350,7 +350,7 @@ public final class StringUtil extends StringUtils { BASE64Encoder encoder = new BASE64Encoder(); return encoder.encode(msg.getBytes()); } catch (Exception e) { - e.printStackTrace(); + // e.printStackTrace(); return EMPTY; } } @@ -385,47 +385,6 @@ public final class StringUtil extends StringUtils { } - /** - * <p>通过给定url获取域名地址.</p> - * <p> - * 获得域名信息需去除 http/https/ftp 与 www 一些头信息,获取有效地址. - * </p> - * <pre> - * StringUtil.getDomain("http://www.baidu.com") = "baidu.com" - * </pre> - * - * @param webSiteUrl url 地址 - * @return 返回截取后的域名地址 - */ - public static String getDomain(String webSiteUrl) { - String url; - - if (isEmpty(webSiteUrl)) { - return EMPTY; - } - - if (webSiteUrl.indexOf("http://") >= 0) { - url = webSiteUrl.substring("http://".length(), webSiteUrl.length()); - } else if (webSiteUrl.indexOf("https://", 0) >= 0) { - url = webSiteUrl.substring("https://".length(), webSiteUrl.length()); - } else if (webSiteUrl.indexOf("ftp://", 0) >= 0) { - url = webSiteUrl.substring("ftp://".length(), webSiteUrl.length()); - } else { - url = webSiteUrl; - } - - if (url.indexOf("/", 0) >= 0) { - url = url.substring(0, url.indexOf("/", 1)); - } - if (url.indexOf("www.") == 0) { - url = url.substring(url.indexOf(".") + 1, url.length()); - } - if (url.indexOf("?") >= 0) { - url = url.substring(0, url.indexOf("?")); - } - - return url; - } /** * <p>按照规则解析字符串得到一个数组.</p> diff --git a/src/main/java/com/mesa/reportservice/util/ziputil.java b/src/main/java/com/mesa/reportservice/util/ziputil.java deleted file mode 100644 index f530f45..0000000 --- a/src/main/java/com/mesa/reportservice/util/ziputil.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.mesa.reportservice.util; - - -import net.jpountz.lz4.*; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; -import org.apache.commons.compress.compressors.xz.XZCompressorInputStream; -import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; -import org.apache.commons.io.output.ByteArrayOutputStream; -import org.xerial.snappy.Snappy; - -import java.io.ByteArrayInputStream; -import java.io.IOException; - -/** - * Created by wk1 on 2020/5/15. - */ -public class ziputil { - - - public static byte[] commonGzipCompress(byte[] data) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(512); - GzipCompressorOutputStream gcos = new GzipCompressorOutputStream(baos); - gcos.write(data); - gcos.close(); - return baos.toByteArray(); - } - - public static byte[] commonGzipDecompress(byte[] data) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(512); - GzipCompressorInputStream gcis = new GzipCompressorInputStream(new ByteArrayInputStream(data)); - int count; - byte[] buffer = new byte[8192]; - while ((count = gcis.read(buffer)) != -1) { - baos.write(buffer, 0, count); - } - gcis.close(); - return baos.toByteArray(); - } - - - - public static byte[] snappyCompress(byte[] data) throws IOException { - return Snappy.compress(data); - } - - public static byte[] snappyDecompress(byte[] data) throws IOException { - return Snappy.uncompress(data); - } - - - public static byte[] commonXzCompress(byte[] data) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); - XZCompressorOutputStream pcos = new XZCompressorOutputStream(baos); - pcos.write(data); - pcos.close(); - return baos.toByteArray(); - } - - public static byte[] commonXzDecompress(byte[] data) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); - XZCompressorInputStream xzis = new XZCompressorInputStream(new ByteArrayInputStream(data)); - int count; - byte[] buffer = new byte[8192]; - while ((count = xzis.read(buffer)) != -1) { - baos.write(buffer, 0, count); - } - xzis.close(); - return baos.toByteArray(); - } - - - public static byte[] lz4Compress(byte[] data) throws IOException { - LZ4Factory factory = LZ4Factory.fastestInstance(); - ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); - LZ4Compressor compressor = factory.fastCompressor(); - LZ4BlockOutputStream compressedOutput = new LZ4BlockOutputStream(byteOutput, 8192, compressor); - compressedOutput.write(data); - compressedOutput.close(); - return byteOutput.toByteArray(); - } - - public static byte[] lz4Decompress(byte[] data) throws IOException { - LZ4Factory factory = LZ4Factory.fastestInstance(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(8192); - LZ4FastDecompressor decompresser = factory.fastDecompressor(); - LZ4BlockInputStream lzis = new LZ4BlockInputStream(new ByteArrayInputStream(data), decompresser); - int count; - byte[] buffer = new byte[8192]; - while ((count = lzis.read(buffer)) != -1) { - baos.write(buffer, 0, count); - } - lzis.close(); - return baos.toByteArray(); - } -} diff --git a/src/main/resources/config/application.properties b/src/main/resources/config/application.properties deleted file mode 100644 index 538c39f..0000000 --- a/src/main/resources/config/application.properties +++ /dev/null @@ -1,120 +0,0 @@ -server.port=9093 -#���½�������ʱ��10s -scan.result.scheduled.plan=0/10 * * * * ? -#��ʱɨ��mysql���ݿ�report_result��ʱ��15s -#ͬʱ��ִ�����߳��� -globle.job_thread=2 -globle.ttl_time=21600600 -#Hbasehttp�Ķ˿� -#Hbase�ı���������ͨ������Ҫ���� -hbase.table=tsg:report_result - -hbase.zookeeper_quorum=192.168.40.193 -hbase.zookeeper_property_clientPort=2181 -hbase.zookeeper_znode_parent=/hbase -hbase.client_retries_number=3 - -hbase.rpc_timeout=100000 -hbase.connect_pool=10 - - -ck.gateway_ip=192.168.41.193:9999 - -#clickhouse�ṩreport�����ip -#ck.task_ip=192.168.40.224:8123 -#clickhouse�����ݿ��� -#ck.task_database=tsg_galaxy_v3 -#clickhouse����������û� -#ck.task_user=tsg_report -#clickhouse������������� -#ck.task_user_password=ceiec2019 -#clickhouse��־����ipͨ��=ck.task_ip -#ck.log_ip=192.168.40.224:8123 -#clickhouse��־������û� -#ck.log_user=tsg_report -#clickhouse��־�û����� -#ck.log_user_password=ceiec2019 -#��������� -http.maxTotal=300 -#������ -http.defaultMaxPerRoute=100 -#�������ӵ��ʱ�� -http.connectTimeout=10000 -#�����ӳ��л�ȡ�����ӵ��ʱ��: -http.connectionRequestTimeout=10000 -#���ݴ�����ʱ�� -http.socketTimeout=21600500 -#�ύ����ǰ���������Ƿ���� -http.staleConnectionCheckEnabled=true - -http.socketTimeoutShort=30000 - - -#�Ƿ�����zookeeper 0����(��Ⱥ) 1����(����) -zookeeper.open=1 -zookeeper.retryCount=5 -zookeeper.elapsedTimeMs=1000 -#zk��Ⱥ��ip -zookeeper.connectString=192.168.40.193:2181 -zookeeper.sessionTimeoutMs=5000 -zookeeper.connectionTimeoutMs=5000 -zookeeper.nameSpace=reportservice -#mariadb��url -spring.datasource.url=jdbc:mariadb://192.168.40.204:3306/treport?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false -#spring.datasource.url=jdbc:mysql://192.168.11.208:3306/ntc-api?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false -#mariadb���û��� -spring.datasource.username=root -#mariadb������ -spring.datasource.password=111111 - -spring.datasource.name=druidDataSource -spring.datasource.type=com.alibaba.druid.pool.DruidDataSource -spring.datasource.driver-class-name=org.mariadb.jdbc.Driver - -#���ü��ͳ�����ص�filters��ȥ�����ؽ���SQL������ͳ�ƣ���wall�����ڷ���ǽ -spring.datasource.druid.filters=stat,wall,slf4j -#���������S -spring.datasource.druid.max-active=30 -#��С������ -spring.datasource.druid.min-idle=1 -#��ʼ�������� -spring.datasource.druid.initial-size=2 -#��ȡ�������ʱʱ�� -spring.datasource.druid.max-wait=600000 -#�����òŽ���һ�μ�⣬�����Ҫ�رյĿ������ӣ���λ�Ǻ��� -spring.datasource.druid.time-between-eviction-runs-millis=60000 -# һ�������ڳ�����С�����ʱ�䣬��λ�Ǻ��� -spring.datasource.druid.min-evictable-idle-time-millis=300000 -#��֤�����Ƿ����,�����ݿ���ִ��һ��sql -spring.datasource.druid.validation-query=select 1 -#��������Ϊtrue����Ӱ�����ܣ����ұ�֤��ȫ�ԡ��������ӵ�ʱ���⣬�������ʱ�����timeBetweenEvictionRunsMillis�� -# ִ��validationQuery��������Ƿ���Ч -spring.datasource.druid.test-while-idle=true -#��������ʱִ��validationQuery��������Ƿ���Ч������������ûή������ -spring.datasource.druid.test-on-borrow=false -#�黹����ʱִ��validationQuery��������Ƿ���Ч������������ûή������ -spring.datasource.druid.test-on-return=false -spring.datasource.druid.connection-properties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 -#�Ƿ���WebStatFilter -spring.datasource.druid.web-stat-filter.enabled=true -#���ò�ͳ����ЩURL(�����ų�һЩ����Ҫ��url) -spring.datasource.druid.web-stat-filter.exclusions=*.js,*.gif,*.jpg,*.png,*.css,*.ico,/druid/* -#�Ƿ���Druid�����Ϣ��ʾҳ�� -spring.datasource.druid.stat-view-servlet.enabled=true -#�������������·�� -spring.datasource.druid.stat-view-servlet.url-pattern=/druid/* -#��ֹ�ֶ����ü������ -spring.datasource.druid.stat-view-servlet.reset-enable=false -#durid-uiҳ���˻����� -spring.datasource.druid.stat-view-servlet.login-username=admin -spring.datasource.druid.stat-view-servlet.login-password=admin -#Spring��أ����ڲ����ӿڵ��õļ��,��Ҫ����aop��ذ� -spring.datasource.druid.aop-patterns=com.mesa.reportservice.controller.*,com.mesa.reportservice.service.*,com.mesa.reportservice.mapper.* - -logging.config: classpath:./config/log4j2-dev.xml - -mybatis.typeAliasesPackage=com.mesa.reportservice.bean -mybatis.mapperLocations=classpath*:/mappers/*.xml -spring.application.name=ReportserviceApplication -management.endpoints.web.exposure.include=* -management.metrics.tags.application=${spring.application.name}
\ No newline at end of file diff --git a/src/main/resources/config/application.yml b/src/main/resources/config/application.yml new file mode 100644 index 0000000..32bc705 --- /dev/null +++ b/src/main/resources/config/application.yml @@ -0,0 +1,124 @@ +#http的端口 +server: + port: 9093 +#更新进度条的时间10s +scan: + result: + scheduled: + plan: 0/15 * * * * ? +#同时间执行是线程数 +globle: + job_thread: 2 +#Hbasehttp的端口 +#Hbase的表名等配置通畅不需要更改 +hbase: + table: tsg:report_result + zookeeper_quorum: 192.168.40.222,192.168.40.223 + zookeeper_property_clientPort: 2182 + zookeeper_znode_parent: /hbase + client_retries_number: 3 + rpc_timeout: 100000 + connect_pool: 10 +#查询网关ip +ck: + gateway_ip: http://192.168.40.223:9998 + +#zk集群的ip +zookeeper: + connectString: 192.168.44.34,192.168.44.35:2181 +#是否启用zookeeper 0启用(集群) 1禁用(单机) + open: 0 + retryCount: 5 + elapsedTimeMs: 10000 + sessionTimeoutMs: 50000 + connectionTimeoutMs: 50000 + nameSpace: reportservice + +#最大连接数 +http: + maxTotal: 300 +#并发数 + defaultMaxPerRoute: 100 +#创建连接的最长时间 + connectTimeout: 10000 +#从连接池中获取到连接的最长时间 + connectionRequestTimeout: 10000 +#数据传输的最长时间 + socketTimeout: 21605000 +#提交请求前测试连接是否可用 + staleConnectionCheckEnabled: true + socketTimeoutShort: 30000 + +#mariadb的url +spring: + application: + name: galaxy_report_service + + datasource: + url: jdbc:mariadb://192.168.40.204:3306/ttreport?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false +#spring.datasource.url=jdbc:mysql://192.168.11.208:3306/ntc-api?serverTimezone=GMT&useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&failOverReadOnly=false +#mariadb的用户名 + username: root +#mariadb的密码 + password: 111111 + +#以下配置不需要更改通常 + name: druidDataSource + type: com.alibaba.druid.pool.DruidDataSource + driver-class-name: org.mariadb.jdbc.Driver + +#配置监控统计拦截的filters,去掉后监控界面SQL无法进行统计,’wall’用于防火墙 + druid: + filters: stat,wall,slf4j + #最大连接数 + max-active: 30 + #最小连接数 + min-idle: 1 + #初始化连接数 + initial-size: 2 + #获取连接最大超时时间 + max-wait: 600000 + #间隔多久才进行一次检测,检测需要关闭的空闲连接,单位是毫秒 + time-between-eviction-runs-millis: 60000 + # 一个连接在池中最小生存的时间,单位是毫秒 + min-evictable-idle-time-millis: 300000 + #验证连接是否可用,在数据库中执行一条sql + validation-query: select 1 + #建议配置为true,不影响性能,并且保证安全性。申请连接的时候检测,如果空闲时间大于timeBetweenEvictionRunsMillis, + # 执行validationQuery检测连接是否有效 + test-while-idle: true + #申请连接时执行validationQuery检测连接是否有效,做了这个配置会降低性能 + test-on-borrow: true + test-on-return: false + connection-properties: druid.stat.mergeSql=true;druid.stat.slowSqlMillis=500 + #是否开启WebStatFilter + web-stat-filter: + enabled: true + #设置不统计哪些URL(用于排除一些不必要的url) + exclusions: "*.js,*.gif,*.jpg,*..ng,*.css,*.ico,/druid/*" + #是否开启Druid监控信息显示页面 + stat-view-servlet: + enabled: true + #甚至浏览器访问路径 + url-pattern: /druid/* + #禁止手动重置监控数据 + reset-enable: false + #durid-ui页面账户密码 + login-username: admin + login-password: admin + #Spring监控,对内部各接口调用的监控,需要导入aop相关包 + aop-patterns: com.mesa.reportservice.controller.*,com.mesa.reportservice.service.*,com.mesa.reportservice.mapper.* +mybatis: + typeAliasesPackage: com.mesa.reportservice.bean + mapperLocations: classpath*:/mappers/*.xml +management: + endpoints: + web: + exposure: + include: "*" + metrics: + tags: + application: galaxy_report_service + +logging: + config: ./config/log4j2-dev.xml
\ No newline at end of file diff --git a/src/test/java/com/mesa/reportservice/ReportserviceApplicationTests.java b/src/test/java/com/mesa/reportservice/ReportserviceApplicationTests.java index 61e9172..ce41cdd 100644 --- a/src/test/java/com/mesa/reportservice/ReportserviceApplicationTests.java +++ b/src/test/java/com/mesa/reportservice/ReportserviceApplicationTests.java @@ -2,7 +2,6 @@ package com.mesa.reportservice; import com.mesa.reportservice.bean.HttpResult; -import com.mesa.reportservice.util.ziputil; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; |
