diff options
| author | zhanghongqing <[email protected]> | 2023-10-24 10:05:27 +0800 |
|---|---|---|
| committer | zhanghongqing <[email protected]> | 2023-10-24 10:05:27 +0800 |
| commit | 8f5bbde5a28e4c25e3d16aaf3c29efed70ef0714 (patch) | |
| tree | 2e0ea2b1fc3172e9fb2aef63a0bebafbed41da7a | |
| parent | 670d49fbd6fa50b40b3610711f04103dce8d43cc (diff) | |
合并IPv4,6定位库，asn库GAL-423
42 files changed, 275 insertions, 295 deletions
diff --git a/galaxy-job-executor/pom.xml b/galaxy-job-executor/pom.xml index 2a3933c..a3f51ac 100644 --- a/galaxy-job-executor/pom.xml +++ b/galaxy-job-executor/pom.xml @@ -62,9 +62,9 @@ <artifactId>spring-kafka</artifactId> </dependency> <dependency> - <groupId>com.zdjizhi</groupId> - <artifactId>galaxy</artifactId> - <version>${galaxy.version}</version> + <groupId>com.geedgenetworks</groupId> + <artifactId>galaxy</artifactId> + <version>1.2</version> <exclusions> <exclusion> <groupId>org.slf4j</groupId> @@ -204,7 +204,7 @@ <JAR_FILE>${project.build.finalName}.xjar</JAR_FILE> </buildArgs> <imageTags> - <imageTag>v1.3.230912</imageTag> + <imageTag>v1.3.230920</imageTag> </imageTags> <resources> <resource> diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/config/StorgeConfig.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/config/StorgeConfig.java index 39a6169..9f18bfa 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/config/StorgeConfig.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/config/StorgeConfig.java @@ -1,9 +1,9 @@ package com.mesalab.executor.core.config; import com.alibaba.nacos.api.config.annotation.NacosValue; +import com.geedgenetworks.utils.StringUtil; import com.google.common.base.Splitter; import com.google.common.collect.Maps; -import com.zdjizhi.utils.StringUtil; import lombok.Data; import org.springframework.context.annotation.Configuration; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/connector/HTTPSource.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/connector/HttpSource.java index a6e59d3..d833462 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/connector/HTTPSource.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/connector/HttpSource.java @@ -11,17 +11,22 @@ import org.apache.commons.lang3.ObjectUtils; import java.util.Map; import java.util.stream.Stream; -public class HTTPSource implements Source { +/** + * @author: zhq + * @date: 2023/9/21 + */ +public class HttpSource implements Source { private HttpParam httpParam; - public HTTPSource(HttpParam httpParam) { + public HttpSource(HttpParam httpParam) { this.httpParam = httpParam; } - public HTTPSource(Map<String, String> sourceParams) { + public HttpSource(Map<String, String> sourceParams) { this.httpParam = BeanUtil.toBean(sourceParams, HttpParam.class); } + @Override public Stream<String> fetchData() { String url = httpParam.getUrl(); @@ -33,7 +38,6 @@ public class HTTPSource implements Source { String result = ""; if ("get".equalsIgnoreCase(httpParam.getMethod())) { result = HttpClientUtils.httpPost(url, JSON.toJSONString(httpParam.getRequestBody())); - } else if ("post".equalsIgnoreCase(httpParam.getMethod())) { result = HttpClientUtils.httpGet(url); } else { @@ -46,31 +50,4 @@ public class HTTPSource implements Source { return Stream.of(result); } - // socks代理服务器的地址与端口 -/* @Deprecated - public static void socketProxy(Map<String, Object> requestParams) { - if (ObjectUtils.isNotEmpty(requestParams.get("proxy"))) { - try { - Properties prop = System.getProperties(); - prop.put("proxySet", true); - prop.setProperty("socksProxyHost", String.valueOf(requestParams.get("proxyHost"))); - prop.setProperty("socksProxyPort", String.valueOf(requestParams.get("proxyPort"))); - } catch (Exception e) { - throw new BusinessException("socket proxy set error " + e.getMessage()); - } - requestParams.remove("proxyHost"); - requestParams.remove("proxyPort"); - } - } - @Deprecated - public static void removeSocketProxy() { - try { - Properties prop = System.getProperties(); - prop.remove("proxySet"); - prop.remove("socksProxyHost"); - prop.remove("socksProxyPort"); - } catch (Exception e) { - throw new BusinessException("Error removing socket proxy: " + e.getMessage()); - } - }*/ }
\ No newline at end of file diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/connector/MysqlSource.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/connector/MysqlSource.java new file mode 100644 index 0000000..daa67d2 --- /dev/null +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/connector/MysqlSource.java @@ -0,0 +1,16 @@ +package com.mesalab.executor.core.connector; + +import com.mesalab.executor.pojo.JDBCParam; + +import java.util.stream.Stream; + +public class MysqlSource implements Source { + + private JDBCParam jdbcParam; + + @Override + public Stream<String> fetchData() { + + return null; + } +} diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/DefaultProducerListener.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/DefaultProducerListener.java index d72debc..81a30d7 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/DefaultProducerListener.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/DefaultProducerListener.java @@ -1,11 +1,11 @@ package com.mesalab.executor.core.utils; + import com.mesalab.executor.exception.BusinessException; +import com.xxl.job.core.log.XxlJobLogger; import org.apache.kafka.clients.producer.RecordMetadata; import org.springframework.kafka.support.ProducerListenerAdapter; import org.springframework.stereotype.Component; -import com.xxl.job.core.log.XxlJobLogger; - /** * 默认生产者监听器 */ diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/HttpClientUtils.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/HttpClientUtils.java index 4a94a73..ac1e6bf 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/HttpClientUtils.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/HttpClientUtils.java @@ -3,10 +3,10 @@ package com.mesalab.executor.core.utils; import cn.hutool.log.Log; import com.dtflys.forest.Forest; import com.dtflys.forest.http.ForestProxy; +import com.geedgenetworks.utils.StringUtil; import com.google.common.collect.Lists; import com.mesalab.executor.core.config.HttpConfig; import com.mesalab.executor.exception.BusinessException; -import com.zdjizhi.utils.StringUtil; import org.apache.commons.io.IOUtils; import org.apache.http.*; import org.apache.http.client.ClientProtocolException; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/JobUtil.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/JobUtil.java index 10b3acd..9b9bba9 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/JobUtil.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/JobUtil.java @@ -14,6 +14,7 @@ import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONNull; import cn.hutool.json.JSONUtil; import cn.hutool.log.Log; +import com.geedgenetworks.utils.IpLookupV2; import com.mesalab.executor.exception.BusinessException; import com.xxl.job.core.log.XxlJobLogger; @@ -258,4 +259,31 @@ public class JobUtil { return DateUtil.format(DateUtil.offsetDay(DateUtil.date(), -days), format); }); } + public static boolean ipKnowledgeNotEmpty(String builtinPath, String userDefinedPath, List<String> ipSample) { + IpLookupV2 ipLookup = new IpLookupV2.Builder(false) + .loadDataFile(builtinPath) + .loadDataFilePrivate(userDefinedPath) + .build(); + for (String sampleIp : ipSample) { + if (ObjectUtil.isEmpty(ipLookup.latLngLookup(sampleIp))) { + JobUtil.infoLog("ip locate library sample validate fail, sampleIp {}, latLng {}", sampleIp, ipLookup.latLngLookup(sampleIp)); + return false; + } + } + return true; + } + + public static boolean asnKnowledgeNotEmpty(String builtinPath, String userDefinedPath, List<String> ipSample) { + IpLookupV2 ipLookup = new IpLookupV2.Builder(false) + .loadAsnDataFile(builtinPath) + .loadAsnDataFilePrivate(userDefinedPath) + .build(); + for (String sampleIp : ipSample) { + if (ObjectUtil.isEmpty(ipLookup.asnLookup(sampleIp))) { + JobUtil.infoLog("asn library sample validate fail, sample ip {}, asn {}", sampleIp, ipLookup.asnLookup(sampleIp)); + return false; + } + } + return true; + } } diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/KafkaUtils.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/KafkaUtils.java index 750a082..0db1190 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/KafkaUtils.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/KafkaUtils.java @@ -1,8 +1,8 @@ package com.mesalab.executor.core.utils; import cn.hutool.log.Log; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; +import com.geedgenetworks.utils.JsonMapper; +import com.geedgenetworks.utils.StringUtil; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.support.SendResult; import org.springframework.util.concurrent.ListenableFuture; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/TypeUtils.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/TypeUtils.java index f8d193f..aeca130 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/TypeUtils.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/TypeUtils.java @@ -18,9 +18,6 @@ import com.alibaba.fastjson.parser.ParserConfig; import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.fastjson.util.FieldInfo; import com.alibaba.fastjson.util.IOUtils; -import com.mesalab.executor.core.connector.JdbcDialect; -import com.mesalab.executor.core.connector.MysqlDialect; -import lombok.NonNull; import java.lang.annotation.Annotation; import java.lang.reflect.*; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/XmlUtil.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/XmlUtil.java index c58c45b..4c38259 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/XmlUtil.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/core/utils/XmlUtil.java @@ -1,12 +1,11 @@ package com.mesalab.executor.core.utils; -import java.io.StringReader; -import java.io.StringWriter; - import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Marshaller; import javax.xml.bind.Unmarshaller; +import java.io.StringReader; +import java.io.StringWriter; public class XmlUtil { diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/exception/BusinessException.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/exception/BusinessException.java index b87d597..e56b352 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/exception/BusinessException.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/exception/BusinessException.java @@ -2,7 +2,10 @@ package com.mesalab.executor.exception; import com.mesalab.executor.core.utils.JobUtil; import com.mesalab.executor.enums.BaseResultEnum; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; /** * 业务异常 diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/CommandJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/CommandJob.java index 1f9baeb..408f9b1 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/CommandJob.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/CommandJob.java @@ -7,6 +7,7 @@ import cn.hutool.json.JSONArray; import cn.hutool.json.JSONUtil; import cn.hutool.log.Log; import com.alibaba.fastjson.JSONPath; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.JobUtil; import com.mesalab.executor.pojo.IpLocateLibraryParam; @@ -16,8 +17,6 @@ import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.IJobHandler; import com.xxl.job.core.handler.annotation.XxlJob; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.IpLookupV2; -import com.zdjizhi.utils.StringUtil; import org.apache.commons.lang3.ObjectUtils; import org.springframework.stereotype.Component; @@ -36,11 +35,6 @@ public class CommandJob { private Log logger = Log.get(); /** - * //1.job execute - * //2.monitoring write log - * //3.result check - * //4.push knowledge - * * @param params * @return */ @@ -49,67 +43,63 @@ public class CommandJob { IpLocateLibraryParam parserParams = parserParams(params); if (ObjectUtils.isEmpty(parserParams)) { - logger.error("params parser error , params is {}", params); - XxlJobLogger.log("params parser error , params is {}", params); + JobUtil.infoLog("params parser error , params is {}", params); return IJobHandler.FAIL; } for (int i = 1; i <= parserParams.getRetry(); i++) { try { int executeResult = JobUtil.executeCommand(parserParams.getCommand()); - XxlJobLogger.log("ip locate library perl {} execute result {}", i, executeResult==0?"success":"fail"); - logger.error("ip locate library perl {} execute result {}", i, executeResult==0?"success":"fail"); + JobUtil.infoLog("ip locate library perl {} execute result {}", i, executeResult == 0 ? "success" : "fail"); //check status - if (executeResult == 0 && checkIpMMDBSample(parserParams)) { - logger.info("ip locate library built success"); - XxlJobLogger.log("ip locate library built success"); - + if (executeResult == 0 && checkIpSample(parserParams)) { + JobUtil.infoLog("ip locate library built success"); if (parserParams.getPushKnowledge()) { //push knowledge - int successCount = 0; - Map<String, String> historyMMDBInfo = historyMMDBInfo(parserParams); - for (KnowledgeBase knowledgeBase : parserParams.getKnowledgeBaseList()) { - - File file = new File(StringUtil.join(parserParams.getPath(), File.separator, knowledgeBase.getName(), ".", knowledgeBase.getFormat())); - String newFile = DigestUtil.sha256Hex(file); - String historyFile = historyMMDBInfo.get(knowledgeBase.getName()); - logger.info("ip locate library name {} ,new file {}, history file {}",knowledgeBase.getName(), newFile, historyFile); - XxlJobLogger.log("ip locate library name {} ,new file {}, history file {}",knowledgeBase.getName(),newFile,historyFile); - if (!newFile.equals(historyFile)) { - knowledgeBase.setSha256(newFile); - String resultStr = HttpClientUtils.httpPostFile(parserParams.getKnowledgeBaseServer(), file, BeanUtil.beanToMap(knowledgeBase)); - logger.info("ip locate library push knowledge result: {},\n message {}", knowledgeBase, resultStr); - XxlJobLogger.log("ip locate library push knowledge result: {},\n message {}", knowledgeBase, resultStr); - if (!"-1".equals(resultStr)) { - Map<String, Object> result = JSONUtil.toBean(resultStr, Map.class); - if (result.get("status") != null && "200".equals(String.valueOf(result.get("status")))) { - successCount++; - } - } - } else { - successCount++; - logger.info("ip locate library {} has no changed, so it is not pushed to the knowledge base", knowledgeBase.getName()); - XxlJobLogger.log("ip locate library {} has no changed, so it is not pushed to the knowledge base", knowledgeBase.getName()); - } - } - if(successCount==parserParams.getKnowledgeBaseList().size()){ + if (pushKnowledge(parserParams) == parserParams.getKnowledgeBaseList().size()) { return ReturnT.SUCCESS; } - } else { return ReturnT.SUCCESS; } } } catch (Exception e) { - XxlJobLogger.log("ip locate library exception {}, retry {} ", e.getMessage(), i); - logger.error("ip locate library exception {}, retry {} ", e.getMessage(), i); + JobUtil.errorLog("ip locate library exception {}, retry {} ", e.getMessage(), i); } } return ReturnT.FAIL; } + private int pushKnowledge(IpLocateLibraryParam parserParams) { + int successCount = 0; + Map<String, String> historyMMDBInfo = historyMMDBInfo(parserParams); + for (KnowledgeBase knowledgeBase : parserParams.getKnowledgeBaseList()) { + + File file = new File(StringUtil.join(parserParams.getPath(), File.separator, knowledgeBase.getName(), ".", knowledgeBase.getFormat())); + String newFile = DigestUtil.sha256Hex(file); + String historyFile = historyMMDBInfo.get(knowledgeBase.getName()); + JobUtil.infoLog("ip locate library name {} ,new file {}, history file {}", knowledgeBase.getName(), newFile, historyFile); + if (!newFile.equals(historyFile)) { + knowledgeBase.setSha256(newFile); + String resultStr = HttpClientUtils.httpPostFile(parserParams.getKnowledgeBaseServer(), file, BeanUtil.beanToMap(knowledgeBase)); + JobUtil.infoLog("ip locate library push knowledge result: {},\n message {}", knowledgeBase, resultStr); + if (!"-1".equals(resultStr)) { + Map<String, Object> result = JSONUtil.toBean(resultStr, Map.class); + if (result.get("status") != null && "200".equals(String.valueOf(result.get("status")))) { + successCount++; + } + } + } else { + successCount++; + JobUtil.infoLog("ip locate library {} not updated.", knowledgeBase.getName()); + } + } + return successCount; + } + /** * 验证历史文件sha256 + * * @param ipLocateLibraryParam * @return */ @@ -126,7 +116,6 @@ public class CommandJob { } return historyInfo; - } /** @@ -135,74 +124,39 @@ public class CommandJob { * @param parserParams * @return */ - public static boolean checkIpMMDBSample(IpLocateLibraryParam parserParams) throws Exception { - - String builtStatusFile = "built-status.json"; - String userStatusFile = "user-status.json"; + public boolean checkIpSample(IpLocateLibraryParam parserParams) throws Exception { + final String builtinStatusFileName = "builtin-status.json"; + final String userStatusFileName = "user-status.json"; String path = parserParams.getPath().concat(File.separator); - MMDBStatus builtMMDBStatus = JobUtil.readJsonFile(path + builtStatusFile, MMDBStatus.class); - Integer builtStatus = builtMMDBStatus.getStatus(); - JobUtil.infoLog("built ip locate library result: status {}, message {}", builtMMDBStatus.getStatus(), builtMMDBStatus.getMessage()); - - MMDBStatus userMMDBStatus = JobUtil.readJsonFile(path + userStatusFile, MMDBStatus.class); - Integer userStatus = userMMDBStatus.getStatus(); - JobUtil.infoLog("user ip locate library result: status {}, message {}", userMMDBStatus.getStatus(), userMMDBStatus.getMessage()); - if (200 != builtStatus || 200 != userStatus) { + + MMDBStatus builtinStatusFile = JobUtil.readJsonFile(path + builtinStatusFileName, MMDBStatus.class); + MMDBStatus userStatusFile = JobUtil.readJsonFile(path + userStatusFileName, MMDBStatus.class); + JobUtil.infoLog("user ip locate library result: status {}, message {}. built ip locate library result: status {}, message {}", + userStatusFile.getStatus(), userStatusFile.getMessage(), builtinStatusFile.getStatus(), builtinStatusFile.getMessage()); + if (200 != builtinStatusFile.getStatus() || 200 != userStatusFile.getStatus()) { return false; } - String v4Built = ""; - String v6Built = ""; - String v4User = ""; - String v6User = ""; - boolean builtContains = false; - boolean userContains = false; + String builtinPath = ""; + String userDefinedPath = ""; + boolean builtinContains = false; + boolean userContains = false; for (KnowledgeBase knowledgeBase : parserParams.getKnowledgeBaseList()) { String filePath = StringUtil.join(path, knowledgeBase.getName(), ".", knowledgeBase.getFormat()); JobUtil.infoLog("ip locate library load path {} ", filePath); - if (knowledgeBase.getName().contains("v4_built")) { - v4Built = filePath; - builtContains= true; - } else if (knowledgeBase.getName().contains("v6_built")) { - v6Built = filePath; - builtContains= true; - } else if (knowledgeBase.getName().contains("v4_user")) { - v4User = filePath; - userContains = true; - } else if (knowledgeBase.getName().contains("v6_user")) { - v6User = filePath; + if (knowledgeBase.getName().contains("built")) { + builtinPath = filePath; + builtinContains = true; + } else if (knowledgeBase.getName().contains("user")) { + userDefinedPath = filePath; userContains = true; } } //测试抽样数据能查到结果说明打包成功 - IpLookupV2 ipLookup = new IpLookupV2.Builder(false) - .loadDataFileV4(v4Built) - .loadDataFileV6(v6Built) - .loadDataFilePrivateV4(v4User) - .loadDataFilePrivateV6(v6User) - .build(); - if(builtContains){ - List<String> builtIpSampleList = Stream.of(builtMMDBStatus.getIpv4Sample(), - builtMMDBStatus.getIpv6Sample()) - .flatMap(Collection::stream).collect(Collectors.toList()); - for (String sampleIp : builtIpSampleList) { - if (ObjectUtil.isEmpty(ipLookup.latLngLookup(sampleIp))) { - JobUtil.infoLog("built ip locate library sample validate fail, sampleIp {}, latLng {}", sampleIp, ipLookup.latLngLookup(sampleIp)); - return false; - } - } - } - if(userContains){ - List<String> userIpSampleList = Stream.of( - userMMDBStatus.getIpv4Sample(), - userMMDBStatus.getIpv6Sample()) - .flatMap(Collection::stream).collect(Collectors.toList()); - for (String sampleIp : userIpSampleList) { - if (ObjectUtil.isEmpty(ipLookup.latLngLookup(sampleIp))) { - JobUtil.infoLog("user ip locate library sample validate fail, sampleIp {}, latLng {}", sampleIp, ipLookup.latLngLookup(sampleIp)); - return false; - } - } + List<String> ipSamples = Stream.of(builtinStatusFile.getIpSample(), userStatusFile.getIpSample()) + .flatMap(Collection::stream).collect(Collectors.toList()); + if (builtinContains || userContains) { + return JobUtil.ipKnowledgeNotEmpty(builtinPath, userDefinedPath, ipSamples); } return true; } diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataExtractJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataExtractJob.java index f053f53..4f8bcdb 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataExtractJob.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataExtractJob.java @@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; import cn.hutool.log.Log; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.executor.core.utils.JobUtil; import com.mesalab.executor.service.DataSinkService; import com.mesalab.executor.service.DataSourceService; @@ -13,7 +14,6 @@ import com.xxl.job.core.handler.IJobHandler; import com.xxl.job.core.handler.annotation.XxlJob; import com.xxl.job.core.log.XxlJobLogger; import com.xxl.job.core.util.ShardingUtil; -import com.zdjizhi.utils.StringUtil; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataflowJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataflowJob.java index 9448b7e..86d827a 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataflowJob.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DataflowJob.java @@ -4,13 +4,15 @@ import cn.hutool.core.io.FileUtil; import cn.hutool.core.net.url.UrlBuilder; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.RandomUtil; -import cn.hutool.core.util.TypeUtil; import cn.hutool.core.util.ZipUtil; import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.log.Log; import com.alibaba.fastjson.util.TypeUtils; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.JsonMapper; +import com.geedgenetworks.utils.SnowflakeId; +import com.geedgenetworks.utils.StringUtil; import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableMap; import com.mesalab.executor.core.config.StorgeConfig; import com.mesalab.executor.core.utils.*; import com.mesalab.executor.exception.BusinessException; @@ -20,10 +22,6 @@ import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.IJobHandler; import com.xxl.job.core.handler.annotation.XxlJob; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.SnowflakeId; -import com.zdjizhi.utils.StringUtil; import io.netty.handler.codec.http.HttpMethod; import org.apache.http.Header; import org.apache.http.message.BasicHeader; @@ -34,10 +32,8 @@ import java.nio.charset.StandardCharsets; import java.text.MessageFormat; import java.util.List; import java.util.Map; -import java.util.Optional; import static com.mesalab.executor.core.utils.Constant.DEFAULT_VSYS_ID_VALUE; -import static com.mesalab.executor.core.utils.Constant.VSYS_ID; @Component public class DataflowJob { diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidCompactJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidCompactJob.java index df3d031..3dbd075 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidCompactJob.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidCompactJob.java @@ -5,6 +5,8 @@ import cn.hutool.log.level.Level; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.databind.JavaType; +import com.geedgenetworks.utils.JsonMapper; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.executor.core.config.StorgeConfig; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.SpringContextUtil; @@ -15,11 +17,8 @@ import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.IJobHandler; import com.xxl.job.core.handler.annotation.XxlJob; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; import org.springframework.stereotype.Component; -import javax.print.DocFlavor; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidMergeHistoryDataJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidMergeHistoryDataJob.java index 795f65e..87285a5 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidMergeHistoryDataJob.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidMergeHistoryDataJob.java @@ -2,24 +2,21 @@ package com.mesalab.executor.jobhandler; import cn.hutool.json.JSONUtil; import cn.hutool.log.Log; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.executor.core.config.StorgeConfig; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.SpringContextUtil; import com.mesalab.executor.core.utils.UrlUtil; import com.mesalab.executor.exception.BusinessException; - import com.mesalab.executor.pojo.DruidNativeMergeParam; import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.IJobHandler; import com.xxl.job.core.handler.annotation.XxlJob; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.StringUtil; import org.springframework.stereotype.Component; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Calendar; -import java.util.Date; import java.util.List; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidNativeQueryJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidNativeQueryJob.java index 63f156c..292c625 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidNativeQueryJob.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/DruidNativeQueryJob.java @@ -1,6 +1,8 @@ package com.mesalab.executor.jobhandler; import cn.hutool.log.Log; +import com.geedgenetworks.utils.JsonMapper; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.KafkaUtils; import com.mesalab.executor.exception.BusinessException; @@ -11,8 +13,6 @@ import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.IJobHandler; import com.xxl.job.core.handler.annotation.XxlJob; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; import io.netty.handler.codec.http.HttpMethod; import org.joda.time.DateTimeZone; import org.joda.time.Interval; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/LogStorageQuotaJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/LogStorageQuotaJob.java index 61a56d4..1768ec4 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/LogStorageQuotaJob.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/LogStorageQuotaJob.java @@ -3,6 +3,10 @@ package com.mesalab.executor.jobhandler; import cn.hutool.core.net.url.UrlBuilder; import cn.hutool.core.thread.ThreadUtil; import cn.hutool.log.Log; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.JsonMapper; +import com.geedgenetworks.utils.StringUtil; +import com.geedgenetworks.utils.TimeConstants; import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Lists; @@ -16,10 +20,6 @@ import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.IJobHandler; import com.xxl.job.core.handler.annotation.XxlJob; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import com.zdjizhi.utils.TimeConstants; import org.apache.commons.lang3.ObjectUtils; import org.apache.http.Header; import org.apache.http.message.BasicHeader; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/LogStorageTtlJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/LogStorageTtlJob.java index 267b898..853c8e4 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/LogStorageTtlJob.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/LogStorageTtlJob.java @@ -5,6 +5,7 @@ import cn.hutool.log.Log; import com.alibaba.fastjson.JSON; import com.alibaba.nacos.api.config.ConfigService; import com.alibaba.nacos.api.exception.NacosException; +import com.geedgenetworks.utils.JsonMapper; import com.google.common.base.Joiner; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -20,7 +21,6 @@ import com.mesalab.executor.service.StorageQuotaService; import com.xxl.job.core.biz.model.ReturnT; import com.xxl.job.core.handler.annotation.XxlJob; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.JsonMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/knowledgeBase/KnowledgeBaseJob.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/knowledgeBase/KnowledgeBaseJob.java index e0b1b44..0cf0b41 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/knowledgeBase/KnowledgeBaseJob.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/jobhandler/knowledgeBase/KnowledgeBaseJob.java @@ -3,7 +3,6 @@ package com.mesalab.executor.jobhandler.knowledgeBase; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONUtil; -import com.alibaba.fastjson.util.TypeUtils; import com.mesalab.executor.core.utils.JobUtil; import com.mesalab.executor.jobhandler.ValidParamUtil; import com.mesalab.executor.service.DataSourceService; @@ -18,6 +17,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static com.xxl.job.core.biz.model.ReturnT.FAIL_CODE; + @Component public class KnowledgeBaseJob { @@ -100,14 +101,14 @@ public class KnowledgeBaseJob { } catch (Exception e) { JobUtil.errorLog(e); - return ReturnT.FAIL; + return new ReturnT<String>(FAIL_CODE, "fail"); } } if (failedTask > 0) { return ReturnT.FAIL; } - return ReturnT.SUCCESS; + return new ReturnT<String>("success"); } private Map builtParam(Map sink, Map dbKm, Map sinkParam) { diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/HttpParam.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/HttpParam.java index bcd4a31..492b421 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/HttpParam.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/HttpParam.java @@ -2,8 +2,8 @@ package com.mesalab.executor.pojo; import cn.hutool.core.util.ObjectUtil; import cn.hutool.json.JSONUtil; +import com.geedgenetworks.utils.StringUtil; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.StringUtil; import io.netty.handler.codec.http.HttpMethod; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/HttpToKafkaParam.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/HttpToKafkaParam.java index f40d011..b92b8ae 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/HttpToKafkaParam.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/HttpToKafkaParam.java @@ -1,13 +1,13 @@ package com.mesalab.executor.pojo; -import java.io.Serializable; -import java.util.Map; - import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import java.io.Serializable; +import java.util.Map; + @Data @Builder @NoArgsConstructor diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/IpLocateLibraryParam.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/IpLocateLibraryParam.java index e5e8f84..67a867a 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/IpLocateLibraryParam.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/IpLocateLibraryParam.java @@ -1,7 +1,10 @@ package com.mesalab.executor.pojo; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; import java.io.Serializable; import java.util.List; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/JDBCParam.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/JDBCParam.java index ce898ac..979a7db 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/JDBCParam.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/JDBCParam.java @@ -10,6 +10,7 @@ import net.sf.jsqlparser.parser.CCJSqlParserManager; import net.sf.jsqlparser.statement.select.Select; import net.sf.jsqlparser.util.TablesNamesFinder; +import java.io.Serializable; import java.io.StringReader; import java.util.List; import java.util.Map; @@ -22,7 +23,7 @@ import java.util.Map; @Data @Builder @NoArgsConstructor -public class JDBCParam { +public class JDBCParam implements Serializable { private String type; private String ip; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/ListAllMyBucketsResult.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/ListAllMyBucketsResult.java index f5b8a5d..8c3350f 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/ListAllMyBucketsResult.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/ListAllMyBucketsResult.java @@ -1,9 +1,8 @@ package com.mesalab.executor.pojo; -import java.util.List; - import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; @XmlRootElement(name = "ListAllMyBucketsResult") diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/MMDBStatus.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/MMDBStatus.java index b19b78b..701030c 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/MMDBStatus.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/pojo/MMDBStatus.java @@ -11,7 +11,6 @@ public class MMDBStatus { private Integer status; private String message; private Long updateTime; - private List<String> ipv4Sample = new ArrayList<>(); - private List<String> ipv6Sample = new ArrayList<>(); + private List<String> ipSample = new ArrayList<>(); } diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/DataSinkService.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/DataSinkService.java index 1bac814..505369f 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/DataSinkService.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/DataSinkService.java @@ -7,12 +7,12 @@ import cn.hutool.core.util.StrUtil; import cn.hutool.log.Log; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.JsonMapper; import com.mesalab.executor.core.utils.DBUtils; import com.mesalab.executor.core.utils.JobUtil; import com.mesalab.executor.pojo.JDBCParam; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.JsonMapper; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/DataSourceService.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/DataSourceService.java index a74966f..46ce458 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/DataSourceService.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/DataSourceService.java @@ -16,6 +16,7 @@ import cn.hutool.log.Log; import cn.hutool.poi.excel.ExcelReader; import cn.hutool.poi.excel.ExcelUtil; import com.alibaba.fastjson.JSON; +import com.geedgenetworks.utils.JsonMapper; import com.mesalab.executor.core.utils.DBUtils; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.JobUtil; @@ -24,7 +25,6 @@ import com.mesalab.executor.exception.BusinessException; import com.mesalab.executor.pojo.HttpParam; import com.mesalab.executor.pojo.JDBCParam; import com.xxl.job.core.handler.IJobHandler; -import com.zdjizhi.utils.JsonMapper; import io.netty.handler.codec.http.HttpMethod; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.commons.lang3.ObjectUtils; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/StorageQuotaService.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/StorageQuotaService.java index 736108a..e7b2123 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/StorageQuotaService.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/StorageQuotaService.java @@ -1,13 +1,13 @@ package com.mesalab.executor.service; import cn.hutool.log.Log; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.JsonMapper; import com.mesalab.executor.core.config.StorgeConfig; import com.mesalab.executor.core.utils.*; import com.mesalab.executor.exception.BusinessException; import com.mesalab.executor.pojo.HosSpace; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.JsonMapper; import org.apache.http.Header; import org.apache.http.message.BasicHeader; import org.springframework.beans.factory.annotation.Value; diff --git a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/knowledgeBase/KnowledgeBaseService.java b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/knowledgeBase/KnowledgeBaseService.java index afb2474..3ee7db3 100644 --- a/galaxy-job-executor/src/main/java/com/mesalab/executor/service/knowledgeBase/KnowledgeBaseService.java +++ b/galaxy-job-executor/src/main/java/com/mesalab/executor/service/knowledgeBase/KnowledgeBaseService.java @@ -1,16 +1,16 @@ package com.mesalab.executor.service.knowledgeBase; import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.file.FileWriter; import cn.hutool.core.util.CharsetUtil; -import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; -import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; import com.alibaba.nacos.api.config.annotation.NacosValue; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.executor.core.utils.DBUtils; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.JobUtil; @@ -19,11 +19,10 @@ import com.mesalab.executor.exception.BusinessException; import com.mesalab.executor.pojo.JDBCParam; import com.mesalab.executor.pojo.KnowledgeBase; import com.mesalab.executor.pojo.MMDBStatus; -import com.zdjizhi.utils.IpLookupV2; -import com.zdjizhi.utils.StringUtil; import org.springframework.stereotype.Service; import java.io.File; +import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -69,7 +68,6 @@ public class KnowledgeBaseService { break; } batchData.clear(); - batchData = null; pageNum++; } JobUtil.infoLog("knowledgeBase file info filename: {},file format: {} , size: {}", tmpFile.getName(), knowledgeBase.getFormat(), tmpFile.length()); @@ -79,9 +77,9 @@ public class KnowledgeBaseService { } catch (Exception e) { JobUtil.errorLog(e); throw new BusinessException(e); - } /*finally { + } finally { FileUtil.del(tmpDir); - }*/ + } return pushStatus; } @@ -149,45 +147,35 @@ public class KnowledgeBaseService { * 区分asn，ip 包含v4 v6 */ public boolean mmdbBuildPush(Map<String, Object> sinkParam) throws BusinessException { - final String workDirCmd = "docker inspect --format '{{range.Mounts}} {{.Source}} {{end}}' galaxy-job-executor |xargs -n 1|grep plugins"; + String url = String.valueOf(sinkParam.get("url")); String params = String.valueOf(sinkParam.get("params")); JSONObject jsonObject = JSONUtil.parseObj(params); KnowledgeBase knowledgeBase = JSONUtil.toBean(jsonObject, KnowledgeBase.class); boolean pushStatus = false; try { - String workDir = JobUtil.exeCmd(workDirCmd); String script = "ip"; - JobUtil.infoLog("get docker work dir {}", workDir); - Integer addrType = knowledgeBase.getType().contains("v6") ? 6 : 4; Boolean ipLocation = true; if (knowledgeBase.getType().contains("asn")) { script = "asn"; ipLocation = false; } - String cmd = "docker run --rm --network=host -v " + workDir + "/mmdb:/mmdb --name perl-mmdb perl:v3.1 sh -c 'cd /mmdb && perl " + script + ".pl " + addrType + " " + knowledgeBase.getId() + "'"; - JobUtil.infoLog("ip locate run docker perl cmd {}", cmd); - - int executeResult = JobUtil.executeCommand(cmd); - JobUtil.infoLog("ip locate knowledgeBase perl execute result {}", executeResult == 0 ? "success" : "fail"); - String filename = StringUtil.join(script, addrType, ".", knowledgeBase.getFormat()); + int executeResult = buildMMDB(knowledgeBase, script); + String filename = StringUtil.join(script, ".", knowledgeBase.getFormat()); if (executeResult == 0) { - if (ipLocation && !checkIpMMDBSample(KNOWLEDGE_WORK_DIR, filename, script)) { + if (ipLocation && !checkSample(KNOWLEDGE_WORK_DIR, filename, script)) { JobUtil.infoLog("ip locate knowledgeBase perl execute or sample validate fail !"); return false; } //push knowledge File file = new File(StringUtil.join(KNOWLEDGE_WORK_DIR, File.separator, filename)); JobUtil.infoLog("knowledgeBase file info filename: {},file format: {} , size: {}", file.getName(), knowledgeBase.getFormat(), file.length()); - if (file.length() > 0) { - //设置特殊id - String konwledgeId = DigestUtil.md5Hex16(knowledgeBase.getId() + knowledgeBase.getType()); - knowledgeBase.setId(konwledgeId); pushStatus = push(knowledgeBase, file, url); } + } } catch (Exception e) { JobUtil.errorLog(e); @@ -196,6 +184,21 @@ public class KnowledgeBaseService { return pushStatus; } + private int buildMMDB(KnowledgeBase knowledgeBase, String script) throws IOException, InterruptedException { + final String workDirCmd = "docker inspect --format '{{range.Mounts}} {{.Source}} {{end}}' galaxy-job-executor |xargs -n 1|grep plugins"; + final String imagesNameCmd = "docker images --format '{{.Repository}}:{{.Tag}}' | grep '^perl:' | sort -r | head -n 1"; + + String workDir = JobUtil.exeCmd(workDirCmd); + String imagesName = JobUtil.exeCmd(imagesNameCmd); + + final String cmd = "docker run --rm --network=host -v " + workDir + "/mmdb:/mmdb --name perl-mmdb "+ imagesName +" sh -c 'cd /mmdb && perl " + script + ".pl " + knowledgeBase.getId() + "'"; + JobUtil.infoLog("ip locate run perl cmd {}", cmd); + + int executeResult = JobUtil.executeCommand(cmd); + JobUtil.infoLog("ip locate knowledgeBase perl execute result {}", executeResult == 0 ? "success" : "fail"); + return executeResult; + } + public String getMetadata(String knowledgeId) { try { ZookeeperUtils zk = new ZookeeperUtils(); @@ -240,34 +243,23 @@ public class KnowledgeBaseService { * @param * @return */ - public static boolean checkIpMMDBSample(String path, String filename, String script) throws Exception { + public static boolean checkSample(String path, String filename, String script) throws Exception { - String builtStatusFile = "built-status.json"; + final String builtinStatusFile = "builtin-status.json"; - MMDBStatus builtMMDBStatus = JobUtil.readJsonFile(path + builtStatusFile, MMDBStatus.class); - JobUtil.infoLog("built ip locate library result: status {}, message {}", builtMMDBStatus.getStatus(), builtMMDBStatus.getMessage()); + MMDBStatus builtMMDBStatus = JobUtil.readJsonFile(path + builtinStatusFile, MMDBStatus.class); + JobUtil.infoLog("builtin ip locate library result: status {}, message {}", builtMMDBStatus.getStatus(), builtMMDBStatus.getMessage()); if (200 != builtMMDBStatus.getStatus()) { - JobUtil.errorLog("built status fail {}", builtMMDBStatus.getStatus()); + JobUtil.errorLog("builtin status fail {}", builtMMDBStatus.getStatus()); return false; } //测试抽样数据能查到结果说明打包成功 - IpLookupV2.Builder builder = new IpLookupV2.Builder(false); - List<String> ipSample = builtMMDBStatus.getIpv4Sample(); - ipSample.addAll(builtMMDBStatus.getIpv6Sample()); - for (String sampleIp : ipSample) { - if ("ip".equals(script)) { - IpLookupV2 ipLookup = builder.loadDataFile(path + File.separator + filename).build(); - if (ObjectUtil.isEmpty(ipLookup.latLngLookup(sampleIp))) { - JobUtil.infoLog("built ip locate library sample validate fail, sampleIp {}, latLng {}", sampleIp, ipLookup.latLngLookup(sampleIp)); - return false; - } - } else if ("asn".equals(script)) { - IpLookupV2 ipLookup = builder.loadAsnDataFile(path + File.separator + filename).build(); - if (ObjectUtil.isEmpty(ipLookup.asnLookup(sampleIp))) { - JobUtil.infoLog("built asn library sample validate fail, sampleIp {}, asn {}", sampleIp, ipLookup.asnLookup(sampleIp)); - return false; - } - } + List<String> ipSample = builtMMDBStatus.getIpSample(); + String samplePath = path + File.separator + filename; + if ("ip".equals(script)) { + return JobUtil.ipKnowledgeNotEmpty(samplePath,"",ipSample); + }else if ("asn".equals(script)) { + return JobUtil.asnKnowledgeNotEmpty(samplePath,"",ipSample); } return true; } diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/CommandJobTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/CommandJobTest.java index 54c0801..3cfc047 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/CommandJobTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/CommandJobTest.java @@ -1,8 +1,11 @@ package com.mesalab.executor.test; import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; +import cn.hutool.core.util.CharsetUtil; import cn.hutool.core.util.ObjectUtil; +import cn.hutool.core.util.StrUtil; import cn.hutool.db.DbUtil; import cn.hutool.db.Entity; import cn.hutool.db.handler.EntityListHandler; @@ -10,13 +13,14 @@ import cn.hutool.db.sql.SqlExecutor; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONUtil; import com.alibaba.fastjson.JSONPath; +import com.geedgenetworks.utils.IpLookupV2; import com.google.common.collect.ImmutableMap; import com.mesalab.executor.core.utils.DBUtils; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.pojo.IpLocateLibraryParam; import com.mesalab.executor.pojo.JDBCParam; import com.mesalab.executor.pojo.KnowledgeBase; -import com.zdjizhi.utils.IpLookupV2; +import org.apache.commons.lang3.time.StopWatch; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -26,6 +30,7 @@ import org.springframework.test.context.junit4.SpringRunner; import java.io.File; import java.sql.*; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @RunWith(SpringRunner.class) @@ -35,26 +40,51 @@ public class CommandJobTest { @Test public void getIPMMDBFile() { IpLookupV2 ipLookup = new IpLookupV2.Builder(false) -// .loadDataFileV4("C:\\code\\galaxy-qgw-service\\dat\\ip_v4_built_in.mmdb") -// .loadDataFileV6("C:\\code\\galaxy-qgw-service\\dat\\ip_v6_built_in.mmdb") -// .loadDataFilePrivateV4("C:\\code\\galaxy-qgw-service\\dat\\ip_v6_built_in.mmdb") -// .loadDataFile("C:\\test\\1.mmdb") - .loadDataFileV4("C:\\home\\tsg\\galaxy\\galaxy-job\\plugins\\mmdb\\1.mmdb") - .loadAsnDataFile("C:\\home\\tsg\\galaxy\\galaxy-job\\plugins\\mmdb\\asn-1.mmdb") + .loadDataFileV4("C:\\test\\knowledge\\ip4.mmdb") +// .loadDataFile("C:\\test\\knowledge\\ip6.mmdb") // .loadDataFilePrivateV6("") .build(); -// String latLngLookup = ipLookup.latLngLookup("0.0.0.0"); -//// System.err.println(latLngLookup); -//// System.err.println(ipLookup.countryLookup("0000:0000:0000:0000:0000:0000:0000:0000")); -// System.err.println(ipLookup.countryLookup("0.0.0.0")); -// System.err.println(ipLookup.cityLookup("192.168.1.1")); - System.err.println(ipLookup.asnLookupDetail("1.0.0.0")); - System.err.println(ipLookup.asnLookupDetail("8.6.146.255")); + System.err.println(ipLookup.infoLookup("2a7:1c44:39f3:1b::")); + System.err.println(ipLookup.infoLookup("2001:240:2440::")); + System.err.println(ipLookup.infoLookup("fe80::b432:5ff:fef9:a3cb")); + System.err.println(ipLookup.infoLookup("fe80::182d:8dff:fe73:fe80")); + System.err.println("#"); + System.err.println(ipLookup.infoLookup("8.6.112.0")); + System.err.println(ipLookup.infoLookup("0.0.0.1")); + System.err.println(ipLookup.infoLookup("223.255.243.241")); + System.err.println(ipLookup.infoLookup("223.255.244.241")); + System.err.println(ipLookup.infoLookup("223.255.246.241")); + System.err.println(ipLookup.infoLookup("59.154.195.156")); + System.err.println("---------------------------"); Assert.assertNotEquals("ip is not found", ObjectUtil.isEmpty("latLngLookup")); + } + @Test + public void getIplocationsTest() { + IpLookupV2 ipLookup = new IpLookupV2.Builder(false) + .loadDataFile("C:\\test\\knowledge\\asn.mmdb") + .build(); + String path = "C:\\test\\knowledge\\cn_ip_location_built_in_202309221601.txt"; + String path2 = "C:\\test\\knowledge\\cn_ip_location_built_in_202309221601.txt"; + File file = FileUtil.file(path); + File file2 = FileUtil.file(path2); + List<String> ipSample = FileUtil.readLines(file, CharsetUtil.CHARSET_UTF_8); + List<String> ipSample2 = FileUtil.readLines(file2, CharsetUtil.CHARSET_UTF_8); + ipSample.addAll(ipSample2); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + int count = 0; + for (String ip : ipSample) { + String trim = StrUtil.trim(ip); + Object o = ipLookup.infoLookupToJSON(trim); + System.err.println(o); + count++; + } + stopWatch.stop(); + System.err.println("解析：" + count+"时间："+stopWatch.getTime(TimeUnit.MILLISECONDS)); } @Test diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DataExtractTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DataExtractTest.java index 9bb7f80..9d1c6b1 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DataExtractTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DataExtractTest.java @@ -27,6 +27,7 @@ import cn.hutool.poi.excel.ExcelReader; import cn.hutool.poi.excel.ExcelUtil; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; +import com.geedgenetworks.utils.DateUtils; import com.mesalab.executor.core.utils.DBUtils; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.JobUtil; @@ -41,7 +42,6 @@ import com.mesalab.executor.service.DataTransformService; import com.xxl.job.core.log.XxlJobLogger; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; -import com.zdjizhi.utils.DateUtils; import org.apache.commons.io.filefilter.FileFilterUtils; import org.apache.groovy.util.Maps; import org.assertj.core.util.Lists; diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DataflowJobTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DataflowJobTest.java index 1ba5bd8..da35503 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DataflowJobTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DataflowJobTest.java @@ -2,13 +2,12 @@ package com.mesalab.executor.test; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; +import com.geedgenetworks.utils.JsonMapper; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.executor.GalaxyExecutorApplication; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.KafkaUtils; import com.mesalab.executor.exception.BusinessException; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; -import lombok.extern.slf4j.Slf4j; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DeleteDataTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DeleteDataTest.java index ec9b467..87678d7 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DeleteDataTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DeleteDataTest.java @@ -1,12 +1,12 @@ package com.mesalab.executor.test; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.JsonMapper; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.mesalab.executor.GalaxyExecutorApplication; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.KafkaUtils; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.JsonMapper; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; import org.junit.runner.RunWith; diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DruidCompactJobTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DruidCompactJobTest.java index 5b9762b..d6de460 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DruidCompactJobTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/DruidCompactJobTest.java @@ -1,14 +1,10 @@ package com.mesalab.executor.test; import com.fasterxml.jackson.databind.JavaType; -import com.mesalab.executor.GalaxyExecutorApplication; -import com.mesalab.executor.core.config.StorgeConfig; +import com.geedgenetworks.utils.JsonMapper; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.executor.core.utils.HttpClientUtils; -import com.mesalab.executor.core.utils.SpringContextUtil; -import com.mesalab.executor.core.utils.UrlUtil; import com.mesalab.executor.pojo.DruidToHttpParam; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; import lombok.extern.slf4j.Slf4j; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/EncryptConfigUtil.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/EncryptConfigUtil.java index 90b3a3a..b93a951 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/EncryptConfigUtil.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/EncryptConfigUtil.java @@ -17,11 +17,10 @@ public class EncryptConfigUtil { //加密所需的salt textEncryptor.setPassword("galaxy"); //要加密的数据（数据库的用户名或密码） - String user = textEncryptor.encrypt("admin"); - String pin = textEncryptor.decrypt("galaxy2019"); - System.err.println("user: "+user); - System.err.println("pin: "+pin); - System.err.println("user: "+textEncryptor.decrypt("iW8ekP1SZC6v/7cfJKAqXXrjApJox+cH")); - System.err.println("pin: "+textEncryptor.decrypt("iw8ekP1S7C6V/7cJKAaXXriApJox+cH")); +// String user = textEncryptor.encrypt("admin"); +// String pin = textEncryptor.decrypt("galaxy2019"); +// System.err.println("user: "+user); +// System.err.println("pin: "+pin); + System.err.println("pin: "+textEncryptor.decrypt("nPNttVO8Tp/G16BFdg/+18dpU04BRnzY")); } }
\ No newline at end of file diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/HttpToStoreTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/HttpToStoreTest.java index bf2d638..d4123e1 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/HttpToStoreTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/HttpToStoreTest.java @@ -11,6 +11,9 @@ import cn.hutool.core.util.ZipUtil; import cn.hutool.crypto.digest.DigestUtil; import cn.hutool.http.HttpUtil; import cn.hutool.log.Log; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.JsonMapper; +import com.geedgenetworks.utils.StringUtil; import com.mesalab.executor.GalaxyExecutorApplication; import com.mesalab.executor.core.config.StorgeConfig; import com.mesalab.executor.core.utils.Constant; @@ -18,9 +21,6 @@ import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.JobUtil; import com.mesalab.executor.pojo.HttpToKafkaParam; import com.xxl.job.core.log.XxlJobLogger; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; import io.netty.handler.codec.http.HttpMethod; import org.apache.http.Header; import org.apache.http.client.methods.CloseableHttpResponse; diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/IpConvert.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/IpConvert.java index c1a3cf7..4bf7afd 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/IpConvert.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/IpConvert.java @@ -1,8 +1,8 @@ package com.mesalab.executor.test; import java.math.BigInteger; -import java.net.InetAddress; -import java.net.UnknownHostException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; /** diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/KnowledgeJobTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/KnowledgeJobTest.java index e1d7dde..b668972 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/KnowledgeJobTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/KnowledgeJobTest.java @@ -2,40 +2,29 @@ package com.mesalab.executor.test; import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.file.FileWriter; -import cn.hutool.core.io.resource.ResourceUtil; import cn.hutool.core.text.csv.*; import cn.hutool.core.util.CharsetUtil; import cn.hutool.core.util.ObjectUtil; -import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSONUtil; -import com.alibaba.fastjson.util.TypeUtils; import com.alibaba.nacos.api.config.annotation.NacosValue; +import com.geedgenetworks.utils.IpLookupV2; import com.google.common.collect.ImmutableMap; -import com.google.common.net.InetAddresses; import com.mesalab.executor.core.utils.HttpClientUtils; import com.mesalab.executor.core.utils.JobUtil; import com.mesalab.executor.core.utils.ZookeeperUtils; import com.mesalab.executor.service.knowledgeBase.KnowledgeBaseService; -import com.zdjizhi.utils.IPUtil; -import com.zdjizhi.utils.IpLookupV2; -import org.apache.tomcat.util.net.IPv6Utils; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.ZooKeeper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; -import java.io.BufferedReader; import java.io.File; -import java.net.Inet4Address; -import java.net.InetAddress; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import static com.mesalab.executor.core.utils.Constant.ZK_KNOWLEDGE_PATH; -import static org.junit.Assert.assertNotNull; @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @@ -149,7 +138,7 @@ public class KnowledgeJobTest { @Test public void asnTest(){ try { - boolean asn = KnowledgeBaseService.checkIpMMDBSample("C:\\test\\knowledge\\", "asn6.mmdb", "asn"); + boolean asn = KnowledgeBaseService.checkSample("C:\\test\\knowledge\\", "asn6.mmdb", "asn"); Assert.assertTrue(asn); } catch (Exception e) { JobUtil.errorLog(e); diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/ShellTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/ShellTest.java index 030448f..2e07948 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/ShellTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/ShellTest.java @@ -1,13 +1,12 @@ package com.mesalab.executor.test; import com.mesalab.executor.GalaxyExecutorApplication; +import com.mesalab.executor.core.utils.HttpClientUtils; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.junit4.SpringRunner; - -import com.mesalab.executor.core.utils.HttpClientUtils; @RunWith(SpringRunner.class) @SpringBootTest(classes = { GalaxyExecutorApplication.class }) @Slf4j diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/SourceJobTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/SourceJobTest.java index 402f048..3ae938b 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/SourceJobTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/SourceJobTest.java @@ -2,6 +2,7 @@ package com.mesalab.executor.test; import cn.hutool.core.date.DateUtil; import cn.hutool.core.util.ReUtil; +import cn.hutool.crypto.digest.DigestUtil; import com.mesalab.executor.pojo.JDBCParam; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; @@ -108,4 +109,10 @@ public class SourceJobTest { System.out.println("Processed string: " + result); } + + public static void main(String[] args) { + + String konwledgeId = DigestUtil.md5Hex16(1 + "cn_ipv6_location_built_in"); + System.err.println(konwledgeId); + } }
\ No newline at end of file diff --git a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/StorageQuotaTest.java b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/StorageQuotaTest.java index 144b2ca..e4387eb 100644 --- a/galaxy-job-executor/src/test/java/com/mesalab/executor/test/StorageQuotaTest.java +++ b/galaxy-job-executor/src/test/java/com/mesalab/executor/test/StorageQuotaTest.java @@ -1,14 +1,14 @@ package com.mesalab.executor.test; +import com.geedgenetworks.utils.DateUtils; +import com.geedgenetworks.utils.JsonMapper; +import com.geedgenetworks.utils.StringUtil; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; import com.mesalab.executor.GalaxyExecutorApplication; import com.mesalab.executor.core.utils.KafkaUtils; import com.mesalab.executor.jobhandler.LogStorageQuotaJob; import com.mesalab.executor.service.StorageQuotaService; -import com.zdjizhi.utils.DateUtils; -import com.zdjizhi.utils.JsonMapper; -import com.zdjizhi.utils.StringUtil; import lombok.extern.slf4j.Slf4j; import org.junit.Test; import org.junit.runner.RunWith; @@ -72,7 +72,7 @@ public class StorageQuotaTest { // String collect = Splitter.on(",").splitToList(a).stream().map(x -> x.substring(x.indexOf("|") + 1)).filter(x-> StringUtil.isNotEmpty(x)).collect(Collectors.joining(",")); List collect = Splitter.on(",").splitToList(filesHosServer).stream().map(x->{ List<String> dc = Splitter.on("|").splitToList(x); - if (dc.size()==2&&StringUtil.isNotBlank(dc.get(1))){ + if (dc.size()==2&& StringUtil.isNotBlank(dc.get(1))){ return ImmutableMap.of(dc.get(0),dc.get(1)); } return null; |