diff options
| author | zhanghongqing <[email protected]> | 2020-12-16 11:17:44 +0800 |
|---|---|---|
| committer | zhanghongqing <[email protected]> | 2020-12-16 11:17:44 +0800 |
| commit | feb19d34980ba458f6c796aa86ffc0d65fe727b3 (patch) | |
| tree | d58382b18ebaa7d32c0e63c365db26d4fcef9d36 | |
| parent | e33ec95108b6666047ed0c3ee21ccb0d1f57c856 (diff) | |
将服务推荐功能从engine抽取到business模块,使用注册中心的方式调用。
43 files changed, 216 insertions, 129 deletions
diff --git a/galaxy-business-api/config/application.yml b/galaxy-business-api/config/application.yml index b958866..b2c994b 100644 --- a/galaxy-business-api/config/application.yml +++ b/galaxy-business-api/config/application.yml @@ -29,4 +29,7 @@ management: show-details: always ## log file path config logging: - config: ./config/logback-spring.xml
\ No newline at end of file + config: ./config/logback-spring.xml +# knowledge +knowledge: + maxrows: 10000
\ No newline at end of file diff --git a/galaxy-business-api/pom.xml b/galaxy-business-api/pom.xml index cb3d2c2..9a6a1d6 100644 --- a/galaxy-business-api/pom.xml +++ b/galaxy-business-api/pom.xml @@ -73,6 +73,10 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>com.github.java-json-tools</groupId> + <artifactId>json-schema-validator</artifactId> + </dependency> </dependencies> <build> diff --git a/galaxy-business-api/src/main/java/com/mesalab/api/common/utils/Constants.java b/galaxy-business-api/src/main/java/com/mesalab/api/common/utils/Constants.java new file mode 100644 index 0000000..e803520 --- /dev/null +++ b/galaxy-business-api/src/main/java/com/mesalab/api/common/utils/Constants.java @@ -0,0 +1,11 @@ +package com.mesalab.api.common.utils; + +/** + * @description: + * @author: zhq + * @create: 2020-12-16 + **/ +public class Constants { + + public static final String GALAXY_QUERY_ENGINE = "http://galaxy-query-engine/v1"; +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/controller/KnowledgeController.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/controller/KnowledgeController.java index f3fef3d..4d33007 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/controller/KnowledgeController.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/controller/KnowledgeController.java @@ -1,11 +1,11 @@ -package com.mesalab.knowledge.controller; +package com.mesalab.api.modules.knowledge.controller; -import com.mesalab.common.base.BaseResult; -import com.mesalab.common.enums.ResultCodeEnum; -import com.mesalab.common.enums.ResultStatusEnum; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.knowledge.enums.QueryTypeEnum; -import com.mesalab.knowledge.service.KnowledgeService; +import com.mesalab.api.common.base.BaseResult; +import com.mesalab.api.common.enums.ResultCodeEnum; +import com.mesalab.api.common.enums.ResultStatusEnum; +import com.mesalab.api.common.exception.BusinessException; +import com.mesalab.api.modules.knowledge.enums.QueryTypeEnum; +import com.mesalab.api.modules.knowledge.service.KnowledgeService; import com.mesalab.tool.utils.StringUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.EnumUtils; @@ -34,8 +34,8 @@ public class KnowledgeController { * @return com.mesalab.common.base.BaseResult **/ @PostMapping(value = "/v1/", produces = "application/json") - public BaseResult knowledgeQuery(HttpServletRequest request, @RequestBody Object object) { - if (EnumUtils.isValidEnum(QueryTypeEnum.class,StringUtil.upperCase(request.getQueryString()))) { + public BaseResult knowledgeQuery(HttpServletRequest request, @RequestBody Object object) { + if (EnumUtils.isValidEnum(QueryTypeEnum.class, StringUtil.upperCase(request.getQueryString()))) { log.info("服务推荐接口, 参数: queryString is {},params is {}", request.getQueryString(), object); return knowledgeService.query(object); } else { diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/DSLObject.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/DSLObject.java index 096bc5e..2885327 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/DSLObject.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/DSLObject.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.entity; +package com.mesalab.api.modules.knowledge.entity; import lombok.Data; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/Match.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/Match.java index 7ad2818..395c2c7 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/Match.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/Match.java @@ -1,7 +1,7 @@ -package com.mesalab.knowledge.entity; +package com.mesalab.api.modules.knowledge.entity; import com.google.common.base.CharMatcher; -import com.mesalab.knowledge.enums.MatchEnum; +import com.mesalab.api.modules.knowledge.enums.MatchEnum; import com.mesalab.tool.utils.StringUtil; import lombok.Data; import org.apache.commons.lang3.EnumUtils; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/Parameters.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/Parameters.java index f770c49..73b208c 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/Parameters.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/Parameters.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.entity; +package com.mesalab.api.modules.knowledge.entity; import lombok.Data; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/QueryExp.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/QueryExp.java index cbc5240..9c823c6 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/QueryExp.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/QueryExp.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.entity; +package com.mesalab.api.modules.knowledge.entity; import lombok.Data; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/Range.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/Range.java index 2ea4780..05fece1 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/Range.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/Range.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.entity; +package com.mesalab.api.modules.knowledge.entity; import lombok.Data; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/Sort.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/Sort.java index 58b6503..fba307a 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/Sort.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/Sort.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.entity; +package com.mesalab.api.modules.knowledge.entity; import lombok.Data; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/arango/BaseDocument.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/arango/BaseDocument.java index 67ff17c..fa3ec48 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/arango/BaseDocument.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/arango/BaseDocument.java @@ -18,7 +18,7 @@ * Copyright holder is ArangoDB GmbH, Cologne, Germany */ -package com.mesalab.knowledge.entity.arango; +package com.mesalab.api.modules.knowledge.entity.arango; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -26,8 +26,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; /** * @author Mark Vollmary diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/arango/BaseEdgeDocument.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/arango/BaseEdgeDocument.java index fdd66d3..2f1c175 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/arango/BaseEdgeDocument.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/arango/BaseEdgeDocument.java @@ -18,14 +18,12 @@ * Copyright holder is ArangoDB GmbH, Cologne, Germany */ -package com.mesalab.knowledge.entity.arango; +package com.mesalab.api.modules.knowledge.entity.arango; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; -import java.util.Map; - /** * @author Mark Vollmary */ diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/arango/IpLearningPath.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/arango/IpLearningPath.java index d5af33e..9cadc10 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/arango/IpLearningPath.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/arango/IpLearningPath.java @@ -1,16 +1,11 @@ -package com.mesalab.knowledge.entity.arango; +package com.mesalab.api.modules.knowledge.entity.arango; -import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; -import org.nutz.json.JsonField; -import java.util.Date; import java.util.List; -import static org.nutz.json.JsonFormat.Function.dateFormat; - /** * @description: for iplearning * @author: zhq diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/arango/SubscriberIdPath.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/arango/SubscriberIdPath.java index d3d217c..6580f68 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/entity/arango/SubscriberIdPath.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/entity/arango/SubscriberIdPath.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.entity.arango; +package com.mesalab.api.modules.knowledge.entity.arango; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/enums/DataSourceEnum.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/enums/DataSourceEnum.java index dd11e86..04aeb39 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/enums/DataSourceEnum.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/enums/DataSourceEnum.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.enums; +package com.mesalab.api.modules.knowledge.enums; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/enums/MatchEnum.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/enums/MatchEnum.java index 5134fac..cd17f73 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/enums/MatchEnum.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/enums/MatchEnum.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.enums; +package com.mesalab.api.modules.knowledge.enums; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/enums/QueryTypeEnum.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/enums/QueryTypeEnum.java index 39c226e..685e87c 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/enums/QueryTypeEnum.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/enums/QueryTypeEnum.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.enums; +package com.mesalab.api.modules.knowledge.enums; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/enums/RangeEnum.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/enums/RangeEnum.java index 09aa862..f871bab 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/enums/RangeEnum.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/enums/RangeEnum.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.enums; +package com.mesalab.api.modules.knowledge.enums; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/service/KnowledgeService.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/service/KnowledgeService.java index 6b42113..1ffa119 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/service/KnowledgeService.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/service/KnowledgeService.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.service; +package com.mesalab.api.modules.knowledge.service; import com.fasterxml.jackson.databind.JsonNode; import com.github.fge.jackson.JsonLoader; @@ -8,13 +8,13 @@ import com.github.fge.jsonschema.core.report.ProcessingReport; import com.github.fge.jsonschema.main.JsonSchema; import com.github.fge.jsonschema.main.JsonSchemaFactory; import com.google.common.base.Splitter; -import com.mesalab.common.base.BaseResult; -import com.mesalab.common.base.BaseResultGenerator; -import com.mesalab.knowledge.entity.*; -import com.mesalab.knowledge.enums.DataSourceEnum; -import com.mesalab.knowledge.enums.MatchEnum; -import com.mesalab.knowledge.enums.RangeEnum; -import com.mesalab.knowledge.strategy.QueryProvider; +import com.mesalab.api.common.base.BaseResult; +import com.mesalab.api.common.base.BaseResultGenerator; +import com.mesalab.api.modules.knowledge.entity.*; +import com.mesalab.api.modules.knowledge.enums.DataSourceEnum; +import com.mesalab.api.modules.knowledge.enums.MatchEnum; +import com.mesalab.api.modules.knowledge.enums.RangeEnum; +import com.mesalab.api.modules.knowledge.strategy.QueryProvider; import com.mesalab.tool.utils.DateUtils; import com.mesalab.tool.utils.JsonMapper; import com.mesalab.tool.utils.StringUtil; @@ -44,7 +44,7 @@ import static java.util.Objects.*; @Service public class KnowledgeService { - @Value("${arango.maxrows}") + @Value("${knowledge.maxrows}") private String maxrows;//查询最大条数 @Autowired @@ -75,7 +75,7 @@ public class KnowledgeService { Parameters parameters = dslQuery.getParameters(); requireNonNull(dslQuery.getDataSource(), "dataSource is null"); - requireNonNull(EnumUtils.getEnum(DataSourceEnum.class,StringUtil.upperCase(dslQuery.getDataSource())), "dataSource no match"); + requireNonNull(EnumUtils.getEnum(DataSourceEnum.class, StringUtil.upperCase(dslQuery.getDataSource())), "dataSource no match"); if (isNull(parameters)) { return; diff --git a/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/BaseQueryProvider.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/BaseQueryProvider.java new file mode 100644 index 0000000..c1852f4 --- /dev/null +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/BaseQueryProvider.java @@ -0,0 +1,31 @@ +package com.mesalab.api.modules.knowledge.strategy; + +import com.google.common.collect.ImmutableMap; +import com.mesalab.api.common.base.BaseResult; +import com.mesalab.api.common.exception.BusinessException; +import com.mesalab.api.common.utils.Constants; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import java.util.List; +import java.util.Map; + +@Component +public class BaseQueryProvider { + + @Autowired + RestTemplate restTemplate; + + private static final String SERVICE_NODE = "/graph/query"; + + public Object queryArango(String aql) { + BaseResult forObject = restTemplate.postForObject(Constants.GALAXY_QUERY_ENGINE.concat(SERVICE_NODE),ImmutableMap.of("query", aql), BaseResult.class); + if (forObject.isSuccess()) { + List<Map> data = (List<Map>) forObject.getData(); + return data; + } else { + throw new BusinessException(forObject.getMessage()); + } + } +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/FqdnProviderImpl.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/FqdnProviderImpl.java index 2cdc0cc..d0d3590 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/FqdnProviderImpl.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/FqdnProviderImpl.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.strategy; +package com.mesalab.api.modules.knowledge.strategy; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -7,12 +7,12 @@ import com.google.common.base.Joiner; import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.mesalab.knowledge.entity.DSLObject; -import com.mesalab.knowledge.entity.Match; -import com.mesalab.knowledge.entity.Parameters; -import com.mesalab.knowledge.entity.Range; -import com.mesalab.knowledge.entity.arango.IpLearningPath; -import com.mesalab.knowledge.enums.RangeEnum; +import com.mesalab.api.modules.knowledge.entity.DSLObject; +import com.mesalab.api.modules.knowledge.entity.Match; +import com.mesalab.api.modules.knowledge.entity.Parameters; +import com.mesalab.api.modules.knowledge.entity.Range; +import com.mesalab.api.modules.knowledge.entity.arango.IpLearningPath; +import com.mesalab.api.modules.knowledge.enums.RangeEnum; import com.mesalab.tool.utils.DateUtils; import com.mesalab.tool.utils.StringUtil; import org.apache.commons.lang3.EnumUtils; @@ -39,7 +39,7 @@ public class FqdnProviderImpl implements QueryProvider { private static final String HTTP = "HTTP"; private static final String DNS = "DNS"; - @Value("${arango.maxrows}") + @Value("${knowledge.maxrows}") private String limit;//查询最大条数 @Autowired @@ -123,7 +123,7 @@ public class FqdnProviderImpl implements QueryProvider { } //循环查询 ,目的为了数量达到limit , 结果数据是否为limit , 1.当查询数量小于limit停止 2.当结果数量=limit停止 - private List<IpLearningPath.IplearningEdges> getDistinctList(StringBuffer aqlsb ,int pageRow,int offSet, int pageSize){ + private List<IpLearningPath.IplearningEdges> getDistinctList(StringBuffer aqlsb , int pageRow, int offSet, int pageSize){ pageRow = pageRow + offSet + pageSize;//每次多查一点点,去重 StringBuffer sb = new StringBuffer(aqlsb); sb.append(" limit ").append(pageRow); diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/IpProviderImpl.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/IpProviderImpl.java index e0c7d68..d2c90fd 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/IpProviderImpl.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/IpProviderImpl.java @@ -1,11 +1,11 @@ -package com.mesalab.knowledge.strategy; +package com.mesalab.api.modules.knowledge.strategy; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.mesalab.knowledge.entity.DSLObject; -import com.mesalab.knowledge.entity.Parameters; -import com.mesalab.knowledge.entity.Sort; +import com.mesalab.api.modules.knowledge.entity.DSLObject; +import com.mesalab.api.modules.knowledge.entity.Parameters; +import com.mesalab.api.modules.knowledge.entity.Sort; import com.mesalab.tool.utils.DateUtils; import com.mesalab.tool.utils.StringUtil; import org.springframework.beans.factory.annotation.Autowired; @@ -26,7 +26,7 @@ import java.util.stream.Collectors; @Component("IP_VIEW") public class IpProviderImpl implements QueryProvider { - @Value("${arango.maxrows}") + @Value("${knowledge.maxrows}") private String limit;//查询最大条数 @Autowired BaseQueryProvider baseQueryProvider; diff --git a/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/QueryProvider.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/QueryProvider.java new file mode 100644 index 0000000..d0d6b19 --- /dev/null +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/QueryProvider.java @@ -0,0 +1,13 @@ +package com.mesalab.api.modules.knowledge.strategy; + + +import com.mesalab.api.modules.knowledge.entity.DSLObject; + +/** + * @Description: 提供查询结果 + */ +public interface QueryProvider { + + Object query(DSLObject.DSLQuery dslQeruy); + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/SubscriberIdProviderImpl.java b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/SubscriberIdProviderImpl.java index 71f2490..6ec334f 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/SubscriberIdProviderImpl.java +++ b/galaxy-business-api/src/main/java/com/mesalab/api/modules/knowledge/strategy/SubscriberIdProviderImpl.java @@ -1,13 +1,13 @@ -package com.mesalab.knowledge.strategy; +package com.mesalab.api.modules.knowledge.strategy; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.CharMatcher; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.mesalab.knowledge.entity.DSLObject; -import com.mesalab.knowledge.entity.Match; -import com.mesalab.knowledge.entity.Parameters; +import com.mesalab.api.modules.knowledge.entity.DSLObject; +import com.mesalab.api.modules.knowledge.entity.Match; +import com.mesalab.api.modules.knowledge.entity.Parameters; import com.mesalab.tool.utils.DateUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -26,7 +26,7 @@ import java.util.Objects; @Component("SUBSCRIBER_ID_VIEW") public class SubscriberIdProviderImpl implements QueryProvider { - @Value("${arango.maxrows}") + @Value("${knowledge.maxrows}") private String limit;//查询最大条数 @Autowired diff --git a/galaxy-query-engine/src/main/resources/dsl-validation.json b/galaxy-business-api/src/main/resources/dsl-validation.json index f54eb05..f54eb05 100644 --- a/galaxy-query-engine/src/main/resources/dsl-validation.json +++ b/galaxy-business-api/src/main/resources/dsl-validation.json diff --git a/galaxy-gateway/config/application.yml b/galaxy-gateway/config/application.yml index c787a07..08382cc 100644 --- a/galaxy-gateway/config/application.yml +++ b/galaxy-gateway/config/application.yml @@ -44,7 +44,7 @@ spring: predicates: - Path=/traffic/** - id: knowledge - uri: lb://galaxy-query-engine + uri: lb://galaxy-business-api predicates: - Path=/knowledge/** - id: query diff --git a/galaxy-query-engine/config/application.yml b/galaxy-query-engine/config/application.yml index e9476f8..b91f179 100644 --- a/galaxy-query-engine/config/application.yml +++ b/galaxy-query-engine/config/application.yml @@ -75,7 +75,6 @@ arango: password: ceiec2018 jwturl: ${arango.server}/_db/${arango.database}/_open/auth queryurl: ${arango.server}/_db/${arango.database}/_api/cursor - maxrows: 10000 socketTimeOut: 300000 ## http pool config http: diff --git a/galaxy-query-engine/pom.xml b/galaxy-query-engine/pom.xml index 5bb0810..ea7cb16 100644 --- a/galaxy-query-engine/pom.xml +++ b/galaxy-query-engine/pom.xml @@ -39,10 +39,6 @@ <artifactId>avro</artifactId> </dependency> <dependency> - <groupId>com.github.java-json-tools</groupId> - <artifactId>json-schema-validator</artifactId> - </dependency> - <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-test</artifactId> <scope>test</scope> diff --git a/galaxy-query-engine/src/main/java/com/mesalab/common/utils/SQLFunctionUtil.java b/galaxy-query-engine/src/main/java/com/mesalab/common/utils/SQLFunctionUtil.java index 8f684d7..d6fed48 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/common/utils/SQLFunctionUtil.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/common/utils/SQLFunctionUtil.java @@ -3,7 +3,7 @@ package com.mesalab.common.utils; import com.google.common.base.Splitter; import com.google.common.collect.Maps; import com.mesalab.common.enums.DBTypeEnum; -import com.mesalab.knowledge.common.utils.HttpConfig; +import com.mesalab.engine.common.config.HttpConfig; import com.mesalab.qgw.model.api.ClickHouseHttpSource; import com.mesalab.qgw.model.api.DruidIoHttpSource; import com.mesalab.qgw.service.impl.HttpClientService; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/common/config/ArangoConfig.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/config/ArangoConfig.java index 9dd84d7..058aca4 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/common/config/ArangoConfig.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/config/ArangoConfig.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.common.config; +package com.mesalab.engine.common.config; import lombok.Data; import org.springframework.beans.factory.annotation.Value; @@ -31,9 +31,6 @@ public class ArangoConfig { @Value("${arango.queryurl}") private String queryurl; - @Value("${arango.maxrows}") - private String maxrows; - private String jwt; } diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/common/utils/HttpConfig.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/config/HttpConfig.java index c7a8acc..441750a 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/common/utils/HttpConfig.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/config/HttpConfig.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.common.utils; +package com.mesalab.engine.common.config; import lombok.Data; import org.springframework.beans.factory.annotation.Value; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/common/utils/Constant.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/Constant.java index 9c776f8..151b9a5 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/common/utils/Constant.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/Constant.java @@ -1,4 +1,4 @@ -package com.mesalab.knowledge.common.utils; +package com.mesalab.engine.common.util; /** * @description: for constant diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/common/utils/JwtCache.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/JwtCache.java index ff1c263..ecc1f24 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/common/utils/JwtCache.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/common/util/JwtCache.java @@ -1,13 +1,13 @@ -package com.mesalab.knowledge.common.utils; - -import java.util.List; -import java.util.concurrent.TimeUnit; +package com.mesalab.engine.common.util; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheStats; import com.mesalab.tool.utils.StringUtil; +import java.util.List; +import java.util.concurrent.TimeUnit; + /** * @description: for jwt cache * @author: zhq diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/GraphQueryController.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/GraphQueryController.java new file mode 100644 index 0000000..1713b81 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/controller/GraphQueryController.java @@ -0,0 +1,44 @@ +package com.mesalab.engine.controller; + +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; +import com.mesalab.engine.common.util.BaseResultUtil; +import com.mesalab.engine.service.GraphEngineService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +/** + * @description: + * @author: zhq + * @create: 2020-12-15 + **/ +@Slf4j +@RestController +@RequestMapping("/v1/graph/") +public class GraphQueryController { + + @Autowired + GraphEngineService graphEngineService; + + @PostMapping("query") + public BaseResult query(@RequestBody Map query) { + BaseResult baseResult; + try { + baseResult = graphEngineService.query(query); + } catch (BusinessException e) { + return BaseResultUtil.failure( + e.getErrorStatus(), + e.getErrorCode(), + e.getErrorMessage()); + } + return baseResult; + } + + +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/engine/service/GraphEngineService.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/GraphEngineService.java new file mode 100644 index 0000000..e116c45 --- /dev/null +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/GraphEngineService.java @@ -0,0 +1,16 @@ +package com.mesalab.engine.service; + +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; + +import java.util.Map; + +/** + * @description: + * @author: zhq + * @create: 2020-12-15 + **/ +public interface GraphEngineService { + + BaseResult query(Map executeSql) throws BusinessException; +} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/BaseQueryProvider.java b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/GraphEngineServiceImpl.java index 366eaf0..b94b6f0 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/BaseQueryProvider.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/engine/service/impl/GraphEngineServiceImpl.java @@ -1,10 +1,12 @@ -package com.mesalab.knowledge.strategy; +package com.mesalab.engine.service.impl; -import com.google.common.collect.ImmutableMap; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.knowledge.common.config.ArangoConfig; -import com.mesalab.knowledge.common.utils.Constant; -import com.mesalab.knowledge.common.utils.JwtCache; +import com.mesalab.engine.common.config.ArangoConfig; +import com.mesalab.engine.common.exception.BusinessException; +import com.mesalab.engine.common.pojo.BaseResult; +import com.mesalab.engine.common.util.BaseResultUtil; +import com.mesalab.engine.common.util.Constant; +import com.mesalab.engine.common.util.JwtCache; +import com.mesalab.engine.service.GraphEngineService; import com.mesalab.qgw.service.impl.HttpClientService; import com.mesalab.tool.utils.JsonMapper; import lombok.extern.slf4j.Slf4j; @@ -12,20 +14,20 @@ import org.apache.http.Header; import org.apache.http.message.BasicHeader; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpHeaders; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; import java.util.HashMap; import java.util.Map; import java.util.Objects; /** - * @description: for base method + * @description: * @author: zhq - * @create: 2020-07-13 + * @create: 2020-12-15 **/ @Slf4j -@Component -public class BaseQueryProvider { +@Service("graphEngineService") +public class GraphEngineServiceImpl implements GraphEngineService { @Autowired ArangoConfig arangoConfig; @@ -33,13 +35,18 @@ public class BaseQueryProvider { @Autowired HttpClientService httpClientService; + @Override + public BaseResult query(Map map) throws BusinessException { + Object jsonResult = queryArango(map); + return BaseResultUtil.success(jsonResult); + } /** * @return java.lang.String * @Author zhq * @Description 获取ArangoJwt * @Date 2020\7\7 0007 **/ - public String getJwt() { + private String getJwt() { Object jwt = JwtCache.get(Constant.ARANGO_CACHE_JWT); if (Objects.nonNull(jwt)) { return String.valueOf(jwt); @@ -64,18 +71,17 @@ public class BaseQueryProvider { * @Param [aql] * @return java.lang.Object **/ - public Object queryArango(String aql) { - Map queryMap = ImmutableMap.of("query", aql); + private Object queryArango(Map query) { Header header = new BasicHeader(HttpHeaders.AUTHORIZATION, getJwt()); String res = ""; try { - res = httpClientService.httpPost(arangoConfig.getQueryurl(), JsonMapper.toJsonString(queryMap), header); + res = httpClientService.httpPost(arangoConfig.getQueryurl(), JsonMapper.toJsonString(query), header); } catch (BusinessException e) { if (e.getErrorMessage() != null && e.getErrorMessage().contains("not authorized to execute this request")) { //刷新jwt JwtCache.remove(Constant.ARANGO_CACHE_JWT); - log.warn("arango jwt 失效后重置"); - res = httpClientService.httpPost(arangoConfig.getQueryurl(), JsonMapper.toJsonString(queryMap), new BasicHeader(HttpHeaders.AUTHORIZATION, getJwt())); + log.warn("arango jwt disable and reset"); + res = httpClientService.httpPost(arangoConfig.getQueryurl(), JsonMapper.toJsonString(query), new BasicHeader(HttpHeaders.AUTHORIZATION, getJwt())); } else { log.error(e.getErrorMessage()); throw e; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/QueryProvider.java b/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/QueryProvider.java deleted file mode 100644 index 902da66..0000000 --- a/galaxy-query-engine/src/main/java/com/mesalab/knowledge/strategy/QueryProvider.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.mesalab.knowledge.strategy; - -import com.mesalab.knowledge.entity.DSLObject; - -/** - * @Description: 提供查询结果 - */ -public interface QueryProvider { - - public Object query(DSLObject.DSLQuery dslQeruy); - -} diff --git a/galaxy-query-engine/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java b/galaxy-query-engine/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java index 9153bd2..8e2ab9a 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/qgw/dialect/ClickHouseDialect.java @@ -7,11 +7,10 @@ import com.mesalab.common.base.BaseResultGenerator; import com.mesalab.common.enums.*; import com.mesalab.common.exception.BusinessException; import com.mesalab.common.utils.*; -import com.mesalab.knowledge.common.utils.HttpConfig; +import com.mesalab.engine.common.config.HttpConfig; import com.mesalab.qgw.model.api.*; import com.mesalab.qgw.service.MetadataService; import com.mesalab.qgw.service.impl.HttpClientService; -import com.mesalab.common.utils.TableFinder; import com.mesalab.tool.utils.Encodes; import com.mesalab.tool.utils.StringUtil; import lombok.extern.slf4j.Slf4j; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java b/galaxy-query-engine/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java index 6cd4a84..68bc368 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/qgw/dialect/DruidDialect.java @@ -14,7 +14,7 @@ import com.mesalab.common.utils.JsonMapper; import com.mesalab.common.utils.SQLFunctionUtil; import com.mesalab.common.utils.SQLHelper; import com.mesalab.common.utils.SpringContextUtil; -import com.mesalab.knowledge.common.utils.HttpConfig; +import com.mesalab.engine.common.config.HttpConfig; import com.mesalab.qgw.model.api.ApiParam; import com.mesalab.qgw.model.api.DruidIoHttpSource; import com.mesalab.qgw.model.api.EngineConfigSource; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java b/galaxy-query-engine/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java index a04a299..9432ff0 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/qgw/monitor/ArangoHealthIndicator.java @@ -1,31 +1,20 @@ package com.mesalab.qgw.monitor; -import com.mesalab.common.configuration.HBaseProperties; -import com.mesalab.common.exception.BusinessException; -import com.mesalab.knowledge.common.config.ArangoConfig; -import com.mesalab.knowledge.common.utils.Constant; -import com.mesalab.knowledge.common.utils.JwtCache; -import com.mesalab.knowledge.strategy.QueryProvider; +import com.mesalab.engine.common.config.ArangoConfig; +import com.mesalab.engine.common.util.Constant; +import com.mesalab.engine.common.util.JwtCache; import com.mesalab.qgw.service.impl.HttpClientService; import com.mesalab.tool.utils.JsonMapper; import com.mesalab.tool.utils.StringUtil; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.http.HttpStatus; import org.apache.http.message.BasicHeader; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.health.AbstractHealthIndicator; import org.springframework.boot.actuate.health.Health; import org.springframework.http.HttpHeaders; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; -import org.springframework.web.client.RestTemplate; import java.util.HashMap; import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; @Component diff --git a/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java b/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java index 9bf422e..178e165 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/HttpClientService.java @@ -2,7 +2,7 @@ package com.mesalab.qgw.service.impl; import com.google.common.collect.Maps; import com.mesalab.common.exception.BusinessException; -import com.mesalab.knowledge.common.utils.HttpConfig; +import com.mesalab.engine.common.config.HttpConfig; import com.mesalab.tool.utils.StringUtil; import lombok.extern.slf4j.Slf4j; import org.apache.http.*; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java index 463916d..4c85787 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/SystemServiceImpl.java @@ -7,7 +7,7 @@ import com.mesalab.common.enums.JobHandlerEnum; import com.mesalab.common.enums.ResultStatusEnum; import com.mesalab.common.exception.BusinessException; import com.mesalab.common.utils.JsonMapper; -import com.mesalab.knowledge.common.utils.HttpConfig; +import com.mesalab.engine.common.config.HttpConfig; import com.mesalab.qgw.model.api.ApiParam; import com.mesalab.qgw.model.api.ClickHouseHttpSource; import com.mesalab.qgw.model.api.JobAdminHttpSource; diff --git a/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/TestSqlServiceImpl.java b/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/TestSqlServiceImpl.java index e0fc6bd..469f858 100644 --- a/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/TestSqlServiceImpl.java +++ b/galaxy-query-engine/src/main/java/com/mesalab/qgw/service/impl/TestSqlServiceImpl.java @@ -10,7 +10,7 @@ import com.mesalab.common.enums.ResultStatusEnum; import com.mesalab.common.exception.BusinessException; import com.mesalab.common.utils.JsonMapper; import com.mesalab.common.utils.SchemaCacheUtils; -import com.mesalab.knowledge.common.utils.HttpConfig; +import com.mesalab.engine.common.config.HttpConfig; import com.mesalab.qgw.model.api.ClickHouseHttpSource; import com.mesalab.tool.utils.Encodes; import com.mesalab.tool.utils.StringUtil; |
