diff options
| author | doufenghu <[email protected]> | 2023-12-23 11:52:20 +0800 |
|---|---|---|
| committer | doufenghu <[email protected]> | 2023-12-23 11:52:20 +0800 |
| commit | d26d478bc168e23be2754e0ecfec81e9855e78ba (patch) | |
| tree | b827ae8571b6edf5892eaef1aefd797261e99b59 /groot-common/src | |
| parent | 493e4f319217f3d7a9212900c1c68225ea442776 (diff) | |
[Improve][Config]删除无用配置,UDF函数UnixTimestamp按文档约束重命名为CurrentUnixTimestamp。
Diffstat (limited to 'groot-common/src')
28 files changed, 299 insertions, 1043 deletions
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfig.java index 775ee13..4fdf0c6 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfig.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfig.java @@ -1,139 +1,29 @@ package com.geedgenetworks.common.config; -import org.yaml.snakeyaml.Yaml; +import lombok.Data; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; +import java.io.Serializable; import java.util.List; import java.util.Map; -import static com.geedgenetworks.common.config.GrootYamlParser.getClassReflect; -import static com.geedgenetworks.common.config.GrootYamlParser.loadYaml; +import static com.google.common.base.Preconditions.checkNotNull; -/** Created by wk on 2021/1/6. */ -public class CommonConfig { - public static Map<String, Object> yamlData; - public static List<String> udfPluginData = new ArrayList<>(); +/** + * Describe the common config of groot stream (grootstream.yaml). + */ +@Data +public class CommonConfig implements Serializable { - static { - try { - Yaml yaml = new Yaml(); - yamlData = - yaml.load( - GrootYamlParser.class - .getClassLoader() - .getResourceAsStream("groot.yaml")); + private List<KnowledgeBaseConfig> knowledgeBaseConfig = CommonConfigOptions.KNOWLEDGE_BASE.defaultValue(); + private Map<String,String> propertiesConfig = CommonConfigOptions.PROPERTIES.defaultValue(); - InputStream inputStreamUdf = GrootYamlParser.class.getClassLoader().getResourceAsStream("groot-platform-plugin"); - if (inputStreamUdf != null) { - BufferedReader br = new BufferedReader(new InputStreamReader(inputStreamUdf)); - String line; - while ((line = br.readLine()) != null) { - // 解析每行并添加到列表 - udfPluginData.add(line); - } - br.close(); - } else { - System.out.println("udf-plugin配置文件不存在"); - - } - } catch (Exception e) { - e.printStackTrace(); - } + public void setKnowledgeBaseConfig(List<KnowledgeBaseConfig> knowledgeBaseConfig) { + checkNotNull(knowledgeBaseConfig, CommonConfigOptions.KNOWLEDGE_BASE + "knowledgeConfig should not be null"); + this.knowledgeBaseConfig = knowledgeBaseConfig; } - public static final Map<String, String> UDF_CLASS_REFLECT = getClassReflect(udfPluginData); - - public static final String NACOS_SERVER_ADDR = - GrootYamlParser.getStringProperty("nacos.server_addr", yamlData); - public static final String NACOS_USERNAME = - GrootYamlParser.getStringProperty("nacos.username", yamlData); - public static final String NACOS_PASSWORD = - GrootYamlParser.getStringProperty("nacos.password", yamlData); - public static final String NACOS_NAMESPACE = - GrootYamlParser.getStringProperty("nacos.namespace", yamlData); - public static final String NACOS_DATA_ID = - GrootYamlParser.getStringProperty("nacos.data_id", yamlData); - public static final String NACOS_GROUP = - GrootYamlParser.getStringProperty("nacos.group", yamlData); - public static final long NACOS_READ_TIMEOUT = - GrootYamlParser.getIntProperty("nacos.read_timeout", yamlData); - - public static final String CONSUL_SERVER_ADDR = - GrootYamlParser.getStringProperty("consul.server_addr", yamlData); - public static final int CONSUL_SERVER_PORT = - GrootYamlParser.getIntProperty("consul.server_port", yamlData); - public static final String CONSUL_TOKEN = - GrootYamlParser.getStringProperty("consul.token", yamlData); - - public static final String HOS_TOKEN = GrootYamlParser.getStringProperty("hos.token", yamlData); - public static final int KNOWLEDGE_FILE_CHECK_NUMBER = - GrootYamlParser.getIntProperty("knowledge.file_check_number", yamlData); - public static final String ID_ARRAY = - GrootYamlParser.getStringProperty("knowledge.id_array", yamlData); - public static final int HTTP_POOL_MAX_CONNECTION = - GrootYamlParser.getIntProperty("http.pool.max_connection", yamlData); - public static final int HTTP_POOL_MAX_PER_ROUTE = - GrootYamlParser.getIntProperty("http.pool.max_per_route", yamlData); - public static final int HTTP_POOL_REQUEST_TIMEOUT = - GrootYamlParser.getIntProperty("http.pool.request_timeout", yamlData); - public static final int HTTP_POOL_CONNECT_TIMEOUT = - GrootYamlParser.getIntProperty("http.pool.connect_timeout", yamlData); - public static final int HTTP_POOL_RESPONSE_TIMEOUT = - GrootYamlParser.getIntProperty("http.pool.response_timeout", yamlData); - public static final String GTPC_FAMILY_NAME = "gtp"; - public static final String RADIUS_FAMILY_NAME = "radius"; - public static final String COMMON_FAMILY_NAME = "common"; - public static final String DEFAULT_RELATIONSHIP_MODULE = "vsys"; - public static final String ZOOKEEPER_QUORUM = - GrootYamlParser.getStringProperty("zookeeper.quorum", yamlData); - public static final long DATA_CENTER_ID_NUM = - GrootYamlParser.getIntProperty("snowid.data_center_id_num", yamlData); - public static final String KNOWLEDGEBASE_FILE_STORAGE_PATH = - GrootYamlParser.getStringProperty("knowledgebase.file_storage_path", yamlData); - public static final String KNOWLEDGEBASE_FILE_STORAGE_TYPE = - GrootYamlParser.getStringProperty("knowledgebase.file_storage_type", yamlData); - public static final String KNOWLEDGEBASE_TYPE_LIST = - GrootYamlParser.getStringProperty("knowledgebase.type_list", yamlData); - public static final String KNOWLEDGEBASE_NAME_LIST = - GrootYamlParser.getStringProperty("knowledgebase.name_list", yamlData); - public static final String HDFS_SERVERS = - GrootYamlParser.getStringProperty("hdfs.servers", yamlData); - /* public static final String IPV4_USER_DEFINED_ID = GrootYamlParser.getStringProperty( "knowledge.ipv4_user_defined_id"); - public static final String IPV6_USER_DEFINED_ID = GrootYamlParser.getStringProperty( "knowledge.ipv6_user_defined_id"); - public static final String IPV4_BUILT_IN_ID = GrootYamlParser.getStringProperty( "knowledge.ipv4_built_in_id"); - public static final String IPV6_BUILT_IN_ID = GrootYamlParser.getStringProperty( "knowledge.ipv6_built_in_id"); - public static final String ASN_V4_ID = GrootYamlParser.getStringProperty( "knowledge.asn_v4_id"); - public static final String ASN_V6_ID = GrootYamlParser.getStringProperty( "knowledge.asn_v6_id");*/ - - public static final Map<String, Object> KNOWLEDGE_IPLOOKUP_MAP = - GrootYamlParser.getMap("knowledge.iplookup", yamlData); - public static final Map<String, Object> KNOWLEDGE_ASNLOOKUP_MAP = - GrootYamlParser.getMap("knowledge.asnlookup", yamlData); - - public static final Integer HBASE_TICK_TUPLE_FREQ_SECS = - GrootYamlParser.getIntProperty("hbase.tick_tuple_freq_secs", yamlData); - public static final Integer HBASE_GTPC_SCAN_MAX_ROWS = - GrootYamlParser.getIntProperty("hbase.gtpc_scan_max_rows", yamlData); - public static final Integer HBASE_RADIUS_SCAN_MAX_ROWS = - GrootYamlParser.getIntProperty("hbase.radius_scan_max_rows", yamlData); - public static final String HBASE_RADIUS_TABLE_NAME = - GrootYamlParser.getStringProperty("hbase.radius_table_name", yamlData); - public static final String HBASE_GTPC_TABLE_NAME = - GrootYamlParser.getStringProperty("hbase.gtpc_table_name", yamlData); - public static final String HBASE_RPC_TIMEOUT = - GrootYamlParser.getStringProperty("hbase.rpc_timeout", yamlData); - public static final String DATA_RELATIONSHIP_MODEL = - GrootYamlParser.getStringProperty("data.relationship_model", yamlData); - - public static void main(String[] args) { - System.out.println(""); - } } diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigDomProcessor.java b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigDomProcessor.java new file mode 100644 index 0000000..6dc9469 --- /dev/null +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigDomProcessor.java @@ -0,0 +1,101 @@ +package com.geedgenetworks.common.config; + +import com.hazelcast.internal.config.AbstractDomConfigProcessor; +import com.hazelcast.logging.ILogger; +import com.hazelcast.logging.Logger; +import lombok.extern.slf4j.Slf4j; +import org.w3c.dom.Node; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.hazelcast.internal.config.DomConfigHelper.*; + +@Slf4j +public class CommonConfigDomProcessor extends AbstractDomConfigProcessor { + private final GrootStreamConfig config; + CommonConfigDomProcessor(boolean domLevel3, GrootStreamConfig config) { + super(domLevel3); + this.config = config; + } + + @Override + public void buildConfig(Node rootNode) { + final CommonConfig commonConfig = config.getCommonConfig(); + for (Node node : childElements(rootNode)) { + String name = cleanNodeName(node); + if (CommonConfigOptions. KNOWLEDGE_BASE.key().equals(name)) { + commonConfig.setKnowledgeBaseConfig(parseKnowledgeBaseConfig(node)); + } else if (CommonConfigOptions.PROPERTIES.key().equals(name)) { + commonConfig.setPropertiesConfig(parsePropertiesConfig(node)); + } else { + log.warn("Unrecognized configuration element: " + name); + } + + } + } + + private Map<String, String> parsePropertiesConfig(Node properties) { + + Map<String, String> propertiesMap = new HashMap<>(); + for (Node node : childElements(properties)) { + String name = cleanNodeName(node); + propertiesMap.put(name,getTextContent(node)); + } + return propertiesMap; + } + + + private List<KnowledgeBaseConfig> parseKnowledgeBaseConfig(Node kbRootNode) { + + List<KnowledgeBaseConfig> knowledgeConfigList = new ArrayList<>(); + for (Node node : childElements(kbRootNode)) { + knowledgeConfigList.add(parseKnowledgeBaseConfigAsObject(node)); + } + return knowledgeConfigList; + } + + private KnowledgeBaseConfig parseKnowledgeBaseConfigAsObject(Node kbNode) { + KnowledgeBaseConfig knowledgeBaseConfig = new KnowledgeBaseConfig(); + for (Node node : childElements(kbNode)) { + String name = cleanNodeName(node); + + if (CommonConfigOptions.KNOWLEDGE_BASE_NAME.key().equals(name)) { + knowledgeBaseConfig.setName(getTextContent(node)); + } else if (CommonConfigOptions.KNOWLEDGE_BASE_TYPE.key().equals(name)) { + knowledgeBaseConfig.setType(getTextContent(node)); + } else if (CommonConfigOptions.KNOWLEDGE_BASE_FILES.key().equals(name)) { + knowledgeBaseConfig.setFiles(parseKnowledgeBaseFilesConfig(node)); + } else if (CommonConfigOptions.KNOWLEDGE_BASE_PROPERTIES.key().equals(name)) { + knowledgeBaseConfig.setProperties(parseKnowledgeBasePropertiesConfig(node)); + } + else{ + log.warn("Unrecognized configuration element: " + name); + } + } + return knowledgeBaseConfig; + } + + private Map<String, String> parseKnowledgeBasePropertiesConfig(Node properties) { + Map<String, String> propertiesMap = new HashMap<>(); + for (Node node : childElements(properties)) { + String name = cleanNodeName(node); + propertiesMap.put(name,getTextContent(node)); + } + return propertiesMap; + } + + private List<String> parseKnowledgeBaseFilesConfig(Node files) { + + List<String> asnLookupConfigList = new ArrayList<>(); + for (Node node : childElements(files)) { + String value = node.getNodeValue(); + asnLookupConfigList.add(value); + } + return asnLookupConfigList; + } + + +} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigLocator.java b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigLocator.java index 3962324..5302cc2 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigLocator.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigLocator.java @@ -5,9 +5,9 @@ import com.hazelcast.internal.config.AbstractConfigLocator; import static com.hazelcast.internal.config.DeclarativeConfigUtil.YAML_ACCEPTED_SUFFIXES; -public final class GrootStreamConfigLocator extends AbstractConfigLocator { +public final class CommonConfigLocator extends AbstractConfigLocator { - public GrootStreamConfigLocator() { + public CommonConfigLocator() { } @Override public boolean locateFromSystemProperty() { diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigOptions.java b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigOptions.java new file mode 100644 index 0000000..9105afe --- /dev/null +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigOptions.java @@ -0,0 +1,64 @@ +package com.geedgenetworks.common.config; + +import com.alibaba.fastjson2.TypeReference; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CommonConfigOptions { + + public static final Option<Map<String, String>> KNOWLEDGE_BASE_PROPERTIES = + Options.key("properties") + .mapType() + .defaultValue(new HashMap<String,String>()) + .withDescription("The properties of knowledgebase"); + public static final Option<String> KNOWLEDGE_BASE_NAME = + Options.key("name") + .stringType() + .defaultValue("") + .withDescription("The name of knowledgebase."); + public static final Option<String> KNOWLEDGE_BASE_TYPE = + Options.key("type") + .stringType() + .defaultValue("") + .withDescription("The type of knowledgebase."); + + public static final Option<List<String>> KNOWLEDGE_BASE_FILES = + Options.key("files") + .listType() + .defaultValue(new ArrayList<String>()) + .withDescription("The files of knowledgebase."); + + + public static final Option<String> KNOWLEDGE_BASE_STORAGE_FS_TYPE = Options.key("fs_type") + .stringType() + .defaultValue("localfile") + .withDescription("The fs type of knowledge base storage."); + + public static final Option<String> KNOWLEDGE_BASE_STORAGE_FS_DEFAULT_PATH = Options.key("fs_default_path") + .stringType() + .defaultValue("") + .withDescription("The default path of knowledge base storage."); + + public static final Option<List<KnowledgeBaseConfig>> KNOWLEDGE_BASE = + Options.key("knowledge_base") + .type(new TypeReference<List<KnowledgeBaseConfig>>() {}) + .noDefaultValue() + .withDescription("The knowledge base configuration."); + + public static final Option<Map<String, String>> PROPERTIES = + Options.key("properties") + .mapType() + .noDefaultValue() + .withDescription("The general properties of grootstream"); + + public static final Option<String> ZOOKEEPER_QUORUM = + Options.key("quorum") + .stringType() + .defaultValue("") + .withDescription("The quorum of zookeeper."); + + +} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigProvider.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigProvider.java index d7f7381..a967ae5 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigProvider.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigProvider.java @@ -26,7 +26,7 @@ public class ConfigProvider { } public static GrootStreamConfig locateAndGetGrootStreamConfig(Properties properties) { - GrootStreamConfigLocator yamlConfigLocator = new GrootStreamConfigLocator(); + CommonConfigLocator yamlConfigLocator = new CommonConfigLocator(); GrootStreamConfig config; validateSuffixInSystemProperty(Constants.SYSPROP_GROOTSTREAM_CONFIG); if (yamlConfigLocator.locateFromSystemProperty()) { diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ConsulConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ConsulConfig.java deleted file mode 100644 index d8d5b7b..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/ConsulConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.geedgenetworks.common.config; - -import lombok.Data; - -import java.io.Serializable; - -import static com.google.common.base.Preconditions.checkNotNull; - -@Data -public class ConsulConfig implements Serializable { - public String serverAddr = ServerConfigOptions.CONSUL_SERVER_ADDR.defaultValue(); - public int serverPort = ServerConfigOptions.CONSUL_SERVER_PORT.defaultValue(); - public String token = ServerConfigOptions.CONSUL_TOKEN.defaultValue(); - - public void setServerAddr(String serverAddr) { - checkNotNull(serverAddr, ServerConfigOptions.CONSUL_SERVER_ADDR + "serverAddr should not be null"); - this.serverAddr = serverAddr; - } - - public void setToken(String token) { - checkNotNull(token, ServerConfigOptions.CONSUL_TOKEN + "token should not be null"); - this.token = token; - } -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java deleted file mode 100644 index a286086..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.geedgenetworks.common.config; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Describe the common config of groot stream (groot.yaml). - */ -@Data -public class EngineConfig implements Serializable { - - - private HttpConPoolConfig httpConPoolConfig = ServerConfigOptions.HTTP_CON_POOL.defaultValue(); - private List<KnowledgeConfig> knowledgeBaseConfig = ServerConfigOptions.KNOWLEDGE_BASE.defaultValue(); - private NacosConfig nacosConfig = ServerConfigOptions.NACOS.defaultValue(); - private ConsulConfig consulConfig = ServerConfigOptions.CONSUL.defaultValue(); - private ZookeeperConfig zookeeperConfig = ServerConfigOptions.ZOOKEEPER.defaultValue(); - private HdfsConfig hdfsConfig = ServerConfigOptions.HDFS.defaultValue(); - private Map<String,String> propertiesConfig = ServerConfigOptions.PROPERTIES.defaultValue(); - - - public void setKnowledgeBaseConfig(List<KnowledgeConfig> knowledgeBaseConfig) { - checkNotNull(knowledgeBaseConfig, ServerConfigOptions.KNOWLEDGE_BASE + "knowledgeConfig should not be null"); - this.knowledgeBaseConfig = knowledgeBaseConfig; - } - - - - - -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfig.java index 00d8319..189b05b 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfig.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfig.java @@ -9,7 +9,7 @@ import java.util.List; @Slf4j public class GrootStreamConfig { - private final EngineConfig engineConfig = new EngineConfig(); + private final CommonConfig commonConfig = new CommonConfig(); private Config hazelcastConfig; @@ -34,7 +34,6 @@ public class GrootStreamConfig { } - /** * Returns the absolute path for `groot-stream.home` based from the system property {@link * GrootStreamProperties#GROOTSTREAM_HOME} @@ -53,8 +52,8 @@ public class GrootStreamConfig { public void setHazelcastConfig(Config hazelcastConfig) { this.hazelcastConfig = hazelcastConfig; } - public EngineConfig getEngineConfig() { - return engineConfig; + public CommonConfig getCommonConfig() { + return commonConfig; } diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigBuilder.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigBuilder.java index 83d7b87..25fc263 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigBuilder.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigBuilder.java @@ -19,13 +19,13 @@ public class GrootStreamConfigBuilder extends AbstractYamlConfigBuilder { private final InputStream in; public GrootStreamConfigBuilder() { - this((GrootStreamConfigLocator) null); + this((CommonConfigLocator) null); } - public GrootStreamConfigBuilder(GrootStreamConfigLocator locator) { + public GrootStreamConfigBuilder(CommonConfigLocator locator) { if (locator == null) { - locator = new GrootStreamConfigLocator(); + locator = new CommonConfigLocator(); locator.locateEverywhere(); } this.in = locator.getIn(); @@ -46,16 +46,16 @@ public class GrootStreamConfigBuilder extends AbstractYamlConfigBuilder { public GrootStreamConfig build(GrootStreamConfig config) { try { - parseAndBuildConfig(config); + parseAndBuildCommonConfig(config); + parseAndBuildUDFPluginConfig(config); + parseAndBuildHazelcastConfig(config); } catch (Exception e) { throw ExceptionUtil.rethrow(e); } - config.setHazelcastConfig(ConfigProvider.locateAndGetMemberConfig(getProperties())); - config.setUDFPluginConfig(ConfigProvider.locateAndGetUDFPluginConfig(getProperties())); return config; } - private void parseAndBuildConfig(GrootStreamConfig config) throws Exception { + private void parseAndBuildCommonConfig(GrootStreamConfig config) throws Exception { YamlMapping yamlRootNode; try { yamlRootNode = (YamlMapping) YamlLoader.load(in); @@ -66,6 +66,7 @@ public class GrootStreamConfigBuilder extends AbstractYamlConfigBuilder { } YamlNode grootStreamRoot = yamlRootNode.childAsMapping(GrootStreamConfigSections.GROOTSTREAM.name); + if (grootStreamRoot == null) { grootStreamRoot = yamlRootNode; } @@ -76,8 +77,19 @@ public class GrootStreamConfigBuilder extends AbstractYamlConfigBuilder { replaceVariables(w3cRootNode); importDocuments(grootStreamRoot); - new GrootStreamDomConfigProcessor(true, config).buildConfig(w3cRootNode); + new CommonConfigDomProcessor(true, config).buildConfig(w3cRootNode); + } + + private void parseAndBuildHazelcastConfig(GrootStreamConfig config){ + config.setHazelcastConfig(ConfigProvider.locateAndGetMemberConfig(getProperties())); } + private void parseAndBuildUDFPluginConfig(GrootStreamConfig config){ + config.setUDFPluginConfig(ConfigProvider.locateAndGetUDFPluginConfig(getProperties())); + } + + + + public GrootStreamConfigBuilder setProperties(Properties properties) { if (properties == null) { properties = System.getProperties(); diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigSections.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigSections.java index d0a35b8..d67c97e 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigSections.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigSections.java @@ -2,8 +2,7 @@ package com.geedgenetworks.common.config; /** Configuration sections for Hazelcast GrootStream shared by YAML based configurations */ enum GrootStreamConfigSections { - GROOTSTREAM("grootstream", false), - ENGINE("engine", false); + GROOTSTREAM("grootstream", false); final String name; final boolean multipleOccurrence; diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamDomConfigProcessor.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamDomConfigProcessor.java deleted file mode 100644 index 61c4855..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamDomConfigProcessor.java +++ /dev/null @@ -1,231 +0,0 @@ -package com.geedgenetworks.common.config; - -import com.hazelcast.internal.config.AbstractDomConfigProcessor; -import com.hazelcast.logging.ILogger; -import com.hazelcast.logging.Logger; -import org.w3c.dom.Node; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.hazelcast.internal.config.DomConfigHelper.*; - -public class GrootStreamDomConfigProcessor extends AbstractDomConfigProcessor { - private static final ILogger LOGGER = Logger.getLogger(GrootStreamDomConfigProcessor.class); - - private final GrootStreamConfig config; - - GrootStreamDomConfigProcessor(boolean domLevel3, GrootStreamConfig config) { - super(domLevel3); - this.config = config; - } - -/* @Override - public void buildConfig(Node rootNode) { - for (Node node : childElements(rootNode)) { - String nodeName = cleanNodeName(node); - if (occurrenceSet.contains(nodeName)) { - throw new InvalidConfigurationException( - "Duplicate '" + nodeName + "' definition found in the configuration."); - } - if (handleNode(node, nodeName)) { - continue; - } - if (!GrootStreamConfigSections.canOccurMultipleTimes(nodeName)) { - occurrenceSet.add(nodeName); - } - } - }*/ - -/* private boolean handleNode(Node node, String name) { - if (GrootStreamConfigSections.GROOTSTREAM.isEqual(name)) { - parseEngineConfig(node, config); - } else { - return true; - } - parseEngineConfig(node, config); - - return false; - }*/ - - public void buildConfig(Node engineNode) { - final EngineConfig engineConfig = config.getEngineConfig(); - for (Node node : childElements(engineNode)) { - String name = cleanNodeName(node); - if (ServerConfigOptions.HTTP_CON_POOL.key().equals(name)) { - engineConfig.setHttpConPoolConfig(parseHttpConPoolConfig(node)); - } else if (ServerConfigOptions. KNOWLEDGE_BASE.key().equals(name)) { - engineConfig.setKnowledgeBaseConfig(parseKnowledgeBaseConfig(node)); - } else if (ServerConfigOptions.NACOS.key().equals(name)) { - engineConfig.setNacosConfig(parseNacosConfig(node)); - } else if (ServerConfigOptions.CONSUL.key().equals(name)) { - engineConfig.setConsulConfig(parseConsulConfig(node)); - } else if (ServerConfigOptions.ZOOKEEPER.key().equals(name)) { - engineConfig.setZookeeperConfig(parseZookeeperConfig(node)); - } else if (ServerConfigOptions.HDFS.key().equals(name)) { - engineConfig.setHdfsConfig(parseHdfsConfig(node)); - } else if (ServerConfigOptions.PROPERTIES.key().equals(name)) { - engineConfig.setPropertiesConfig(parsePropertiesConfig(node)); - } else { - LOGGER.warning("Unrecognized configuration element: " + name); - - } - - } - } - - private Map<String, String> parsePropertiesConfig(Node properties) { - - Map<String, String> propertiesMap = new HashMap<>(); - for (Node node : childElements(properties)) { - String name = cleanNodeName(node); - propertiesMap.put(name,getTextContent(node)); - } - return propertiesMap; - } - - - private ZookeeperConfig parseZookeeperConfig(Node zookeeperNode) { - ZookeeperConfig zookeeperConfig = new ZookeeperConfig(); - for (Node node : childElements(zookeeperNode)) { - String name = cleanNodeName(node); - if (ServerConfigOptions.ZOOKEEPER_QUORUM.key().equals(name)) { - zookeeperConfig.setQuorum(getTextContent(node)); - } else { - LOGGER.warning("Unrecognized configuration element: " + name); - } - } - return zookeeperConfig; - } - - private HdfsConfig parseHdfsConfig(Node hdfsNode) { - HdfsConfig hdfsConfig = new HdfsConfig(); - for (Node node : childElements(hdfsNode)) { - String name = cleanNodeName(node); - if (ServerConfigOptions.HDFS_SERVERS.key().equals(name)) { - hdfsConfig.setServers(getTextContent(node)); - } else { - LOGGER.warning("Unrecognized configuration element: " + name); - } - } - return hdfsConfig; - } - - private ConsulConfig parseConsulConfig(Node consulNode) { - ConsulConfig consulConfig = new ConsulConfig(); - for (Node node : childElements(consulNode)) { - String name = cleanNodeName(node); - if (ServerConfigOptions.CONSUL_SERVER_ADDR.key().equals(name)) { - consulConfig.setServerAddr(getTextContent(node)); - } else if (ServerConfigOptions.CONSUL_SERVER_PORT.key().equals(name)) { - consulConfig.setServerPort(getIntegerValue(ServerConfigOptions.CONSUL_SERVER_PORT.key(), getTextContent(node))); - } else if (ServerConfigOptions.CONSUL_TOKEN.key().equals(name)) { - consulConfig.setToken(getTextContent(node)); - } else { - LOGGER.warning("Unrecognized configuration element: " + name); - } - } - return consulConfig; - } - - private NacosConfig parseNacosConfig(Node nacosNode) { - NacosConfig nacosConfig = new NacosConfig(); - for (Node node : childElements(nacosNode)) { - String name = cleanNodeName(node); - if (ServerConfigOptions.NACOS_SERVER_ADDR.key().equals(name)) { - nacosConfig.setServerAddr(getTextContent(node)); - } else if (ServerConfigOptions.NACOS_NAMESPACE.key().equals(name)) { - nacosConfig.setNamespace(getTextContent(node)); - } else if (ServerConfigOptions.NACOS_GROUP.key().equals(name)) { - nacosConfig.setGroup(getTextContent(node)); - } else if (ServerConfigOptions.NACOS_DATA_ID.key().equals(name)) { - nacosConfig.setDataId(getTextContent(node)); - } else if (ServerConfigOptions.NACOS_READ_TIMEOUT.key().equals(name)) { - nacosConfig.setReadTimeout(getIntegerValue(ServerConfigOptions.NACOS_READ_TIMEOUT.key(), getTextContent(node))); - } else if (ServerConfigOptions.NACOS_USERNAME.key().equals(name)) { - nacosConfig.setUserName(getTextContent(node)); - } else if (ServerConfigOptions.NACOS_PASSWORD.key().equals(name)) { - nacosConfig.setPassword(getTextContent(node)); - } else { - LOGGER.warning("Unrecognized configuration element: " + name); - } - } - return nacosConfig; - } - - - private HttpConPoolConfig parseHttpConPoolConfig(Node httpConPoolNode) { - HttpConPoolConfig httpConPoolConfig = new HttpConPoolConfig(); - for (Node node : childElements(httpConPoolNode)) { - String name = cleanNodeName(node); - if (ServerConfigOptions.HTTP_CON_POOL_MAX_TOTAL.key().equals(name)) { - httpConPoolConfig.setMaxTotal(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_MAX_TOTAL.key(), getTextContent(node))); - } else if (ServerConfigOptions.HTTP_CON_POOL_MAX_PER_ROUTE.key().equals(name)) { - httpConPoolConfig.setMaxPerRoute(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_MAX_PER_ROUTE.key(), getTextContent(node))); - } else if (ServerConfigOptions.HTTP_CON_POOL_CONNECTION_REQUEST_TIMEOUT.key().equals(name)) { - httpConPoolConfig.setConnectionRequestTimeout(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_CONNECTION_REQUEST_TIMEOUT.key(), getTextContent(node))); - } else if (ServerConfigOptions.HTTP_CON_POOL_CONNECT_TIMEOUT.key().equals(name)) { - httpConPoolConfig.setConnectTimeout(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_CONNECT_TIMEOUT.key(), getTextContent(node))); - } else if (ServerConfigOptions.HTTP_CON_POOL_SOCKET_TIMEOUT.key().equals(name)) { - httpConPoolConfig.setSocketTimeout(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_SOCKET_TIMEOUT.key(), getTextContent(node))); - } else { - LOGGER.warning("Unrecognized configuration element: " + name); - } - } - return httpConPoolConfig; - - } - - private List<KnowledgeConfig> parseKnowledgeBaseConfig(Node knowledgeBaseNode) { - - List<KnowledgeConfig> knowledgeConfigList = new ArrayList<>(); - for (Node node : childElements(knowledgeBaseNode)) { - knowledgeConfigList.add(parseKnowledgeConfig(node)); - } - return knowledgeConfigList; - } - - private KnowledgeConfig parseKnowledgeConfig(Node asnLookupNode) { - KnowledgeConfig knowledgeConfig = new KnowledgeConfig(); - for (Node node : childElements(asnLookupNode)) { - String name = cleanNodeName(node); - - if (ServerConfigOptions.KNOWLEDGE_BASE_NAME.key().equals(name)) { - knowledgeConfig.setName(getTextContent(node)); - } else if (ServerConfigOptions.KNOWLEDGE_BASE_TYPE.key().equals(name)) { - knowledgeConfig.setType(getTextContent(node)); - } else if (ServerConfigOptions.KNOWLEDGE_BASE_FILES.key().equals(name)) { - knowledgeConfig.setFiles(parseFilesConfig(node)); - } else if (ServerConfigOptions.KNOWLEDGE_BASE_PROPERTIES.key().equals(name)) { - knowledgeConfig.setProperties(parseKnowledgePropertiesConfig(node)); - } - else{ - LOGGER.warning("Unrecognized configuration element: " + name); - } - } - return knowledgeConfig; - } - - private Map<String, String> parseKnowledgePropertiesConfig(Node properties) { - Map<String, String> propertiesMap = new HashMap<>(); - for (Node node : childElements(properties)) { - String name = cleanNodeName(node); - propertiesMap.put(name,getTextContent(node)); - } - return propertiesMap; - } - - private List<String> parseFilesConfig(Node files) { - - List<String> asnLookupConfigList = new ArrayList<>(); - for (Node node : childElements(files)) { - String value = node.getNodeValue(); - asnLookupConfigList.add(value); - } - return asnLookupConfigList; - } - - -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootYamlParser.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootYamlParser.java deleted file mode 100644 index 2fbb680..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootYamlParser.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.geedgenetworks.common.config; - -import cn.hutool.core.io.IoUtil; -import cn.hutool.setting.yaml.YamlUtil; -import lombok.NonNull; - -import java.io.InputStream; -import java.lang.reflect.Method; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public final class GrootYamlParser { - - public static String getStringProperty(String key, Map<String, Object> yamlData) { - - return getYamlValue(yamlData, key, String.class); - } - - public static Integer getIntProperty(String key, Map<String, Object> yamlData) { - - return getYamlValue(yamlData, key, Integer.class); - } - - public static Long getLongProperty(String key, Map<String, Object> yamlData) { - - return getYamlValue(yamlData, key, Long.class); - } - - public static <T> T getYamlValue(Map<String, Object> data, String key, Class<T> valueType) { - - String[] keys = key.split("\\."); - - for (int i = 0; i < keys.length - 1; i++) { - Object value = data.get(keys[i]); - if (value == null || !(value instanceof Map)) { - return null; - } - data = (Map<String, Object>) value; - } - - return data.containsKey(keys[keys.length - 1]) - ? valueType.cast(data.get(keys[keys.length - 1])) - : null; - } - - public static Map<String, String> getClassReflect(List<String> plugins) { - - Map<String, String> classReflect = new HashMap<>(); - - for (String classPath : plugins) { - - Class cls = null; - try { - cls = Class.forName(classPath); - Method method = cls.getMethod("functionName"); - Object object = cls.newInstance(); - String result = (String) method.invoke(object); - classReflect.put(result, classPath); - System.out.println("Returned Value: " + result); - } catch (Exception e) { - e.printStackTrace(); - } - } - return classReflect; - } - - public static Map<String, Object> loadYaml(@NonNull String filePath) { - return loadYaml(Paths.get(filePath)); - } - public static Map<String, Object> loadYaml(@NonNull Path filePath) { - return loadYaml(IoUtil.toStream(filePath.toFile())); - } - public static Map<String, Object> loadYaml(InputStream input) { - return YamlUtil.load(input, Map.class); - } - - - private static Object getValue(String key, Map<String, Object> data) { - if (data.containsKey(key)) { - return data.get(key); - } else { - for (Object value : data.values()) { - if (value instanceof Map) { - Object result = getValue(key, (Map<String, Object>) value); - if (result != null) { - return result; - } - } - } - } - return null; - } - - public static List<Object> getList(String key, Map<String, Object> yamlData) { - Object value = getValue(key, yamlData); - if (value instanceof List) { - return (List<Object>) value; - } - return null; - } - - public static Map<String, Object> getMap(String key, Map<String, Object> yamlData) { - Object value = getValue(key, yamlData); - if (value instanceof Map) { - return (Map<String, Object>) value; - } - return null; - } - - public static void main(String[] args) { - System.out.println(); - } -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/HdfsConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/HdfsConfig.java deleted file mode 100644 index 1323f54..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/HdfsConfig.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.geedgenetworks.common.config; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class HdfsConfig implements Serializable { - private String servers = ServerConfigOptions.HDFS_SERVERS.defaultValue(); -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/HosConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/HosConfig.java deleted file mode 100644 index 903d6cd..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/HosConfig.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.geedgenetworks.common.config; - -import lombok.Data; - -import java.io.Serializable; - -import static com.google.common.base.Preconditions.checkNotNull; - -@Data -public class HosConfig implements Serializable { - public String token = ServerConfigOptions.HOS_TOKEN.defaultValue(); - - public void setToken(String token) { - checkNotNull(token, ServerConfigOptions.HOS_TOKEN + "token should not be null"); - this.token = token; - } - -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/HttpConPoolConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/HttpConPoolConfig.java deleted file mode 100644 index c0a0e74..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/HttpConPoolConfig.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.geedgenetworks.common.config; - -import lombok.Data; - -import java.io.Serializable; - -import static com.google.common.base.Preconditions.checkArgument; - -@Data -public class HttpConPoolConfig implements Serializable { - - public static final long MIN_TIMEOUT_MS = 10; - public int maxTotal = ServerConfigOptions.HTTP_CON_POOL_MAX_TOTAL.defaultValue(); - public int maxPerRoute = ServerConfigOptions.HTTP_CON_POOL_MAX_PER_ROUTE.defaultValue(); - public int connectionRequestTimeout = ServerConfigOptions.HTTP_CON_POOL_CONNECTION_REQUEST_TIMEOUT.defaultValue(); - public int connectTimeout = ServerConfigOptions.HTTP_CON_POOL_CONNECT_TIMEOUT.defaultValue(); - public int socketTimeout = ServerConfigOptions.HTTP_CON_POOL_SOCKET_TIMEOUT.defaultValue(); - - public void setSocketTimeout(int socketTimeout) { - checkArgument(socketTimeout > MIN_TIMEOUT_MS, ServerConfigOptions.HTTP_CON_POOL_SOCKET_TIMEOUT + "socketTimeout should be greater than " + MIN_TIMEOUT_MS); - this.socketTimeout = socketTimeout; - } -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeBaseConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeBaseConfig.java new file mode 100644 index 0000000..c02df61 --- /dev/null +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeBaseConfig.java @@ -0,0 +1,34 @@ +package com.geedgenetworks.common.config; + +import com.geedgenetworks.utils.StringUtil; +import lombok.Data; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; + +@Data +public class KnowledgeBaseConfig implements Serializable { + private String name = CommonConfigOptions.KNOWLEDGE_BASE_NAME.defaultValue(); + private String type = CommonConfigOptions.KNOWLEDGE_BASE_TYPE.defaultValue(); + private Map<String, String> properties = CommonConfigOptions.KNOWLEDGE_BASE_PROPERTIES.defaultValue(); + private List<String> files = CommonConfigOptions.KNOWLEDGE_BASE_FILES.defaultValue(); + + public void setType(String type) { + checkArgument(StringUtil.isNotBlank(type) && Arrays.asList("asnlookup", "geoiplookup").contains(type.toLowerCase()) + , CommonConfigOptions.KNOWLEDGE_BASE_TYPE + "Type should be asnlookup or geoiplookup"); + this.type = type; + } + + public void setFiles(List<String> files) { + checkArgument(files != null && files.size() > 0, + CommonConfigOptions.KNOWLEDGE_BASE_FILES + "files should not be null"); + this.files = files; + } + + + +} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeConfig.java deleted file mode 100644 index cc407e4..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeConfig.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.geedgenetworks.common.config; - -import lombok.Data; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -@Data -public class KnowledgeConfig implements Serializable { - private String name = ServerConfigOptions.KNOWLEDGE_BASE_NAME.defaultValue(); - private String type = ServerConfigOptions.KNOWLEDGE_BASE_TYPE.defaultValue(); - private Map<String, String> properties = ServerConfigOptions.KNOWLEDGE_BASE_PROPERTIES.defaultValue(); - private List<String> files = ServerConfigOptions.KNOWLEDGE_BASE_FILES.defaultValue(); -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/NacosConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/NacosConfig.java deleted file mode 100644 index 2c1f7e7..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/NacosConfig.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.geedgenetworks.common.config; - -import lombok.Data; - -import java.io.Serializable; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; - -@Data -public class NacosConfig implements Serializable { - private String serverAddr = ServerConfigOptions.NACOS_SERVER_ADDR.defaultValue(); - private String userName = ServerConfigOptions.NACOS_USERNAME.defaultValue(); - private String password = ServerConfigOptions.NACOS_PASSWORD.defaultValue(); - private String namespace = ServerConfigOptions.NACOS_NAMESPACE.defaultValue(); - private String dataId = ServerConfigOptions.NACOS_DATA_ID.defaultValue(); - private String group = ServerConfigOptions.NACOS_GROUP.defaultValue(); - private int readTimeout = ServerConfigOptions.NACOS_READ_TIMEOUT.defaultValue(); - - public void setReadTimeout(int readTimeout) { - checkArgument(readTimeout > 0, ServerConfigOptions.NACOS_READ_TIMEOUT + "readTimeout should be greater than 0"); - this.readTimeout = readTimeout; - } - - public void setServerAddr(String serverAddr) { - checkNotNull(serverAddr, ServerConfigOptions.NACOS_SERVER_ADDR + "serverAddr should not be null"); - this.serverAddr = serverAddr; - } - - public void setDataId(String dataId) { - checkNotNull(dataId, ServerConfigOptions.NACOS_DATA_ID + "dataId should not be null"); - this.dataId = dataId; - } -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java deleted file mode 100644 index 2c03b21..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java +++ /dev/null @@ -1,206 +0,0 @@ -package com.geedgenetworks.common.config; - -import com.alibaba.fastjson2.TypeReference; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class ServerConfigOptions { - - - public static final Option<Integer> HTTP_CON_POOL_MAX_TOTAL = - Options.key("max_total") - .intType() - .defaultValue(100) - .withDescription("The max connections of http con pool."); - - public static final Option<Integer> HTTP_CON_POOL_MAX_PER_ROUTE = - Options.key("max_per_route") - .intType() - .defaultValue(10) - .withDescription("The max per route of http con pool."); - - public static final Option<Integer> HTTP_CON_POOL_CONNECTION_REQUEST_TIMEOUT = - Options.key("connection_request_timeout") - .intType() - .defaultValue(10000) - .withDescription("The connection request timeout of http con pool."); - public static final Option<Integer> HTTP_CON_POOL_CONNECT_TIMEOUT = - Options.key("connect_timeout") - .intType() - .defaultValue(10000) - .withDescription("The connect timeout of http con pool."); - - public static final Option<Integer> HTTP_CON_POOL_SOCKET_TIMEOUT = - Options.key("socket_timeout") - .intType() - .defaultValue(60000) - .withDescription("The socket timeout of http con pool."); - - public static final Option<HttpConPoolConfig> HTTP_CON_POOL = Options.key("http_con_pool") - .type(new TypeReference<HttpConPoolConfig>() {}) - .defaultValue(new HttpConPoolConfig()) - .withDescription("The http con pool configuration."); - - public static final Option<Map<String, String>> KNOWLEDGE_BASE_PROPERTIES = - Options.key("properties") - .mapType() - .defaultValue(new HashMap<String,String>()) - .withDescription("The properties of knowledgebase"); - public static final Option<String> KNOWLEDGE_BASE_NAME = - Options.key("name") - .stringType() - .defaultValue("") - .withDescription("The name of knowledgebase."); - public static final Option<String> KNOWLEDGE_BASE_TYPE = - Options.key("type") - .stringType() - .defaultValue("") - .withDescription("The type of knowledgebase."); - - public static final Option<List<String>> KNOWLEDGE_BASE_FILES = - Options.key("files") - .listType() - .defaultValue(new ArrayList<>()) - .withDescription("The files of knowledgebase."); - - - public static final Option<String> KNOWLEDGE_BASE_STORAGE_FS_TYPE = Options.key("fs_type") - .stringType() - .defaultValue("localfile") - .withDescription("The fs type of knowledge base storage."); - - public static final Option<String> KNOWLEDGE_BASE_STORAGE_FS_DEFAULT_PATH = Options.key("fs_default_path") - .stringType() - .defaultValue("") - .withDescription("The default path of knowledge base storage."); - - - - public static final Option<List<KnowledgeConfig>> KNOWLEDGE_BASE = - Options.key("knowledge_base") - .type(new TypeReference<List<KnowledgeConfig>>() {}) - .defaultValue(new ArrayList<>()) - .withDescription("The knowledge base configuration."); - - public static final Option<String> SNOWFLAKE_DATA_CENTER_ID = - Options.key("data_center_id_num") - .stringType() - .defaultValue("") - .withDescription("The data center id of snowflake."); - - public static final Option<SnowflakeConfig> SNOWFLAKE = - Options.key("snowflake") - .type(new TypeReference<SnowflakeConfig>() {}) - .defaultValue(new SnowflakeConfig()) - .withDescription("The snowflake configuration."); - - - public static final Option<String> NACOS_SERVER_ADDR = - Options.key("server_addr") - .stringType() - .defaultValue("") - .withDescription("The server address of nacos."); - public static final Option<String> NACOS_USERNAME = - Options.key("username") - .stringType() - .defaultValue("nacos") - .withDescription("The username of nacos."); - public static final Option<String> NACOS_PASSWORD = - Options.key("password") - .stringType() - .defaultValue("nacos") - .withDescription("The password of nacos."); - public static final Option<String> NACOS_NAMESPACE = - Options.key("namespace") - .stringType() - .defaultValue("public") - .withDescription("The namespace of nacos."); - public static final Option<String> NACOS_DATA_ID = - Options.key("data_id") - .stringType() - .defaultValue("") - .withDescription("The data id of nacos."); - public static final Option<String> NACOS_GROUP = - Options.key("group") - .stringType() - .defaultValue("DEFAULT_GROUP") - .withDescription("The group of nacos."); - public static final Option<Integer> NACOS_READ_TIMEOUT = - Options.key("read_timeout") - .intType() - .defaultValue(5000) - .withDescription("The read timeout of nacos."); - - public static final Option<NacosConfig> NACOS = - Options.key("nacos") - .type(new TypeReference<NacosConfig>() {}) - .defaultValue(new NacosConfig()) - .withDescription("The nacos configuration."); - - public static final Option<String> CONSUL_SERVER_ADDR = Options.key("server_addr") - .stringType() - .defaultValue("") - .withDescription("The server address of consul."); - public static final Option<Integer> CONSUL_SERVER_PORT = - Options.key("server_port") - .intType() - .defaultValue(8500) - .withDescription("The server port of consul."); - public static final Option<String> CONSUL_TOKEN = - Options.key("token") - .stringType() - .defaultValue("") - .withDescription("The token of consul."); - - public static final Option<ConsulConfig> CONSUL = - Options.key("consul") - .type(new TypeReference<ConsulConfig>() {}) - .defaultValue(new ConsulConfig()) - .withDescription("The consul configuration."); - - public static final Option<String> HOS_TOKEN = - Options.key("token") - .stringType() - .defaultValue("") - .withDescription("The token of hos."); - - public static final Option<HosConfig> HOS = - Options.key("hos") - .type(new TypeReference<HosConfig>() {}) - .defaultValue(new HosConfig()) - .withDescription("The hos configuration."); - - - public static final Option<String> ZOOKEEPER_QUORUM = - Options.key("quorum") - .stringType() - .defaultValue("") - .withDescription("The quorum of zookeeper."); - - public static final Option<ZookeeperConfig> ZOOKEEPER = - Options.key("zookeeper") - .type(new TypeReference<ZookeeperConfig>() {}) - .defaultValue(new ZookeeperConfig()) - .withDescription("The zookeeper configuration."); - - public static final Option<String> HDFS_SERVERS = - Options.key("servers") - .stringType() - .defaultValue("") - .withDescription("The servers of hdfs."); - - public static final Option<HdfsConfig> HDFS = Options.key("hdfs") - .type(new TypeReference<HdfsConfig>() {}) - .defaultValue(new HdfsConfig()) - .withDescription("The hdfs configuration."); - - public static final Option<Map<String, String>> PROPERTIES = - Options.key("properties") - .mapType() - .defaultValue(new HashMap<String,String>()) - .withDescription("The properties of grootstream"); - -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/SnowflakeConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/SnowflakeConfig.java deleted file mode 100644 index 59e84b6..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/SnowflakeConfig.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.geedgenetworks.common.config; - -import lombok.Data; - -import java.io.Serializable; - -@Data -public class SnowflakeConfig implements Serializable { - private String dataCenterId = ServerConfigOptions.SNOWFLAKE_DATA_CENTER_ID.defaultValue(); -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/StorageConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/StorageConfig.java deleted file mode 100644 index 64ef74c..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/StorageConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.geedgenetworks.common.config; - -import com.geedgenetworks.utils.StringUtil; -import lombok.Data; - -import java.io.Serializable; -import java.util.Arrays; - -import static com.google.common.base.Preconditions.checkArgument; - -@Data -public class StorageConfig implements Serializable { - private String fsType = ServerConfigOptions.KNOWLEDGE_BASE_STORAGE_FS_TYPE.defaultValue(); - private String fsDefaultPath = ServerConfigOptions.KNOWLEDGE_BASE_STORAGE_FS_DEFAULT_PATH.defaultValue(); - - public void setFsType(String fsType) { - checkArgument(StringUtil.isNotBlank(fsType) && Arrays.asList("hdfs", "localfile").contains(fsType.toLowerCase()) - , ServerConfigOptions.KNOWLEDGE_BASE_STORAGE_FS_TYPE + "fsType should be hdfs or localfile"); - this.fsType = fsType; - } -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/TypesafeConfigUtils.java b/groot-common/src/main/java/com/geedgenetworks/common/config/TypesafeConfigUtils.java index a8a6fa5..59c6088 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/TypesafeConfigUtils.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/TypesafeConfigUtils.java @@ -20,7 +20,7 @@ public final class TypesafeConfigUtils { * @param source config source * @param prefix config prefix * @param keepPrefix true if keep prefix - * @deprecated use org.apache.seatunnel.api.configuration.Option interface instead + * @deprecated */ @Deprecated public static Config extractSubConfig(Config source, String prefix, boolean keepPrefix) { diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ZookeeperConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ZookeeperConfig.java deleted file mode 100644 index 77c8be7..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/ZookeeperConfig.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.geedgenetworks.common.config; - -import java.io.Serializable; - -import static com.google.common.base.Preconditions.checkNotNull; - -public class ZookeeperConfig implements Serializable { - private String quorum = ServerConfigOptions.ZOOKEEPER_QUORUM.defaultValue(); - - public void setQuorum(String quorum) { - checkNotNull(quorum, ServerConfigOptions.ZOOKEEPER_QUORUM + "quorum should not be null"); - this.quorum = quorum; - } - -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/utils/HttpClientUtils.java b/groot-common/src/main/java/com/geedgenetworks/common/utils/HttpClientUtils.java index c68555f..fa68843 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/utils/HttpClientUtils.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/utils/HttpClientUtils.java @@ -1,6 +1,7 @@ package com.geedgenetworks.common.utils; -import com.geedgenetworks.common.config.CommonConfig; +import com.geedgenetworks.common.config.Option; +import com.geedgenetworks.common.config.Options; import com.geedgenetworks.utils.StringUtil; import org.apache.commons.io.IOUtils; import org.apache.http.*; @@ -11,7 +12,6 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.client.utils.URIBuilder; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ConnectTimeoutException; @@ -41,7 +41,6 @@ import java.nio.charset.StandardCharsets; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.security.cert.X509Certificate; -import java.util.Map; public class HttpClientUtils { /** 全局连接池对象 */ @@ -50,7 +49,13 @@ public class HttpClientUtils { private final Logger logger = LoggerFactory.getLogger(HttpClientUtils.class); - public static final String ERROR_MESSAGE = "-1"; + private static final String ERROR_MESSAGE = "-1"; + + private static final int DEFAULT_MAX_TOTAL = 400; + private static final int DEFAULT_MAX_PER_ROUTE = 10; + private static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT = 10000; + private static final int DEFAULT_CONNECT_TIMEOUT = 10000; + private static final int DEFAULT_SOCKET_TIMEOUT = 60000; /* * 静态代码块配置连接池信息 @@ -58,9 +63,9 @@ public class HttpClientUtils { static { // 设置最大连接数 - CONN_MANAGER.setMaxTotal(CommonConfig.HTTP_POOL_MAX_CONNECTION); + CONN_MANAGER.setMaxTotal(DEFAULT_MAX_TOTAL); // 设置每个连接的路由数 - CONN_MANAGER.setDefaultMaxPerRoute(CommonConfig.HTTP_POOL_MAX_PER_ROUTE); + CONN_MANAGER.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE); } /** @@ -97,9 +102,9 @@ public class HttpClientUtils { PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); // 设置最大连接数 - connManager.setMaxTotal(CommonConfig.HTTP_POOL_MAX_CONNECTION); + connManager.setMaxTotal(DEFAULT_MAX_TOTAL); // 设置每个连接的路由数 - connManager.setDefaultMaxPerRoute(CommonConfig.HTTP_POOL_MAX_PER_ROUTE); + connManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE); return connManager; } catch (KeyManagementException | NoSuchAlgorithmException e) { throw new RuntimeException(e.getMessage()); @@ -116,11 +121,11 @@ public class HttpClientUtils { RequestConfig requestConfig = RequestConfig.custom() // 获取连接超时时间 - .setConnectionRequestTimeout(CommonConfig.HTTP_POOL_REQUEST_TIMEOUT) + .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT) // 请求超时时间 - .setConnectTimeout(CommonConfig.HTTP_POOL_CONNECT_TIMEOUT) + .setConnectTimeout(DEFAULT_CONNECT_TIMEOUT) // 响应超时时间 - .setSocketTimeout(CommonConfig.HTTP_POOL_RESPONSE_TIMEOUT) + .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT) .build(); /* @@ -198,7 +203,7 @@ public class HttpClientUtils { String msg = ERROR_MESSAGE; // 获取客户端连接对象 - CloseableHttpClient httpClient = getHttpClient(CommonConfig.HTTP_POOL_RESPONSE_TIMEOUT); + CloseableHttpClient httpClient = getHttpClient(DEFAULT_SOCKET_TIMEOUT); CloseableHttpResponse response = null; try { @@ -253,7 +258,7 @@ public class HttpClientUtils { public String httpPost(URI uri, String requestBody, Header... headers) { String msg = ERROR_MESSAGE; // 获取客户端连接对象 - CloseableHttpClient httpClient = getHttpClient(CommonConfig.HTTP_POOL_RESPONSE_TIMEOUT); + CloseableHttpClient httpClient = getHttpClient(DEFAULT_SOCKET_TIMEOUT); // 创建POST请求对象 CloseableHttpResponse response = null; diff --git a/groot-common/src/main/resources/groot.yaml b/groot-common/src/main/resources/groot.yaml deleted file mode 100644 index a5bc3f9..0000000 --- a/groot-common/src/main/resources/groot.yaml +++ /dev/null @@ -1,74 +0,0 @@ - -nacos: - server_addr: 192.168.44.12:8848 - username: nacos - password: nacos - namespace: test - data_id: knowledge_base.json - group: DEFAULT_GROUP - read_timeout: 5000 - -consul: - server_addr: 192.168.41.30 - server_port: 8500 - token: c989b503-1d74-f49c-5698-e90fe461e938 - -hos: - token: c21f969b5f03d33d43e04f8f136e7682 - - -knowledge: - file_check_number: 3 - # iplookup: -# TSG: -# ip_v4_user_defined_id: acf1db8589c5e277-2d8cb5cf98593ed8 - # ip_v6_user_defined_id: acf1db8589c5e277-a442cce2d726eddf - # ip_v4_built_in_id: acf1db8589c5e277-42014a5735551e3e - # ip_v6_built_in_id: acf1db8589c5e277-8a9a0362c1320877 -# CN: - # ip_v4_user_defined_id: acf1db8589c5e277-2d8cb5cf98593ed8 - # ip_v6_user_defined_id: acf1db8589c5e277-a442cce2d726eddf - # ip_v4_built_in_id: acf1db8589c5e277-10798bc00aed2135 - # ip_v6_built_in_id: acf1db8589c5e277-8a9a0362c1320877 - # asn_v4_id: 7ce2f9890950ba90-fcc25696bf11a8a0 - # asn_v6_id: 7ce2f9890950ba90-71f13b3736863ddb - asnlookup: - TSG: - #asn_v4_id: 7ce2f9890950ba90-fe9081f96d1d8ca8 - #asn_v6_id: 7ce2f9890950ba90-c901e886e2d89a30 - asn_v4_id: 7ce2f9890950ba90-fcc25696bf11a8a0 - asn_v6_id: 7ce2f9890950ba90-71f13b3736863ddb - # CN: - # ip_v4_user_defined_id: acf1db8589c5e277-2d8cb5cf98593ed8 - # ip_v6_user_defined_id: acf1db8589c5e277-a442cce2d726eddf - # ip_v4_built_in_id: acf1db8589c5e277-10798bc00aed2135 -# ip_v6_built_in_id: acf1db8589c5e277-8a9a0362c1320877 -# asn_v4_id: 7ce2f9890950ba90-fcc25696bf11a8a0 -# asn_v6_id: 7ce2f9890950ba90-71f13b3736863ddb -http: - pool: - max_connection: 400 - max_per_route: 60 - request_timeout: 60000 - connect_timeout: 60000 - response_timeout: 60000 - -snowid: - data_center_id_num: 1 -zookeeper: - quorum: 192.168.44.12:2181 - -knowledgebase: - file_storage_path: /knowledgebase/ETL-SESSION-RECORD-COMPLETED/ - file_storage_type: hdfs -hdfs: - servers: 192.168.44.11:9000,192.168.44.14:9000 -hbase: - tick_tuple_freq_secs: - gtpc_scan_max_rows: 10000 - radius_scan_max_rows: 10000 - radius_table_name: RADIUS-TABLE - gtpc_table_name: GTPC-TABLE - rpc_timeout: -data: - relationship_model:
\ No newline at end of file diff --git a/groot-common/src/main/resources/udf.plugins b/groot-common/src/main/resources/udf.plugins index d22c057..07867f6 100644 --- a/groot-common/src/main/resources/udf.plugins +++ b/groot-common/src/main/resources/udf.plugins @@ -3,9 +3,9 @@ com.geedgenetworks.core.udf.Drop com.geedgenetworks.core.udf.AsnLookup com.geedgenetworks.core.udf.Eval com.geedgenetworks.core.udf.JsonExtract -com.geedgenetworks.core.udf.UnixTimestamp +com.geedgenetworks.core.udf.CurrentUnixTimestamp com.geedgenetworks.core.udf.Domain com.geedgenetworks.core.udf.DecodeBase64 com.geedgenetworks.core.udf.GeoIpLookup com.geedgenetworks.core.udf.PathCombine -com.geedgenetworks.core.udf.UnixTimestampConverter +com.geedgenetworks.core.udf.UnixTimestampConverter
\ No newline at end of file diff --git a/groot-common/src/test/java/com/geedgenetworks/common/config/YamlGrootStreamConfigParserTest.java b/groot-common/src/test/java/com/geedgenetworks/common/config/YamlGrootStreamConfigParserTest.java index 6e42ae4..9a122ed 100644 --- a/groot-common/src/test/java/com/geedgenetworks/common/config/YamlGrootStreamConfigParserTest.java +++ b/groot-common/src/test/java/com/geedgenetworks/common/config/YamlGrootStreamConfigParserTest.java @@ -3,23 +3,25 @@ package com.geedgenetworks.common.config; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Arrays; + public class YamlGrootStreamConfigParserTest { @Test public void testGrootStreamConfig() { - GrootStreamConfigLocator yamlConfigLocator = new GrootStreamConfigLocator(); + CommonConfigLocator commonConfigLocator = new CommonConfigLocator(); GrootStreamConfig config; - if (yamlConfigLocator.locateInWorkDirOrOnClasspath()) { + if (commonConfigLocator.locateInWorkDirOrOnClasspath()) { // 2. Try loading YAML config from the working directory or from the classpath - config = new GrootStreamConfigBuilder(yamlConfigLocator).setProperties(null).build(); + config = new GrootStreamConfigBuilder(commonConfigLocator).build(); } else { throw new RuntimeException("can't find yaml in resources"); } Assertions.assertNotNull(config); - Assertions.assertNotNull(config.getEngineConfig().getNacosConfig().getServerAddr()); - Assertions.assertTrue(config.getEngineConfig().getHttpConPoolConfig().getSocketTimeout() > 0); - Assertions.assertNotNull(config.getEngineConfig().getKnowledgeBaseConfig()); - // Assertions.assertEquals("hos", config.getEngineConfig().getKnowledgeBaseConfig().get(0).getProperties().get("fs_type")); + Assertions.assertNotNull(config.getCommonConfig().getKnowledgeBaseConfig()); + Assertions.assertTrue(config.getCommonConfig().getKnowledgeBaseConfig().size() > 0); + Assertions.assertTrue(Arrays.asList("asnlookup", "geoiplookup").contains(config.getCommonConfig().getKnowledgeBaseConfig().get(0).getType())); + Assertions.assertTrue(config.getUDFPluginConfig().size() > 0); } diff --git a/groot-common/src/test/resources/grootstream.yaml b/groot-common/src/test/resources/grootstream.yaml index c233936..4aa7ef9 100644 --- a/groot-common/src/test/resources/grootstream.yaml +++ b/groot-common/src/test/resources/grootstream.yaml @@ -1,51 +1,34 @@ grootstream: - engine: - tick_tuple_freq_secs: 90 - gtpc_scan_max_rows: 10000 - radius_scan_max_rows: 10000 - radius_table_name: RADIUS-TABLE - gtpc_table_name: GTPC-TABLE - hbase_rpc_timeout: 60000 - http_con_pool: - max_total: 400 - max_per_route: 60 - connection_request_timeout: 60000 - connect_timeout: 60000 - socket_timeout: 60000 - knowledge_base: - - name: tsg_asnlookup - type: asnlookup - properties: - fs_type: hos - fs_default_path: http://path - files: - default_mmdb_file: 7ce2f9890950ba90-fcc25696bf11a8a0 - user_defined_mmdb_file: 7ce2f9890950ba90-71f13b3736863ddb - - name: CN_asnlookup - type: asnlookup - files: - default_mmdb_file: 7ce2f9890950ba90-fcc25696bf11a8a0 - user_defined_mmdb_file: 7ce2f9890950ba90-71f13b3736863ddb - snowflake: - data_center_id_num: 1 - nacos: - server_addr: 192.168.44.12:8848 - username: nacos - password: nacos - namespace: test - data_id: knowledge_base.json - group: DEFAULT_GROUP - read_timeout: 5000 - consul: - server_addr: 192.168.41.30 - server_port: 8500 - token: c989b503-1d74-f49c-5698-e90fe461e938 - hos: - token: c21f969b5f03d33d43e04f8f136e7682 - zookeeper: - quorum: 192.168.44.12:2181 - hdfs: - servers: 192.168.44.11:9000,192.168.44.14:9000 + knowledge_base: + - name: tsg_asnlookup + type: asnlookup + properties: + fs_type: hos + fs_default_path: http://path + files: + - http://192.168.44.12:9098/hos/knowledge_base_bucket/757732ce-8214-4c34-aea2-aca6c51a7e82-YXNuX2J1aWx0aW4=.mmdb + + - name: tsg_geoiplookup + type: geoiplookup + files: + - 7ce2f9890950ba90-fcc25696bf11a8a0 + - 7ce2f9890950ba90-71f13b3736863ddb + properties: + hos.path: http://192.168.44.12:8089 + hos.bucket.name.traffic_file: traffic_file_bucket + hos.bucket.name.troubleshooting_file: troubleshooting_file_bucket + + + + + + + + + + + + |
