summaryrefslogtreecommitdiff
path: root/groot-common/src
diff options
context:
space:
mode:
authordoufenghu <[email protected]>2023-12-23 11:52:20 +0800
committerdoufenghu <[email protected]>2023-12-23 11:52:20 +0800
commitd26d478bc168e23be2754e0ecfec81e9855e78ba (patch)
treeb827ae8571b6edf5892eaef1aefd797261e99b59 /groot-common/src
parent493e4f319217f3d7a9212900c1c68225ea442776 (diff)
[Improve][Config]删除无用配置,UDF函数UnixTimestamp按文档约束重命名为CurrentUnixTimestamp。
Diffstat (limited to 'groot-common/src')
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfig.java136
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigDomProcessor.java101
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigLocator.java (renamed from groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigLocator.java)4
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigOptions.java64
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/ConfigProvider.java2
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/ConsulConfig.java24
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java36
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfig.java7
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigBuilder.java28
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigSections.java3
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamDomConfigProcessor.java231
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/GrootYamlParser.java116
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/HdfsConfig.java10
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/HosConfig.java18
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/HttpConPoolConfig.java23
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeBaseConfig.java34
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeConfig.java15
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/NacosConfig.java34
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java206
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/SnowflakeConfig.java10
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/StorageConfig.java21
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/TypesafeConfigUtils.java2
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/config/ZookeeperConfig.java15
-rw-r--r--groot-common/src/main/java/com/geedgenetworks/common/utils/HttpClientUtils.java31
-rw-r--r--groot-common/src/main/resources/groot.yaml74
-rw-r--r--groot-common/src/main/resources/udf.plugins4
-rw-r--r--groot-common/src/test/java/com/geedgenetworks/common/config/YamlGrootStreamConfigParserTest.java16
-rw-r--r--groot-common/src/test/resources/grootstream.yaml77
28 files changed, 299 insertions, 1043 deletions
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfig.java
index 775ee13..4fdf0c6 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfig.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfig.java
@@ -1,139 +1,29 @@
package com.geedgenetworks.common.config;
-import org.yaml.snakeyaml.Yaml;
+import lombok.Data;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
+import java.io.Serializable;
import java.util.List;
import java.util.Map;
-import static com.geedgenetworks.common.config.GrootYamlParser.getClassReflect;
-import static com.geedgenetworks.common.config.GrootYamlParser.loadYaml;
+import static com.google.common.base.Preconditions.checkNotNull;
-/** Created by wk on 2021/1/6. */
-public class CommonConfig {
- public static Map<String, Object> yamlData;
- public static List<String> udfPluginData = new ArrayList<>();
+/**
+ * Describe the common config of groot stream (grootstream.yaml).
+ */
+@Data
+public class CommonConfig implements Serializable {
- static {
- try {
- Yaml yaml = new Yaml();
- yamlData =
- yaml.load(
- GrootYamlParser.class
- .getClassLoader()
- .getResourceAsStream("groot.yaml"));
+ private List<KnowledgeBaseConfig> knowledgeBaseConfig = CommonConfigOptions.KNOWLEDGE_BASE.defaultValue();
+ private Map<String,String> propertiesConfig = CommonConfigOptions.PROPERTIES.defaultValue();
- InputStream inputStreamUdf = GrootYamlParser.class.getClassLoader().getResourceAsStream("groot-platform-plugin");
- if (inputStreamUdf != null) {
- BufferedReader br = new BufferedReader(new InputStreamReader(inputStreamUdf));
- String line;
- while ((line = br.readLine()) != null) {
- // 解析每行并添加到列表
- udfPluginData.add(line);
- }
- br.close();
- } else {
- System.out.println("udf-plugin配置文件不存在");
-
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
+ public void setKnowledgeBaseConfig(List<KnowledgeBaseConfig> knowledgeBaseConfig) {
+ checkNotNull(knowledgeBaseConfig, CommonConfigOptions.KNOWLEDGE_BASE + "knowledgeConfig should not be null");
+ this.knowledgeBaseConfig = knowledgeBaseConfig;
}
- public static final Map<String, String> UDF_CLASS_REFLECT = getClassReflect(udfPluginData);
-
- public static final String NACOS_SERVER_ADDR =
- GrootYamlParser.getStringProperty("nacos.server_addr", yamlData);
- public static final String NACOS_USERNAME =
- GrootYamlParser.getStringProperty("nacos.username", yamlData);
- public static final String NACOS_PASSWORD =
- GrootYamlParser.getStringProperty("nacos.password", yamlData);
- public static final String NACOS_NAMESPACE =
- GrootYamlParser.getStringProperty("nacos.namespace", yamlData);
- public static final String NACOS_DATA_ID =
- GrootYamlParser.getStringProperty("nacos.data_id", yamlData);
- public static final String NACOS_GROUP =
- GrootYamlParser.getStringProperty("nacos.group", yamlData);
- public static final long NACOS_READ_TIMEOUT =
- GrootYamlParser.getIntProperty("nacos.read_timeout", yamlData);
-
- public static final String CONSUL_SERVER_ADDR =
- GrootYamlParser.getStringProperty("consul.server_addr", yamlData);
- public static final int CONSUL_SERVER_PORT =
- GrootYamlParser.getIntProperty("consul.server_port", yamlData);
- public static final String CONSUL_TOKEN =
- GrootYamlParser.getStringProperty("consul.token", yamlData);
-
- public static final String HOS_TOKEN = GrootYamlParser.getStringProperty("hos.token", yamlData);
- public static final int KNOWLEDGE_FILE_CHECK_NUMBER =
- GrootYamlParser.getIntProperty("knowledge.file_check_number", yamlData);
- public static final String ID_ARRAY =
- GrootYamlParser.getStringProperty("knowledge.id_array", yamlData);
- public static final int HTTP_POOL_MAX_CONNECTION =
- GrootYamlParser.getIntProperty("http.pool.max_connection", yamlData);
- public static final int HTTP_POOL_MAX_PER_ROUTE =
- GrootYamlParser.getIntProperty("http.pool.max_per_route", yamlData);
- public static final int HTTP_POOL_REQUEST_TIMEOUT =
- GrootYamlParser.getIntProperty("http.pool.request_timeout", yamlData);
- public static final int HTTP_POOL_CONNECT_TIMEOUT =
- GrootYamlParser.getIntProperty("http.pool.connect_timeout", yamlData);
- public static final int HTTP_POOL_RESPONSE_TIMEOUT =
- GrootYamlParser.getIntProperty("http.pool.response_timeout", yamlData);
- public static final String GTPC_FAMILY_NAME = "gtp";
- public static final String RADIUS_FAMILY_NAME = "radius";
- public static final String COMMON_FAMILY_NAME = "common";
- public static final String DEFAULT_RELATIONSHIP_MODULE = "vsys";
- public static final String ZOOKEEPER_QUORUM =
- GrootYamlParser.getStringProperty("zookeeper.quorum", yamlData);
- public static final long DATA_CENTER_ID_NUM =
- GrootYamlParser.getIntProperty("snowid.data_center_id_num", yamlData);
- public static final String KNOWLEDGEBASE_FILE_STORAGE_PATH =
- GrootYamlParser.getStringProperty("knowledgebase.file_storage_path", yamlData);
- public static final String KNOWLEDGEBASE_FILE_STORAGE_TYPE =
- GrootYamlParser.getStringProperty("knowledgebase.file_storage_type", yamlData);
- public static final String KNOWLEDGEBASE_TYPE_LIST =
- GrootYamlParser.getStringProperty("knowledgebase.type_list", yamlData);
- public static final String KNOWLEDGEBASE_NAME_LIST =
- GrootYamlParser.getStringProperty("knowledgebase.name_list", yamlData);
- public static final String HDFS_SERVERS =
- GrootYamlParser.getStringProperty("hdfs.servers", yamlData);
- /* public static final String IPV4_USER_DEFINED_ID = GrootYamlParser.getStringProperty( "knowledge.ipv4_user_defined_id");
- public static final String IPV6_USER_DEFINED_ID = GrootYamlParser.getStringProperty( "knowledge.ipv6_user_defined_id");
- public static final String IPV4_BUILT_IN_ID = GrootYamlParser.getStringProperty( "knowledge.ipv4_built_in_id");
- public static final String IPV6_BUILT_IN_ID = GrootYamlParser.getStringProperty( "knowledge.ipv6_built_in_id");
- public static final String ASN_V4_ID = GrootYamlParser.getStringProperty( "knowledge.asn_v4_id");
- public static final String ASN_V6_ID = GrootYamlParser.getStringProperty( "knowledge.asn_v6_id");*/
-
- public static final Map<String, Object> KNOWLEDGE_IPLOOKUP_MAP =
- GrootYamlParser.getMap("knowledge.iplookup", yamlData);
- public static final Map<String, Object> KNOWLEDGE_ASNLOOKUP_MAP =
- GrootYamlParser.getMap("knowledge.asnlookup", yamlData);
-
- public static final Integer HBASE_TICK_TUPLE_FREQ_SECS =
- GrootYamlParser.getIntProperty("hbase.tick_tuple_freq_secs", yamlData);
- public static final Integer HBASE_GTPC_SCAN_MAX_ROWS =
- GrootYamlParser.getIntProperty("hbase.gtpc_scan_max_rows", yamlData);
- public static final Integer HBASE_RADIUS_SCAN_MAX_ROWS =
- GrootYamlParser.getIntProperty("hbase.radius_scan_max_rows", yamlData);
- public static final String HBASE_RADIUS_TABLE_NAME =
- GrootYamlParser.getStringProperty("hbase.radius_table_name", yamlData);
- public static final String HBASE_GTPC_TABLE_NAME =
- GrootYamlParser.getStringProperty("hbase.gtpc_table_name", yamlData);
- public static final String HBASE_RPC_TIMEOUT =
- GrootYamlParser.getStringProperty("hbase.rpc_timeout", yamlData);
- public static final String DATA_RELATIONSHIP_MODEL =
- GrootYamlParser.getStringProperty("data.relationship_model", yamlData);
-
- public static void main(String[] args) {
- System.out.println("");
- }
}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigDomProcessor.java b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigDomProcessor.java
new file mode 100644
index 0000000..6dc9469
--- /dev/null
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigDomProcessor.java
@@ -0,0 +1,101 @@
+package com.geedgenetworks.common.config;
+
+import com.hazelcast.internal.config.AbstractDomConfigProcessor;
+import com.hazelcast.logging.ILogger;
+import com.hazelcast.logging.Logger;
+import lombok.extern.slf4j.Slf4j;
+import org.w3c.dom.Node;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static com.hazelcast.internal.config.DomConfigHelper.*;
+
+@Slf4j
+public class CommonConfigDomProcessor extends AbstractDomConfigProcessor {
+ private final GrootStreamConfig config;
+ CommonConfigDomProcessor(boolean domLevel3, GrootStreamConfig config) {
+ super(domLevel3);
+ this.config = config;
+ }
+
+ @Override
+ public void buildConfig(Node rootNode) {
+ final CommonConfig commonConfig = config.getCommonConfig();
+ for (Node node : childElements(rootNode)) {
+ String name = cleanNodeName(node);
+ if (CommonConfigOptions. KNOWLEDGE_BASE.key().equals(name)) {
+ commonConfig.setKnowledgeBaseConfig(parseKnowledgeBaseConfig(node));
+ } else if (CommonConfigOptions.PROPERTIES.key().equals(name)) {
+ commonConfig.setPropertiesConfig(parsePropertiesConfig(node));
+ } else {
+ log.warn("Unrecognized configuration element: " + name);
+ }
+
+ }
+ }
+
+ private Map<String, String> parsePropertiesConfig(Node properties) {
+
+ Map<String, String> propertiesMap = new HashMap<>();
+ for (Node node : childElements(properties)) {
+ String name = cleanNodeName(node);
+ propertiesMap.put(name,getTextContent(node));
+ }
+ return propertiesMap;
+ }
+
+
+ private List<KnowledgeBaseConfig> parseKnowledgeBaseConfig(Node kbRootNode) {
+
+ List<KnowledgeBaseConfig> knowledgeConfigList = new ArrayList<>();
+ for (Node node : childElements(kbRootNode)) {
+ knowledgeConfigList.add(parseKnowledgeBaseConfigAsObject(node));
+ }
+ return knowledgeConfigList;
+ }
+
+ private KnowledgeBaseConfig parseKnowledgeBaseConfigAsObject(Node kbNode) {
+ KnowledgeBaseConfig knowledgeBaseConfig = new KnowledgeBaseConfig();
+ for (Node node : childElements(kbNode)) {
+ String name = cleanNodeName(node);
+
+ if (CommonConfigOptions.KNOWLEDGE_BASE_NAME.key().equals(name)) {
+ knowledgeBaseConfig.setName(getTextContent(node));
+ } else if (CommonConfigOptions.KNOWLEDGE_BASE_TYPE.key().equals(name)) {
+ knowledgeBaseConfig.setType(getTextContent(node));
+ } else if (CommonConfigOptions.KNOWLEDGE_BASE_FILES.key().equals(name)) {
+ knowledgeBaseConfig.setFiles(parseKnowledgeBaseFilesConfig(node));
+ } else if (CommonConfigOptions.KNOWLEDGE_BASE_PROPERTIES.key().equals(name)) {
+ knowledgeBaseConfig.setProperties(parseKnowledgeBasePropertiesConfig(node));
+ }
+ else{
+ log.warn("Unrecognized configuration element: " + name);
+ }
+ }
+ return knowledgeBaseConfig;
+ }
+
+ private Map<String, String> parseKnowledgeBasePropertiesConfig(Node properties) {
+ Map<String, String> propertiesMap = new HashMap<>();
+ for (Node node : childElements(properties)) {
+ String name = cleanNodeName(node);
+ propertiesMap.put(name,getTextContent(node));
+ }
+ return propertiesMap;
+ }
+
+ private List<String> parseKnowledgeBaseFilesConfig(Node files) {
+
+ List<String> asnLookupConfigList = new ArrayList<>();
+ for (Node node : childElements(files)) {
+ String value = node.getNodeValue();
+ asnLookupConfigList.add(value);
+ }
+ return asnLookupConfigList;
+ }
+
+
+}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigLocator.java b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigLocator.java
index 3962324..5302cc2 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigLocator.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigLocator.java
@@ -5,9 +5,9 @@ import com.hazelcast.internal.config.AbstractConfigLocator;
import static com.hazelcast.internal.config.DeclarativeConfigUtil.YAML_ACCEPTED_SUFFIXES;
-public final class GrootStreamConfigLocator extends AbstractConfigLocator {
+public final class CommonConfigLocator extends AbstractConfigLocator {
- public GrootStreamConfigLocator() {
+ public CommonConfigLocator() {
}
@Override
public boolean locateFromSystemProperty() {
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigOptions.java b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigOptions.java
new file mode 100644
index 0000000..9105afe
--- /dev/null
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/CommonConfigOptions.java
@@ -0,0 +1,64 @@
+package com.geedgenetworks.common.config;
+
+import com.alibaba.fastjson2.TypeReference;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class CommonConfigOptions {
+
+ public static final Option<Map<String, String>> KNOWLEDGE_BASE_PROPERTIES =
+ Options.key("properties")
+ .mapType()
+ .defaultValue(new HashMap<String,String>())
+ .withDescription("The properties of knowledgebase");
+ public static final Option<String> KNOWLEDGE_BASE_NAME =
+ Options.key("name")
+ .stringType()
+ .defaultValue("")
+ .withDescription("The name of knowledgebase.");
+ public static final Option<String> KNOWLEDGE_BASE_TYPE =
+ Options.key("type")
+ .stringType()
+ .defaultValue("")
+ .withDescription("The type of knowledgebase.");
+
+ public static final Option<List<String>> KNOWLEDGE_BASE_FILES =
+ Options.key("files")
+ .listType()
+ .defaultValue(new ArrayList<String>())
+ .withDescription("The files of knowledgebase.");
+
+
+ public static final Option<String> KNOWLEDGE_BASE_STORAGE_FS_TYPE = Options.key("fs_type")
+ .stringType()
+ .defaultValue("localfile")
+ .withDescription("The fs type of knowledge base storage.");
+
+ public static final Option<String> KNOWLEDGE_BASE_STORAGE_FS_DEFAULT_PATH = Options.key("fs_default_path")
+ .stringType()
+ .defaultValue("")
+ .withDescription("The default path of knowledge base storage.");
+
+ public static final Option<List<KnowledgeBaseConfig>> KNOWLEDGE_BASE =
+ Options.key("knowledge_base")
+ .type(new TypeReference<List<KnowledgeBaseConfig>>() {})
+ .noDefaultValue()
+ .withDescription("The knowledge base configuration.");
+
+ public static final Option<Map<String, String>> PROPERTIES =
+ Options.key("properties")
+ .mapType()
+ .noDefaultValue()
+ .withDescription("The general properties of grootstream");
+
+ public static final Option<String> ZOOKEEPER_QUORUM =
+ Options.key("quorum")
+ .stringType()
+ .defaultValue("")
+ .withDescription("The quorum of zookeeper.");
+
+
+}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigProvider.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigProvider.java
index d7f7381..a967ae5 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigProvider.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigProvider.java
@@ -26,7 +26,7 @@ public class ConfigProvider {
}
public static GrootStreamConfig locateAndGetGrootStreamConfig(Properties properties) {
- GrootStreamConfigLocator yamlConfigLocator = new GrootStreamConfigLocator();
+ CommonConfigLocator yamlConfigLocator = new CommonConfigLocator();
GrootStreamConfig config;
validateSuffixInSystemProperty(Constants.SYSPROP_GROOTSTREAM_CONFIG);
if (yamlConfigLocator.locateFromSystemProperty()) {
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ConsulConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ConsulConfig.java
deleted file mode 100644
index d8d5b7b..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/ConsulConfig.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import lombok.Data;
-
-import java.io.Serializable;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-@Data
-public class ConsulConfig implements Serializable {
- public String serverAddr = ServerConfigOptions.CONSUL_SERVER_ADDR.defaultValue();
- public int serverPort = ServerConfigOptions.CONSUL_SERVER_PORT.defaultValue();
- public String token = ServerConfigOptions.CONSUL_TOKEN.defaultValue();
-
- public void setServerAddr(String serverAddr) {
- checkNotNull(serverAddr, ServerConfigOptions.CONSUL_SERVER_ADDR + "serverAddr should not be null");
- this.serverAddr = serverAddr;
- }
-
- public void setToken(String token) {
- checkNotNull(token, ServerConfigOptions.CONSUL_TOKEN + "token should not be null");
- this.token = token;
- }
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java
deleted file mode 100644
index a286086..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import lombok.Data;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-/**
- * Describe the common config of groot stream (groot.yaml).
- */
-@Data
-public class EngineConfig implements Serializable {
-
-
- private HttpConPoolConfig httpConPoolConfig = ServerConfigOptions.HTTP_CON_POOL.defaultValue();
- private List<KnowledgeConfig> knowledgeBaseConfig = ServerConfigOptions.KNOWLEDGE_BASE.defaultValue();
- private NacosConfig nacosConfig = ServerConfigOptions.NACOS.defaultValue();
- private ConsulConfig consulConfig = ServerConfigOptions.CONSUL.defaultValue();
- private ZookeeperConfig zookeeperConfig = ServerConfigOptions.ZOOKEEPER.defaultValue();
- private HdfsConfig hdfsConfig = ServerConfigOptions.HDFS.defaultValue();
- private Map<String,String> propertiesConfig = ServerConfigOptions.PROPERTIES.defaultValue();
-
-
- public void setKnowledgeBaseConfig(List<KnowledgeConfig> knowledgeBaseConfig) {
- checkNotNull(knowledgeBaseConfig, ServerConfigOptions.KNOWLEDGE_BASE + "knowledgeConfig should not be null");
- this.knowledgeBaseConfig = knowledgeBaseConfig;
- }
-
-
-
-
-
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfig.java
index 00d8319..189b05b 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfig.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfig.java
@@ -9,7 +9,7 @@ import java.util.List;
@Slf4j
public class GrootStreamConfig {
- private final EngineConfig engineConfig = new EngineConfig();
+ private final CommonConfig commonConfig = new CommonConfig();
private Config hazelcastConfig;
@@ -34,7 +34,6 @@ public class GrootStreamConfig {
}
-
/**
* Returns the absolute path for `groot-stream.home` based from the system property {@link
* GrootStreamProperties#GROOTSTREAM_HOME}
@@ -53,8 +52,8 @@ public class GrootStreamConfig {
public void setHazelcastConfig(Config hazelcastConfig) {
this.hazelcastConfig = hazelcastConfig;
}
- public EngineConfig getEngineConfig() {
- return engineConfig;
+ public CommonConfig getCommonConfig() {
+ return commonConfig;
}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigBuilder.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigBuilder.java
index 83d7b87..25fc263 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigBuilder.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigBuilder.java
@@ -19,13 +19,13 @@ public class GrootStreamConfigBuilder extends AbstractYamlConfigBuilder {
private final InputStream in;
public GrootStreamConfigBuilder() {
- this((GrootStreamConfigLocator) null);
+ this((CommonConfigLocator) null);
}
- public GrootStreamConfigBuilder(GrootStreamConfigLocator locator) {
+ public GrootStreamConfigBuilder(CommonConfigLocator locator) {
if (locator == null) {
- locator = new GrootStreamConfigLocator();
+ locator = new CommonConfigLocator();
locator.locateEverywhere();
}
this.in = locator.getIn();
@@ -46,16 +46,16 @@ public class GrootStreamConfigBuilder extends AbstractYamlConfigBuilder {
public GrootStreamConfig build(GrootStreamConfig config) {
try {
- parseAndBuildConfig(config);
+ parseAndBuildCommonConfig(config);
+ parseAndBuildUDFPluginConfig(config);
+ parseAndBuildHazelcastConfig(config);
} catch (Exception e) {
throw ExceptionUtil.rethrow(e);
}
- config.setHazelcastConfig(ConfigProvider.locateAndGetMemberConfig(getProperties()));
- config.setUDFPluginConfig(ConfigProvider.locateAndGetUDFPluginConfig(getProperties()));
return config;
}
- private void parseAndBuildConfig(GrootStreamConfig config) throws Exception {
+ private void parseAndBuildCommonConfig(GrootStreamConfig config) throws Exception {
YamlMapping yamlRootNode;
try {
yamlRootNode = (YamlMapping) YamlLoader.load(in);
@@ -66,6 +66,7 @@ public class GrootStreamConfigBuilder extends AbstractYamlConfigBuilder {
}
YamlNode grootStreamRoot =
yamlRootNode.childAsMapping(GrootStreamConfigSections.GROOTSTREAM.name);
+
if (grootStreamRoot == null) {
grootStreamRoot = yamlRootNode;
}
@@ -76,8 +77,19 @@ public class GrootStreamConfigBuilder extends AbstractYamlConfigBuilder {
replaceVariables(w3cRootNode);
importDocuments(grootStreamRoot);
- new GrootStreamDomConfigProcessor(true, config).buildConfig(w3cRootNode);
+ new CommonConfigDomProcessor(true, config).buildConfig(w3cRootNode);
+ }
+
+ private void parseAndBuildHazelcastConfig(GrootStreamConfig config){
+ config.setHazelcastConfig(ConfigProvider.locateAndGetMemberConfig(getProperties()));
}
+ private void parseAndBuildUDFPluginConfig(GrootStreamConfig config){
+ config.setUDFPluginConfig(ConfigProvider.locateAndGetUDFPluginConfig(getProperties()));
+ }
+
+
+
+
public GrootStreamConfigBuilder setProperties(Properties properties) {
if (properties == null) {
properties = System.getProperties();
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigSections.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigSections.java
index d0a35b8..d67c97e 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigSections.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamConfigSections.java
@@ -2,8 +2,7 @@ package com.geedgenetworks.common.config;
/** Configuration sections for Hazelcast GrootStream shared by YAML based configurations */
enum GrootStreamConfigSections {
- GROOTSTREAM("grootstream", false),
- ENGINE("engine", false);
+ GROOTSTREAM("grootstream", false);
final String name;
final boolean multipleOccurrence;
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamDomConfigProcessor.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamDomConfigProcessor.java
deleted file mode 100644
index 61c4855..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootStreamDomConfigProcessor.java
+++ /dev/null
@@ -1,231 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import com.hazelcast.internal.config.AbstractDomConfigProcessor;
-import com.hazelcast.logging.ILogger;
-import com.hazelcast.logging.Logger;
-import org.w3c.dom.Node;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static com.hazelcast.internal.config.DomConfigHelper.*;
-
-public class GrootStreamDomConfigProcessor extends AbstractDomConfigProcessor {
- private static final ILogger LOGGER = Logger.getLogger(GrootStreamDomConfigProcessor.class);
-
- private final GrootStreamConfig config;
-
- GrootStreamDomConfigProcessor(boolean domLevel3, GrootStreamConfig config) {
- super(domLevel3);
- this.config = config;
- }
-
-/* @Override
- public void buildConfig(Node rootNode) {
- for (Node node : childElements(rootNode)) {
- String nodeName = cleanNodeName(node);
- if (occurrenceSet.contains(nodeName)) {
- throw new InvalidConfigurationException(
- "Duplicate '" + nodeName + "' definition found in the configuration.");
- }
- if (handleNode(node, nodeName)) {
- continue;
- }
- if (!GrootStreamConfigSections.canOccurMultipleTimes(nodeName)) {
- occurrenceSet.add(nodeName);
- }
- }
- }*/
-
-/* private boolean handleNode(Node node, String name) {
- if (GrootStreamConfigSections.GROOTSTREAM.isEqual(name)) {
- parseEngineConfig(node, config);
- } else {
- return true;
- }
- parseEngineConfig(node, config);
-
- return false;
- }*/
-
- public void buildConfig(Node engineNode) {
- final EngineConfig engineConfig = config.getEngineConfig();
- for (Node node : childElements(engineNode)) {
- String name = cleanNodeName(node);
- if (ServerConfigOptions.HTTP_CON_POOL.key().equals(name)) {
- engineConfig.setHttpConPoolConfig(parseHttpConPoolConfig(node));
- } else if (ServerConfigOptions. KNOWLEDGE_BASE.key().equals(name)) {
- engineConfig.setKnowledgeBaseConfig(parseKnowledgeBaseConfig(node));
- } else if (ServerConfigOptions.NACOS.key().equals(name)) {
- engineConfig.setNacosConfig(parseNacosConfig(node));
- } else if (ServerConfigOptions.CONSUL.key().equals(name)) {
- engineConfig.setConsulConfig(parseConsulConfig(node));
- } else if (ServerConfigOptions.ZOOKEEPER.key().equals(name)) {
- engineConfig.setZookeeperConfig(parseZookeeperConfig(node));
- } else if (ServerConfigOptions.HDFS.key().equals(name)) {
- engineConfig.setHdfsConfig(parseHdfsConfig(node));
- } else if (ServerConfigOptions.PROPERTIES.key().equals(name)) {
- engineConfig.setPropertiesConfig(parsePropertiesConfig(node));
- } else {
- LOGGER.warning("Unrecognized configuration element: " + name);
-
- }
-
- }
- }
-
- private Map<String, String> parsePropertiesConfig(Node properties) {
-
- Map<String, String> propertiesMap = new HashMap<>();
- for (Node node : childElements(properties)) {
- String name = cleanNodeName(node);
- propertiesMap.put(name,getTextContent(node));
- }
- return propertiesMap;
- }
-
-
- private ZookeeperConfig parseZookeeperConfig(Node zookeeperNode) {
- ZookeeperConfig zookeeperConfig = new ZookeeperConfig();
- for (Node node : childElements(zookeeperNode)) {
- String name = cleanNodeName(node);
- if (ServerConfigOptions.ZOOKEEPER_QUORUM.key().equals(name)) {
- zookeeperConfig.setQuorum(getTextContent(node));
- } else {
- LOGGER.warning("Unrecognized configuration element: " + name);
- }
- }
- return zookeeperConfig;
- }
-
- private HdfsConfig parseHdfsConfig(Node hdfsNode) {
- HdfsConfig hdfsConfig = new HdfsConfig();
- for (Node node : childElements(hdfsNode)) {
- String name = cleanNodeName(node);
- if (ServerConfigOptions.HDFS_SERVERS.key().equals(name)) {
- hdfsConfig.setServers(getTextContent(node));
- } else {
- LOGGER.warning("Unrecognized configuration element: " + name);
- }
- }
- return hdfsConfig;
- }
-
- private ConsulConfig parseConsulConfig(Node consulNode) {
- ConsulConfig consulConfig = new ConsulConfig();
- for (Node node : childElements(consulNode)) {
- String name = cleanNodeName(node);
- if (ServerConfigOptions.CONSUL_SERVER_ADDR.key().equals(name)) {
- consulConfig.setServerAddr(getTextContent(node));
- } else if (ServerConfigOptions.CONSUL_SERVER_PORT.key().equals(name)) {
- consulConfig.setServerPort(getIntegerValue(ServerConfigOptions.CONSUL_SERVER_PORT.key(), getTextContent(node)));
- } else if (ServerConfigOptions.CONSUL_TOKEN.key().equals(name)) {
- consulConfig.setToken(getTextContent(node));
- } else {
- LOGGER.warning("Unrecognized configuration element: " + name);
- }
- }
- return consulConfig;
- }
-
- private NacosConfig parseNacosConfig(Node nacosNode) {
- NacosConfig nacosConfig = new NacosConfig();
- for (Node node : childElements(nacosNode)) {
- String name = cleanNodeName(node);
- if (ServerConfigOptions.NACOS_SERVER_ADDR.key().equals(name)) {
- nacosConfig.setServerAddr(getTextContent(node));
- } else if (ServerConfigOptions.NACOS_NAMESPACE.key().equals(name)) {
- nacosConfig.setNamespace(getTextContent(node));
- } else if (ServerConfigOptions.NACOS_GROUP.key().equals(name)) {
- nacosConfig.setGroup(getTextContent(node));
- } else if (ServerConfigOptions.NACOS_DATA_ID.key().equals(name)) {
- nacosConfig.setDataId(getTextContent(node));
- } else if (ServerConfigOptions.NACOS_READ_TIMEOUT.key().equals(name)) {
- nacosConfig.setReadTimeout(getIntegerValue(ServerConfigOptions.NACOS_READ_TIMEOUT.key(), getTextContent(node)));
- } else if (ServerConfigOptions.NACOS_USERNAME.key().equals(name)) {
- nacosConfig.setUserName(getTextContent(node));
- } else if (ServerConfigOptions.NACOS_PASSWORD.key().equals(name)) {
- nacosConfig.setPassword(getTextContent(node));
- } else {
- LOGGER.warning("Unrecognized configuration element: " + name);
- }
- }
- return nacosConfig;
- }
-
-
- private HttpConPoolConfig parseHttpConPoolConfig(Node httpConPoolNode) {
- HttpConPoolConfig httpConPoolConfig = new HttpConPoolConfig();
- for (Node node : childElements(httpConPoolNode)) {
- String name = cleanNodeName(node);
- if (ServerConfigOptions.HTTP_CON_POOL_MAX_TOTAL.key().equals(name)) {
- httpConPoolConfig.setMaxTotal(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_MAX_TOTAL.key(), getTextContent(node)));
- } else if (ServerConfigOptions.HTTP_CON_POOL_MAX_PER_ROUTE.key().equals(name)) {
- httpConPoolConfig.setMaxPerRoute(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_MAX_PER_ROUTE.key(), getTextContent(node)));
- } else if (ServerConfigOptions.HTTP_CON_POOL_CONNECTION_REQUEST_TIMEOUT.key().equals(name)) {
- httpConPoolConfig.setConnectionRequestTimeout(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_CONNECTION_REQUEST_TIMEOUT.key(), getTextContent(node)));
- } else if (ServerConfigOptions.HTTP_CON_POOL_CONNECT_TIMEOUT.key().equals(name)) {
- httpConPoolConfig.setConnectTimeout(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_CONNECT_TIMEOUT.key(), getTextContent(node)));
- } else if (ServerConfigOptions.HTTP_CON_POOL_SOCKET_TIMEOUT.key().equals(name)) {
- httpConPoolConfig.setSocketTimeout(getIntegerValue(ServerConfigOptions.HTTP_CON_POOL_SOCKET_TIMEOUT.key(), getTextContent(node)));
- } else {
- LOGGER.warning("Unrecognized configuration element: " + name);
- }
- }
- return httpConPoolConfig;
-
- }
-
- private List<KnowledgeConfig> parseKnowledgeBaseConfig(Node knowledgeBaseNode) {
-
- List<KnowledgeConfig> knowledgeConfigList = new ArrayList<>();
- for (Node node : childElements(knowledgeBaseNode)) {
- knowledgeConfigList.add(parseKnowledgeConfig(node));
- }
- return knowledgeConfigList;
- }
-
- private KnowledgeConfig parseKnowledgeConfig(Node asnLookupNode) {
- KnowledgeConfig knowledgeConfig = new KnowledgeConfig();
- for (Node node : childElements(asnLookupNode)) {
- String name = cleanNodeName(node);
-
- if (ServerConfigOptions.KNOWLEDGE_BASE_NAME.key().equals(name)) {
- knowledgeConfig.setName(getTextContent(node));
- } else if (ServerConfigOptions.KNOWLEDGE_BASE_TYPE.key().equals(name)) {
- knowledgeConfig.setType(getTextContent(node));
- } else if (ServerConfigOptions.KNOWLEDGE_BASE_FILES.key().equals(name)) {
- knowledgeConfig.setFiles(parseFilesConfig(node));
- } else if (ServerConfigOptions.KNOWLEDGE_BASE_PROPERTIES.key().equals(name)) {
- knowledgeConfig.setProperties(parseKnowledgePropertiesConfig(node));
- }
- else{
- LOGGER.warning("Unrecognized configuration element: " + name);
- }
- }
- return knowledgeConfig;
- }
-
- private Map<String, String> parseKnowledgePropertiesConfig(Node properties) {
- Map<String, String> propertiesMap = new HashMap<>();
- for (Node node : childElements(properties)) {
- String name = cleanNodeName(node);
- propertiesMap.put(name,getTextContent(node));
- }
- return propertiesMap;
- }
-
- private List<String> parseFilesConfig(Node files) {
-
- List<String> asnLookupConfigList = new ArrayList<>();
- for (Node node : childElements(files)) {
- String value = node.getNodeValue();
- asnLookupConfigList.add(value);
- }
- return asnLookupConfigList;
- }
-
-
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootYamlParser.java b/groot-common/src/main/java/com/geedgenetworks/common/config/GrootYamlParser.java
deleted file mode 100644
index 2fbb680..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/GrootYamlParser.java
+++ /dev/null
@@ -1,116 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import cn.hutool.core.io.IoUtil;
-import cn.hutool.setting.yaml.YamlUtil;
-import lombok.NonNull;
-
-import java.io.InputStream;
-import java.lang.reflect.Method;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public final class GrootYamlParser {
-
- public static String getStringProperty(String key, Map<String, Object> yamlData) {
-
- return getYamlValue(yamlData, key, String.class);
- }
-
- public static Integer getIntProperty(String key, Map<String, Object> yamlData) {
-
- return getYamlValue(yamlData, key, Integer.class);
- }
-
- public static Long getLongProperty(String key, Map<String, Object> yamlData) {
-
- return getYamlValue(yamlData, key, Long.class);
- }
-
- public static <T> T getYamlValue(Map<String, Object> data, String key, Class<T> valueType) {
-
- String[] keys = key.split("\\.");
-
- for (int i = 0; i < keys.length - 1; i++) {
- Object value = data.get(keys[i]);
- if (value == null || !(value instanceof Map)) {
- return null;
- }
- data = (Map<String, Object>) value;
- }
-
- return data.containsKey(keys[keys.length - 1])
- ? valueType.cast(data.get(keys[keys.length - 1]))
- : null;
- }
-
- public static Map<String, String> getClassReflect(List<String> plugins) {
-
- Map<String, String> classReflect = new HashMap<>();
-
- for (String classPath : plugins) {
-
- Class cls = null;
- try {
- cls = Class.forName(classPath);
- Method method = cls.getMethod("functionName");
- Object object = cls.newInstance();
- String result = (String) method.invoke(object);
- classReflect.put(result, classPath);
- System.out.println("Returned Value: " + result);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- return classReflect;
- }
-
- public static Map<String, Object> loadYaml(@NonNull String filePath) {
- return loadYaml(Paths.get(filePath));
- }
- public static Map<String, Object> loadYaml(@NonNull Path filePath) {
- return loadYaml(IoUtil.toStream(filePath.toFile()));
- }
- public static Map<String, Object> loadYaml(InputStream input) {
- return YamlUtil.load(input, Map.class);
- }
-
-
- private static Object getValue(String key, Map<String, Object> data) {
- if (data.containsKey(key)) {
- return data.get(key);
- } else {
- for (Object value : data.values()) {
- if (value instanceof Map) {
- Object result = getValue(key, (Map<String, Object>) value);
- if (result != null) {
- return result;
- }
- }
- }
- }
- return null;
- }
-
- public static List<Object> getList(String key, Map<String, Object> yamlData) {
- Object value = getValue(key, yamlData);
- if (value instanceof List) {
- return (List<Object>) value;
- }
- return null;
- }
-
- public static Map<String, Object> getMap(String key, Map<String, Object> yamlData) {
- Object value = getValue(key, yamlData);
- if (value instanceof Map) {
- return (Map<String, Object>) value;
- }
- return null;
- }
-
- public static void main(String[] args) {
- System.out.println();
- }
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/HdfsConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/HdfsConfig.java
deleted file mode 100644
index 1323f54..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/HdfsConfig.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import lombok.Data;
-
-import java.io.Serializable;
-
-@Data
-public class HdfsConfig implements Serializable {
- private String servers = ServerConfigOptions.HDFS_SERVERS.defaultValue();
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/HosConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/HosConfig.java
deleted file mode 100644
index 903d6cd..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/HosConfig.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import lombok.Data;
-
-import java.io.Serializable;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-@Data
-public class HosConfig implements Serializable {
- public String token = ServerConfigOptions.HOS_TOKEN.defaultValue();
-
- public void setToken(String token) {
- checkNotNull(token, ServerConfigOptions.HOS_TOKEN + "token should not be null");
- this.token = token;
- }
-
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/HttpConPoolConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/HttpConPoolConfig.java
deleted file mode 100644
index c0a0e74..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/HttpConPoolConfig.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import lombok.Data;
-
-import java.io.Serializable;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-@Data
-public class HttpConPoolConfig implements Serializable {
-
- public static final long MIN_TIMEOUT_MS = 10;
- public int maxTotal = ServerConfigOptions.HTTP_CON_POOL_MAX_TOTAL.defaultValue();
- public int maxPerRoute = ServerConfigOptions.HTTP_CON_POOL_MAX_PER_ROUTE.defaultValue();
- public int connectionRequestTimeout = ServerConfigOptions.HTTP_CON_POOL_CONNECTION_REQUEST_TIMEOUT.defaultValue();
- public int connectTimeout = ServerConfigOptions.HTTP_CON_POOL_CONNECT_TIMEOUT.defaultValue();
- public int socketTimeout = ServerConfigOptions.HTTP_CON_POOL_SOCKET_TIMEOUT.defaultValue();
-
- public void setSocketTimeout(int socketTimeout) {
- checkArgument(socketTimeout > MIN_TIMEOUT_MS, ServerConfigOptions.HTTP_CON_POOL_SOCKET_TIMEOUT + "socketTimeout should be greater than " + MIN_TIMEOUT_MS);
- this.socketTimeout = socketTimeout;
- }
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeBaseConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeBaseConfig.java
new file mode 100644
index 0000000..c02df61
--- /dev/null
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeBaseConfig.java
@@ -0,0 +1,34 @@
+package com.geedgenetworks.common.config;
+
+import com.geedgenetworks.utils.StringUtil;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+@Data
+public class KnowledgeBaseConfig implements Serializable {
+ private String name = CommonConfigOptions.KNOWLEDGE_BASE_NAME.defaultValue();
+ private String type = CommonConfigOptions.KNOWLEDGE_BASE_TYPE.defaultValue();
+ private Map<String, String> properties = CommonConfigOptions.KNOWLEDGE_BASE_PROPERTIES.defaultValue();
+ private List<String> files = CommonConfigOptions.KNOWLEDGE_BASE_FILES.defaultValue();
+
+ public void setType(String type) {
+ checkArgument(StringUtil.isNotBlank(type) && Arrays.asList("asnlookup", "geoiplookup").contains(type.toLowerCase())
+ , CommonConfigOptions.KNOWLEDGE_BASE_TYPE + "Type should be asnlookup or geoiplookup");
+ this.type = type;
+ }
+
+ public void setFiles(List<String> files) {
+ checkArgument(files != null && files.size() > 0,
+ CommonConfigOptions.KNOWLEDGE_BASE_FILES + "files should not be null");
+ this.files = files;
+ }
+
+
+
+}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeConfig.java
deleted file mode 100644
index cc407e4..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/KnowledgeConfig.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import lombok.Data;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-
-@Data
-public class KnowledgeConfig implements Serializable {
- private String name = ServerConfigOptions.KNOWLEDGE_BASE_NAME.defaultValue();
- private String type = ServerConfigOptions.KNOWLEDGE_BASE_TYPE.defaultValue();
- private Map<String, String> properties = ServerConfigOptions.KNOWLEDGE_BASE_PROPERTIES.defaultValue();
- private List<String> files = ServerConfigOptions.KNOWLEDGE_BASE_FILES.defaultValue();
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/NacosConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/NacosConfig.java
deleted file mode 100644
index 2c1f7e7..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/NacosConfig.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import lombok.Data;
-
-import java.io.Serializable;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-@Data
-public class NacosConfig implements Serializable {
- private String serverAddr = ServerConfigOptions.NACOS_SERVER_ADDR.defaultValue();
- private String userName = ServerConfigOptions.NACOS_USERNAME.defaultValue();
- private String password = ServerConfigOptions.NACOS_PASSWORD.defaultValue();
- private String namespace = ServerConfigOptions.NACOS_NAMESPACE.defaultValue();
- private String dataId = ServerConfigOptions.NACOS_DATA_ID.defaultValue();
- private String group = ServerConfigOptions.NACOS_GROUP.defaultValue();
- private int readTimeout = ServerConfigOptions.NACOS_READ_TIMEOUT.defaultValue();
-
- public void setReadTimeout(int readTimeout) {
- checkArgument(readTimeout > 0, ServerConfigOptions.NACOS_READ_TIMEOUT + "readTimeout should be greater than 0");
- this.readTimeout = readTimeout;
- }
-
- public void setServerAddr(String serverAddr) {
- checkNotNull(serverAddr, ServerConfigOptions.NACOS_SERVER_ADDR + "serverAddr should not be null");
- this.serverAddr = serverAddr;
- }
-
- public void setDataId(String dataId) {
- checkNotNull(dataId, ServerConfigOptions.NACOS_DATA_ID + "dataId should not be null");
- this.dataId = dataId;
- }
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java
deleted file mode 100644
index 2c03b21..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java
+++ /dev/null
@@ -1,206 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import com.alibaba.fastjson2.TypeReference;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class ServerConfigOptions {
-
-
- public static final Option<Integer> HTTP_CON_POOL_MAX_TOTAL =
- Options.key("max_total")
- .intType()
- .defaultValue(100)
- .withDescription("The max connections of http con pool.");
-
- public static final Option<Integer> HTTP_CON_POOL_MAX_PER_ROUTE =
- Options.key("max_per_route")
- .intType()
- .defaultValue(10)
- .withDescription("The max per route of http con pool.");
-
- public static final Option<Integer> HTTP_CON_POOL_CONNECTION_REQUEST_TIMEOUT =
- Options.key("connection_request_timeout")
- .intType()
- .defaultValue(10000)
- .withDescription("The connection request timeout of http con pool.");
- public static final Option<Integer> HTTP_CON_POOL_CONNECT_TIMEOUT =
- Options.key("connect_timeout")
- .intType()
- .defaultValue(10000)
- .withDescription("The connect timeout of http con pool.");
-
- public static final Option<Integer> HTTP_CON_POOL_SOCKET_TIMEOUT =
- Options.key("socket_timeout")
- .intType()
- .defaultValue(60000)
- .withDescription("The socket timeout of http con pool.");
-
- public static final Option<HttpConPoolConfig> HTTP_CON_POOL = Options.key("http_con_pool")
- .type(new TypeReference<HttpConPoolConfig>() {})
- .defaultValue(new HttpConPoolConfig())
- .withDescription("The http con pool configuration.");
-
- public static final Option<Map<String, String>> KNOWLEDGE_BASE_PROPERTIES =
- Options.key("properties")
- .mapType()
- .defaultValue(new HashMap<String,String>())
- .withDescription("The properties of knowledgebase");
- public static final Option<String> KNOWLEDGE_BASE_NAME =
- Options.key("name")
- .stringType()
- .defaultValue("")
- .withDescription("The name of knowledgebase.");
- public static final Option<String> KNOWLEDGE_BASE_TYPE =
- Options.key("type")
- .stringType()
- .defaultValue("")
- .withDescription("The type of knowledgebase.");
-
- public static final Option<List<String>> KNOWLEDGE_BASE_FILES =
- Options.key("files")
- .listType()
- .defaultValue(new ArrayList<>())
- .withDescription("The files of knowledgebase.");
-
-
- public static final Option<String> KNOWLEDGE_BASE_STORAGE_FS_TYPE = Options.key("fs_type")
- .stringType()
- .defaultValue("localfile")
- .withDescription("The fs type of knowledge base storage.");
-
- public static final Option<String> KNOWLEDGE_BASE_STORAGE_FS_DEFAULT_PATH = Options.key("fs_default_path")
- .stringType()
- .defaultValue("")
- .withDescription("The default path of knowledge base storage.");
-
-
-
- public static final Option<List<KnowledgeConfig>> KNOWLEDGE_BASE =
- Options.key("knowledge_base")
- .type(new TypeReference<List<KnowledgeConfig>>() {})
- .defaultValue(new ArrayList<>())
- .withDescription("The knowledge base configuration.");
-
- public static final Option<String> SNOWFLAKE_DATA_CENTER_ID =
- Options.key("data_center_id_num")
- .stringType()
- .defaultValue("")
- .withDescription("The data center id of snowflake.");
-
- public static final Option<SnowflakeConfig> SNOWFLAKE =
- Options.key("snowflake")
- .type(new TypeReference<SnowflakeConfig>() {})
- .defaultValue(new SnowflakeConfig())
- .withDescription("The snowflake configuration.");
-
-
- public static final Option<String> NACOS_SERVER_ADDR =
- Options.key("server_addr")
- .stringType()
- .defaultValue("")
- .withDescription("The server address of nacos.");
- public static final Option<String> NACOS_USERNAME =
- Options.key("username")
- .stringType()
- .defaultValue("nacos")
- .withDescription("The username of nacos.");
- public static final Option<String> NACOS_PASSWORD =
- Options.key("password")
- .stringType()
- .defaultValue("nacos")
- .withDescription("The password of nacos.");
- public static final Option<String> NACOS_NAMESPACE =
- Options.key("namespace")
- .stringType()
- .defaultValue("public")
- .withDescription("The namespace of nacos.");
- public static final Option<String> NACOS_DATA_ID =
- Options.key("data_id")
- .stringType()
- .defaultValue("")
- .withDescription("The data id of nacos.");
- public static final Option<String> NACOS_GROUP =
- Options.key("group")
- .stringType()
- .defaultValue("DEFAULT_GROUP")
- .withDescription("The group of nacos.");
- public static final Option<Integer> NACOS_READ_TIMEOUT =
- Options.key("read_timeout")
- .intType()
- .defaultValue(5000)
- .withDescription("The read timeout of nacos.");
-
- public static final Option<NacosConfig> NACOS =
- Options.key("nacos")
- .type(new TypeReference<NacosConfig>() {})
- .defaultValue(new NacosConfig())
- .withDescription("The nacos configuration.");
-
- public static final Option<String> CONSUL_SERVER_ADDR = Options.key("server_addr")
- .stringType()
- .defaultValue("")
- .withDescription("The server address of consul.");
- public static final Option<Integer> CONSUL_SERVER_PORT =
- Options.key("server_port")
- .intType()
- .defaultValue(8500)
- .withDescription("The server port of consul.");
- public static final Option<String> CONSUL_TOKEN =
- Options.key("token")
- .stringType()
- .defaultValue("")
- .withDescription("The token of consul.");
-
- public static final Option<ConsulConfig> CONSUL =
- Options.key("consul")
- .type(new TypeReference<ConsulConfig>() {})
- .defaultValue(new ConsulConfig())
- .withDescription("The consul configuration.");
-
- public static final Option<String> HOS_TOKEN =
- Options.key("token")
- .stringType()
- .defaultValue("")
- .withDescription("The token of hos.");
-
- public static final Option<HosConfig> HOS =
- Options.key("hos")
- .type(new TypeReference<HosConfig>() {})
- .defaultValue(new HosConfig())
- .withDescription("The hos configuration.");
-
-
- public static final Option<String> ZOOKEEPER_QUORUM =
- Options.key("quorum")
- .stringType()
- .defaultValue("")
- .withDescription("The quorum of zookeeper.");
-
- public static final Option<ZookeeperConfig> ZOOKEEPER =
- Options.key("zookeeper")
- .type(new TypeReference<ZookeeperConfig>() {})
- .defaultValue(new ZookeeperConfig())
- .withDescription("The zookeeper configuration.");
-
- public static final Option<String> HDFS_SERVERS =
- Options.key("servers")
- .stringType()
- .defaultValue("")
- .withDescription("The servers of hdfs.");
-
- public static final Option<HdfsConfig> HDFS = Options.key("hdfs")
- .type(new TypeReference<HdfsConfig>() {})
- .defaultValue(new HdfsConfig())
- .withDescription("The hdfs configuration.");
-
- public static final Option<Map<String, String>> PROPERTIES =
- Options.key("properties")
- .mapType()
- .defaultValue(new HashMap<String,String>())
- .withDescription("The properties of grootstream");
-
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/SnowflakeConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/SnowflakeConfig.java
deleted file mode 100644
index 59e84b6..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/SnowflakeConfig.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import lombok.Data;
-
-import java.io.Serializable;
-
-@Data
-public class SnowflakeConfig implements Serializable {
- private String dataCenterId = ServerConfigOptions.SNOWFLAKE_DATA_CENTER_ID.defaultValue();
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/StorageConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/StorageConfig.java
deleted file mode 100644
index 64ef74c..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/StorageConfig.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import com.geedgenetworks.utils.StringUtil;
-import lombok.Data;
-
-import java.io.Serializable;
-import java.util.Arrays;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-@Data
-public class StorageConfig implements Serializable {
- private String fsType = ServerConfigOptions.KNOWLEDGE_BASE_STORAGE_FS_TYPE.defaultValue();
- private String fsDefaultPath = ServerConfigOptions.KNOWLEDGE_BASE_STORAGE_FS_DEFAULT_PATH.defaultValue();
-
- public void setFsType(String fsType) {
- checkArgument(StringUtil.isNotBlank(fsType) && Arrays.asList("hdfs", "localfile").contains(fsType.toLowerCase())
- , ServerConfigOptions.KNOWLEDGE_BASE_STORAGE_FS_TYPE + "fsType should be hdfs or localfile");
- this.fsType = fsType;
- }
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/TypesafeConfigUtils.java b/groot-common/src/main/java/com/geedgenetworks/common/config/TypesafeConfigUtils.java
index a8a6fa5..59c6088 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/TypesafeConfigUtils.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/TypesafeConfigUtils.java
@@ -20,7 +20,7 @@ public final class TypesafeConfigUtils {
* @param source config source
* @param prefix config prefix
* @param keepPrefix true if keep prefix
- * @deprecated use org.apache.seatunnel.api.configuration.Option interface instead
+ * @deprecated
*/
@Deprecated
public static Config extractSubConfig(Config source, String prefix, boolean keepPrefix) {
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ZookeeperConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ZookeeperConfig.java
deleted file mode 100644
index 77c8be7..0000000
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/ZookeeperConfig.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package com.geedgenetworks.common.config;
-
-import java.io.Serializable;
-
-import static com.google.common.base.Preconditions.checkNotNull;
-
-public class ZookeeperConfig implements Serializable {
- private String quorum = ServerConfigOptions.ZOOKEEPER_QUORUM.defaultValue();
-
- public void setQuorum(String quorum) {
- checkNotNull(quorum, ServerConfigOptions.ZOOKEEPER_QUORUM + "quorum should not be null");
- this.quorum = quorum;
- }
-
-}
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/utils/HttpClientUtils.java b/groot-common/src/main/java/com/geedgenetworks/common/utils/HttpClientUtils.java
index c68555f..fa68843 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/utils/HttpClientUtils.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/utils/HttpClientUtils.java
@@ -1,6 +1,7 @@
package com.geedgenetworks.common.utils;
-import com.geedgenetworks.common.config.CommonConfig;
+import com.geedgenetworks.common.config.Option;
+import com.geedgenetworks.common.config.Options;
import com.geedgenetworks.utils.StringUtil;
import org.apache.commons.io.IOUtils;
import org.apache.http.*;
@@ -11,7 +12,6 @@ import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.HttpClientContext;
-import org.apache.http.client.utils.URIBuilder;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ConnectTimeoutException;
@@ -41,7 +41,6 @@ import java.nio.charset.StandardCharsets;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
-import java.util.Map;
public class HttpClientUtils {
/** 全局连接池对象 */
@@ -50,7 +49,13 @@ public class HttpClientUtils {
private final Logger logger = LoggerFactory.getLogger(HttpClientUtils.class);
- public static final String ERROR_MESSAGE = "-1";
+ private static final String ERROR_MESSAGE = "-1";
+
+ private static final int DEFAULT_MAX_TOTAL = 400;
+ private static final int DEFAULT_MAX_PER_ROUTE = 10;
+ private static final int DEFAULT_CONNECTION_REQUEST_TIMEOUT = 10000;
+ private static final int DEFAULT_CONNECT_TIMEOUT = 10000;
+ private static final int DEFAULT_SOCKET_TIMEOUT = 60000;
/*
* 静态代码块配置连接池信息
@@ -58,9 +63,9 @@ public class HttpClientUtils {
static {
// 设置最大连接数
- CONN_MANAGER.setMaxTotal(CommonConfig.HTTP_POOL_MAX_CONNECTION);
+ CONN_MANAGER.setMaxTotal(DEFAULT_MAX_TOTAL);
// 设置每个连接的路由数
- CONN_MANAGER.setDefaultMaxPerRoute(CommonConfig.HTTP_POOL_MAX_PER_ROUTE);
+ CONN_MANAGER.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE);
}
/**
@@ -97,9 +102,9 @@ public class HttpClientUtils {
PoolingHttpClientConnectionManager connManager =
new PoolingHttpClientConnectionManager(socketFactoryRegistry);
// 设置最大连接数
- connManager.setMaxTotal(CommonConfig.HTTP_POOL_MAX_CONNECTION);
+ connManager.setMaxTotal(DEFAULT_MAX_TOTAL);
// 设置每个连接的路由数
- connManager.setDefaultMaxPerRoute(CommonConfig.HTTP_POOL_MAX_PER_ROUTE);
+ connManager.setDefaultMaxPerRoute(DEFAULT_MAX_PER_ROUTE);
return connManager;
} catch (KeyManagementException | NoSuchAlgorithmException e) {
throw new RuntimeException(e.getMessage());
@@ -116,11 +121,11 @@ public class HttpClientUtils {
RequestConfig requestConfig =
RequestConfig.custom()
// 获取连接超时时间
- .setConnectionRequestTimeout(CommonConfig.HTTP_POOL_REQUEST_TIMEOUT)
+ .setConnectionRequestTimeout(DEFAULT_CONNECTION_REQUEST_TIMEOUT)
// 请求超时时间
- .setConnectTimeout(CommonConfig.HTTP_POOL_CONNECT_TIMEOUT)
+ .setConnectTimeout(DEFAULT_CONNECT_TIMEOUT)
// 响应超时时间
- .setSocketTimeout(CommonConfig.HTTP_POOL_RESPONSE_TIMEOUT)
+ .setSocketTimeout(DEFAULT_SOCKET_TIMEOUT)
.build();
/*
@@ -198,7 +203,7 @@ public class HttpClientUtils {
String msg = ERROR_MESSAGE;
// 获取客户端连接对象
- CloseableHttpClient httpClient = getHttpClient(CommonConfig.HTTP_POOL_RESPONSE_TIMEOUT);
+ CloseableHttpClient httpClient = getHttpClient(DEFAULT_SOCKET_TIMEOUT);
CloseableHttpResponse response = null;
try {
@@ -253,7 +258,7 @@ public class HttpClientUtils {
public String httpPost(URI uri, String requestBody, Header... headers) {
String msg = ERROR_MESSAGE;
// 获取客户端连接对象
- CloseableHttpClient httpClient = getHttpClient(CommonConfig.HTTP_POOL_RESPONSE_TIMEOUT);
+ CloseableHttpClient httpClient = getHttpClient(DEFAULT_SOCKET_TIMEOUT);
// 创建POST请求对象
CloseableHttpResponse response = null;
diff --git a/groot-common/src/main/resources/groot.yaml b/groot-common/src/main/resources/groot.yaml
deleted file mode 100644
index a5bc3f9..0000000
--- a/groot-common/src/main/resources/groot.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-
-nacos:
- server_addr: 192.168.44.12:8848
- username: nacos
- password: nacos
- namespace: test
- data_id: knowledge_base.json
- group: DEFAULT_GROUP
- read_timeout: 5000
-
-consul:
- server_addr: 192.168.41.30
- server_port: 8500
- token: c989b503-1d74-f49c-5698-e90fe461e938
-
-hos:
- token: c21f969b5f03d33d43e04f8f136e7682
-
-
-knowledge:
- file_check_number: 3
- # iplookup:
-# TSG:
-# ip_v4_user_defined_id: acf1db8589c5e277-2d8cb5cf98593ed8
- # ip_v6_user_defined_id: acf1db8589c5e277-a442cce2d726eddf
- # ip_v4_built_in_id: acf1db8589c5e277-42014a5735551e3e
- # ip_v6_built_in_id: acf1db8589c5e277-8a9a0362c1320877
-# CN:
- # ip_v4_user_defined_id: acf1db8589c5e277-2d8cb5cf98593ed8
- # ip_v6_user_defined_id: acf1db8589c5e277-a442cce2d726eddf
- # ip_v4_built_in_id: acf1db8589c5e277-10798bc00aed2135
- # ip_v6_built_in_id: acf1db8589c5e277-8a9a0362c1320877
- # asn_v4_id: 7ce2f9890950ba90-fcc25696bf11a8a0
- # asn_v6_id: 7ce2f9890950ba90-71f13b3736863ddb
- asnlookup:
- TSG:
- #asn_v4_id: 7ce2f9890950ba90-fe9081f96d1d8ca8
- #asn_v6_id: 7ce2f9890950ba90-c901e886e2d89a30
- asn_v4_id: 7ce2f9890950ba90-fcc25696bf11a8a0
- asn_v6_id: 7ce2f9890950ba90-71f13b3736863ddb
- # CN:
- # ip_v4_user_defined_id: acf1db8589c5e277-2d8cb5cf98593ed8
- # ip_v6_user_defined_id: acf1db8589c5e277-a442cce2d726eddf
- # ip_v4_built_in_id: acf1db8589c5e277-10798bc00aed2135
-# ip_v6_built_in_id: acf1db8589c5e277-8a9a0362c1320877
-# asn_v4_id: 7ce2f9890950ba90-fcc25696bf11a8a0
-# asn_v6_id: 7ce2f9890950ba90-71f13b3736863ddb
-http:
- pool:
- max_connection: 400
- max_per_route: 60
- request_timeout: 60000
- connect_timeout: 60000
- response_timeout: 60000
-
-snowid:
- data_center_id_num: 1
-zookeeper:
- quorum: 192.168.44.12:2181
-
-knowledgebase:
- file_storage_path: /knowledgebase/ETL-SESSION-RECORD-COMPLETED/
- file_storage_type: hdfs
-hdfs:
- servers: 192.168.44.11:9000,192.168.44.14:9000
-hbase:
- tick_tuple_freq_secs:
- gtpc_scan_max_rows: 10000
- radius_scan_max_rows: 10000
- radius_table_name: RADIUS-TABLE
- gtpc_table_name: GTPC-TABLE
- rpc_timeout:
-data:
- relationship_model: \ No newline at end of file
diff --git a/groot-common/src/main/resources/udf.plugins b/groot-common/src/main/resources/udf.plugins
index d22c057..07867f6 100644
--- a/groot-common/src/main/resources/udf.plugins
+++ b/groot-common/src/main/resources/udf.plugins
@@ -3,9 +3,9 @@ com.geedgenetworks.core.udf.Drop
com.geedgenetworks.core.udf.AsnLookup
com.geedgenetworks.core.udf.Eval
com.geedgenetworks.core.udf.JsonExtract
-com.geedgenetworks.core.udf.UnixTimestamp
+com.geedgenetworks.core.udf.CurrentUnixTimestamp
com.geedgenetworks.core.udf.Domain
com.geedgenetworks.core.udf.DecodeBase64
com.geedgenetworks.core.udf.GeoIpLookup
com.geedgenetworks.core.udf.PathCombine
-com.geedgenetworks.core.udf.UnixTimestampConverter
+com.geedgenetworks.core.udf.UnixTimestampConverter \ No newline at end of file
diff --git a/groot-common/src/test/java/com/geedgenetworks/common/config/YamlGrootStreamConfigParserTest.java b/groot-common/src/test/java/com/geedgenetworks/common/config/YamlGrootStreamConfigParserTest.java
index 6e42ae4..9a122ed 100644
--- a/groot-common/src/test/java/com/geedgenetworks/common/config/YamlGrootStreamConfigParserTest.java
+++ b/groot-common/src/test/java/com/geedgenetworks/common/config/YamlGrootStreamConfigParserTest.java
@@ -3,23 +3,25 @@ package com.geedgenetworks.common.config;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.Arrays;
+
public class YamlGrootStreamConfigParserTest {
@Test
public void testGrootStreamConfig() {
- GrootStreamConfigLocator yamlConfigLocator = new GrootStreamConfigLocator();
+ CommonConfigLocator commonConfigLocator = new CommonConfigLocator();
GrootStreamConfig config;
- if (yamlConfigLocator.locateInWorkDirOrOnClasspath()) {
+ if (commonConfigLocator.locateInWorkDirOrOnClasspath()) {
// 2. Try loading YAML config from the working directory or from the classpath
- config = new GrootStreamConfigBuilder(yamlConfigLocator).setProperties(null).build();
+ config = new GrootStreamConfigBuilder(commonConfigLocator).build();
} else {
throw new RuntimeException("can't find yaml in resources");
}
Assertions.assertNotNull(config);
- Assertions.assertNotNull(config.getEngineConfig().getNacosConfig().getServerAddr());
- Assertions.assertTrue(config.getEngineConfig().getHttpConPoolConfig().getSocketTimeout() > 0);
- Assertions.assertNotNull(config.getEngineConfig().getKnowledgeBaseConfig());
- // Assertions.assertEquals("hos", config.getEngineConfig().getKnowledgeBaseConfig().get(0).getProperties().get("fs_type"));
+ Assertions.assertNotNull(config.getCommonConfig().getKnowledgeBaseConfig());
+ Assertions.assertTrue(config.getCommonConfig().getKnowledgeBaseConfig().size() > 0);
+ Assertions.assertTrue(Arrays.asList("asnlookup", "geoiplookup").contains(config.getCommonConfig().getKnowledgeBaseConfig().get(0).getType()));
+ Assertions.assertTrue(config.getUDFPluginConfig().size() > 0);
}
diff --git a/groot-common/src/test/resources/grootstream.yaml b/groot-common/src/test/resources/grootstream.yaml
index c233936..4aa7ef9 100644
--- a/groot-common/src/test/resources/grootstream.yaml
+++ b/groot-common/src/test/resources/grootstream.yaml
@@ -1,51 +1,34 @@
grootstream:
- engine:
- tick_tuple_freq_secs: 90
- gtpc_scan_max_rows: 10000
- radius_scan_max_rows: 10000
- radius_table_name: RADIUS-TABLE
- gtpc_table_name: GTPC-TABLE
- hbase_rpc_timeout: 60000
- http_con_pool:
- max_total: 400
- max_per_route: 60
- connection_request_timeout: 60000
- connect_timeout: 60000
- socket_timeout: 60000
- knowledge_base:
- - name: tsg_asnlookup
- type: asnlookup
- properties:
- fs_type: hos
- fs_default_path: http://path
- files:
- default_mmdb_file: 7ce2f9890950ba90-fcc25696bf11a8a0
- user_defined_mmdb_file: 7ce2f9890950ba90-71f13b3736863ddb
- - name: CN_asnlookup
- type: asnlookup
- files:
- default_mmdb_file: 7ce2f9890950ba90-fcc25696bf11a8a0
- user_defined_mmdb_file: 7ce2f9890950ba90-71f13b3736863ddb
- snowflake:
- data_center_id_num: 1
- nacos:
- server_addr: 192.168.44.12:8848
- username: nacos
- password: nacos
- namespace: test
- data_id: knowledge_base.json
- group: DEFAULT_GROUP
- read_timeout: 5000
- consul:
- server_addr: 192.168.41.30
- server_port: 8500
- token: c989b503-1d74-f49c-5698-e90fe461e938
- hos:
- token: c21f969b5f03d33d43e04f8f136e7682
- zookeeper:
- quorum: 192.168.44.12:2181
- hdfs:
- servers: 192.168.44.11:9000,192.168.44.14:9000
+ knowledge_base:
+ - name: tsg_asnlookup
+ type: asnlookup
+ properties:
+ fs_type: hos
+ fs_default_path: http://path
+ files:
+ - http://192.168.44.12:9098/hos/knowledge_base_bucket/757732ce-8214-4c34-aea2-aca6c51a7e82-YXNuX2J1aWx0aW4=.mmdb
+
+ - name: tsg_geoiplookup
+ type: geoiplookup
+ files:
+ - 7ce2f9890950ba90-fcc25696bf11a8a0
+ - 7ce2f9890950ba90-71f13b3736863ddb
+ properties:
+ hos.path: http://192.168.44.12:8089
+ hos.bucket.name.traffic_file: traffic_file_bucket
+ hos.bucket.name.troubleshooting_file: troubleshooting_file_bucket
+
+
+
+
+
+
+
+
+
+
+
+