[improve][core][common] 增加定时任务工具类，部分udf优化，tsg任务所有功能配置，grootstream全局参数修改（暂时只修改配置文件，没有去掉engine待讨论）

5 files changed, 53 insertions, 59 deletions

diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java b/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java
index 3591ce6..a286086 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/EngineConfig.java
@@ -4,6 +4,7 @@ import lombok.Data;
 
 import java.io.Serializable;
 import java.util.List;
+import java.util.Map;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
@@ -20,6 +21,7 @@ public class EngineConfig implements Serializable {
     private ConsulConfig consulConfig = ServerConfigOptions.CONSUL.defaultValue();
     private ZookeeperConfig zookeeperConfig = ServerConfigOptions.ZOOKEEPER.defaultValue();
     private HdfsConfig hdfsConfig = ServerConfigOptions.HDFS.defaultValue();
+    private Map<String,String> propertiesConfig = ServerConfigOptions.PROPERTIES.defaultValue();
 
 
     public void setKnowledgeBaseConfig(List<KnowledgeConfig> knowledgeBaseConfig) {
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java
index 04bc7ec..2c03b21 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/ServerConfigOptions.java
@@ -197,6 +197,10 @@ public class ServerConfigOptions {
             .defaultValue(new HdfsConfig())
             .withDescription("The hdfs configuration.");
 
-
+    public static final Option<Map<String, String>> PROPERTIES =
+            Options.key("properties")
+                    .mapType()
+                    .defaultValue(new HashMap<String,String>())
+                    .withDescription("The properties of grootstream");
 
 }
diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/YamlGrootStreamDomConfigProcessor.java b/groot-common/src/main/java/com/geedgenetworks/common/config/YamlGrootStreamDomConfigProcessor.java
index ccdd58a..843863f 100644
--- a/groot-common/src/main/java/com/geedgenetworks/common/config/YamlGrootStreamDomConfigProcessor.java
+++ b/groot-common/src/main/java/com/geedgenetworks/common/config/YamlGrootStreamDomConfigProcessor.java
@@ -23,7 +23,7 @@ public class YamlGrootStreamDomConfigProcessor extends AbstractDomConfigProcesso
         this.config = config;
     }
 
-    @Override
+/*    @Override
     public void buildConfig(Node rootNode) {
         for (Node node : childElements(rootNode)) {
             String nodeName = cleanNodeName(node);
@@ -38,18 +38,20 @@ public class YamlGrootStreamDomConfigProcessor extends AbstractDomConfigProcesso
                 occurrenceSet.add(nodeName);
             }
         }
-    }
+    }*/
 
-    private boolean handleNode(Node node, String name) {
-        if (GrootStreamConfigSections.ENGINE.isEqual(name)) {
+/*    private boolean handleNode(Node node, String name) {
+        if (GrootStreamConfigSections.GROOTSTREAM.isEqual(name)) {
             parseEngineConfig(node, config);
         } else {
             return true;
         }
+        parseEngineConfig(node, config);
+
         return false;
-    }
+    }*/
 
-    private void parseEngineConfig(Node engineNode, GrootStreamConfig config) {
+    public void buildConfig(Node engineNode) {
         final EngineConfig engineConfig = config.getEngineConfig();
         for (Node node : childElements(engineNode)) {
             String name = cleanNodeName(node);
@@ -65,6 +67,8 @@ public class YamlGrootStreamDomConfigProcessor extends AbstractDomConfigProcesso
                 engineConfig.setZookeeperConfig(parseZookeeperConfig(node));
             } else if (ServerConfigOptions.HDFS.key().equals(name)) {
                 engineConfig.setHdfsConfig(parseHdfsConfig(node));
+            } else if (ServerConfigOptions.PROPERTIES.key().equals(name)) {
+                engineConfig.setPropertiesConfig(parsePropertiesConfig(node));
             } else {
                 LOGGER.warning("Unrecognized configuration element: " + name);
 
@@ -73,6 +77,16 @@ public class YamlGrootStreamDomConfigProcessor extends AbstractDomConfigProcesso
         }
     }
 
+    private Map<String, String> parsePropertiesConfig(Node properties) {
+
+            Map<String, String> propertiesMap = new HashMap<>();
+            for (Node node : childElements(properties)) {
+                String name = cleanNodeName(node);
+                propertiesMap.put(name,getTextContent(node));
+            }
+            return propertiesMap;
+    }
+
 
     private ZookeeperConfig parseZookeeperConfig(Node zookeeperNode) {
         ZookeeperConfig zookeeperConfig = new ZookeeperConfig();
diff --git a/groot-common/src/main/resources/groot-platform-plugin b/groot-common/src/main/resources/groot-platform-plugin
index bebf1e9..d22c057 100644
--- a/groot-common/src/main/resources/groot-platform-plugin
+++ b/groot-common/src/main/resources/groot-platform-plugin
@@ -6,4 +6,6 @@ com.geedgenetworks.core.udf.JsonExtract
 com.geedgenetworks.core.udf.UnixTimestamp
 com.geedgenetworks.core.udf.Domain
 com.geedgenetworks.core.udf.DecodeBase64
-com.geedgenetworks.core.udf.GeoIpLookup
\ No newline at end of file
+com.geedgenetworks.core.udf.GeoIpLookup
+com.geedgenetworks.core.udf.PathCombine
+com.geedgenetworks.core.udf.UnixTimestampConverter
diff --git a/groot-common/src/main/resources/grootstream.yaml b/groot-common/src/main/resources/grootstream.yaml
index d0906bc..7ce881c 100644
--- a/groot-common/src/main/resources/grootstream.yaml
+++ b/groot-common/src/main/resources/grootstream.yaml
@@ -1,53 +1,25 @@
 grootstream:
-  engine:
-    tick_tuple_freq_secs: 90
-    gtpc_scan_max_rows: 10000
-    radius_scan_max_rows: 10000
-    radius_table_name: RADIUS-TABLE
-    gtpc_table_name: GTPC-TABLE
-    hbase_rpc_timeout: 60000
-    http_con_pool:
-      max_total: 400
-      max_per_route: 60
-      connection_request_timeout: 60000
-      connect_timeout: 60000
-      socket_timeout: 60000
-    knowledge_base:
-      - name: tsg_asnlookup
-        type: asnlookup
-        properties:
-          fs_type: hos
-          fs_default_path: http://path
-        files:
-          default_mmdb_file: 7ce2f9890950ba90-fcc25696bf11a8a0
-          user_defined_mmdb_file: 7ce2f9890950ba90-71f13b3736863ddb
-      - name: tsg_geoiplookup
-        type: geoiplookup
-        files:
-          default_mmdb_file: 7ce2f9890950ba90-fcc25696bf11a8a0
-          user_defined_mmdb_file: 7ce2f9890950ba90-71f13b3736863ddb
-
-    snowflake:
-      data_center_id_num: 1
-    nacos:
-      server_addr: 192.168.44.12:8848
-      username: nacos
-      password: nacos
-      namespace: test
-      data_id: knowledge_base.json
-      group: DEFAULT_GROUP
-      read_timeout: 5000
-    consul:
-      server_addr: 192.168.41.30
-      server_port: 8500
-      token: c989b503-1d74-f49c-5698-e90fe461e938
-    hos:
-      token: c21f969b5f03d33d43e04f8f136e7682
-    zookeeper:
-      quorum: 192.168.44.12:2181
-    hdfs:
-      servers: 192.168.44.11:9000,192.168.44.14:9000
-
-
-
 
+  http_con_pool:
+    max_total: 400
+    max_per_route: 60
+    connection_request_timeout: 60000
+    connect_timeout: 60000
+    socket_timeout: 60000
+  knowledge_base:
+    - name: tsg_asnlookup
+      type: asnlookup
+      properties:
+        fs_type: hos
+        fs_default_path: http://path
+      files:
+        7ce2f9890950ba90-fcc25696bf11a8a0
+        7ce2f9890950ba90-71f13b3736863ddb
+    - name: tsg_geoiplookup
+      type: geoiplookup
+      files:
+        7ce2f9890950ba90-fcc25696bf11a8a0
+        7ce2f9890950ba90-71f13b3736863ddb
+  properties:
+    hos.path: //path
+    hos.traffic.file.bucket: bucket
\ No newline at end of file