Merge branch 'release/1.7.0' into 'master'v1.7.0master

[feature][bootstrap][common]node新增tags属性用于分流，需要与downstream相对应。rules中name标签修改为t...

See merge request galaxy/platform/groot-stream!128

2 files changed, 85 insertions, 6 deletions

diff --git a/config/template/grootstream_job_template.yaml b/config/template/grootstream_job_template.yaml
index 0ca2d68..b26fbb2 100644
--- a/config/template/grootstream_job_template.yaml
+++ b/config/template/grootstream_job_template.yaml
@@ -10,7 +10,7 @@ sources: # [object] Define connector source
     type: kafka  # [string] Source Type
     schema: # [object] Source Schema, config through fields or local_file or url. if not set schema, all fields(Map<String, Object>) will be output.
       #fields: "struct<log_id:bigint, recv_time:bigint,client_ip: string>"
-      local_file: /../schema/kafka_source_schema.json  # [string] Local File Path for Schema
+      local_file: $GROOT_HOME/config/dat/schema/kafka_source_schema.json  # [string] Local File Path for Schema
       #url: http:// # [string] URL for Schema
     properties: # [object] Kafka source properties
       topic: SESSION-RECORD  # [string] Topic Name, consumer will subscribe this topic.
@@ -45,9 +45,9 @@ sources: # [object] Define connector source
 
       kafka.security.protocol: SSL
       kafka.ssl.endpoint.identification.algorithm: ""
-      kafka.ssl.keystore.location: /data/tsg/olap/flink/topology/data/keystore.jks
+      kafka.ssl.keystore.location: $GROOT_HOME/config/dat/keystore.jks
       kafka.ssl.keystore.password: 86cf0e2ffba3f541a6c6761313e5cc7e
-      kafka.ssl.truststore.location: /data/tsg/olap/flink/topology/data/truststore.jks
+      kafka.ssl.truststore.location: $GROOT_HOME/config/dat/truststore.jks
       kafka.ssl.truststore.password: 86cf0e2ffba3f541a6c6761313e5cc7e
       kafka.ssl.key.password: 86cf0e2ffba3f541a6c6761313e5cc7e
       #kafka.security.protocol: SASL_PLAINTEXT
@@ -100,7 +100,7 @@ sources: # [object] Define connector source
       data: CIin2awGEICAoLC/hYzKAhoEQkFTRSCch8z3wtqEhAQo6o/Xmc0xMMCy15nNMTjWIkDRCEiIp9msBlCIp9msBloIMjE0MjYwMDNg//8DaP//A3JqeyJ0YWdzIjpbeyJ0YWciOiJkYXRhX2NlbnRlciIsInZhbHVlIjoiY2VudGVyLXh4Zy05MTQwIn0seyJ0YWciOiJkZXZpY2VfZ3JvdXAiLCJ2YWx1ZSI6Imdyb3VwLXh4Zy05MTQwIn1dfXoPY2VudGVyLXh4Zy05MTQwggEOZ3JvdXAteHhnLTkxNDCKAQ0xOTIuMTY4LjQwLjgxkAEEmAEBoAEBqAGQwAGyAQdbMSwxLDJd4gEDt+gY4gINMTkyLjU2LjE1MS44MOgCoeYD8gIHV2luZG93c/oCGOe+juWbvS5Vbmtub3duLlVua25vd24uLrIDDTE5Mi41Ni4yMjIuOTO4A/ZwwgMFTGludXjKAxjnvo7lm70uVW5rbm93bi5Vbmtub3duLi6SBAN0Y3CaBBFFVEhFUk5FVC5JUHY0LlRDULAMBLgMBcAM9gHIDJEOoA2AAagN8cr+jgKwDezksIAPwg0RYTI6ZmE6ZGM6NTY6Yzc6YjPKDRE0ODo3Mzo5Nzo5NjozODoyMNINETQ4OjczOjk3Ojk2OjM4OjIw2g0RYTI6ZmE6ZGM6NTY6Yzc6YjM=
       type: base64
       format: protobuf
-      protobuf.descriptor.file.path: ..\session_record_test.desc
+      protobuf.descriptor.file.path: $GROOT_HOME/config/dat/schema/session_record_test.desc
       protobuf.message.name: SessionRecord
 
   ipfix_source: # [object] IPFIX source connector name, must be unique. It used to define the source node of the job topology.
@@ -328,6 +328,7 @@ processing_pipelines: # [object] Define Processors for processing pipelines.
     group_by_fields: [ recv_time, sled_ip ] # [array of string] Group By Fields
     window_type: tumbling_processing_time # [string] Window Type, tumbling_processing_time, tumbling_event_time, sliding_processing_time, sliding_event_time
     window_size: 60
+    mini_batch: true # [boolean] Enable Local Aggregation, default is false
     functions:
       - function: NUMBER_SUM
         lookup_fields: [ received_bytes, sent_bytes ]
@@ -420,7 +421,7 @@ sinks: # [object] Define connector sink
   clickhouse_sink: # [object] ClickHouse sink connector name, must be unique. It used to define the sink node of the job topology.
     type: clickhouse
     schema:
-      local_file: /../schema/clickhouse_sink_schema.json
+      local_file: $GROOT_HOME/config/dat/schema/clickhouse_sink_schema.json
     properties:
       host: 127.0.0.1:9001
       table: inline_source_test_local
@@ -445,9 +446,11 @@ application: # [object] Application Configuration
   env: # [object] Define job runtime environment variables
     name: inline-to-print-job # [string] Job Name
     parallelism: 3 # [number] Job-Level Parallelism
-    shade.identifier: default # [string] Shade Identifier, Using to encrypt and decrypt sensitive configuration. Support enum: default, aes, base64. if set default, it will not encrypt and decrypt sensitive configuration.
+    shade.identifier: default # [string] Config Shade Identifier, Using to encrypt and decrypt sensitive configuration. Support enum: default, aes, base64. if set default, it will not encrypt and decrypt sensitive configuration.
+    kms.type: local # [string] Key Management Service Type, default is local. Support enum: local, vault.
     pipeline:
       object-reuse: true # [boolean] Object Reuse, default is false
+
   topology: # [array of object] Node List. It will be used build data flow for job dag graph.
     - name: inline_source # [string] Node Name, must be unique. It will be used as the name of the corresponding Flink operator. eg. kafka_source the processor type as SOURCE.
       #parallelism: 1 # [number] Operator-Level Parallelism.
diff --git a/config/template/mock_schema/session_record_mock_desc.json b/config/template/mock_schema/session_record_mock_desc.json
index c8c4acf..90060a6 100644
--- a/config/template/mock_schema/session_record_mock_desc.json
+++ b/config/template/mock_schema/session_record_mock_desc.json
@@ -115,12 +115,43 @@
             "end": "103.144.108.255"
           },
           {
+            "name": "client_ip_tags",
+            "type": "String",
+            "array": true,
+            "options": [
+              "Country:United States",
+              "ASN:63278",
+              "Cloud Provider:IBM Cloud",
+              "Country Code:US",
+              "CDN Provider:Light CDN",
+              "ASN:6423"
+
+            ],
+            "arrayLenMin":1,
+            "arrayLenMax":5
+          },
+          {
             "name": "server_ip",
             "type": "IPv4",
             "start": "1.0.0.0",
             "end": "162.105.10.255"
           },
           {
+            "name": "server_ip_tags",
+            "type": "String",
+            "array": true,
+            "options": [
+              "Country:China",
+              "ASN:15169",
+              "Cloud Provider:Alibaba Cloud",
+              "Country Code:CN",
+              "CDN Provider:Akamai",
+              "Super Administrative Area:Guangdong"
+            ],
+            "arrayLenMin":1,
+            "arrayLenMax":5
+          },
+          {
             "name": "c2s_ttl",
             "type": "Number",
             "options": [
@@ -167,12 +198,43 @@
             "end": "162.105.10.255"
           },
           {
+            "name": "client_ip_tags",
+            "type": "String",
+            "array": true,
+            "options": [
+              "Country:China",
+              "ASN:15169",
+              "Cloud Provider:Alibaba Cloud",
+              "Country Code:CN",
+              "CDN Provider:Akamai",
+              "Super Administrative Area:Guangdong"
+            ],
+            "arrayLenMin":1,
+            "arrayLenMax":5
+          },
+          {
             "name": "server_ip",
             "type": "IPv4",
             "start": "103.144.108.1",
             "end": "103.144.108.255"
           },
           {
+            "name": "server_ip_tags",
+            "type": "String",
+            "array": true,
+            "options": [
+              "Country:United States",
+              "ASN:63278",
+              "Cloud Provider:IBM Cloud",
+              "Country Code:US",
+              "CDN Provider:Light CDN",
+              "ASN:6423"
+
+            ],
+            "arrayLenMin":1,
+            "arrayLenMax":5
+          },
+          {
             "name": "c2s_ttl",
             "type": "Number",
             "options": [
@@ -340,6 +402,20 @@
             "nullRate": 0.1
           },
           {
+            "name": "server_fqdn_tags",
+            "type": "String",
+            "array": true,
+            "options": [
+              "Category Name:Entertainment and Arts",
+              "IoC:Malware",
+              "Category Name:Home and Garden",
+              "Category Name:Translation",
+              "IoC:Spam"
+            ],
+            "arrayLenMin":1,
+            "arrayLenMax":5
+          },
+          {
             "name": "server_port",
             "type": "Number",
             "options": [