代码优化，使用定时线程池刷新写入clickhouse

1 files changed, 19 insertions, 12 deletions

diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties
index 3f47eda..4214242 100644
--- a/properties/service_flow_config.properties
+++ b/properties/service_flow_config.properties
@@ -1,25 +1,25 @@
 #--------------------------------\u5730\u5740\u914D\u7F6E------------------------------#
 #\u7BA1\u7406kafka\u5730\u5740,\u591A\u53F0\u9017\u53F7\u8FDE\u63A5ip1:9094,ip2:9094
-source.kafka.servers=192.168.44.85:9094,192.168.44.86:9094,192.168.44.87:9094
+source.kafka.servers=192.168.44.12:9094
 
 #\u7BA1\u7406\u8F93\u51FAkafka\u5730\u5740
-sink.kafka.servers=
+sink.kafka.servers=192.168.44.12:9094
 #--------------------------------HTTP/\u5B9A\u4F4D\u5E93/ssl------------------------------#
 tools.library=
 #--------------------------------Kafka\u6D88\u8D39/\u751F\u4EA7\u914D\u7F6E------------------------------#
 #\u8BFB\u53D6topic,\u5B58\u50A8\u8BE5spout id\u7684\u6D88\u8D39offset\u4FE1\u606F\uFF0C\u53EF\u901A\u8FC7\u8BE5\u62D3\u6251\u547D\u540D;\u5177\u4F53\u5B58\u50A8offset\u7684\u4F4D\u7F6E\uFF0C\u786E\u5B9A\u4E0B\u6B21\u8BFB\u53D6\u4E0D\u91CD\u590D\u7684\u6570\u636E\uFF1B
-group.id=KNOWLEDGE-GROUP5
+group.id=KNOWLEDGE-GROUPtest
 #--------------------------------topology\u914D\u7F6E------------------------------#
 #consumer \u5E76\u884C\u5EA6
 source.parallelism=1
 #\u8F6C\u6362\u51FD\u6570\u5E76\u884C\u5EA6
 transform.parallelism=1
 #kafka producer \u5E76\u884C\u5EA6
-sink.parallelism=3
+sink.parallelism=1
 
 #--------------------------------\u4E1A\u52A1\u914D\u7F6E------------------------------#
 #1 connection\u65E5\u5FD7 \uFF0C2 dns\u65E5\u5FD7
-log.type=2
+log.type=1
 
 #\u751F\u4EA7\u8005\u538B\u7F29\u6A21\u5F0F none or snappy
 producer.kafka.compression.type=none
@@ -27,6 +27,10 @@ producer.kafka.compression.type=none
 source.kafka.topic.connection=CONNECTION-RECORD-LOG
 source.kafka.topic.sketch=CONNECTION-SKETCH-RECORD-LOG
 source.kafka.topic.dns=DNS-RECORD-LOG
+#kafka\u56DE\u5199\u7EDF\u8BA1\u6570\u636E
+sink.kafka.topic.relation.connection=CONNECTION-RELATION-LOG
+sink.kafka.topic.relation.dns=DNS-RELATION-LOG
+
 #\u5199\u5165clickhouse\u672C\u5730\u8868
 sink.ck.table.connection=connection_record_log_local
 sink.ck.table.sketch=connection_sketch_record_log_local
@@ -43,7 +47,8 @@ sink.arangodb.table.r.nx.domain2domain=R_NX_DOMAIN2DOMAIN
 #\u4F7F\u7528flink\u5165\u5E93\u539F\u59CB\u65E5\u5FD70\uFF1A\u5426\uFF0C1\uFF1A\u662F
 sink.ck.raw.log.insert.open=1
 #clickhouse\u914D\u7F6E\uFF0C\u591A\u4E2A\u9017\u53F7\u8FDE\u63A5 ip1:8123,ip2:8123
-ck.hosts=192.168.44.85:8123,192.168.44.86:8123,192.168.44.87:8123
+ck.hosts=192.168.44.12:8123
+# ,192.168.44.86:8123,192.168.44.87:8123
 ck.database=tsg_galaxy_v3
 ck.username=tsg_insert
 ck.pin=galaxy2019
@@ -51,19 +56,19 @@ ck.pin=galaxy2019
 ck.connection.timeout=10000
 ck.socket.timeout=600000
 #clickhouse\u5165\u5E93\u6279\u91CF\u5355\u4F4D\u6761
-ck.batch=100000
+ck.batch=20000
 #clickhouse\u5165\u5E93\u524D\u79EF\u7D2F\u6279\u91CF\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2
-sink.ck.batch.delay.time=30000
+sink.ck.batch.delay.time=3000
 
 #flink \u65E5\u5FD7\u5EF6\u8FDF\u8D85\u65F6\u65F6\u95F4
 flink.watermark.max.delay.time=60
 #ck relation\u7EDF\u8BA1\u65F6\u95F4\u95F4\u9694 \u5355\u4F4Ds
-log.aggregate.duration=30
+log.aggregate.duration=10
 #arangodb \u7EDF\u8BA1\u65F6\u95F4\u95F4\u9694 \u5355\u4F4Ds
-log.aggregate.duration.graph=600
+log.aggregate.duration.graph=30
 
 #arangoDB\u53C2\u6570\u914D\u7F6E
-arangodb.host=192.168.44.83
+arangodb.host=192.168.44.12
 arangodb.port=8529
 arangodb.user=root
 arangodb.password=galaxy_2019
@@ -72,4 +77,6 @@ arangodb.batch=10000
 arangodb.ttl=3600
 arangodb.thread.pool.number=10
 #\u6279\u91CF\u7D2F\u8BA1\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2ms
-sink.arangodb.batch.delay.time=1000
\ No newline at end of file
+sink.arangodb.batch.delay.time=1000
+
+aggregate.max.value.length=18
\ No newline at end of file