优化代码：增加入库批处理数量参数

1 files changed, 40 insertions, 45 deletions

diff --git a/properties/service_flow_config.properties b/properties/service_flow_config.properties
index 4de148b..7fef174 100644
--- a/properties/service_flow_config.properties
+++ b/properties/service_flow_config.properties
@@ -1,64 +1,59 @@
 #--------------------------------\u5730\u5740\u914D\u7F6E------------------------------#
-#\u7BA1\u7406kafka\u5730\u5740
-source.kafka.servers=192.168.45.102:9092
+#\u7BA1\u7406kafka\u5730\u5740,\u591A\u53F0\u9017\u53F7\u8FDE\u63A5ip1:9094,ip2:9094
+source.kafka.servers=192.168.45.102:9094
 
 #\u7BA1\u7406\u8F93\u51FAkafka\u5730\u5740
-sink.kafka.servers=192.168.45.102:9092
-
-#zookeeper \u5730\u5740 \u7528\u4E8E\u914D\u7F6Elog_id
-zookeeper.servers=192.168.45.102:2181
-
-#--------------------------------HTTP/\u5B9A\u4F4D\u5E93------------------------------#
-#\u5B9A\u4F4D\u5E93\u5730\u5740
-tools.library=D:\\workerspace\\dat\\
-
+sink.kafka.servers=
+#--------------------------------HTTP/\u5B9A\u4F4D\u5E93/ssl------------------------------#
+tools.library=
 #--------------------------------Kafka\u6D88\u8D39/\u751F\u4EA7\u914D\u7F6E------------------------------#
 #\u8BFB\u53D6topic,\u5B58\u50A8\u8BE5spout id\u7684\u6D88\u8D39offset\u4FE1\u606F\uFF0C\u53EF\u901A\u8FC7\u8BE5\u62D3\u6251\u547D\u540D;\u5177\u4F53\u5B58\u50A8offset\u7684\u4F4D\u7F6E\uFF0C\u786E\u5B9A\u4E0B\u6B21\u8BFB\u53D6\u4E0D\u91CD\u590D\u7684\u6570\u636E\uFF1B
-group.id=knowledge-group
-
+group.id=KNOWLEDGE-GROUP
 #--------------------------------topology\u914D\u7F6E------------------------------#
-
 #consumer \u5E76\u884C\u5EA6
-source.parallelism=1
+source.parallelism=12
 #\u8F6C\u6362\u51FD\u6570\u5E76\u884C\u5EA6
-transform.parallelism=1
+transform.parallelism=12
 #kafka producer \u5E76\u884C\u5EA6
-sink.parallelism=1
-#\u6570\u636E\u4E2D\u5FC3\uFF0C\u53D6\u503C\u8303\u56F4(0-31)
-data.center.id.num=0
+sink.parallelism=12
 
-#--------------------------------\u9ED8\u8BA4\u503C\u914D\u7F6E------------------------------#
+#--------------------------------\u4E1A\u52A1\u914D\u7F6E------------------------------#
 #1 connection\u65E5\u5FD7 \uFF0C2 dns\u65E5\u5FD7
-log.need.complete=2
+log.type=1
 
 #\u751F\u4EA7\u8005\u538B\u7F29\u6A21\u5F0F none or snappy
 producer.kafka.compression.type=none
-
-source.kafka.topic.connection=connection_record_log
-source.kafka.topic.sketch=connection_sketch_record_log
-source.kafka.topic.dns=dns_record_log
-
+#kafka\u6570\u636E\u6E90topic
+source.kafka.topic.connection=CONNECTION-RECORD-LOG
+source.kafka.topic.sketch=CONNECTION-SKETCH-RECORD-LOG
+source.kafka.topic.dns=DNS-RECORD-LOG
+#\u5199\u5165clickhouse\u672C\u5730\u8868
 sink.ck.table.connection=connection_record_log_local
 sink.ck.table.sketch=connection_sketch_record_log_local
 sink.ck.table.dns=dns_record_log_local
 sink.ck.table.relation.connection=connection_relation_log_local
 sink.ck.table.relation.dns=dns_relation_log_local
+#\u5199arangodb\u8868
+sink.arangodb.table.r.visit.ip2ip=R_VISIT_IP2IP
+sink.arangodb.table.r.cname.domain2domain=R_CNAME_DOMAIN2DOMAIN
+sink.arangodb.table.r.mx.domain2domain=R_MX_DOMAIN2DOMAIN
+sink.arangodb.table.r.resolve.domain2ip=R_RESOLVE_DOMAIN2IP
+sink.arangodb.table.r.nx.domain2domain=R_NX_DOMAIN2DOMAIN
 
-sink.arango.table.r.visit.ip2ip=R_VISIT_IP2IP
-sink.arango.table.r.cname.domain2domain=R_CNAME_DOMAIN2DOMAIN
-sink.arango.table.r.mx.domain2domain=R_MX_DOMAIN2DOMAIN
-sink.arango.table.r.resolve.domain2ip=R_RESOLVE_DOMAIN2IP
-sink.arango.table.r.nx.domain2domain=R_NX_DOMAIN2DOMAIN
-
-#clickhouse \u5165\u5E93
+#\u4F7F\u7528flink\u5165\u5E93\u539F\u59CB\u65E5\u5FD7\uFF0C\u6216\u8005\u9009\u62E9\u5B89\u88C5gohangout\u5165\u5E93 0\uFF1A\u5426\uFF0C1\uFF1A\u662F
+sink.ck.raw.log.insert.open=1
+#clickhouse\u914D\u7F6E\uFF0C\u591A\u4E2A\u9017\u53F7\u8FDE\u63A5 ip1:8123,ip2:8123
 ck.hosts=192.168.45.102:8123,192.168.45.102:8123
 ck.database=tsg_galaxy_v3
 ck.username=default
 ck.pin=galaxy2019
-#\u5355\u4F4D\u6BEB\u79D2
+#\u8D85\u65F6\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2
 ck.connection.timeout=10000
 ck.socket.timeout=300000
-ck.batch=10000
+#clickhouse\u5165\u5E93\u6279\u91CF\u5355\u4F4D\u6761
+ck.batch=10
+#clickhouse\u5165\u5E93\u524D\u79EF\u7D2F\u6279\u91CF\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2
+sink.ck.batch.delay.time=1000
 
 #flink \u65E5\u5FD7\u5EF6\u8FDF\u8D85\u65F6\u65F6\u95F4
 flink.watermark.max.delay.time=50
@@ -68,13 +63,13 @@ log.aggregate.duration=5
 log.aggregate.duration.graph=5
 
 #arangoDB\u53C2\u6570\u914D\u7F6E
-arangoDB.host=192.168.45.102
-arangoDB.port=8529
-arangoDB.user=root
-arangoDB.password=galaxy_2019
-arangoDB.DB.name=knowledge
-arangoDB.batch=100000
-arangoDB.ttl=3600
-thread.pool.number=10
-
-sink.batch.delay.time=5
+arangodb.host=192.168.45.102
+arangodb.port=8529
+arangodb.user=root
+arangodb.password=galaxy_2019
+arangodb.db.name=knowledge
+arangodb.batch=100000
+arangodb.ttl=3600
+arangodb.thread.pool.number=10
+#\u6279\u91CF\u7D2F\u8BA1\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2ms
+sink.arangodb.batch.delay.time=5
\ No newline at end of file