#--------------------------------\u5730\u5740\u914D\u7F6E------------------------------# #\u7BA1\u7406kafka\u5730\u5740,\u591A\u53F0\u9017\u53F7\u8FDE\u63A5ip1:9094,ip2:9094 source.kafka.servers=192.168.44.12:9092 #\u7BA1\u7406\u8F93\u51FAkafka\u5730\u5740 sink.kafka.servers=192.168.44.12:9092 #--------------------------------HTTP/\u5B9A\u4F4D\u5E93/ssl------------------------------# tools.library= #--------------------------------Kafka\u6D88\u8D39/\u751F\u4EA7\u914D\u7F6E------------------------------# #\u8BFB\u53D6topic,\u5B58\u50A8\u8BE5spout id\u7684\u6D88\u8D39offset\u4FE1\u606F\uFF0C\u53EF\u901A\u8FC7\u8BE5\u62D3\u6251\u547D\u540D;\u5177\u4F53\u5B58\u50A8offset\u7684\u4F4D\u7F6E\uFF0C\u786E\u5B9A\u4E0B\u6B21\u8BFB\u53D6\u4E0D\u91CD\u590D\u7684\u6570\u636E\uFF1B group.id=KNOWLEDGE-GROUP-20220928 #--------------------------------topology\u914D\u7F6E------------------------------# #consumer \u5E76\u884C\u5EA6 source.parallelism=1 #\u8F6C\u6362\u51FD\u6570\u5E76\u884C\u5EA6 transform.parallelism=1 #kafka producer \u5E76\u884C\u5EA6 sink.parallelism=1 #--------------------------------\u4E1A\u52A1\u914D\u7F6E------------------------------# #1 connection\u65E5\u5FD7 \uFF0C2 dns\u65E5\u5FD7 log.type=2 #\u751F\u4EA7\u8005\u538B\u7F29\u6A21\u5F0F none or snappy producer.kafka.compression.type=none #kafka\u6570\u636E\u6E90topic source.kafka.topic.connection=CONNECTION-RECORD-LOG source.kafka.topic.sketch=CONNECTION-SKETCH-RECORD-LOG source.kafka.topic.dns=DNS-RECORD-LOG #kafka\u56DE\u5199\u7EDF\u8BA1\u6570\u636E sink.kafka.topic.relation.connection=CONNECTION-RELATION-LOG sink.kafka.topic.relation.dns=DNS-RELATION-LOG #\u5199\u5165clickhouse\u672C\u5730\u8868 sink.ck.table.connection=CONNECTION-RECORD-COMPLETED-LOG sink.ck.table.sketch=CONNECTION-SKETCH-RECORD-COMPLETED-LOG sink.ck.table.dns=DNS-RECORD-COMPLETED-LOG sink.ck.table.relation.connection=connection_relation_log_local sink.ck.table.relation.dns=dns_relation_log_local #\u5199arangodb\u8868 sink.arangodb.table.r.visit.ip2ip=R_VISIT_IP2IP sink.arangodb.table.r.cname.domain2domain=R_CNAME_DOMAIN2DOMAIN sink.arangodb.table.r.mx.domain2domain=R_MX_DOMAIN2DOMAIN sink.arangodb.table.r.resolve.domain2ip=R_RESOLVE_DOMAIN2IP sink.arangodb.table.r.nx.domain2domain=R_NX_DOMAIN2DOMAIN #\u5165\u5E93\u539F\u59CB\u65E5\u5FD71\uFF1Aflink\uFF0C2\uFF1Akafka , 3:\u53EA\u7EDF\u8BA1\u65E5\u5FD7 sink.ck.raw.log.insert.open=2 #clickhouse\u914D\u7F6E\uFF0C\u591A\u4E2A\u9017\u53F7\u8FDE\u63A5 ip1:8123,ip2:8123 ck.hosts=192.168.44.12:8123 # ,192.168.44.86:8123,192.168.44.87:8123 ck.database=tsg_galaxy_v3 ck.username=tsg_insert ck.pin=galaxy2019 #\u8D85\u65F6\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2 ck.connection.timeout=10000 ck.socket.timeout=600000 #clickhouse\u5165\u5E93\u6279\u91CF\u5355\u4F4D\u6761 ck.batch=100000 #clickhouse\u5165\u5E93\u524D\u79EF\u7D2F\u6279\u91CF\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2 sink.ck.batch.delay.time=30000 #flink \u65E5\u5FD7\u5EF6\u8FDF\u8D85\u65F6\u65F6\u95F4 flink.watermark.max.delay.time=60 #ck relation\u7EDF\u8BA1\u65F6\u95F4\u95F4\u9694 \u5355\u4F4Ds log.aggregate.duration=30 #arangodb \u7EDF\u8BA1\u65F6\u95F4\u95F4\u9694 \u5355\u4F4Ds log.aggregate.duration.graph=10 #arangoDB\u53C2\u6570\u914D\u7F6E arangodb.host=192.168.44.12 arangodb.port=8529 arangodb.user=root arangodb.password=galaxy_2019 arangodb.db.name=knowledge arangodb.batch=50000 arangodb.ttl=3600 arangodb.thread.pool.number=10 #\u6279\u91CF\u7D2F\u8BA1\u65F6\u95F4\u5355\u4F4D\u6BEB\u79D2ms sink.arangodb.batch.delay.time=10000 aggregate.max.value.length=18 #\u662F\u5426\u5165ip2ip\u8868 0\uFF1A\u5426\uFF0C 1:\u662F sink.arangodb.raw.log.insert.open=0