blob: 0bf92f87021746d19ea39b37f6f8b3314359d4cf (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
|
flink.job.name=agg_traffic_file_chunk_combine
#kafka source配置
#9092为无验证 9095为ssl 9094为sasl
source.kafka.broker=192.168.44.12:9092
source.kafka.topic=TRAFFIC-FILE-STREAM-RECORD
source.kafka.group.id=test
#earliest从头开始 latest最新
source.kafka.auto.offset.reset=latest
source.kafka.session.timeout.ms=60000
#每次拉取操作从分区中获取的最大记录数
source.kafka.max.poll.records=1000
#消费者从单个分区中一次性获取的最大字节数
source.kafka.max.partition.fetch.bytes=31457280
source.kafka.enable.auto.commit=true
#kafka SASL验证用户名
source.kafka.user=admin
#kafka SASL及SSL验证密码
source.kafka.pin=galaxy2019
#SSL需要
source.kafka.tools.library=/opt/tsg/olap/topology/data/
map.filter.expression=FileChunk.offset <= 1073741824
#窗口相关配置
combiner.window.parallelism=1
#窗口大小，单位秒
combiner.window.size=30
#combiner.window.enable.last.chunk.trigger=true
#sink相关参数
sink.parallelism=1
#可选hos、oss、hbase
sink.type=hos
sink.async=true
#sink.filter.expression=
#限流配置
#sink.rate.limit.threshold=0
#sink.rate.limit.exclusion.expression=FileChunk.fileType == "eml"
#hos sink相关配置
#访问nginx或单个hos配置为ip:port；访问多个hos，配置为ip1:port,ip2:port...
sink.hos.endpoint=192.168.44.12:8186
sink.hos.bucket=traffic_file_bucket
sink.hos.token=c21f969b5f03d33d43e04f8f136e7682
#sink.hos.batch.size=1048576
#sink.hos.batch.interval.ms=10000
#http相关配置
sink.http.client.retries.number=3
sink.http.client.max.total=20
sink.http.client.max.per.route=10
sink.http.client.connect.timeout.ms=10000
sink.http.client.request.timeout.ms=10000
sink.http.client.socket.timeout.ms=60000
#hbase sink相关配置
sink.hbase.zookeeper=192.168.44.12
sink.hbase.retries.number=10
sink.hbase.rpc.timeout.ms=600000
sink.hbase.client.write.buffer=10971520
sink.hbase.client.ipc.pool.size=3
sink.hbase.batch.size=1048576
sink.hbase.batch.interval.ms=10000
#oss sink相关配置
#可以多个地址，ip1:port,ip2:port...
#sink.oss.endpoint=192.168.44.12:8186
#sink.oss.filter.expression=FileChunk.offset == 0 && FileChunk.lastChunkFlag == 1
#sink.oss.cache.time.ms=30000
#sink.oss.cache.size=1073741824
#文件关联元信息相关配置
#source.kafka.file.meta.session.topic=SESSION-RECORD
#source.kafka.file.meta.proxy.topic=PROXY-EVENT
#source.kafka.file.meta.group.id=file_chunk_combine_1
#file.meta.filter.expression=FileChunk.meta.fileId.contains("_9")
|
