diff options
| author | wanglihui <[email protected]> | 2020-11-10 16:59:39 +0800 |
|---|---|---|
| committer | wanglihui <[email protected]> | 2020-11-10 16:59:39 +0800 |
| commit | b62131dacdb8812d81db2fc59820e3071739e410 (patch) | |
| tree | b355eccbd24d306cb128ddfb125adca9a51dd009 /ip-learning-spark/src/main/resources | |
| parent | db5ca9db08227b87d2975229b9757304245de47f (diff) | |
Diffstat (limited to 'ip-learning-spark/src/main/resources')
| -rw-r--r-- | ip-learning-spark/src/main/resources/application.properties | 32 |
1 files changed, 13 insertions, 19 deletions
diff --git a/ip-learning-spark/src/main/resources/application.properties b/ip-learning-spark/src/main/resources/application.properties index c2e81ea..9b3cee4 100644 --- a/ip-learning-spark/src/main/resources/application.properties +++ b/ip-learning-spark/src/main/resources/application.properties @@ -1,45 +1,39 @@ #spark任务配置 -spark.sql.shuffle.partitions=5 +spark.sql.shuffle.partitions=10 spark.executor.memory=4g spark.app.name=test spark.network.timeout=300s -repartitionNumber=36 spark.serializer=org.apache.spark.serializer.KryoSerializer master=local[*] #spark读取clickhouse配置 -spark.read.clickhouse.url=jdbc:clickhouse://192.168.40.186:8123/tsg_galaxy_v3 +spark.read.clickhouse.url=jdbc:clickhouse://192.168.44.67:8123/tsg_galaxy_v3 spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver spark.read.clickhouse.user=default -spark.read.clickhouse.password=111111 -spark.read.clickhouse.numPartitions=144 +spark.read.clickhouse.password=ceiec2019 +spark.read.clickhouse.numPartitions=5 spark.read.clickhouse.fetchsize=10000 -spark.read.clickhouse.partitionColumn=common_recv_time +spark.read.clickhouse.partitionColumn=LAST_FOUND_TIME clickhouse.socket.timeout=300000 #arangoDB配置 arangoDB.host=192.168.40.182 arangoDB.port=8529 arangoDB.user=upsert arangoDB.password=ceiec2018 -#arangoDB.DB.name=insert_iplearn_index -arangoDB.DB.name=iplearn_media_domain +arangoDB.DB.name=ip-learning-test-0 +#arangoDB.DB.name=iplearn_media_domain arangoDB.ttl=3600 -thread.pool.number=5 +thread.pool.number=10 #读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围 -clickhouse.time.limit.type=0 -read.clickhouse.max.time=1571245220 -read.clickhouse.min.time=1571245210 +clickhouse.time.limit.type=1 +read.clickhouse.max.time=1603785961 +read.clickhouse.min.time=1603354682 -#读取arangoDB时间范围方式，0：正常读；1：指定时间范围 -arango.time.limit.type=0 -read.arango.max.time=1571245320 -read.arango.min.time=1571245200 - -arangoDB.read.limit= +arangoDB.read.limit=1 update.arango.batch=10000 distinct.client.ip.num=10000 recent.count.hour=24 -update.interval=10800 +update.interval=3600 |