diff options
Diffstat (limited to 'MSH-PIC/deployment configuration')
| -rw-r--r-- | MSH-PIC/deployment configuration/tsg/MSH_config.zip | bin | 0 -> 5271 bytes | |||
| -rw-r--r-- | MSH-PIC/deployment configuration/tsg/components.yml | 217 | ||||
| -rw-r--r-- | MSH-PIC/deployment configuration/tsg/config.yml | 98 | ||||
| -rw-r--r-- | MSH-PIC/deployment configuration/tsg/hosts | 101 | ||||
| -rw-r--r-- | MSH-PIC/deployment configuration/tsg/services.yml | 39 |
5 files changed, 455 insertions, 0 deletions
diff --git a/MSH-PIC/deployment configuration/tsg/MSH_config.zip b/MSH-PIC/deployment configuration/tsg/MSH_config.zip Binary files differnew file mode 100644 index 0000000..ff5b492 --- /dev/null +++ b/MSH-PIC/deployment configuration/tsg/MSH_config.zip diff --git a/MSH-PIC/deployment configuration/tsg/components.yml b/MSH-PIC/deployment configuration/tsg/components.yml new file mode 100644 index 0000000..7c4d1ba --- /dev/null +++ b/MSH-PIC/deployment configuration/tsg/components.yml @@ -0,0 +1,217 @@ +#===========================Apache Zookeeper configuration===============================# +#The zookeeper JVM heap size MB,The -Xmx value must be greater than or equal to 1024 considering the running capacity。 +zookeeper_java_opt: '-Xmx4096m -Xms1024m' + +#===========================Apache Druid configuration===============================# +#Druid's MariaDB database name +mariadb_druid_database: druid + +#The maxmium size of segment data,bytes +#The sum of available disk space across these locations is set as the default value for property +#druid.server.maxSize:which controls the total size of segment data that can be assigned by the Coordinator to a Historical. +server_disk_maxsize: 5000000000000 + +#The maxmium size of cache segment data,bytes +#druid.segmentCache.locations specifies locations where segment data can be stored on the Historical. +#A greater proportion of segments can be kept in memory, allowing for better query performance. +segmentCache_max_size: 300000000000 + +#========historical configuration======== +#JVM heap size,MB +historical_mem: 512 + +#The maximum size of direct memory,MB +historical_MaxDirectMemorySize: 512 + +#Buffer size,Bytes +#druid.processing.buffer.sizeBytes,controls the size of the off-heap buffers allocated to the processing threads. +#The TopN and GroupBy queries use these buffers to store intermediate computed results. +historical_buffer_sizeBytes: 50000000 + +#druid.processing.numMergeBuffers +#The number of direct memory buffers available for merging query results. +historical_numMergeBuffers: 4 + +#druid.processing.numThreads +#The number of processing threads to have available for parallel processing of segments. +#It should generally be set to (number of cores - 1) +historical_numThreads: 5 + +#========middlemanager configuration======== +#A string of -X Java options to pass to the peon's(worker) JVM. +#druid.indexer.runner.javaOpts,JVM configuration for each task execution +middlemanager_runner_javaOpts: '-Xms512m -Xmx512m -XX:MaxDirectMemorySize=512m' + +#druid.processing.numMergeBuffers +#The number of direct memory buffers available for merging query results. +middlemanager_numMergeBuffers: 2 + +#Buffer size,Bytes +#druid.indexer.fork.property.druid.processing.buffer.sizeBytes,controls the size of the off-heap buffers allocated to the processing threads. +##The TopN and GroupBy queries use these buffers to store intermediate computed results. +middlemanager_buffer_sizeBytes: 20000000 + +#druid.indexer.fork.property.druid.processing.numThreads +#The number of processing threads to have available for parallel processing of segments. +##It should generally be set to (number of cores - 1) +middlemanager_numThreads: 1 + +#========coordinator configuration======== +#coordinator-overlord JVM heap size,MB +coordinator_mem: 512 + +#========broker configuration======== +#JVM heap size,MB +broker_mem: 512 + +#The maximum size of direct memory,MB +broker_MaxDirectMemorySize: 1024 + +#Buffer size,Bytes +##druid.processing.buffer.sizeBytes,controls the size of the off-heap buffers allocated to the processing threads. +##The TopN and GroupBy queries use these buffers to store intermediate computed results. +broker_sizeBytes: 50000000 + +#druid.processing.numMergeBuffers +#The number of direct memory buffers available for merging query results. +broker_numMergeBuffers: 6 + +#druid.processing.numThreads +#The number of processing threads to have available for parallel processing of segments. +#It should generally be set to (number of cores - 1) +broker_numThreads: 1 + +#===========================Hadoop configuration===============================# +#---------------------------------HDFS config----------------------------# +#namenode JVM heap size MB +#The -Xmx value must be greater than or equal to 512 considering the running capacity. +namenode_java_opt: '-Xmx10240m -Xms10240m' + +#datanode JVM heap size MB +#The -Xmx value must be greater than or equal to 512 considering the running capacity. +datanode_java_opt: '-Xmx5120m -Xms5120m' + +#journalnode JVM heap size MB +#The -Xmx value must be greater than or equal to 1024 considering the running capacity. +journal_java_opt: '-Xmx1024m -Xms1024m' + +#zkfc JVM heap size MB +#The -Xmx value must be greater than or equal to 512 considering the running capacity. +zkfc_java_opt: '-Xmx1024m -Xms1024m' + +#The number of server threads for the namenode. +#dfs.namenode.handler.count +#It should generally be set to 20*log2(cluster size) +namenode_handlers: 30 + +#The number of server threads for the datanode +#dfs.datanode.handler.count +datanode_handlers: 40 + +#---------------------------------Yarn config----------------------------# +#ResourceManager JVM heap size,Unit in megabytes(MB). +resource_manager_java_opt: '-Xmx2048m -Xms1024m' + +#NodeManager JVM heap size,Unit in megabytes(MB). +node_manager_java_opt: '-Xmx2048m -Xms1024m' + +#The NodeManager maximum amount of memory that can be allocated for a single container request. Unit in megabytes(MB). +nodemanager_mem: 61440 + +#The maximum number of virtual CPU cores that can be allocated for a single container request. +#It's generally equal to the number of CPU cores. +nodemanager_cores: 48 + +#parameter is used to specify the maximum percentage of resources available for applications in the cluster. +#It limits the resource percentage that can be used by the ApplicationMaster (AM). +resource_scheduler_capacity_percent: 0.5 + +#===========================HBase Configuration===============================# +#Hmaster JVM heap size,MB +hmaster_java_opt: '-Xmx2048m -Xms2048m' + +#Hregionserver JVM heap size,MB +hregion_java_opt: '-Xmx20480m -Xms20480m -Xmn128m' + +#The number of server threads for the regionserver +#It should generally be set to (number of cores - 1) +#hbase.regionserver.handler.count +regionserverhandlers: 40 + +#Maximum HStoreFile size. +#If any one of a column families' HStoreFiles has grown to exceed this value, the hosting HRegion is split in two. +hregion_max_filesize: 10737418240 + +#The HBase resource isolation function is used to group tables for storage. +#open: 1 , close: 0 +hbase_enable_rsgroup: 0 + +#===========================Apache Kafka Configuration===============================# +#Kafka JVM heap size,MB +kafka_java_opt: '-Xmx16384m -Xms4096m' + +#The minimum age of a log file to be eligible for deletion due to age +#hours,default 168 hours +log_reten_hours: 168 + +#A size-based retention policy for logs. Segments are pruned from the log unless the remaining +# segments drop below topic_max_bytes. Functions independently of log_reten_hours. +topic_max_bytes: 10737418240 + +#Record topic partition num, whose default value is the number of clusters. +#If a value is greater than the default value and is an integer multiple of it, this value can be used as the record topic partition num. +record_topic_partition: 24 +#===========================Mariadb configuration=============================== +#Buffer pool size,MB +mariadb_innodb_buffer_pool_size: 2048 + +#mariadb port +galaxy_mariadb_port: 3306 + +#Mariadb username +galaxy_mariadb_username: root + +#===========================Spark configuration===============================# +#spark worker JVM heap size,MB +spark_worker_mem: 1024 + +#The number of processing threads of worker +spark_worker_cores: 30 + +#===========================Nacos Configuration===============================# +#A string of -X Java options to the Nacos +nacos_java_opt: '-Xmx1024m -Xms1024m -Xmn256m' + +#Nacos's MaraiDB database name +mariadb_nacos_database: nacos + +#===========================Flink Configuration================================# +#JobManager JVM heap size,MB +jobmanager_memory_size: 1024 + +#taskmanager Network buffer size,MB +taskmanager_memory_network_min: 64 +taskmanager_memory_network_max: 128 + +#Taskmanager direct memory,MB +taskmanager_memory_managed_size: 10 + +#TaskManager JVM heap size,MB +taskmanager_memory_size: 1024 + +#Taskmanager JVM metaspace size,MB +taskmanager_jvm_metaspace_size: 384 + +#Taskmanager Framework Off-Heap,MB +taskmanager_memory_framework_offheap_size: 128 + +#The number of slots for taskmanager +taskmanager_numberOfTaskSlots: 1 + +#===========================Clickhouse Configuration================================# +#Clickhouse node max memory use,KB +clickhouse_max_memory: 150000000000 + +#Number of threads performing background operations in the Mutiation table engine +clickhouse_background_pool_size: 16 + diff --git a/MSH-PIC/deployment configuration/tsg/config.yml b/MSH-PIC/deployment configuration/tsg/config.yml new file mode 100644 index 0000000..184b5a1 --- /dev/null +++ b/MSH-PIC/deployment configuration/tsg/config.yml @@ -0,0 +1,98 @@ +#============================Galaxy OLAP Configuration===============================# +#The default installation location +install_path: /home/tsg/olap + +#The default data storage location,use storing application data,logs and configuration files +data_path: /home/tsg/olap + +#Are you allowed to uninstall existed componets +#Note:Used to repeatedly install or remove old version components.Only supports OLAP componets. +allowed_unload: "yes" + +#Use commas (,) to separate the network segments that the firewall allows to access +firewall_allow_network_segment: 192.168.10.0/24,192.168.20.0/24,192.168.30.0/24 + +#Name of the currently deployed data center. +#This parameter is used to configure the UI navigation domain name of the OLAP component. +data_center_name: MSH + +#============================Central Management Configuration===============================# +#Central Management MariaDB IP,used for services of galaxy-qgw-service and galaxy-report-service. +cm_mariadb_host: 127.0.0.1 + +#Central Management MariaDB Port +cm_mariadb_port: 3306 + +#Central Management MariaDB database name +cm_mariadb_database: tsg-bifang + +#CM mariadb password +#Sensitive data is encrypted since version 22.01, and clear text pin will no longer appear in configuration files. +#This is default pin,If the pin change Contact the administrator for processing. +cm_mariadb_pin: ENC(iW8ekP1SZC6v/7cfJKAqXXrjApJox+cH) + +#============================Token Configuration===============================# +#Central Management Server Token,Each environment Token is different. +#Requires communication with CM developers. +#used for services of DOS-DETECTION-APPLICATION and IP-Locate-Library. +cm_server_token: aa2bdec5518ad131f71944b13ce5c298&1& + +#Galaxy-hos-service Server Token,Each environment Token is different. +#For the Token generation method, see STEP4 in OLAP Cluster Installation. +#used for services of galaxy-hos-service/galaxy-job-service/packet_dump. +hos_server_token: f5c5186ba4874182b33b9b2b2b6e3f77 + +#Encrypt Galaxy-hos-service Server Token,Each environment Token is different. +#For the Token generation method, see STEP4 in OLAP Cluster Installation. +hos_servrt_token_encrypt: M8BbPaTywYw1/NyRY6TAVnqPzx7Nae92BVBcHoYi3pL9/o6kunHqpW3E50LO/XEL + +#============================Keepalived Configuration===============================# +#=======Gateway Keepalived Configuration=======# +#Specify the interface virtual IP address +#It used to configure high availability for OLAP query engine +#Only one IP address can be configured on the same network segment +gateway_keepalive_host: 192.168.20.252 + +#Specify the network interface to which the virtual address is assigned +gateway_keepalive_interface: em1 + +#The virtual router ID must be unique to each VRRP instance that you define +#No special requirements, this value does not need to be modified +gateway_keepalive_router_id: 61 + +#=======HOS Keepalived Configuration=======# +#Specify the interface virtual IP address +#It used to configure high availability for HOS service +#Only one IP address can be configured on the same network segment +hos_keepalive_host: 192.168.20.251 + +#Specify the network interface to which the virtual address is assigned +hos_keepalive_interface: em1 + +#The virtual router ID must be unique to each VRRP instance that you define +#No special requirements, this value does not need to be modified +hos_keepalive_router_id: 62 + +#=======Mariadb Keepalived Configuration=======# +#Specify the interface virtual IP address +#It used to configure high availability for Mariadb service +#Only one IP address can be configured on the same network segment +mariadb_keepalive_host: 127.0.0.1 + +#Specify the network interface to which the virtual address is assigned +mariadb_keepalive_interface: em1 + +#The virtual router ID must be unique to each VRRP instance that you define +#No special requirements, this value does not need to be modified +mariadb_keepalive_router_id: 63 + +#============================Kafka Configuration===============================# +#Kafka source brokers,kafkaip1:9094,kafkaip2:9094,kafkaip3:9094..... +#The default source and sink servers are the same +#It is used for getting data out of Kafka +kafka_source_servers: 192.168.20.193:9094,192.168.20.194:9094,192.168.20.195:9094 + +#Kafka sink brokers,kafkaip1:9094,kafkaip2:9094,kafkaip3:9094..... +#The default source and sink servers are the same +#It is used for sending data into kafka +kafka_sink_servers: 192.168.20.223:9094,192.168.20.224:9094,192.168.20.225:9094 diff --git a/MSH-PIC/deployment configuration/tsg/hosts b/MSH-PIC/deployment configuration/tsg/hosts new file mode 100644 index 0000000..fac59ab --- /dev/null +++ b/MSH-PIC/deployment configuration/tsg/hosts @@ -0,0 +1,101 @@ +#=================================Galaxy OLAP deployment checklist====================================== +#Mariadb,master to master. +#Maximum 2 servers. +[mariadb] + +#Include components: galaxy-job-service +#At least 2 servers +[job] + +#Include components: galaxy-qgw-service +#At least 2 servers +[qgw] + +#Include components: galaxy-report-service +#At least 2 servers +[report] + +#galaxy-hos-service,galaxy-hos-keepalive +#At least 2 servers +[hos] +192.168.20.193 +192.168.20.194 + +#Include components:galaxy-gateway-keepalive,galaxy-gateway-nginx,chproxy +#At least 2 servers,cluster servers>2,galaxy-gateway-keepalive on the first and second server. +[gateway_load_balancer] + +#Include components:galaxy-hos-keepalive and galaxy-hos-nginx +#At least 2 servers,cluster servers>2,galaxy-hos-keepalive on the first and second server. +[hos_load_balancer] +192.168.20.193 +192.168.20.194 + +#ArangoDB,only support single server deployment +[arangodb] + +#Alibaba Nacos +#At least 3 servers +[nacos] + +#Apache Zookeeper +#At least 3 servers,and it is strongly recommended that you have an odd number of servers. +[zookeeper] +192.168.20.193 +192.168.20.194 +192.168.20.195 + +#Apache Kafka +#By default,install Kafka-Manager on the first server. +[kafka] +192.168.20.193 +192.168.20.194 +192.168.20.195 + +#Apache Flink +#By default,install master process on the first and second server.All servers install worker process. +[flink] +192.168.20.193 +192.168.20.194 +192.168.20.195 + +#Apache Hadoop +[hadoop] +192.168.20.193 +192.168.20.194 +192.168.20.195 + +#Apache HBase +[hbase] +192.168.20.193 +192.168.20.194 +192.168.20.195 + +#Apache Spark +#By default,cluster servers<=3,all servers install worker process.cluster servers>3,only install master process on first server. +[spark] + +#Galaxy-Gohangout +[gohangout] +192.168.20.193 +192.168.20.194 +192.168.20.195 + +#Apache Druid +#By default,cluster servers>4,only install Query process on first and second server. +[druid] + +#Clickhouse +#By default,install Query process on first and second server. +[clickhouse] +192.168.20.193 +192.168.20.194 +192.168.20.195 + +#DPI packet_dump +[packet_dump] +192.168.20.193 + +#Superset,only support single server deployment +[superset] + diff --git a/MSH-PIC/deployment configuration/tsg/services.yml b/MSH-PIC/deployment configuration/tsg/services.yml new file mode 100644 index 0000000..aae93d1 --- /dev/null +++ b/MSH-PIC/deployment configuration/tsg/services.yml @@ -0,0 +1,39 @@ +#=================Galaxy-hos-service Configuration=================# +#JVM heap size +hos_java_opts: "-Xmx10240m -Xms10240m -Xmn512m" + +#Whether to enable SSL. +#open: 1 , close: 0 +hos_enable_ssl: 0 + +#Download files quickly,Used for HBase with a memory larger than 20GB. +#open: 1 , close: 0 +hos_quick_download: 0 + +#Nacos namespace +hos_config_namespace: MSH + +#=================Galaxy-qgw-service Configuration=================# +#JVM heap size +qgw_java_opts: "-Xmx5120m -Xms1024m" + +#Nacos namespace +qgw_config_namespace: MSH + +#=================Galaxy-job-service Configuration=================# +#JVM heap size +job_java_opts: "-Xmx2048m -Xms1024m" + +#Galaxy-job-service's MariaDB database name +mariadb_job_database: xxl_job + +#Nacos namespace +job_config_namespace: MSH + +#================Galaxy-report-service Configuration=================# +#JVM heap size +report_java_opts: "-Xmx1024m -Xms1024m" + +#Nacos namespace +report_config_namespace: MSH + |