diff options
Diffstat (limited to 'TWA-PIC/hbase/conf')
| -rw-r--r-- | TWA-PIC/hbase/conf/backup-masters | 2 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/core-site.xml | 58 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/hadoop-metrics2-hbase.properties | 44 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/hbase-env.cmd | 83 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/hbase-env.sh | 143 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/hbase-policy.xml | 53 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/hbase-site.xml | 205 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/hdfs-site.xml | 142 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/log4j-hbtop.properties | 27 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/log4j.properties | 124 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/regionservers | 3 | ||||
| -rw-r--r-- | TWA-PIC/hbase/conf/yarn-site.xml | 224 |
12 files changed, 1108 insertions, 0 deletions
diff --git a/TWA-PIC/hbase/conf/backup-masters b/TWA-PIC/hbase/conf/backup-masters new file mode 100644 index 0000000..96a29ea --- /dev/null +++ b/TWA-PIC/hbase/conf/backup-masters @@ -0,0 +1,2 @@ +192.168.30.194 +192.168.30.195 diff --git a/TWA-PIC/hbase/conf/core-site.xml b/TWA-PIC/hbase/conf/core-site.xml new file mode 100644 index 0000000..62c06ae --- /dev/null +++ b/TWA-PIC/hbase/conf/core-site.xml @@ -0,0 +1,58 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + <property> + <name>fs.defaultFS</name> + <value>hdfs://ns1</value> + </property> + <property> + <name>hadoop.tmp.dir</name> + <value>file:/home/tsg/olap/hadoop/tmp</value> + </property> + <property> + <name>io.file.buffer.size</name> + <value>131702</value> + </property> + <property> + <name>hadoop.proxyuser.root.hosts</name> + <value>*</value> + </property> + <property> + <name>hadoop.proxyuser.root.groups</name> + <value>*</value> + </property> + <property> + <name>hadoop.logfile.size</name> + <value>10000000</value> + <description>The max size of each log file</description> + </property> + <property> + <name>hadoop.logfile.count</name> + <value>1</value> + <description>The max number of log files</description> + </property> + <property> + <name>ha.zookeeper.quorum</name> + <value>192.168.30.193:2181,192.168.30.194:2181,192.168.30.195:2181</value> + </property> + <property> + <name>ipc.client.connect.timeout</name> + <value>90000</value> + </property> +</configuration> diff --git a/TWA-PIC/hbase/conf/hadoop-metrics2-hbase.properties b/TWA-PIC/hbase/conf/hadoop-metrics2-hbase.properties new file mode 100644 index 0000000..4c7dbbe --- /dev/null +++ b/TWA-PIC/hbase/conf/hadoop-metrics2-hbase.properties @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +*.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink +# default sampling period +*.period=10 + +# Below are some examples of sinks that could be used +# to monitor different hbase daemons. + +# hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file-all.filename=all.metrics + +# hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file0.context=hmaster +# hbase.sink.file0.filename=master.metrics + +# hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file1.context=thrift-one +# hbase.sink.file1.filename=thrift-one.metrics + +# hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file2.context=thrift-two +# hbase.sink.file2.filename=thrift-one.metrics + +# hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink +# hbase.sink.file3.context=rest +# hbase.sink.file3.filename=rest.metrics diff --git a/TWA-PIC/hbase/conf/hbase-env.cmd b/TWA-PIC/hbase/conf/hbase-env.cmd new file mode 100644 index 0000000..1f1c3e3 --- /dev/null +++ b/TWA-PIC/hbase/conf/hbase-env.cmd @@ -0,0 +1,83 @@ +@rem/** +@rem * Licensed to the Apache Software Foundation (ASF) under one +@rem * or more contributor license agreements. See the NOTICE file +@rem * distributed with this work for additional information +@rem * regarding copyright ownership. The ASF licenses this file +@rem * to you under the Apache License, Version 2.0 (the +@rem * "License"); you may not use this file except in compliance +@rem * with the License. You may obtain a copy of the License at +@rem * +@rem * http://www.apache.org/licenses/LICENSE-2.0 +@rem * +@rem * Unless required by applicable law or agreed to in writing, software +@rem * distributed under the License is distributed on an "AS IS" BASIS, +@rem * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem * See the License for the specific language governing permissions and +@rem * limitations under the License. +@rem */ + +@rem Set environment variables here. + +@rem The java implementation to use. Java 1.8+ required. +@rem set JAVA_HOME=c:\apps\java + +@rem Extra Java CLASSPATH elements. Optional. +@rem set HBASE_CLASSPATH= + +@rem The maximum amount of heap to use. Default is left to JVM default. +@rem set HBASE_HEAPSIZE=1000 + +@rem Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +@rem offheap, set the value to "8G". +@rem set HBASE_OFFHEAPSIZE=1000 + +@rem For example, to allocate 8G of offheap, to 8G: +@rem etHBASE_OFFHEAPSIZE=8G + +@rem Extra Java runtime options. +@rem Below are what we set by default. May only work with SUN JVM. +@rem For more on why as well as other possible settings, +@rem see http://hbase.apache.org/book.html#performance +@rem JDK6 on Windows has a known bug for IPv6, use preferIPv4Stack unless JDK7. +@rem @rem See TestIPv6NIOServerSocketChannel. +set HBASE_OPTS=%HBASE_OPTS% "-XX:+UseConcMarkSweepGC" "-Djava.net.preferIPv4Stack=true" + +@rem Uncomment below to enable java garbage collection logging for the server-side processes +@rem this enables basic gc logging for the server processes to the .out file +@rem set SERVER_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" %HBASE_GC_OPTS% + +@rem this enables gc logging using automatic GC log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. Either use this set of options or the one above +@rem set SERVER_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" "-XX:+UseGCLogFileRotation" "-XX:NumberOfGCLogFiles=1" "-XX:GCLogFileSize=512M" %HBASE_GC_OPTS% + +@rem Uncomment below to enable java garbage collection logging for the client processes in the .out file. +@rem set CLIENT_GC_OPTS="-verbose:gc" "-XX:+PrintGCDetails" "-XX:+PrintGCDateStamps" %HBASE_GC_OPTS% + +@rem Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS) +@rem set HBASE_USE_GC_LOGFILE=true + +@rem Uncomment and adjust to enable JMX exporting +@rem See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +@rem More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +@rem +@rem set HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false" "-Dcom.sun.management.jmxremote.authenticate=false" +@rem set HBASE_MASTER_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10101" +@rem set HBASE_REGIONSERVER_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10102" +@rem set HBASE_THRIFT_OPTS=%HBASE_JMX_BASE% "-Dcom.sun.management.jmxremote.port=10103" +@rem set HBASE_ZOOKEEPER_OPTS=%HBASE_JMX_BASE% -Dcom.sun.management.jmxremote.port=10104" + +@rem File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +@rem set HBASE_REGIONSERVERS=%HBASE_HOME%\conf\regionservers + +@rem Where log files are stored. $HBASE_HOME/logs by default. +@rem set HBASE_LOG_DIR=%HBASE_HOME%\logs + +@rem A string representing this instance of hbase. $USER by default. +@rem set HBASE_IDENT_STRING=%USERNAME% + +@rem Seconds to sleep between slave commands. Unset by default. This +@rem can be useful in large clusters, where, e.g., slave rsyncs can +@rem otherwise arrive faster than the master can service them. +@rem set HBASE_SLAVE_SLEEP=0.1 + +@rem Tell HBase whether it should manage it's own instance of ZooKeeper or not. +@rem set HBASE_MANAGES_ZK=true diff --git a/TWA-PIC/hbase/conf/hbase-env.sh b/TWA-PIC/hbase/conf/hbase-env.sh new file mode 100644 index 0000000..b3d225e --- /dev/null +++ b/TWA-PIC/hbase/conf/hbase-env.sh @@ -0,0 +1,143 @@ +# +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# This script sets variables multiple times over the course of starting an hbase process, +# so try to keep things idempotent unless you want to take an even deeper look +# into the startup scripts (bin/hbase, etc.) + +# The java implementation to use. Java 1.7+ required. +export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_73 + +# Extra Java CLASSPATH elements. Optional. +# export HBASE_CLASSPATH= + +# The maximum amount of heap to use. Default is left to JVM default. +#export HBASE_HEAPSIZE={heap} + +# Uncomment below if you intend to use off heap cache. For example, to allocate 8G of +# offheap, set the value to "8G". +#export HBASE_OFFHEAPSIZE=5G + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="-XX:+UseConcMarkSweepGC " + +# Configure PermSize. Only needed in JDK7. You can safely remove it for JDK8+ +export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xmx20480m -Xms20480m -Xmn128m -Xss256k -XX:MetaspaceSize=512m -XX:MaxMetaspaceSize=512m -XX:SurvivorRatio=2 -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -XX:MaxTenuringThreshold=15 -XX:+UseCMSCompactAtFullCollection -XX:CMSFullGCsBeforeCompaction=1 -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:-DisableExplicitGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/home/tsg/olap/hbase-2.2.3/logs/gc-regionserver-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/tsg/olap/hbase-2.2.3/logs/" + +export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" + +export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Xmx2048m -Xms2048m -Xss256k -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -XX:SurvivorRatio=2 -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -XX:MaxTenuringThreshold=15 -XX:+UseCMSCompactAtFullCollection -XX:CMSFullGCsBeforeCompaction=1 -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70 -XX:-DisableExplicitGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/home/tsg/olap/hbase-2.2.3/logs/gc-master-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=100M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/home/tsg/olap/hbase-2.2.3/logs/ -javaagent:/home/tsg/olap/hbase-2.2.3/monitor/jmx_prometheus_javaagent-0.12.0.jar=9907:/home/tsg/olap/hbase-2.2.3/monitor/hbase.yaml" + +export HBASE_REGIONSERVER_JMX_OPTS="$HBASE_JMX_BASE -javaagent:/home/tsg/olap/hbase-2.2.3/monitor/jmx_prometheus_javaagent-0.12.0.jar=9908:/home/tsg/olap/hbase-2.2.3/monitor/hbase.yaml" + +# Uncomment one of the below three options to enable java garbage collection logging for the server-side processes. + +# This enables basic gc logging to the .out file. +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH>" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH> -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# Uncomment one of the below three options to enable java garbage collection logging for the client processes. + +# This enables basic gc logging to the .out file. +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps" + +# This enables basic gc logging to its own file. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH>" + +# This enables basic GC logging to its own file with automatic log rolling. Only applies to jdk 1.6.0_34+ and 1.7.0_2+. +# If FILE-PATH is not replaced, the log file(.gc) would still be generated in the HBASE_LOG_DIR . +# export CLIENT_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:<FILE-PATH> -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=1 -XX:GCLogFileSize=512M" + +# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations +# needed setting up off-heap block caching. + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX +# section in HBase Reference Guide for instructions. + +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" +# export HBASE_REST_OPTS="$HBASE_REST_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10105" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# Uncomment and adjust to keep all the Region Server pages mapped to be memory resident +#HBASE_REGIONSERVER_MLOCK=true +#HBASE_REGIONSERVER_UID="hbase" + +# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. +# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +export HBASE_LOG_DIR=/home/tsg/olap/hbase-2.2.3/logs + +# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +export HBASE_PID_DIR=/home/tsg/olap/hbase-2.2.3/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +export HBASE_MANAGES_ZK=false + +# The default log rolling policy is RFA, where the log file is rolled as per the size defined for the +# RFA appender. Please refer to the log4j.properties file to see more details on this appender. +# In case one needs to do log rolling on a date change, one should set the environment property +# HBASE_ROOT_LOGGER to "<DESIRED_LOG LEVEL>,DRFA". +# For example: +#HBASE_ROOT_LOGGER=INFO,DRFA +HBASE_ROOT_LOGGER=ERROR,DRFA +# The reason for changing default to RFA is to avoid the boundary case of filling out disk space as +# DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context. diff --git a/TWA-PIC/hbase/conf/hbase-policy.xml b/TWA-PIC/hbase/conf/hbase-policy.xml new file mode 100644 index 0000000..bf47240 --- /dev/null +++ b/TWA-PIC/hbase/conf/hbase-policy.xml @@ -0,0 +1,53 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration> + <property> + <name>security.client.protocol.acl</name> + <value>*</value> + <description>ACL for ClientProtocol and AdminProtocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + </property> + + <property> + <name>security.admin.protocol.acl</name> + <value>*</value> + <description>ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + </property> + + <property> + <name>security.masterregion.protocol.acl</name> + <value>*</value> + <description>ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed.</description> + </property> +</configuration> diff --git a/TWA-PIC/hbase/conf/hbase-site.xml b/TWA-PIC/hbase/conf/hbase-site.xml new file mode 100644 index 0000000..1f070ea --- /dev/null +++ b/TWA-PIC/hbase/conf/hbase-site.xml @@ -0,0 +1,205 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>hbase.rootdir</name> + <value>hdfs://ns1/hbase</value> + </property> + + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + </property> + + <property> + <name>hbase.zookeeper.quorum</name> + <value>192.168.30.193,192.168.30.194,192.168.30.195</value> + </property> + + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>2181</value> + </property> + + <property> + <name>hbase.master.info.port</name> + <value>60010</value> + </property> + +<!-- + <property> + <name>hbase.client.keyvalue.maxsize</name> + <value>1073741824</value> + </property> +--> + + <property> + <name>hbase.server.keyvalue.maxsize</name> + <value>5368709120</value> + </property> + + <property> + <name>zookeeper.znode.parent</name> + <value>/hbase</value> + </property> + + <property> + <name>hbase.rpc.timeout</name> + <value>300000</value> + </property> + + <property> + <name>zookeeper.session.timeout</name> + <value>300000</value> + </property> + + <!--小于该值的文件将在mob compaction中合并--> + <property> + <name>hbase.mob.compaction.mergeable.threshold</name> + <value>1342177280</value> + </property> + + <property> + <name>hbase.mob.file.cache.size</name> + <value>1000</value> + </property> + + <!--mob cache回收缓存周期--> + <property> + <name>hbase.mob.cache.evict.period</name> + <value>3600</value> + </property> + + <!--mob cache回收之后cache中保留文件个数比例，cache数量超过hbase.mob.file.cache.size会回收--> + <property> + <name>hbase.mob.cache.evict.remain.ratio</name> + <value>0.5f</value> + </property> + + <!--开启mob--> + <property> + <name>hfile.format.version</name> + <value>3</value> + </property> + + <property> + <name>hbase.hregion.memstore.flush.size</name> + <value>534217728</value> + </property> + + <!-- flush线程数 --> + <property> + <name>hbase.hstore.flusher.count</name> + <value>8</value> + </property> + + <property> + <name>hbase.regionserver.global.memstore.size.lower.limit</name> + <value>0.4</value> + </property> + + <property> + <name>hbase.regionserver.global.memstore.size</name> + <value>0.45</value> + </property> + + <property> + <name>hfile.block.cache.size</name> + <value>0.3</value> + </property> + + <property> + <name>hbase.hregion.memstore.block.multiplier</name> + <value>10</value> + </property> + + <property> + <name>hbase.ipc.server.max.callqueue.length</name> + <value>1073741824</value> + </property> + + <property> + <name>hbase.regionserver.handler.count</name> + <value>40</value> + <description>Count of RPC Listener instances spun up on RegionServers. + Same property is used by the Master for count of master handlers.</description> + </property> + + <property> + <name>hbase.zookeeper.property.maxClientCnxns</name> + <value>1000</value> + </property> + + <property> + <name>hbase.ipc.max.request.size</name> + <value>1173741824</value> + </property> + + <property> + <name>hbase.hstore.blockingWaitTime</name> + <value>30000</value> + </property> + <property> + <name>hbase.hstore.blockingStoreFiles</name> + <value>100</value> + </property> + + <!--split参数--> + <property> + <name>hbase.hregion.max.filesize</name> + <value>107374182400</value> + </property> + + <property> + <name>hbase.regionserver.regionSplitLimit</name> + <value>1000</value> + </property> + +<!-- phoenix --> + <property> + <name>phoenix.schema.isNamespaceMappingEnabled</name> + <value>true</value> + </property> + <property> + <name>phoenix.schema.mapSystemTablesToNamespace</name> + <value>true</value> + </property> + +<!-- RsGroup --> + <property> + <name>hbase.coprocessor.master.classes</name> + <value>org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint</value> + </property> + + <property> + <name>hbase.master.loadbalancer.class</name> + <value>org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer</value> + </property> + +<!--表region自动平衡--> + <property> + <name>hbase.master.loadbalance.bytable</name> + <value>true</value> + </property> + +</configuration> diff --git a/TWA-PIC/hbase/conf/hdfs-site.xml b/TWA-PIC/hbase/conf/hdfs-site.xml new file mode 100644 index 0000000..cf332eb --- /dev/null +++ b/TWA-PIC/hbase/conf/hdfs-site.xml @@ -0,0 +1,142 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + <property> + <name>dfs.namenode.name.dir</name> + <value>file:/home/tsg/olap/hadoop/dfs/name</value> + </property> + <property> + <name>dfs.datanode.data.dir</name> + <value>file:/home/tsg/olap/hadoop/dfs/data</value> + </property> + <property> + <name>dfs.replication</name> + <value>2</value> + </property> + <property> + <name>dfs.webhdfs.enabled</name> + <value>true</value> + </property> + <property> + <name>dfs.permissions</name> + <value>false</value> + </property> + <property> + <name>dfs.permissions.enabled</name> + <value>false</value> + </property> + <property> + <name>dfs.nameservices</name> + <value>ns1</value> + </property> + <property> + <name>dfs.blocksize</name> + <value>134217728</value> + </property> + <property> + <name>dfs.ha.namenodes.ns1</name> + <value>nn1,nn2</value> + </property> + <!-- nn1的RPC通信地址，nn1所在地址 --> + <property> + <name>dfs.namenode.rpc-address.ns1.nn1</name> + <value>192.168.30.193:9000</value> + </property> + <!-- nn1的http通信地址，外部访问地址 --> + <property> + <name>dfs.namenode.http-address.ns1.nn1</name> + <value>192.168.30.193:50070</value> + </property> + <!-- nn2的RPC通信地址，nn2所在地址 --> + <property> + <name>dfs.namenode.rpc-address.ns1.nn2</name> + <value>192.168.30.194:9000</value> + </property> + <!-- nn2的http通信地址，外部访问地址 --> + <property> + <name>dfs.namenode.http-address.ns1.nn2</name> + <value>192.168.30.194:50070</value> + </property> + <!-- 指定NameNode的元数据在JournalNode日志上的存放位置(一般和zookeeper部署在一起) --> + <property> + <name>dfs.namenode.shared.edits.dir</name> + <value>qjournal://192.168.30.193:8485;192.168.30.194:8485;192.168.30.195:8485/ns1</value> + </property> + <!-- 指定JournalNode在本地磁盘存放数据的位置 --> + <property> + <name>dfs.journalnode.edits.dir</name> + <value>/home/tsg/olap/hadoop/journal</value> + </property> + <!--客户端通过代理访问namenode，访问文件系统，HDFS 客户端与Active 节点通信的Java 类，使用其确定Active 节点是否活跃 --> + <property> + <name>dfs.client.failover.proxy.provider.ns1</name> + <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value> + </property> + <!--这是配置自动切换的方法，有多种使用方法，具体可以看官网，在文末会给地址，这里是远程登录杀死的方法 --> + <property> + <name>dfs.ha.fencing.methods</name> + <value>sshfence</value> + <value>shell(true)</value> + </property> + <!-- 这个是使用sshfence隔离机制时才需要配置ssh免登陆 --> + <property> + <name>dfs.ha.fencing.ssh.private-key-files</name> + <value>/root/.ssh/id_rsa</value> + </property> + <!-- 配置sshfence隔离机制超时时间，这个属性同上，如果你是用脚本的方法切换，这个应该是可以不配置的 --> + <property> + <name>dfs.ha.fencing.ssh.connect-timeout</name> + <value>30000</value> + </property> + <!-- 这个是开启自动故障转移，如果你没有自动故障转移，这个可以先不配 --> + <property> + <name>dfs.ha.automatic-failover.enabled</name> + <value>true</value> + </property> + <property> + <name>dfs.datanode.max.transfer.threads</name> + <value>8192</value> + </property> + <!-- namenode处理RPC请求线程数，增大该值资源占用不大 --> + <property> + <name>dfs.namenode.handler.count</name> + <value>30</value> + </property> + <!-- datanode处理RPC请求线程数，增大该值会占用更多内存 --> + <property> + <name>dfs.datanode.handler.count</name> + <value>40</value> + </property> + <!-- balance时可占用的带宽 --> + <property> + <name>dfs.balance.bandwidthPerSec</name> + <value>104857600</value> + </property> + <!-- 磁盘预留空间，该空间不会被hdfs占用，单位字节--> + <property> + <name>dfs.datanode.du.reserved</name> + <value>53687091200</value> + </property> + <!-- datanode与namenode连接超时时间，单位毫秒 2 * heartbeat.recheck.interval + 30000 --> + <property> + <name>heartbeat.recheck.interval</name> + <value>100000</value> + </property> +</configuration> + diff --git a/TWA-PIC/hbase/conf/log4j-hbtop.properties b/TWA-PIC/hbase/conf/log4j-hbtop.properties new file mode 100644 index 0000000..1c93b0f --- /dev/null +++ b/TWA-PIC/hbase/conf/log4j-hbtop.properties @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=WARN,console +log4j.threshold=WARN + +# console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ssZ}{UTC}] %-5p [%t] %c{2}: %m%n + +# ZooKeeper will still put stuff at WARN +log4j.logger.org.apache.zookeeper=ERROR diff --git a/TWA-PIC/hbase/conf/log4j.properties b/TWA-PIC/hbase/conf/log4j.properties new file mode 100644 index 0000000..b9863ee --- /dev/null +++ b/TWA-PIC/hbase/conf/log4j.properties @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Define some default values that can be overridden by system properties +hbase.root.logger=ERROR,console +hbase.security.logger=ERROR,console +hbase.log.dir=. +hbase.log.file=hbase.log +hbase.log.level=ERROR + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ssZ}] %-5p [%t] %c{2}: %.1000m%n + +# Rolling File Appender properties +hbase.log.maxfilesize=256MB +hbase.log.maxbackupindex=20 + +# Rolling File Appender +log4j.appender.RFA=org.apache.log4j.RollingFileAppender +log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file} + +log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize} +log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex} + +log4j.appender.RFA.layout=org.apache.log4j.PatternLayout +log4j.appender.RFA.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ssZ}{UTC}] %-5p [%t] %c{2}: %.1000m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +hbase.security.log.maxfilesize=256MB +hbase.security.log.maxbackupindex=20 +log4j.appender.RFAS=org.apache.log4j.RollingFileAppender +log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize} +log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex} +log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.RFAS.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ssZ}{UTC}] %p %c: %.1000m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE +#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.visibility.VisibilityController=TRACE + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=[%d{yyyy-MM-dd HH:mm:ssZ}{UTC}] %-5p [%t] %c{2}: %.1000m%n + +log4j.appender.asyncconsole=org.apache.hadoop.hbase.AsyncConsoleAppender +log4j.appender.asyncconsole.target=System.err + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=${hbase.log.level} +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=${hbase.log.level} +log4j.logger.org.apache.hadoop.hbase.META=${hbase.log.level} +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=${hbase.log.level} +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKWatcher=${hbase.log.level} +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT +# Enable this to get detailed connection error/retry logging. +# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE + + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of hbase:meta messages +# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=INFO + +# EventCounter +# Add "EventCounter" to rootlogger if you want to use this +# Uncomment the line below to add EventCounter information +# log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter + +# Prevent metrics subsystem start/stop messages (HBASE-17722) +log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN +log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN +log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN diff --git a/TWA-PIC/hbase/conf/regionservers b/TWA-PIC/hbase/conf/regionservers new file mode 100644 index 0000000..5288815 --- /dev/null +++ b/TWA-PIC/hbase/conf/regionservers @@ -0,0 +1,3 @@ +192.168.30.193 +192.168.30.194 +192.168.30.195 diff --git a/TWA-PIC/hbase/conf/yarn-site.xml b/TWA-PIC/hbase/conf/yarn-site.xml new file mode 100644 index 0000000..fd262c9 --- /dev/null +++ b/TWA-PIC/hbase/conf/yarn-site.xml @@ -0,0 +1,224 @@ +<?xml version="1.0"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<configuration> + + <property> + <name>yarn.nodemanager.aux-services</name> + <value>mapreduce_shuffle</value> + </property> + + <property> + <name>yarn.resourcemanager.ha.enabled</name> + <value>true</value> + </property> + + <!--声明两台resourcemanager的地址--> + <property> + <name>yarn.resourcemanager.cluster-id</name> + <value>rsmcluster</value> + </property> + + <property> + <name>yarn.resourcemanager.ha.rm-ids</name> + <value>rsm1,rsm2</value> + </property> + + <!-- 配置rm1--> + <!-- 配置rm1 hostname--> + <property> + <name>yarn.resourcemanager.hostname.rsm1</name> + <value>192.168.30.193</value> + </property> + + <!-- 配置rm1 web application--> + <property> + <name>yarn.resourcemanager.webapp.address.rsm1</name> + <value>192.168.30.193:8080</value> + </property> + + <!-- 配置rm1 调度端口，默认8030--> + <property> + <name>yarn.resourcemanager.scheduler.address.rsm1</name> + <value>192.168.30.193:8030</value> + </property> + + <!-- 默认端口8031--> + <property> + <name>yarn.resourcemanager.resource-tracker.address.rsm1</name> + <value>192.168.30.193:8031</value> + </property> + + <!-- 配置rm1 应用程序管理器接口的地址端口，默认8032--> + <property> + <name>yarn.resourcemanager.address.rsm1</name> + <value>192.168.30.193:8032</value> + </property> + + <!-- 配置rm1 管理端口，默认8033--> + <property> + <name>yarn.resourcemanager.admin.address.rsm1</name> + <value>192.168.30.193:8033</value> + </property> + + <property> + <name>yarn.resourcemanager.ha.admin.address.rsm1</name> + <value>192.168.30.193:23142</value> + </property> + + <!-- 配置rm2--> + <property> + <name>yarn.resourcemanager.hostname.rsm2</name> + <value>192.168.30.194</value> + </property> + + <property> + <name>yarn.resourcemanager.webapp.address.rsm2</name> + <value>192.168.30.194:8080</value> + </property> + + <property> + <name>yarn.resourcemanager.scheduler.address.rsm2</name> + <value>192.168.30.194:8030</value> + </property> + + <property> + <name>yarn.resourcemanager.resource-tracker.address.rsm2</name> + <value>192.168.30.194:8031</value> + </property> + + <property> + <name>yarn.resourcemanager.address.rsm2</name> + <value>192.168.30.194:8032</value> + </property> + + <property> + <name>yarn.resourcemanager.admin.address.rsm2</name> + <value>192.168.30.194:8033</value> + </property> + + <property> + <name>yarn.resourcemanager.ha.admin.address.rsm2</name> + <value>192.168.30.194:23142</value> + </property> + + <!--指定zookeeper集群的地址--> + <property> + <name>yarn.resourcemanager.zk-address</name> + <value>192.168.30.193:2181,192.168.30.194:2181,192.168.30.195:2181</value> + </property> + + <!--启用自动恢复，当任务进行一半，rm坏掉，就要启动自动恢复，默认是false--> + <property> + <name>yarn.resourcemanager.recovery.enabled</name> + <value>true</value> + </property> + + <!--启用Nodemanager自动恢复，默认是false--> + <property> + <name>yarn.nodemanager.recovery.enabled</name> + <value>true</value> + </property> + + <!--配置NodeManager保存运行状态的本地文件系统目录路径 --> + <property> + <name>yarn.nodemanager.recovery.dir</name> + <value>/home/tsg/olap/hadoop-2.7.1/yarn</value> + </property> + + <property> + <name>yarn.resourcemanager.store.class</name> + <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> + </property> + + <!--配置nm可用的RPC地址，默认${yarn.nodemanager.hostname}:0，为临时端口。集群重启后，nm与rm连接的端口会变化，这里指定端口，保障nm restart功能 --> + <property> + <name>yarn.nodemanager.address</name> + <value>${yarn.nodemanager.hostname}:9923</value> + </property> + + <property> + <name>yarn.log-aggregation-enable</name> + <value>true</value> + </property> + + <property> + <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name> + <value>3600</value> + </property> + + <property> + <name>yarn.nodemanager.remote-app-log-dir</name> + <value>/home/tsg/olap/hadoop-2.7.1/logs/app-logs/</value> + </property> + + <!--NM可以为容器分配的物理内存量，以MB为单位 ，默认8192--> + <property> + <name>yarn.nodemanager.resource.memory-mb</name> + <value>61440</value> + </property> + + <!-- RM上每个容器请求的最小分配，以mb为单位，默认1024--> + <property> + <name>yarn.scheduler.minimum-allocation-mb</name> + <value>1024</value> + </property> + + <!-- RM上每个容器请求的最大分配，以mb为单位，一般设置为 yarn.nodemanager.resource.memory-mb 一致，默认8192--> + <property> + <name>yarn.scheduler.maximum-allocation-mb</name> + <value>61440</value> + </property> + + <!--可为容器分配的vcore数。RM调度器在为容器分配资源时使用它。这不是用来限制YARN容器使用的物理内核的数量，默认8，一般配置为服务器cpu总核数一致 --> + <property> + <name>yarn.nodemanager.resource.cpu-vcores</name> + <value>48</value> + </property> + + <!--RM上每个容器请求的最小分配(以虚拟CPU内核为单位) ,默认1--> + <property> + <name>yarn.scheduler.minimum-allocation-vcores</name> + <value>1</value> + </property> + + <!--RM上每个容器请求的最大分配(以虚拟CPU内核为单位) ,默认32，一般配置为略小于yarn.nodemanager.resource.cpu-vcores，同时指定任务的slot不应超过该值--> + <property> + <name>yarn.scheduler.maximum-allocation-vcores</name> + <value>48</value> + </property> + + <property> + <name>yarn.nodemanager.vmem-check-enabled</name> + <value>false</value> + </property> + + <property> + <name>yarn.nodemanager.pmem-check-enabled</name> + <value>false</value> + </property> + + <!--ApplicationMaster重启次数，配置HA后默认为2，生产环境可增大该值--> + <property> + <name>yarn.resourcemanager.am.max-attempts</name> + <value>10000</value> + </property> + + <property> + <name>yarn.log.server.url</name> + <value>http://192.168.30.193:19888/jobhistory/logs</value> + </property> + +</configuration> + |