diff options
Diffstat (limited to 'MPE/kafka/bin')
60 files changed, 1849 insertions, 0 deletions
diff --git a/MPE/kafka/bin/connect-distributed.sh b/MPE/kafka/bin/connect-distributed.sh new file mode 100644 index 0000000..99cd27b --- /dev/null +++ b/MPE/kafka/bin/connect-distributed.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 [-daemon] connect-distributed.properties" + exit 1 +fi + +base_dir=$(dirname $0) + +if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties" +fi + +EXTRA_ARGS=${EXTRA_ARGS-'-name connectDistributed'} + +COMMAND=$1 +case $COMMAND in + -daemon) + EXTRA_ARGS="-daemon "$EXTRA_ARGS + shift + ;; + *) + ;; +esac + +exec $(dirname $0)/kafka-run-class.sh $EXTRA_ARGS org.apache.kafka.connect.cli.ConnectDistributed "$@" diff --git a/MPE/kafka/bin/connect-standalone.sh b/MPE/kafka/bin/connect-standalone.sh new file mode 100644 index 0000000..623562a --- /dev/null +++ b/MPE/kafka/bin/connect-standalone.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 [-daemon] connect-standalone.properties" + exit 1 +fi + +base_dir=$(dirname $0) + +if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/connect-log4j.properties" +fi + +EXTRA_ARGS=${EXTRA_ARGS-'-name connectStandalone'} + +COMMAND=$1 +case $COMMAND in + -daemon) + EXTRA_ARGS="-daemon "$EXTRA_ARGS + shift + ;; + *) + ;; +esac + +exec $(dirname $0)/kafka-run-class.sh $EXTRA_ARGS org.apache.kafka.connect.cli.ConnectStandalone "$@" diff --git a/MPE/kafka/bin/create_topic.sh b/MPE/kafka/bin/create_topic.sh new file mode 100644 index 0000000..74ef16e --- /dev/null +++ b/MPE/kafka/bin/create_topic.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +source /etc/profile + +kafka-operation.sh list > /data/tsg/olap/kafka_2.11-1.0.0/topic_list + +function compare(){ +RECORD_PARTITION=3 +res=$(( 24 % 3 )) +if [ $res = 0 ];then + if [ $RECORD_PARTITION -le "24" ];then + RECORD_PARTITION=24 + fi +fi +} + +function createTopic(){ +REPLICATION=$1 +PARTION_NUMS=$2 +TOPIC_NAME=$3 + +HAS_TOPIC=`cat /data/tsg/olap/kafka_2.11-1.0.0/topic_list | grep -wx "$TOPIC_NAME" | wc -l` + +if [ $HAS_TOPIC -eq '0' ];then + kafka-topics.sh '--create' '--zookeeper' '192.168.20.221:2181,192.168.20.222:2181,192.168.20.223:2181/kafka' '--replication-factor' $REPLICATION '--partitions' $PARTION_NUMS '--topic' $TOPIC_NAME +fi + +} + +compare + +#泛收类型日志，因量级原因不增加副本 +createTopic 1 3 TRANSACTION-RECORD +createTopic 1 3 TRANSACTION-RECORD-COMPLETED +createTopic 1 3 INTERIM-SESSION-RECORD +createTopic 1 3 INTERIM-SESSION-RECORD-COMPLETED +createTopic 1 3 SESSION-RECORD +createTopic 1 3 SESSION-RECORD-COMPLETED + +#业务相关日志，数据量较少且重要；增加副本 +createTopic 2 3 ACTIVE-DEFENCE-EVENT +createTopic 2 3 ACTIVE-DEFENCE-EVENT-COMPLETED +createTopic 2 3 DOS-EVENT +createTopic 2 3 DOS-SKETCH-RECORD +createTopic 2 3 GTPC-RECORD +createTopic 2 3 GTPC-RECORD-COMPLETED +createTopic 2 3 PROXY-EVENT +createTopic 2 3 PROXY-EVENT-COMPLETED +createTopic 2 3 RADIUS-RECORD +createTopic 2 3 RADIUS-RECORD-COMPLETED +createTopic 2 3 SECURITY-EVENT +createTopic 2 3 SECURITY-EVENT-COMPLETED +createTopic 2 3 SYS-PACKET-CAPTURE-EVENT +createTopic 2 3 SYS-PACKET-CAPTURE-EVENT-COMPLETED +createTopic 2 3 VOIP-RECORD +createTopic 2 3 VOIP-RECORD-COMPLETED +createTopic 2 3 BGP-RECORD +createTopic 2 3 BGP-RECORD-COMPLETED + +#功能端使用的Topic +createTopic 2 3 INTERNAL-RTP-RECORD +createTopic 2 3 PXY-EXCH-INTERMEDIA-CERT +createTopic 2 3 INTERNAL-PACKET-CAPTURE-EVENT +createTopic 2 3 SECURITY-PACKET-CAPTURE-RECORD + +#分析日志(预聚合） +createTopic 2 3 RADIUS-ONFF +createTopic 2 3 SYS-STORAGE +createTopic 2 3 VOIP-CONVERSATION-RECORD +createTopic 2 3 TRAFFIC-TOP-DESTINATION-IP-METRICS +createTopic 2 3 POLICY-RULE-METRICS +createTopic 2 3 NETWORK-TRAFFIC-METRICS +createTopic 2 3 TRAFFIC-TOP-METRICS diff --git a/MPE/kafka/bin/dae-kafka.sh b/MPE/kafka/bin/dae-kafka.sh new file mode 100644 index 0000000..a156076 --- /dev/null +++ b/MPE/kafka/bin/dae-kafka.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +source /etc/profile + +BASE_DIR=/data/tsg/olap +VERSION=kafka_2.11-1.0.0 + +function checkLogFiles(){ +if [ ! -d "$BASE_DIR/$VERSION/logs/reslogs/" ];then + mkdir -p $BASE_DIR/$VERSION/logs/reslogs +fi + +if [ ! -f "$BASE_DIR/$VERSION/logs/restart_sum" ];then + echo 0 > $BASE_DIR/$VERSION/logs/restart_sum +fi +} + +checkLogFiles + +function set_log(){ +OLD_NUM=`cat $BASE_DIR/$VERSION/logs/restart_sum` +RESTART_NUM=`expr $OLD_NUM + 1` +echo $RESTART_NUM > $BASE_DIR/$VERSION/logs/restart_sum +if [ $OLD_NUM -eq "0" ];then + echo "`date "+%Y-%m-%d %H:%M:%S"` - kafka服务初次启动" >> $BASE_DIR/$VERSION/logs/restart.log +else + RESLOG_NAME=restart_log_`date "+%Y%m%d_%H%M%S"` + echo "`date "+%Y-%m-%d %H:%M:%S"` - kafka服务异常重启 - 重启次数 -> $RESTART_NUM ;错误日志归纳文件路径:$BASE_DIR/$VERSION/reslogs/$RESLOG_NAME" >> $BASE_DIR/$VERSION/logs/restart.log + tail -n 50000 $BASE_DIR/$VERSION/logs/server.log | egrep "ERROR|WARN" >> $BASE_DIR/$VERSION/logs/reslogs/$RESLOG_NAME +fi +} + + +while true ; do +PROCESS=`jps | grep -w Kafka | grep -v grep |wc -l` +PORT=`netstat -anlp | egrep "9092|9094|9095" | grep "LISTEN" | wc -l` + +if [ $PORT -ne "3" ];then + if [ $PROCESS -lt "1" ];then + JMX_PORT=9191 nohup $BASE_DIR/$VERSION/bin/kafka-server-start.sh $BASE_DIR/$VERSION/config/server.properties > /dev/null 2>&1 & + set_log + fi +#else +# echo "`date "+%Y-%m-%d %H:%M:%S"` - Kafka端口未监听进程存在，判断为僵尸进程，开始kill本机Kafka进程" >> $BASE_DIR/$VERSION/logs/restart.log +# jps | grep Kafka | awk '{print $1}' | xargs kill -9 +fi + +sleep 60 +done diff --git a/MPE/kafka/bin/kafka-acls.sh b/MPE/kafka/bin/kafka-acls.sh new file mode 100644 index 0000000..8fa6554 --- /dev/null +++ b/MPE/kafka/bin/kafka-acls.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.AclCommand "$@" diff --git a/MPE/kafka/bin/kafka-broker-api-versions.sh b/MPE/kafka/bin/kafka-broker-api-versions.sh new file mode 100644 index 0000000..4f560a0 --- /dev/null +++ b/MPE/kafka/bin/kafka-broker-api-versions.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.BrokerApiVersionsCommand "$@" diff --git a/MPE/kafka/bin/kafka-configs.sh b/MPE/kafka/bin/kafka-configs.sh new file mode 100644 index 0000000..2f9eb8c --- /dev/null +++ b/MPE/kafka/bin/kafka-configs.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.ConfigCommand "$@" diff --git a/MPE/kafka/bin/kafka-console-consumer.sh b/MPE/kafka/bin/kafka-console-consumer.sh new file mode 100644 index 0000000..f30e891 --- /dev/null +++ b/MPE/kafka/bin/kafka-console-consumer.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then +export KAFKA_HEAP_OPTS="-Djava.security.auth.login.config=/data/tsg/olap/kafka_2.11-1.0.0/config/kafka_client_jaas.conf -Xmx512M" +# export KAFKA_HEAP_OPTS="-Xmx512M" +fi + +exec $(dirname $0)/kafka-run-class.sh kafka.tools.ConsoleConsumer "$@" diff --git a/MPE/kafka/bin/kafka-console-producer.sh b/MPE/kafka/bin/kafka-console-producer.sh new file mode 100644 index 0000000..1cf224f --- /dev/null +++ b/MPE/kafka/bin/kafka-console-producer.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then +export KAFKA_HEAP_OPTS="-Djava.security.auth.login.config=/data/tsg/olap/kafka_2.11-1.0.0/config/kafka_client_jaas.conf -Xmx512M" +# export KAFKA_HEAP_OPTS="-Xmx512M" +fi +exec $(dirname $0)/kafka-run-class.sh kafka.tools.ConsoleProducer "$@" diff --git a/MPE/kafka/bin/kafka-consumer-groups.sh b/MPE/kafka/bin/kafka-consumer-groups.sh new file mode 100644 index 0000000..feb063d --- /dev/null +++ b/MPE/kafka/bin/kafka-consumer-groups.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.ConsumerGroupCommand "$@" diff --git a/MPE/kafka/bin/kafka-consumer-perf-test.sh b/MPE/kafka/bin/kafka-consumer-perf-test.sh new file mode 100644 index 0000000..77cda72 --- /dev/null +++ b/MPE/kafka/bin/kafka-consumer-perf-test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx512M" +fi +exec $(dirname $0)/kafka-run-class.sh kafka.tools.ConsumerPerformance "$@" diff --git a/MPE/kafka/bin/kafka-delete-records.sh b/MPE/kafka/bin/kafka-delete-records.sh new file mode 100644 index 0000000..8726f91 --- /dev/null +++ b/MPE/kafka/bin/kafka-delete-records.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.DeleteRecordsCommand "$@" diff --git a/MPE/kafka/bin/kafka-log-dirs.sh b/MPE/kafka/bin/kafka-log-dirs.sh new file mode 100644 index 0000000..dc16edc --- /dev/null +++ b/MPE/kafka/bin/kafka-log-dirs.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.LogDirsCommand "$@" diff --git a/MPE/kafka/bin/kafka-mirror-maker.sh b/MPE/kafka/bin/kafka-mirror-maker.sh new file mode 100644 index 0000000..981f271 --- /dev/null +++ b/MPE/kafka/bin/kafka-mirror-maker.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.tools.MirrorMaker "$@" diff --git a/MPE/kafka/bin/kafka-operation.sh b/MPE/kafka/bin/kafka-operation.sh new file mode 100644 index 0000000..eeff1b1 --- /dev/null +++ b/MPE/kafka/bin/kafka-operation.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +source /etc/profile +BASE_DIR=$KAFKA_HOME +IP_LIST=192.168.20.223:9094 +ZK_LISR=192.168.20.221:2181,192.168.20.222:2181,192.168.20.223:2181/kafka +partitions=1 +case $1 in + producer) + exec $BASE_DIR/bin/kafka-console-producer.sh '--broker-list' $IP_LIST '--topic' $2 '--producer.config' /data/tsg/olap/kafka_2.11-1.0.0/config/sasl-config.properties + ;; + consumer) + exec $BASE_DIR/bin/kafka-console-consumer.sh '--bootstrap-server' $IP_LIST '--topic' $2 '--consumer.config' /data/tsg/olap/kafka_2.11-1.0.0/config/sasl-config.properties + ;; + consumer-begin) + exec $BASE_DIR/bin/kafka-console-consumer.sh '--bootstrap-server' $IP_LIST '--topic' $2 '--from-beginning' '--consumer.config' /data/tsg/olap/kafka_2.11-1.0.0/config/sasl-config.properties + ;; + create) + exec $BASE_DIR/bin/kafka-topics.sh '--create' '--zookeeper' $ZK_LISR '--replication-factor' 1 '--partitions' $partitions '--topic' $2 + ;; + list) + exec $BASE_DIR/bin/kafka-topics.sh '--list' '--zookeeper' $ZK_LISR + ;; + start) + JMX_PORT=9191 nohup $BASE_DIR/bin/kafka-server-start.sh $BASE_DIR/config/server.properties > /dev/null 2>&1 & + ;; + stop) + exec $BASE_DIR/bin/kafka-server-stop.sh + ;; + group) + exec $BASE_DIR/bin/kafka-consumer-groups.sh '--bootstrap-server' $IP_LIST '--describe' '--group' $2 + ;; + election-leader) + exec $BASE_DIR/bin/kafka-preferred-replica-election.sh '--zookeeper' $ZK_LISR + ;; + *) + echo 'Usage: kafka-operation.sh {producer|consumer|consumer-begin|create} {topic-name}' + echo 'Status: kafka-operation.sh {start|stop|list}' + echo 'maintenance: kafka-operation.sh {election-leader}' +esac diff --git a/MPE/kafka/bin/kafka-preferred-replica-election.sh b/MPE/kafka/bin/kafka-preferred-replica-election.sh new file mode 100644 index 0000000..638a92a --- /dev/null +++ b/MPE/kafka/bin/kafka-preferred-replica-election.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.PreferredReplicaLeaderElectionCommand "$@" diff --git a/MPE/kafka/bin/kafka-producer-perf-test.sh b/MPE/kafka/bin/kafka-producer-perf-test.sh new file mode 100644 index 0000000..73a6288 --- /dev/null +++ b/MPE/kafka/bin/kafka-producer-perf-test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx512M" +fi +exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance "$@" diff --git a/MPE/kafka/bin/kafka-reassign-partitions.sh b/MPE/kafka/bin/kafka-reassign-partitions.sh new file mode 100644 index 0000000..4c7f1bc --- /dev/null +++ b/MPE/kafka/bin/kafka-reassign-partitions.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.ReassignPartitionsCommand "$@" diff --git a/MPE/kafka/bin/kafka-replay-log-producer.sh b/MPE/kafka/bin/kafka-replay-log-producer.sh new file mode 100644 index 0000000..bba3241 --- /dev/null +++ b/MPE/kafka/bin/kafka-replay-log-producer.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.tools.ReplayLogProducer "$@" diff --git a/MPE/kafka/bin/kafka-replica-verification.sh b/MPE/kafka/bin/kafka-replica-verification.sh new file mode 100644 index 0000000..4960836 --- /dev/null +++ b/MPE/kafka/bin/kafka-replica-verification.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.tools.ReplicaVerificationTool "$@" diff --git a/MPE/kafka/bin/kafka-run-class.sh b/MPE/kafka/bin/kafka-run-class.sh new file mode 100644 index 0000000..bc765de --- /dev/null +++ b/MPE/kafka/bin/kafka-run-class.sh @@ -0,0 +1,272 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 [-daemon] [-name servicename] [-loggc] classname [opts]" + exit 1 +fi + +# CYGINW == 1 if Cygwin is detected, else 0. +if [[ $(uname -a) =~ "CYGWIN" ]]; then + CYGWIN=1 +else + CYGWIN=0 +fi + +if [ -z "$INCLUDE_TEST_JARS" ]; then + INCLUDE_TEST_JARS=false +fi + +# Exclude jars not necessary for running commands. +regex="(-(test|src|scaladoc|javadoc)\.jar|jar.asc)$" +should_include_file() { + if [ "$INCLUDE_TEST_JARS" = true ]; then + return 0 + fi + file=$1 + if [ -z "$(echo "$file" | egrep "$regex")" ] ; then + return 0 + else + return 1 + fi +} + +base_dir=$(dirname $0)/.. + +if [ -z "$SCALA_VERSION" ]; then + SCALA_VERSION=2.11.11 +fi + +if [ -z "$SCALA_BINARY_VERSION" ]; then + SCALA_BINARY_VERSION=$(echo $SCALA_VERSION | cut -f 1-2 -d '.') +fi + +# run ./gradlew copyDependantLibs to get all dependant jars in a local dir +shopt -s nullglob +for dir in "$base_dir"/core/build/dependant-libs-${SCALA_VERSION}*; +do + if [ -z "$CLASSPATH" ] ; then + CLASSPATH="$dir/*" + else + CLASSPATH="$CLASSPATH:$dir/*" + fi +done + +for file in "$base_dir"/examples/build/libs/kafka-examples*.jar; +do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi +done + +for file in "$base_dir"/clients/build/libs/kafka-clients*.jar; +do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi +done + +for file in "$base_dir"/streams/build/libs/kafka-streams*.jar; +do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi +done + +for file in "$base_dir"/streams/examples/build/libs/kafka-streams-examples*.jar; +do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi +done + +for file in "$base_dir"/streams/build/dependant-libs-${SCALA_VERSION}/rocksdb*.jar; +do + CLASSPATH="$CLASSPATH":"$file" +done + +for file in "$base_dir"/tools/build/libs/kafka-tools*.jar; +do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi +done + +for dir in "$base_dir"/tools/build/dependant-libs-${SCALA_VERSION}*; +do + CLASSPATH="$CLASSPATH:$dir/*" +done + +for cc_pkg in "api" "transforms" "runtime" "file" "json" "tools" +do + for file in "$base_dir"/connect/${cc_pkg}/build/libs/connect-${cc_pkg}*.jar; + do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi + done + if [ -d "$base_dir/connect/${cc_pkg}/build/dependant-libs" ] ; then + CLASSPATH="$CLASSPATH:$base_dir/connect/${cc_pkg}/build/dependant-libs/*" + fi +done + +# classpath addition for release +for file in "$base_dir"/libs/*; +do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi +done + +for file in "$base_dir"/core/build/libs/kafka_${SCALA_BINARY_VERSION}*.jar; +do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi +done +shopt -u nullglob + +if [ -z "$CLASSPATH" ] ; then + echo "Classpath is empty. Please build the project first e.g. by running './gradlew jar -Pscala_version=$SCALA_VERSION'" + exit 1 +fi + +# JMX settings +if [ -z "$KAFKA_JMX_OPTS" ]; then + KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " +fi + +# JMX port to use +if [ $JMX_PORT ]; then + KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Dcom.sun.management.jmxremote.port=$JMX_PORT " +fi + +# Log directory to use +if [ "x$LOG_DIR" = "x" ]; then + LOG_DIR="$base_dir/logs" +fi + +# Log4j settings +if [ -z "$KAFKA_LOG4J_OPTS" ]; then + # Log to console. This is a tool. + LOG4J_DIR="$base_dir/config/tools-log4j.properties" + # If Cygwin is detected, LOG4J_DIR is converted to Windows format. + (( CYGWIN )) && LOG4J_DIR=$(cygpath --path --mixed "${LOG4J_DIR}") + KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:${LOG4J_DIR}" +else + # create logs directory + if [ ! -d "$LOG_DIR" ]; then + mkdir -p "$LOG_DIR" + fi +fi + +# If Cygwin is detected, LOG_DIR is converted to Windows format. +(( CYGWIN )) && LOG_DIR=$(cygpath --path --mixed "${LOG_DIR}") +KAFKA_LOG4J_OPTS="-Dkafka.logs.dir=$LOG_DIR $KAFKA_LOG4J_OPTS" + +# Generic jvm settings you want to add +if [ -z "$KAFKA_OPTS" ]; then + KAFKA_OPTS="" +fi + +# Set Debug options if enabled +if [ "x$KAFKA_DEBUG" != "x" ]; then + + # Use default ports + DEFAULT_JAVA_DEBUG_PORT="5005" + + if [ -z "$JAVA_DEBUG_PORT" ]; then + JAVA_DEBUG_PORT="$DEFAULT_JAVA_DEBUG_PORT" + fi + + # Use the defaults if JAVA_DEBUG_OPTS was not set + DEFAULT_JAVA_DEBUG_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=${DEBUG_SUSPEND_FLAG:-n},address=$JAVA_DEBUG_PORT" + if [ -z "$JAVA_DEBUG_OPTS" ]; then + JAVA_DEBUG_OPTS="$DEFAULT_JAVA_DEBUG_OPTS" + fi + + echo "Enabling Java debug options: $JAVA_DEBUG_OPTS" + KAFKA_OPTS="$JAVA_DEBUG_OPTS $KAFKA_OPTS" +fi + +# Which java to use +if [ -z "$JAVA_HOME" ]; then + JAVA="java" +else + JAVA="$JAVA_HOME/bin/java" +fi + +# Memory options +if [ -z "$KAFKA_HEAP_OPTS" ]; then + KAFKA_HEAP_OPTS="-Xmx256M" +fi + +# JVM performance options +if [ -z "$KAFKA_JVM_PERFORMANCE_OPTS" ]; then +# KAFKA_JVM_PERFORMANCE_OPTS="-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -Djava.awt.headless=true" + KAFKA_JVM_PERFORMANCE_OPTS="-server -XX:MetaspaceSize=96m -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1HeapRegionSize=16M -XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80 -XX:+ExplicitGCInvokesConcurrent -Djava.awt.headless=true" +fi + + +while [ $# -gt 0 ]; do + COMMAND=$1 + case $COMMAND in + -name) + DAEMON_NAME=$2 + CONSOLE_OUTPUT_FILE=$LOG_DIR/$DAEMON_NAME.out + shift 2 + ;; + -loggc) + if [ -z "$KAFKA_GC_LOG_OPTS" ]; then + GC_LOG_ENABLED="true" + fi + shift + ;; + -daemon) + DAEMON_MODE="true" + shift + ;; + *) + break + ;; + esac +done + +# GC options +GC_FILE_SUFFIX='-gc.log' +GC_LOG_FILE_NAME='' +if [ "x$GC_LOG_ENABLED" = "xtrue" ]; then + GC_LOG_FILE_NAME=$DAEMON_NAME$GC_FILE_SUFFIX + # the first segment of the version number, which is '1' for releases before Java 9 + # it then becomes '9', '10', ... + JAVA_MAJOR_VERSION=$($JAVA -version 2>&1 | sed -E -n 's/.* version "([^.-]*).*"/\1/p') + if [[ "$JAVA_MAJOR_VERSION" -ge "9" ]] ; then + KAFKA_GC_LOG_OPTS="-Xlog:gc*:file=$LOG_DIR/$GC_LOG_FILE_NAME:time,tags:filecount=10,filesize=102400" + else + KAFKA_GC_LOG_OPTS="-Xloggc:$LOG_DIR/$GC_LOG_FILE_NAME -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M" + fi +fi + +# If Cygwin is detected, classpath is converted to Windows format. +(( CYGWIN )) && CLASSPATH=$(cygpath --path --mixed "${CLASSPATH}") + +# Launch mode +if [ "x$DAEMON_MODE" = "xtrue" ]; then + nohup $JAVA $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS $KAFKA_JMX_OPTS $KAFKA_LOG4J_OPTS -cp $CLASSPATH $KAFKA_OPTS "$@" > "$CONSOLE_OUTPUT_FILE" 2>&1 < /dev/null & +else + exec $JAVA $KAFKA_HEAP_OPTS $KAFKA_JVM_PERFORMANCE_OPTS $KAFKA_GC_LOG_OPTS $KAFKA_JMX_OPTS $KAFKA_LOG4J_OPTS -cp $CLASSPATH $KAFKA_OPTS "$@" +fi diff --git a/MPE/kafka/bin/kafka-server-start.sh b/MPE/kafka/bin/kafka-server-start.sh new file mode 100644 index 0000000..b5ea4b3 --- /dev/null +++ b/MPE/kafka/bin/kafka-server-start.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export KAFKA_OPTS="-Djava.security.auth.login.config=/data/tsg/olap/kafka_2.11-1.0.0/config/kafka_server_jaas.conf -javaagent:/data/tsg/olap/kafka_2.11-1.0.0/monitor/jmx_prometheus_javaagent-0.12.0.jar=9901:/data/tsg/olap/kafka_2.11-1.0.0/monitor/kafka.yml" + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 [-daemon] server.properties [--override property=value]*" + exit 1 +fi +base_dir=$(dirname $0) + +if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" +fi + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx16384m -Xms4096m" +fi + +EXTRA_ARGS=${EXTRA_ARGS-'-name kafkaServer -loggc'} + +COMMAND=$1 +case $COMMAND in + -daemon) + EXTRA_ARGS="-daemon "$EXTRA_ARGS + shift + ;; + *) + ;; +esac + +exec $base_dir/kafka-run-class.sh $EXTRA_ARGS kafka.Kafka "$@" diff --git a/MPE/kafka/bin/kafka-server-start.sh.bak b/MPE/kafka/bin/kafka-server-start.sh.bak new file mode 100644 index 0000000..5a53126 --- /dev/null +++ b/MPE/kafka/bin/kafka-server-start.sh.bak @@ -0,0 +1,44 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 [-daemon] server.properties [--override property=value]*" + exit 1 +fi +base_dir=$(dirname $0) + +if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" +fi + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx1G -Xms1G" +fi + +EXTRA_ARGS=${EXTRA_ARGS-'-name kafkaServer -loggc'} + +COMMAND=$1 +case $COMMAND in + -daemon) + EXTRA_ARGS="-daemon "$EXTRA_ARGS + shift + ;; + *) + ;; +esac + +exec $base_dir/kafka-run-class.sh $EXTRA_ARGS kafka.Kafka "$@" diff --git a/MPE/kafka/bin/kafka-server-stop.sh b/MPE/kafka/bin/kafka-server-stop.sh new file mode 100644 index 0000000..d3c660c --- /dev/null +++ b/MPE/kafka/bin/kafka-server-stop.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +PIDS=$(ps ax | grep -i 'kafka\.Kafka' | grep java | grep -v grep | awk '{print $1}') + +if [ -z "$PIDS" ]; then + echo "No kafka server to stop" + exit 1 +else + kill -s TERM $PIDS +fi + diff --git a/MPE/kafka/bin/kafka-simple-consumer-shell.sh b/MPE/kafka/bin/kafka-simple-consumer-shell.sh new file mode 100644 index 0000000..27e386a --- /dev/null +++ b/MPE/kafka/bin/kafka-simple-consumer-shell.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.tools.SimpleConsumerShell "$@" diff --git a/MPE/kafka/bin/kafka-streams-application-reset.sh b/MPE/kafka/bin/kafka-streams-application-reset.sh new file mode 100644 index 0000000..3363732 --- /dev/null +++ b/MPE/kafka/bin/kafka-streams-application-reset.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx512M" +fi + +exec $(dirname $0)/kafka-run-class.sh kafka.tools.StreamsResetter "$@" diff --git a/MPE/kafka/bin/kafka-topics.sh b/MPE/kafka/bin/kafka-topics.sh new file mode 100644 index 0000000..ad6a2d4 --- /dev/null +++ b/MPE/kafka/bin/kafka-topics.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.TopicCommand "$@" diff --git a/MPE/kafka/bin/kafka-verifiable-consumer.sh b/MPE/kafka/bin/kafka-verifiable-consumer.sh new file mode 100644 index 0000000..852847d --- /dev/null +++ b/MPE/kafka/bin/kafka-verifiable-consumer.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx512M" +fi +exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.VerifiableConsumer "$@" diff --git a/MPE/kafka/bin/kafka-verifiable-producer.sh b/MPE/kafka/bin/kafka-verifiable-producer.sh new file mode 100644 index 0000000..b59bae7 --- /dev/null +++ b/MPE/kafka/bin/kafka-verifiable-producer.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx512M" +fi +exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.VerifiableProducer "$@" diff --git a/MPE/kafka/bin/kflogdelete.sh b/MPE/kafka/bin/kflogdelete.sh new file mode 100644 index 0000000..3bdc891 --- /dev/null +++ b/MPE/kafka/bin/kflogdelete.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +#只保留最近30天的日志 +#将此脚本加载到系统定时任务中 /etc/crontab +#脚本会读取环境变量，固需要配置环境变量。 +source /etc/profile + +kafka_dir=/data/tsg/olap/kafka_2.11-1.0.0 + +find /data/tsg/olap/kafka_2.11-1.0.0/logs/ -mtime +30 -name "*.log.*" -exec rm -rf {} \; + diff --git a/MPE/kafka/bin/set_kafka_env.sh b/MPE/kafka/bin/set_kafka_env.sh new file mode 100644 index 0000000..b2caf0e --- /dev/null +++ b/MPE/kafka/bin/set_kafka_env.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo -e "\n#kafka\nexport KAFKA_HOME=/data/tsg/olap/kafka_2.11-1.0.0\nexport PATH=\$KAFKA_HOME/bin:\$PATH" >> /etc/profile.d/kafka.sh +chmod +x /etc/profile.d/kafka.sh +source /etc/profile + +keeppath='/etc/init.d/keepkafalive' +if [ -x $keeppath ];then + chkconfig --add keepkafalive + chkconfig keepkafalive on + service keepkafalive start && sleep 5 + kafka_dae=`ps -ef | grep dae-kafka.sh | grep -v grep | wc -l` + if [ $kafka_dae -eq "0" ];then + nohup /data/tsg/olap/kafka_2.11-1.0.0/bin/dae-kafka.sh > /dev/null 2>&1 & + fi +fi + diff --git a/MPE/kafka/bin/trogdor.sh b/MPE/kafka/bin/trogdor.sh new file mode 100644 index 0000000..b211209 --- /dev/null +++ b/MPE/kafka/bin/trogdor.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +usage() { + cat <<EOF +The Trogdor fault injector. + +Usage: + $0 [action] [options] + +Actions: + agent: Run the trogdor agent. + coordinator: Run the trogdor coordinator. + client: Run the client which communicates with the trogdor coordinator. + agent-client: Run the client which communicates with the trogdor agent. + help: This help message. +EOF +} + +if [[ $# -lt 1 ]]; then + usage + exit 0 +fi +action="${1}" +shift +CLASS="" +case ${action} in + agent) CLASS="org.apache.kafka.trogdor.agent.Agent";; + coordinator) CLASS="org.apache.kafka.trogdor.coordinator.Coordinator";; + client) CLASS="org.apache.kafka.trogdor.coordinator.CoordinatorClient";; + agent-client) CLASS="org.apache.kafka.trogdor.agent.AgentClient";; + help) usage; exit 0;; + *) echo "Unknown action '${action}'. Type '$0 help' for help."; exit 1;; +esac + +export INCLUDE_TEST_JARS=1 +exec $(dirname $0)/kafka-run-class.sh "${CLASS}" "$@" diff --git a/MPE/kafka/bin/windows/connect-distributed.bat b/MPE/kafka/bin/windows/connect-distributed.bat new file mode 100644 index 0000000..aaa3c41 --- /dev/null +++ b/MPE/kafka/bin/windows/connect-distributed.bat @@ -0,0 +1,34 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +IF [%1] EQU [] ( + echo USAGE: %0 connect-distributed.properties + EXIT /B 1 +) + +SetLocal +rem Using pushd popd to set BASE_DIR to the absolute path +pushd %~dp0..\.. +set BASE_DIR=%CD% +popd + +rem Log4j settings +IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] ( + set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/tools-log4j.properties +) + +"%~dp0kafka-run-class.bat" org.apache.kafka.connect.cli.ConnectDistributed %* +EndLocal diff --git a/MPE/kafka/bin/windows/connect-standalone.bat b/MPE/kafka/bin/windows/connect-standalone.bat new file mode 100644 index 0000000..54cc11f --- /dev/null +++ b/MPE/kafka/bin/windows/connect-standalone.bat @@ -0,0 +1,34 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +IF [%1] EQU [] ( + echo USAGE: %0 connect-standalone.properties + EXIT /B 1 +) + +SetLocal +rem Using pushd popd to set BASE_DIR to the absolute path +pushd %~dp0..\.. +set BASE_DIR=%CD% +popd + +rem Log4j settings +IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] ( + set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/tools-log4j.properties +) + +"%~dp0kafka-run-class.bat" org.apache.kafka.connect.cli.ConnectStandalone %* +EndLocal diff --git a/MPE/kafka/bin/windows/kafka-acls.bat b/MPE/kafka/bin/windows/kafka-acls.bat new file mode 100644 index 0000000..8f0be85 --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-acls.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.admin.AclCommand %* diff --git a/MPE/kafka/bin/windows/kafka-broker-api-versions.bat b/MPE/kafka/bin/windows/kafka-broker-api-versions.bat new file mode 100644 index 0000000..f7ec72d --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-broker-api-versions.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +%~dp0kafka-run-class.bat kafka.admin.BrokerApiVersionsCommand %* diff --git a/MPE/kafka/bin/windows/kafka-configs.bat b/MPE/kafka/bin/windows/kafka-configs.bat new file mode 100644 index 0000000..3792a5d --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-configs.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.admin.ConfigCommand %* diff --git a/MPE/kafka/bin/windows/kafka-console-consumer.bat b/MPE/kafka/bin/windows/kafka-console-consumer.bat new file mode 100644 index 0000000..bbbd336 --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-console-consumer.bat @@ -0,0 +1,20 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +SetLocal +set KAFKA_HEAP_OPTS=-Xmx512M +"%~dp0kafka-run-class.bat" kafka.tools.ConsoleConsumer %* +EndLocal diff --git a/MPE/kafka/bin/windows/kafka-console-producer.bat b/MPE/kafka/bin/windows/kafka-console-producer.bat new file mode 100644 index 0000000..e1834bc --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-console-producer.bat @@ -0,0 +1,20 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +SetLocal +set KAFKA_HEAP_OPTS=-Xmx512M +"%~dp0kafka-run-class.bat" kafka.tools.ConsoleProducer %* +EndLocal diff --git a/MPE/kafka/bin/windows/kafka-consumer-groups.bat b/MPE/kafka/bin/windows/kafka-consumer-groups.bat new file mode 100644 index 0000000..e027b9e --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-consumer-groups.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.admin.ConsumerGroupCommand %* diff --git a/MPE/kafka/bin/windows/kafka-consumer-offset-checker.bat b/MPE/kafka/bin/windows/kafka-consumer-offset-checker.bat new file mode 100644 index 0000000..2baa1b8 --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-consumer-offset-checker.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.tools.ConsumerOffsetChecker %* diff --git a/MPE/kafka/bin/windows/kafka-consumer-perf-test.bat b/MPE/kafka/bin/windows/kafka-consumer-perf-test.bat new file mode 100644 index 0000000..606c784 --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-consumer-perf-test.bat @@ -0,0 +1,20 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +SetLocal +set KAFKA_HEAP_OPTS=-Xmx512M -Xms512M +"%~dp0kafka-run-class.bat" kafka.tools.ConsumerPerformance %* +EndLocal diff --git a/MPE/kafka/bin/windows/kafka-mirror-maker.bat b/MPE/kafka/bin/windows/kafka-mirror-maker.bat new file mode 100644 index 0000000..a1fae45 --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-mirror-maker.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.tools.MirrorMaker %* diff --git a/MPE/kafka/bin/windows/kafka-preferred-replica-election.bat b/MPE/kafka/bin/windows/kafka-preferred-replica-election.bat new file mode 100644 index 0000000..f9f0014 --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-preferred-replica-election.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.admin.PreferredReplicaLeaderElectionCommand %* diff --git a/MPE/kafka/bin/windows/kafka-producer-perf-test.bat b/MPE/kafka/bin/windows/kafka-producer-perf-test.bat new file mode 100644 index 0000000..917d211 --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-producer-perf-test.bat @@ -0,0 +1,20 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +SetLocal +set KAFKA_HEAP_OPTS=-Xmx512M +"%~dp0kafka-run-class.bat" org.apache.kafka.tools.ProducerPerformance %* +EndLocal diff --git a/MPE/kafka/bin/windows/kafka-reassign-partitions.bat b/MPE/kafka/bin/windows/kafka-reassign-partitions.bat new file mode 100644 index 0000000..62b710d --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-reassign-partitions.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.admin.ReassignPartitionsCommand %* diff --git a/MPE/kafka/bin/windows/kafka-replay-log-producer.bat b/MPE/kafka/bin/windows/kafka-replay-log-producer.bat new file mode 100644 index 0000000..7b51302 --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-replay-log-producer.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.tools.ReplayLogProducer %* diff --git a/MPE/kafka/bin/windows/kafka-replica-verification.bat b/MPE/kafka/bin/windows/kafka-replica-verification.bat new file mode 100644 index 0000000..bf4805d --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-replica-verification.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.tools.ReplicaVerificationTool %* diff --git a/MPE/kafka/bin/windows/kafka-run-class.bat b/MPE/kafka/bin/windows/kafka-run-class.bat new file mode 100644 index 0000000..c56f82c --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-run-class.bat @@ -0,0 +1,191 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +setlocal enabledelayedexpansion + +IF [%1] EQU [] ( + echo USAGE: %0 classname [opts] + EXIT /B 1 +) + +rem Using pushd popd to set BASE_DIR to the absolute path +pushd %~dp0..\.. +set BASE_DIR=%CD% +popd + +IF ["%SCALA_VERSION%"] EQU [""] ( + set SCALA_VERSION=2.11.11 +) + +IF ["%SCALA_BINARY_VERSION%"] EQU [""] ( + for /f "tokens=1,2 delims=." %%a in ("%SCALA_VERSION%") do ( + set FIRST=%%a + set SECOND=%%b + if ["!SECOND!"] EQU [""] ( + set SCALA_BINARY_VERSION=!FIRST! + ) else ( + set SCALA_BINARY_VERSION=!FIRST!.!SECOND! + ) + ) +) + +rem Classpath addition for kafka-core dependencies +for %%i in ("%BASE_DIR%\core\build\dependant-libs-%SCALA_VERSION%\*.jar") do ( + call :concat "%%i" +) + +rem Classpath addition for kafka-examples +for %%i in ("%BASE_DIR%\examples\build\libs\kafka-examples*.jar") do ( + call :concat "%%i" +) + +rem Classpath addition for kafka-clients +for %%i in ("%BASE_DIR%\clients\build\libs\kafka-clients*.jar") do ( + call :concat "%%i" +) + +rem Classpath addition for kafka-streams +for %%i in ("%BASE_DIR%\streams\build\libs\kafka-streams*.jar") do ( + call :concat "%%i" +) + +rem Classpath addition for kafka-streams-examples +for %%i in ("%BASE_DIR%\streams\examples\build\libs\kafka-streams-examples*.jar") do ( + call :concat "%%i" +) + +for %%i in ("%BASE_DIR%\streams\build\dependant-libs-%SCALA_VERSION%\rocksdb*.jar") do ( + call :concat "%%i" +) + +rem Classpath addition for kafka tools +for %%i in ("%BASE_DIR%\tools\build\libs\kafka-tools*.jar") do ( + call :concat "%%i" +) + +for %%i in ("%BASE_DIR%\tools\build\dependant-libs-%SCALA_VERSION%\*.jar") do ( + call :concat "%%i" +) + +for %%p in (api runtime file json tools) do ( + for %%i in ("%BASE_DIR%\connect\%%p\build\libs\connect-%%p*.jar") do ( + call :concat "%%i" + ) + if exist "%BASE_DIR%\connect\%%p\build\dependant-libs\*" ( + call :concat "%BASE_DIR%\connect\%%p\build\dependant-libs\*" + ) +) + +rem Classpath addition for release +for %%i in ("%BASE_DIR%\libs\*") do ( + call :concat "%%i" +) + +rem Classpath addition for core +for %%i in ("%BASE_DIR%\core\build\libs\kafka_%SCALA_BINARY_VERSION%*.jar") do ( + call :concat "%%i" +) + +rem JMX settings +IF ["%KAFKA_JMX_OPTS%"] EQU [""] ( + set KAFKA_JMX_OPTS=-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false +) + +rem JMX port to use +IF ["%JMX_PORT%"] NEQ [""] ( + set KAFKA_JMX_OPTS=%KAFKA_JMX_OPTS% -Dcom.sun.management.jmxremote.port=%JMX_PORT% +) + +rem Log directory to use +IF ["%LOG_DIR%"] EQU [""] ( + set LOG_DIR="%BASE_DIR~%/logs" +) + +rem Log4j settings +IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] ( + set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%BASE_DIR%/config/tools-log4j.properties +) ELSE ( + rem create logs directory + IF not exist "%LOG_DIR%" ( + mkdir "%LOG_DIR%" + ) +) + +set KAFKA_LOG4J_OPTS=-Dkafka.logs.dir="%LOG_DIR%" "%KAFKA_LOG4J_OPTS%" + +rem Generic jvm settings you want to add +IF ["%KAFKA_OPTS%"] EQU [""] ( + set KAFKA_OPTS= +) + +set DEFAULT_JAVA_DEBUG_PORT=5005 +set DEFAULT_DEBUG_SUSPEND_FLAG=n +rem Set Debug options if enabled +IF ["%KAFKA_DEBUG%"] NEQ [""] ( + + + IF ["%JAVA_DEBUG_PORT%"] EQU [""] ( + set JAVA_DEBUG_PORT=%DEFAULT_JAVA_DEBUG_PORT% + ) + + IF ["%DEBUG_SUSPEND_FLAG%"] EQU [""] ( + set DEBUG_SUSPEND_FLAG=%DEFAULT_DEBUG_SUSPEND_FLAG% + ) + set DEFAULT_JAVA_DEBUG_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=!DEBUG_SUSPEND_FLAG!,address=!JAVA_DEBUG_PORT! + + IF ["%JAVA_DEBUG_OPTS%"] EQU [""] ( + set JAVA_DEBUG_OPTS=!DEFAULT_JAVA_DEBUG_OPTS! + ) + + echo Enabling Java debug options: !JAVA_DEBUG_OPTS! + set KAFKA_OPTS=!JAVA_DEBUG_OPTS! !KAFKA_OPTS! +) + +rem Which java to use +IF ["%JAVA_HOME%"] EQU [""] ( + set JAVA=java +) ELSE ( + set JAVA="%JAVA_HOME%/bin/java" +) + +rem Memory options +IF ["%KAFKA_HEAP_OPTS%"] EQU [""] ( + set KAFKA_HEAP_OPTS=-Xmx256M +) + +rem JVM performance options +IF ["%KAFKA_JVM_PERFORMANCE_OPTS%"] EQU [""] ( + set KAFKA_JVM_PERFORMANCE_OPTS=-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -Djava.awt.headless=true +) + +IF not defined CLASSPATH ( + echo Classpath is empty. Please build the project first e.g. by running 'gradlew jarAll' + EXIT /B 2 +) + +set COMMAND=%JAVA% %KAFKA_HEAP_OPTS% %KAFKA_JVM_PERFORMANCE_OPTS% %KAFKA_JMX_OPTS% %KAFKA_LOG4J_OPTS% -cp %CLASSPATH% %KAFKA_OPTS% %* +rem echo. +rem echo %COMMAND% +rem echo. +%COMMAND% + +goto :eof +:concat +IF not defined CLASSPATH ( + set CLASSPATH="%~1" +) ELSE ( + set CLASSPATH=%CLASSPATH%;"%~1" +) diff --git a/MPE/kafka/bin/windows/kafka-server-start.bat b/MPE/kafka/bin/windows/kafka-server-start.bat new file mode 100644 index 0000000..8624eda --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-server-start.bat @@ -0,0 +1,38 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +IF [%1] EQU [] ( + echo USAGE: %0 server.properties + EXIT /B 1 +) + +SetLocal +IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] ( + set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%~dp0../../config/log4j.properties +) +IF ["%KAFKA_HEAP_OPTS%"] EQU [""] ( + rem detect OS architecture + wmic os get osarchitecture | find /i "32-bit" >nul 2>&1 + IF NOT ERRORLEVEL 1 ( + rem 32-bit OS + set KAFKA_HEAP_OPTS=-Xmx512M -Xms512M + ) ELSE ( + rem 64-bit OS + set KAFKA_HEAP_OPTS=-Xmx1G -Xms1G + ) +) +"%~dp0kafka-run-class.bat" kafka.Kafka %* +EndLocal diff --git a/MPE/kafka/bin/windows/kafka-server-stop.bat b/MPE/kafka/bin/windows/kafka-server-stop.bat new file mode 100644 index 0000000..676577c --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-server-stop.bat @@ -0,0 +1,18 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +wmic process where (commandline like "%%kafka.Kafka%%" and not name="wmic.exe") delete +rem ps ax | grep -i 'kafka.Kafka' | grep -v grep | awk '{print $1}' | xargs kill -SIGTERM diff --git a/MPE/kafka/bin/windows/kafka-simple-consumer-shell.bat b/MPE/kafka/bin/windows/kafka-simple-consumer-shell.bat new file mode 100644 index 0000000..8836128 --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-simple-consumer-shell.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.tools.SimpleConsumerShell %* diff --git a/MPE/kafka/bin/windows/kafka-topics.bat b/MPE/kafka/bin/windows/kafka-topics.bat new file mode 100644 index 0000000..677b09d --- /dev/null +++ b/MPE/kafka/bin/windows/kafka-topics.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +"%~dp0kafka-run-class.bat" kafka.admin.TopicCommand %* diff --git a/MPE/kafka/bin/windows/zookeeper-server-start.bat b/MPE/kafka/bin/windows/zookeeper-server-start.bat new file mode 100644 index 0000000..f201a58 --- /dev/null +++ b/MPE/kafka/bin/windows/zookeeper-server-start.bat @@ -0,0 +1,30 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +IF [%1] EQU [] ( + echo USAGE: %0 zookeeper.properties + EXIT /B 1 +) + +SetLocal +IF ["%KAFKA_LOG4J_OPTS%"] EQU [""] ( + set KAFKA_LOG4J_OPTS=-Dlog4j.configuration=file:%~dp0../../config/log4j.properties +) +IF ["%KAFKA_HEAP_OPTS%"] EQU [""] ( + set KAFKA_HEAP_OPTS=-Xmx512M -Xms512M +) +"%~dp0kafka-run-class.bat" org.apache.zookeeper.server.quorum.QuorumPeerMain %* +EndLocal diff --git a/MPE/kafka/bin/windows/zookeeper-server-stop.bat b/MPE/kafka/bin/windows/zookeeper-server-stop.bat new file mode 100644 index 0000000..8b57dd8 --- /dev/null +++ b/MPE/kafka/bin/windows/zookeeper-server-stop.bat @@ -0,0 +1,17 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +wmic process where (commandline like "%%zookeeper%%" and not name="wmic.exe") delete diff --git a/MPE/kafka/bin/windows/zookeeper-shell.bat b/MPE/kafka/bin/windows/zookeeper-shell.bat new file mode 100644 index 0000000..ea3c398 --- /dev/null +++ b/MPE/kafka/bin/windows/zookeeper-shell.bat @@ -0,0 +1,22 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +IF [%1] EQU [] ( + echo USAGE: %0 zookeeper_host:port[/path] [args...] + EXIT /B 1 +) + +"%~dp0kafka-run-class.bat" org.apache.zookeeper.ZooKeeperMain -server %* diff --git a/MPE/kafka/bin/zookeeper-security-migration.sh b/MPE/kafka/bin/zookeeper-security-migration.sh new file mode 100644 index 0000000..722bde7 --- /dev/null +++ b/MPE/kafka/bin/zookeeper-security-migration.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +exec $(dirname $0)/kafka-run-class.sh kafka.admin.ZkSecurityMigrator "$@" diff --git a/MPE/kafka/bin/zookeeper-server-start.sh b/MPE/kafka/bin/zookeeper-server-start.sh new file mode 100644 index 0000000..bd9c114 --- /dev/null +++ b/MPE/kafka/bin/zookeeper-server-start.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 [-daemon] zookeeper.properties" + exit 1 +fi +base_dir=$(dirname $0) + +if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then + export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties" +fi + +if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then + export KAFKA_HEAP_OPTS="-Xmx512M -Xms512M" +fi + +EXTRA_ARGS=${EXTRA_ARGS-'-name zookeeper -loggc'} + +COMMAND=$1 +case $COMMAND in + -daemon) + EXTRA_ARGS="-daemon "$EXTRA_ARGS + shift + ;; + *) + ;; +esac + +exec $base_dir/kafka-run-class.sh $EXTRA_ARGS org.apache.zookeeper.server.quorum.QuorumPeerMain "$@" diff --git a/MPE/kafka/bin/zookeeper-server-stop.sh b/MPE/kafka/bin/zookeeper-server-stop.sh new file mode 100644 index 0000000..f771064 --- /dev/null +++ b/MPE/kafka/bin/zookeeper-server-stop.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +PIDS=$(ps ax | grep java | grep -i QuorumPeerMain | grep -v grep | awk '{print $1}') + +if [ -z "$PIDS" ]; then + echo "No zookeeper server to stop" + exit 1 +else + kill -s TERM $PIDS +fi + diff --git a/MPE/kafka/bin/zookeeper-shell.sh b/MPE/kafka/bin/zookeeper-shell.sh new file mode 100644 index 0000000..95007fa --- /dev/null +++ b/MPE/kafka/bin/zookeeper-shell.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 zookeeper_host:port[/path] [args...]" + exit 1 +fi + +exec $(dirname $0)/kafka-run-class.sh org.apache.zookeeper.ZooKeeperMain -server "$@" |