P19 23.07 online-configP19

1 files changed, 67 insertions, 0 deletions

diff --git a/MSH-PIC/flink/topology/completion/start.sh b/MSH-PIC/flink/topology/completion/start.sh
new file mode 100644
index 0000000..28760f5
--- /dev/null
+++ b/MSH-PIC/flink/topology/completion/start.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+#启动storm任务脚本
+source /etc/profile
+#任务jar所在目录
+BASE_DIR=$(pwd)
+
+#######################参数配置####################################
+#yarn任务运行模式 per-job 或 session
+TASK_MODE="per-job"
+#更新jar的名字
+PRIMORDIAL='log-completion-schema-230607-FastJson2.jar'
+#jar name
+JAR_NAME='log-completion-schema_tmp.jar'
+
+MAIN_CLASS="com.zdjizhi.topology.LogFlowWriteTopology"
+SESSION_CLUSTER="Flink session cluster"
+CONFIG_NAME="service_flow_config.properties"
+JOBMANAGER_MEMORY="1024m"
+TASKMANAGER_MEMORY="4096m"
+TASK_SLOTS=3
+#######################参数配置####################################
+
+APPLICATION_ID=$(yarn application -list | grep "$SESSION_CLUSTER" | awk '{print $1}')
+
+yes | cp -r $PRIMORDIAL $JAR_NAME
+
+#cd $BASE_DIR
+jar -xvf $BASE_DIR/$JAR_NAME $CONFIG_NAME
+function read_dir() {
+    for file in $(ls $1); do
+        if [ -d $1"/"$file ]; then
+            read_dir $1"/"$file
+        else
+                if [[ -z $TASK_MODE || $TASK_MODE == "per-job" ]]; then
+                    num=$(yarn application -list | grep $file | wc -l)
+                    if [ $num -eq "0" ]; then
+                        cat $1$file >$BASE_DIR/$CONFIG_NAME
+                        jar -uvf $BASE_DIR/$JAR_NAME $CONFIG_NAME
+			if [[ $file == "ETL-PROXY-EVENT-COMPLETED" || $file == "ETL-SECURITY-EVENT-COMPLETED" ]]; then
+                          flink run -t yarn-per-job -Djobmanager.memory.process.size=2048m -Dtaskmanager.memory.process.size=5120m -Dyarn.application.name=$file -Dtaskmanager.numberOfTaskSlots=8 -d -c $MAIN_CLASS $BASE_DIR/$JAR_NAME $file
+                        else
+                          flink run -t yarn-per-job -Djobmanager.memory.process.size=$JOBMANAGER_MEMORY -Dtaskmanager.memory.process.size=$TASKMANAGER_MEMORY -Dyarn.application.name=$file -Dtaskmanager.numberOfTaskSlots=$TASK_SLOTS -d -c $MAIN_CLASS $BASE_DIR/$JAR_NAME $file
+                        sleep 10
+                        fi
+                    fi
+                fi
+                if [[ -n $APPLICATION_ID && (-z $TASK_MODE || $TASK_MODE == "session") ]]; then
+                    num=$(flink list | grep "$file" | grep -v flink | wc -l)
+                    if [ $num -eq "0" ]; then
+                        cat $1$file >$BASE_DIR/$CONFIG_NAME
+                        jar -uvf $BASE_DIR/$JAR_NAME $CONFIG_NAME
+                        #session
+                        flink run -t yarn-session -Dyarn.application.id=$APPLICATION_ID -d -c $MAIN_CLASS $BASE_DIR/$JAR_NAME $file
+                        sleep 10
+                    fi
+                fi
+            fi
+    done
+}
+if [ $# != 1 ]; then
+    echo "usage: ./startall.sh [Configuration path]"
+    exit 1
+fi
+#读取第一个参数 为配置文件目录名称
+read_dir $1
+rm -rf $JAR_NAME
+