提交POC测试时log-stream和file-sync程序配置P19-POC

5 files changed, 185 insertions, 0 deletions

diff --git a/file-sync-service/volumes/p19-file-sync-service/config/application.yml b/file-sync-service/volumes/p19-file-sync-service/config/application.yml
new file mode 100644
index 0000000..71570b2
--- /dev/null
+++ b/file-sync-service/volumes/p19-file-sync-service/config/application.yml
@@ -0,0 +1,21 @@
+nacos:
+  config:
+    type: yaml
+    server-addr: 10.3.60.2:8848
+    namespace: prod
+    data-id: p19-file-sync-service.yaml
+    auto-refresh: true
+    group: Galaxy
+    username: nacos
+    password: nacos
+    bootstrap:
+      enable: true
+      log:
+        enable: true
+  discovery:
+    service-name: hos
+spring:
+  profiles:
+    active: dev
+logging:
+  config: ./config/log4j2-dev.xml
diff --git a/file-sync-service/volumes/p19-file-sync-service/config/log4j2-dev.xml b/file-sync-service/volumes/p19-file-sync-service/config/log4j2-dev.xml
new file mode 100644
index 0000000..76a4e15
--- /dev/null
+++ b/file-sync-service/volumes/p19-file-sync-service/config/log4j2-dev.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+
+    <!--日志打印相关参数配置-->
+    <Properties>
+        <!--每5M压缩日志文件-->
+        <property name="LOG_SIZE">100M</property>
+        <!--最多产生10个压缩文件-->
+        <property name="LOG_NUMS">5</property>
+        <!--日志打印等级-->
+        <property name="LOG_LEVEL">info</property>
+        <!--日志文件路径-->
+        <property name="LOG_PATH">logs</property>
+        <!--日志文件名称-->
+        <property name="LOG_FILE_NAME">galaxy-fsync-service</property>
+        <!--日志打印格式-->
+        <property name="LOG_PATTERN">[%d{yyyy-MM-dd HH:mm:ss}] [%p] [Thread:%t] %l %x - %m%n</property>
+    </Properties>
+
+    <appenders>
+        <Console name="consoleSystemOutAppender" target="SYSTEM_OUT">
+            <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+            <PatternLayout pattern="${LOG_PATTERN}"/>
+        </Console>
+
+        <RollingFile name="rollingFileAllAppender"
+                     fileName="${LOG_PATH}/${LOG_FILE_NAME}.log"
+                     filePattern="${LOG_PATH}/history/$${date:yyyy-MM-dd}/${LOG_FILE_NAME}-%d{yyyy-MM-dd}-%i.log.gz">
+            <PatternLayout pattern="${LOG_PATTERN}"/>
+            <Policies>
+                <SizeBasedTriggeringPolicy size="${LOG_SIZE}"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+            </Policies>
+            <Filters>
+                <ThresholdFilter level="all" onMatch="ACCEPT" onMismatch="DENY"/>
+            </Filters>
+            <DefaultRolloverStrategy max="${LOG_NUMS}">
+                <Delete basePath="${LOG_PATH}/history" maxDepth="1">
+                    <IfFileName glob="*.log.gz">
+                        <IfLastModified age="90d">
+                            <IfAny>
+                                <IfAccumulatedFileSize exceeds="200 GB" />
+                            </IfAny>
+                        </IfLastModified>
+                    </IfFileName>
+                </Delete>
+            </DefaultRolloverStrategy>
+        </RollingFile>
+    </appenders>
+    <loggers>
+        <root level="${LOG_LEVEL}">
+            <appender-ref ref="consoleSystemOutAppender"/>
+            <appender-ref ref="rollingFileAllAppender"/>
+        </root>
+    </loggers>
+</configuration>
diff --git a/file-sync-service/workspace/p19-file-sync-service/config/p19-file-sync-service.yaml b/file-sync-service/workspace/p19-file-sync-service/config/p19-file-sync-service.yaml
new file mode 100644
index 0000000..0e2fdd8
--- /dev/null
+++ b/file-sync-service/workspace/p19-file-sync-service/config/p19-file-sync-service.yaml
@@ -0,0 +1,38 @@
+spring:
+  application:
+    name: P19FileSyncServiceApplication
+server:
+  port: 8189
+#最大线程数
+thread:
+  maxSize: 1000
+http:
+  connectTimeout: 10000
+  connectionRequestTimeout: 10000
+  defaultMaxPerRoute: 1000
+  maxTotal: 2000
+  socketTimeout: 180000
+  staleConnectionCheckEnabled: true
+kafka:
+  consumer:
+    #kafka地址
+    servers: 10.3.60.2:9094,10.3.60.3:9094,10.3.60.4:9094
+    topic: FILE-TO-OSS
+    group_id: 1
+    auto_commit_interval: 100
+    #latest，消费最新消息，保证每个组一个消费者消费同一条消息；earliest，从头开始消费消息
+    auto_offset_reset: latest
+    #是否是批量消费
+    batch_listener: true
+    #批量消费条数
+    pool_record: 1000
+    #并发数，设置和topic分区数一致
+    concurrency: 3
+    enable_auto_commit: false
+    poll_timeout: 1500
+    session_timeout: 60000
+    #开启验证的用户名和密码
+    sasl_password: galaxy2019
+    sasl_username: admin
+logging:
+  config: ./config/log4j2-dev.xml
diff --git a/file-sync-service/workspace/p19-file-sync-service/docker-compose.yml b/file-sync-service/workspace/p19-file-sync-service/docker-compose.yml
new file mode 100644
index 0000000..5f4bdfd
--- /dev/null
+++ b/file-sync-service/workspace/p19-file-sync-service/docker-compose.yml
@@ -0,0 +1,15 @@
+version: '2'
+
+services:
+  galaxy-hos-service:
+    image: p19-file-sync-service:21.12.01
+    container_name: p19-file-sync-service
+    environment:
+      JAVA_OPTS: "-Xms2g -Xmx2g -Xmn256m -Xss256k -XX:MetaspaceSize=256m -XX:MaxPermSize=256m -XX:SurvivorRatio=2 -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -XX:MaxTenuringThreshold=15 -XX:+UseCMSCompactAtFullCollection -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=70"
+    volumes:
+      - "/home/tsg/olap/galaxy/volumes/p19-file-sync-service/config:/home/tsg/galaxy/p19-file-sync-service/config"
+      - "/home/tsg/olap/galaxy/volumes/p19-file-sync-service/logs:/home/tsg/galaxy/p19-file-sync-service/logs"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "/etc/timezone:/etc/timezone:ro"
+    restart: always
+    network_mode: "host"
diff --git a/file-sync-service/workspace/p19-file-sync-service/push_config.sh b/file-sync-service/workspace/p19-file-sync-service/push_config.sh
new file mode 100644
index 0000000..25c32b3
--- /dev/null
+++ b/file-sync-service/workspace/p19-file-sync-service/push_config.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+nacos_server=$1
+config_path_root=$2
+app_name=p19-file-sync-service
+tenant=prod
+
+nacos_push_url="http://$nacos_server:8848/nacos/v1/cs/configs?username=nacos&password=nacos&tenant=$tenant&group=Galaxy"
+
+shell_resu=1
+
+#1.获取config 路径
+config_path_list=$(readlink -f $config_path_root)
+
+function push(){
+if [ "$suffix" = "yaml" ]||[ "$suffix" = "json" ]||[ "$suffix" = "text" ]||[ "$suffix" = "properties" ];then
+  result=$(curl --retry-delay 3 --retry 3 -sw '%{http_code}' -o /dev/null --request POST "$nacos_push_url&dataId=$data_id&appName=$app_name&type=$suffix" --data-urlencode content="`cat $config_path_file`")
+  if [[ $result -eq '200' ]];then
+   check=`curl -sw '%{http_code}' -o /dev/null -X GET "$nacos_push_url&dataId=$data_id"`
+   if [[ $check -ne '200' ]];then
+     shell_resu=0
+   fi
+  fi
+fi
+}
+
+
+function pushConfig(){
+#2.app文件夹文件
+for config_path in  $config_path_list
+do
+if [ -d $config_path ];then
+
+  for file in `ls $config_path`
+  do
+  config_path_file=$config_path/$file
+  if [ -f $config_path_file ];then
+    data_id=${config_path_file##*/}
+    suffix=${config_path_file##*.}
+    if [ "$suffix" = "sql" ];then
+      suffix="text"
+    fi
+  
+    if [ "$suffix" = "yml" ];then
+      suffix="yaml"
+    fi
+#3.提交config配置
+        push
+  fi
+  done
+fi
+done
+}
+pushConfig
+echo $shell_resu