feat: add ci

8 files changed, 533 insertions, 0 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..1cba265
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,74 @@
+# docker镜像
+image: git.mesalab.cn:7443/nezha/nz-build-env:1.2
+# 定义全局变量
+variables:
+  MINIO_HOST: 'http://192.168.44.36:2020/'
+  MINIO_USER: 'admin'
+  MINIO_PWD: "Nezha@02!"
+  MAVEN_REPO: "/etc/maven/repository"
+# 依赖的docker服务
+# services:
+#  - mariadb
+# - redis
+# 开始执行脚本前所需执行脚本
+before_script:
+  - echo "begin ci"
+# 脚本执行完后的钩子，执行所需脚本
+after_script:
+  - echo "end ci"
+# 该ci pipeline适合的场景
+stages:
+  - build
+  - test
+# maven setting /usr/share/maven/conf/settings.xml
+cache:
+  paths:
+    - $MAVEN_REPO
+
+# 定义的任务
+build_rpm:
+  stage: build
+  # 所需执行的脚本
+  script:
+    - env | sort
+    - pwd 
+    - chmod +x ./tools/package.sh
+    - ./tools/package.sh
+  artifacts:
+    name: "$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA"
+    when: on_success
+    paths:
+      - ./target/*.xjar
+      - ./target/xjar
+    expire_in: 1 week
+  # 在哪个分支上可用
+  only:
+    - /^rel-.*$/i
+  # 指定哪个ci runner跑该工作
+  tags:
+    - nezha
+
+# 定义的任务
+dev_build:
+  stage: test
+  # 所需执行的脚本
+  script:
+    - env | sort
+    - pwd
+    - mvn clean install -Dxjar.password=111111 -Dxjar.excludes=/db/*,/static/**/*
+    - cd ./target && go build xjar.go && cd ..
+    - chmod +x ./tools/package.sh
+    - ./tools/package.sh
+  artifacts:
+    name: "$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA"
+    when: on_success
+    paths:
+      - ./target/*.xjar
+      - ./target/xjar
+    expire_in: 1 week
+  # 在哪个分支上可用
+  only:
+    - /^dev-.*$/i
+  # 指定哪个ci runner跑该工作
+  tags:
+    - nezha
\ No newline at end of file
diff --git a/tools/afterinstall.sh b/tools/afterinstall.sh
new file mode 100644
index 0000000..0bdf7b3
--- /dev/null
+++ b/tools/afterinstall.sh
@@ -0,0 +1,190 @@
+#!/bin/sh
+
+
+# Find Java
+if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
+    JAVA_EXE="$JAVA_HOME/bin/java"
+elif type -p java > /dev/null 2>&1; then
+    JAVA_EXE=$(type -p java)
+elif [[ -x "/usr/bin/java" ]];  then
+    JAVA_EXE="/usr/bin/java"
+else
+    echo "Unable to find Java"
+    JAVA_EXE="java"
+#    exit 1
+fi
+
+# install
+if [ 1 -eq $1 ];then
+  cat > /usr/lib/systemd/system/nz-agent.service <<EOF
+[Unit]
+Description=nz-agent
+After=network.target
+[Service]
+WorkingDirectory=/opt/nezha/nz-agent
+ExecStart=/opt/nezha/nz-agent/xjar ${JAVA_EXE} -Dnz-agent.dir=/opt/nezha/nz-agent -jar /opt/nezha/nz-agent/nz-agent.xjar
+RestartSec=5s
+Restart=always
+[Install]
+WantedBy=multi-user.target
+EOF
+
+  cat > /usr/lib/systemd/system/prometheus.service <<"EOF"
+[Unit]
+Description=prometheus
+After=network.target
+[Service]
+WorkingDirectory=/opt/nezha/nz-agent/prometheus
+EnvironmentFile=/opt/nezha/nz-agent/prometheus/config.conf
+ExecStart=/opt/nezha/nz-agent/prometheus/prometheus $OPTION
+RestartSec=10s
+Restart=always
+LimitNOFILE=524288
+[Install]
+WantedBy=multi-user.target
+EOF
+  cat > /opt/nezha/nz-agent/prometheus/config.conf <<EOF
+OPTION=" --config.file='/opt/nezha/nz-agent/prometheus/prometheus.yml' --web.listen-address='127.0.0.1:10091' --storage.tsdb.path='/opt/nezha/data' --web.enable-admin-api --web.enable-lifecycle --storage.tsdb.retention.time=15d "
+EOF
+  cat > /usr/lib/systemd/system/snmp-exporter.service <<"EOF"
+[Unit]
+Description=snmp-exporter
+After=network.target
+[Service]
+EnvironmentFile=/opt/nezha/nz-agent/snmp_exporter/config.conf
+ExecStart=/opt/nezha/nz-agent/snmp_exporter/snmp_exporter $OPTION
+RestartSec=10s
+Restart=always
+LimitNOFILE=524288
+[Install]
+WantedBy=multi-user.target
+EOF
+  cat >/opt/nezha/nz-agent/snmp_exporter/config.conf <<EOF
+OPTION="--web.listen-address='127.0.0.1:19116' --config.file='/opt/nezha/nz-agent/snmp_exporter/snmp.yml' "
+EOF
+
+
+  cat > /usr/lib/systemd/system/blackbox-exporter.service <<"EOF"
+[Unit]
+Description=blackbox_exporter
+After=network.target
+[Service]
+WorkingDirectory=/opt/nezha/nz-agent/blackbox_exporter
+EnvironmentFile=/opt/nezha/nz-agent/blackbox_exporter/config.conf
+ExecStart=/opt/nezha/nz-agent/blackbox_exporter/blackbox_exporter $OPTION
+RestartSec=10s
+Restart=always
+LimitNOFILE=524288
+[Install]
+WantedBy=multi-user.target
+EOF
+  cat >/opt/nezha/nz-agent/blackbox_exporter/config.conf <<EOF
+OPTION="--web.listen-address='127.0.0.1:19115' --config.file='/opt/nezha/nz-agent/blackbox_exporter/blackbox.yml' "
+EOF
+
+  chmod 755 /opt/nezha/nz-agent/cortex/cortex-linux-amd64
+  cat > /usr/lib/systemd/system/cortex.service <<"EOF"
+[Unit]
+Description=cortex
+After=network.target
+[Service]
+WorkingDirectory=/opt/nezha/nz-agent/cortex
+EnvironmentFile=/opt/nezha/nz-agent/cortex/config.conf
+ExecStart=/opt/nezha/nz-agent/cortex/cortex-linux-amd64 $OPTION
+RestartSec=10s
+Restart=always
+LimitNOFILE=524288
+[Install]
+WantedBy=multi-user.target
+EOF
+  cat >/opt/nezha/nz-agent/cortex/config.conf <<EOF
+OPTION=" -config.file=/opt/nezha/nz-agent/cortex/cortex.yml"
+EOF
+  chmod 755 /opt/nezha/nz-agent/loki/loki-linux-amd64
+  cat > /usr/lib/systemd/system/loki.service <<"EOF"
+[Unit]
+Description=loki
+After=network.target
+[Service]
+WorkingDirectory=/opt/nezha/nz-agent/loki
+EnvironmentFile=/opt/nezha/nz-agent/loki/config.conf
+ExecStart=/opt/nezha/nz-agent/loki/loki-linux-amd64 $OPTION
+RestartSec=10s
+Restart=always
+LimitNOFILE=524288
+[Install]
+WantedBy=multi-user.target
+EOF
+  cat >/opt/nezha/nz-agent/loki/config.conf <<EOF
+OPTION=" -config.file=/opt/nezha/nz-agent/loki/loki.yml"
+EOF
+  systemctl daemon-reload
+  systemctl enable nz-agent && systemctl restart nz-agent
+  systemctl enable prometheus && systemctl restart prometheus
+  systemctl enable snmp-exporter && systemctl restart snmp-exporter
+  systemctl enable blackbox-exporter && systemctl restart blackbox-exporter
+  systemctl enable cortex && systemctl restart cortex
+  systemctl enable loki && systemctl restart loki
+  echo 'install nz-agent success !'
+fi
+compareMD5()
+{
+  if [ ! -f $1 ] || [ ! -f $2 ];then
+		echo 1
+		return 1
+	fi
+	
+	local f1MD5=`md5sum $1|awk '{print $1}'`
+  local f2MD5=`md5sum $2|awk '{print $1}'`
+
+  if [ ${f1MD5} = ${f2MD5} ];then
+      echo 0
+      return 0
+  else
+      echo 1
+      return 1
+  fi
+}
+# update
+if [ 2 -eq $1 ];then
+
+  cat > /opt/nezha/nz-agent/prometheus/config.conf <<EOF
+OPTION=" --config.file='/opt/nezha/nz-agent/prometheus/prometheus.yml' --web.listen-address='127.0.0.1:10091' --storage.tsdb.path='/opt/nezha/data' --web.enable-admin-api --web.enable-lifecycle --storage.tsdb.retention.time=15d "
+EOF
+
+  cat >/opt/nezha/nz-agent/snmp_exporter/config.conf <<EOF
+OPTION="--web.listen-address='127.0.0.1:19116' --config.file='/opt/nezha/nz-agent/snmp_exporter/snmp.yml' "
+EOF
+
+  cat >/opt/nezha/nz-agent/blackbox_exporter/config.conf <<EOF
+OPTION="--web.listen-address='127.0.0.1:19115' --config.file='/opt/nezha/nz-agent/blackbox_exporter/blackbox.yml' "
+EOF
+
+  chmod 755 /opt/nezha/nz-agent/cortex/cortex-linux-amd64
+  cat >/opt/nezha/nz-agent/cortex/config.conf <<EOF
+OPTION=" -config.file=/opt/nezha/nz-agent/cortex/cortex.yml"
+EOF
+
+  chmod 755 /opt/nezha/nz-agent/loki/loki-linux-amd64
+  cat >/opt/nezha/nz-agent/loki/config.conf <<EOF
+OPTION=" -config.file=/opt/nezha/nz-agent/loki/loki.yml"
+EOF
+
+
+	AGENT_PATH=/opt/nezha/nz-agent
+
+	TMP_PATH=/tmp/nezha/nz-agent
+	echo 'move backup config file...'
+	if [ -d $TMP_PATH ];then
+		for i in $(ls $AGENT_PATH/config);do
+			if [ 1 -eq `compareMD5 $AGENT_PATH/config/$i $TMP_PATH/config/$i` ];then
+				echo 'return config file '${i}
+				#mv -f $AGENT_PATH/config/$i $AGENT_PATH/config/$i
+				cp -f $TMP_PATH/config/$i $AGENT_PATH/config/$i
+				#cat $AGENT_PATH/config/token.auth'.oldconfig' > $AGENT_PATH/config/token.auth && rm -f $AGENT_PATH/config/token.auth'.oldconfig'
+			fi	
+		done	
+	fi
+	systemctl restart nz-agent
+fi
+
diff --git a/tools/afterremove.sh b/tools/afterremove.sh
new file mode 100644
index 0000000..b725742
--- /dev/null
+++ b/tools/afterremove.sh
@@ -0,0 +1,26 @@
+#!/bin/sh
+if [ 0 -eq $1 ];then
+
+	echo 'start remove nz-agent module from disk...'
+
+	systemctl stop nz-agent.service
+	systemctl stop prometheus.service
+	systemctl stop snmp-exporter.service
+	systemctl stop blackbox-exporter.service
+	systemctl stop cortex.service
+	systemctl stop loki.service
+
+	rm -rf /opt/nezha/nz-agent
+
+	systemctl disable nz-agent.service
+	systemctl disable prometheus.service
+	systemctl disable snmp-exporter.service
+	systemctl disable blackbox-exporter.service
+	systemctl disable cortex.service
+	systemctl disable loki.service
+	rm -rf /usr/lib/systemd/system/{nz-agent.service,prometheus.service,snmp-exporter.service,blackbox-exporter.service,cortex.service,loki.service}
+
+	systemctl daemon-reload
+
+	echo 'uninstall success!'
+fi
diff --git a/tools/beforeinstall.sh b/tools/beforeinstall.sh
new file mode 100644
index 0000000..cd6a80f
--- /dev/null
+++ b/tools/beforeinstall.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+AGENT_PATH=/opt/nezha/nz-agent
+
+if [ 2 -eq $1 ];then
+	
+	if [ ! -d ${AGENT_PATH}/config ];then
+		exit 0
+	fi
+	systemctl stop nz-agent
+	TMP_PATH=/tmp/nezha/nz-agent
+	rm -rf $TMP_PATH
+
+	mkdir -p $TMP_PATH
+	cp -rf $AGENT_PATH/config $TMP_PATH
+	echo 'backup config file from '${AGENT_PATH}
+	
+	if [ 0 -lt $(rpm -aq nz-agent|wc -w) ];then
+		echo 'clean before install...'
+		rm -rf $AGENT_PATH
+	fi
+fi
+
+
diff --git a/tools/blackbox.yml b/tools/blackbox.yml
new file mode 100644
index 0000000..217ad1e
--- /dev/null
+++ b/tools/blackbox.yml
@@ -0,0 +1,34 @@
+modules:
+  http_2xx:
+    prober: http
+  http_post_2xx:
+    prober: http
+    http:
+      method: POST
+  tcp_connect:
+    prober: tcp
+  pop3s_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^+OK"
+      tls: true
+      tls_config:
+        insecure_skip_verify: false
+  ssh_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-"
+  irc_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: "NICK prober"
+      - send: "USER prober prober prober :prober"
+      - expect: "PING :([^ ]+)"
+        send: "PONG ${1}"
+      - expect: "^:[^ ]+ 001"
+  ping:
+    prober: icmp
+    timeout: 5s
\ No newline at end of file
diff --git a/tools/cortex.yml b/tools/cortex.yml
new file mode 100644
index 0000000..ff429e4
--- /dev/null
+++ b/tools/cortex.yml
@@ -0,0 +1,59 @@
+auth_enabled: false
+server:
+  http_listen_port: 10093
+  grpc_listen_port: 10094
+distributor:
+  shard_by_all_labels: true
+  pool:
+    health_check_ingesters: true
+
+ingester_client:
+  grpc_client_config:
+    max_recv_msg_size: 104857600
+    max_send_msg_size: 104857600
+    grpc_compression: gzip
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory
+storage:
+  engine: blocks
+
+blocks_storage:
+  tsdb:
+    dir: /opt/nezha/nz-agent/cortex/tsdb
+
+  bucket_store:
+    sync_dir: /opt/nezha/nz-agent/cortex/tsdb-sync
+  backend: filesystem
+  filesystem:
+    dir: /opt/nezha/nz-agent/cortex/data/tsdb
+
+store_gateway:
+  sharding_enabled: true
+  sharding_ring:
+    kvstore:
+      store: inmemory
+
+compactor:
+  data_dir: /opt/nezha/nz-agent/cortex/compactor
+  sharding_ring:
+    kvstore:
+      store: inmemory
+
+frontend_worker:
+  match_max_concurrent: true
+
+ruler:
+  enable_api: true
+  enable_sharding: true
+  ring:
+    num_tokens: 512
+    kvstore:
+      store: inmemory
+
+ruler_storage:
+  backend: local
+  local:
+    directory: /opt/nezha/nz-agent/cortex/rules
\ No newline at end of file
diff --git a/tools/loki.yml b/tools/loki.yml
new file mode 100644
index 0000000..11c4203
--- /dev/null
+++ b/tools/loki.yml
@@ -0,0 +1,38 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 13100
+  grpc_listen_port: 10095
+
+ingester:
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+
+schema_config:
+  configs:
+  - from: 2021-09-15
+    store: boltdb
+    object_store: filesystem
+    schema: v11
+    index:
+      prefix: index_
+      period: 168h
+
+storage_config:
+  boltdb:
+    directory: /tmp/loki/index
+
+  filesystem:
+    directory: /tmp/loki/chunks
+
+limits_config:
+  enforce_metric_name: false
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
\ No newline at end of file
diff --git a/tools/package.sh b/tools/package.sh
new file mode 100644
index 0000000..23acad2
--- /dev/null
+++ b/tools/package.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+set -vx
+CUR_PWD=`pwd`
+
+PACKAGE_NAME=nz-agent
+BRANCH_ARRAY=(${CI_COMMIT_REF_NAME//-/ })
+PACKAGE_VERSION=${BRANCH_ARRAY[1]}.$CI_COMMIT_SHORT_SHA
+ITERATION=Beta
+if [[ "${BRANCH_ARRAY[0]}" == "rel" ]] ; then 
+  ITERATION='Release';
+fi
+RPM_FULL_NAME=${PACKAGE_NAME}-${PACKAGE_VERSION}-${ITERATION}.x86_64.rpm
+
+# depends 文件名
+PROMETHEUS_FILENAME=prometheus-2.34.0.linux-amd64.tar.gz
+SNMP_EXPORTER_FILENAME=snmp_exporter-0.20.0.linux-amd64.tar.gz
+BLACKBOX_EXPORTER_FILENAME=blackbox_exporter-0.20.0.linux-amd64.tar.gz
+LOKI_FILENAME=loki-linux-amd64.zip
+CORTEX_FILENAME=cortex-linux-amd64
+
+
+# 初始化 minio
+mc alias set nz $MINIO_HOST $MINIO_USER $MINIO_PWD
+
+BUILD_PATH=$CUR_PWD/build
+RPM_AGENT_PATH=$BUILD_PATH/opt/nezha/nz-agent
+RPM_PROMETHEUS_PATH=$BUILD_PATH/opt/nezha/nz-agentprometheus
+RPM_CORTEX_PATH=$BUILD_PATH/opt/nezha/nz-agent/cortex
+RPM_LOKI_PATH=$BUILD_PATH/opt/nezha/nz-agent/loki
+RPM_SNMP_EXPORTER_PATH=$BUILD_PATH/opt/nezha/nz-agent/snmp_exporter
+RPM_BLACKBOX_EXPORTER_PATH=$BUILD_PATH/opt/nezha/nz-agent/blackbox_exporter
+
+
+mkdir -p $RPM_AGENT_PATH $RPM_PROMETHEUS_PATH $RPM_CORTEX_PATH $RPM_LOKI_PATH $RPM_SNMP_EXPORTER_PATH $RPM_BLACKBOX_EXPORTER_PATH
+mkdir -p $RPM_AGENT_PATH/config
+
+echo 'packaging  begin ...'
+# mvn 编译 
+mvn clean install -Dxjar.password=111111 -Dxjar.excludes=/db/*,/static/**/*
+cd ./target && go build xjar.go 
+cd $CUR_PWD 
+# 设置可执行权限
+chmod +x ./target/xjar
+cp -f ./target/nz-agent.xjar $RPM_TALON_PATH
+cp -f ./target/xjar $RPM_TALON_PATH
+cp -f ./src/main/resources/{application-prod.yml,logback-spring.xml,config/auth.yml,config/token.auth,config/lokiPushUrls.yml}  ${RPM_AGENT_PATH}/config
+sed -i 's/<property name="log.path" value=".*"/<property name="log.path" value="\/var\/log\/nezha\/nz-agent\/"/g' ${RPM_AGENT_PATH}/config/logback-spring.xml
+
+# 准备 prometheus 编译目录
+mc cp nz/depends/prometheus/$PROMETHEUS_FILENAME ./
+tar -xzf ./$PROMETHEUS_FILENAME --strip-components 1 -C $RPM_PROMETHEUS_PATH
+echo '' > $RPM_PROMETHEUS_PATH/rule.yml
+
+# 准备 cortex 编译目录
+mc cp nz/depends/cortex/$CORTEX_FILENAME ./
+unzip -o $CORTEX_FILENAME -d ${RPM_CORTEX_PATH}
+cp -f ./tools/cortex.yml $RPM_CORTEX_PATH
+
+# 准备 loki 编译目录
+mc cp nz/depends/loki/$LOKI_FILENAME ./
+unzip -o $LOKI_FILENAME -d ${RPM_LOKI_PATH}
+cp -f ./tools/loki.yml $RPM_LOKI_PATH
+
+# 准备 blackbox_exporter 编译目录
+mc cp nz/depends/exporter/$BLACKBOX_EXPORTER_FILENAME ./
+tar -xzf ./$BLACKBOX_EXPORTER_FILENAME --strip-components 1 -C $RPM_BLACKBOX_EXPORTER_PATH
+cp -f ./tools/blackbox.yml $RPM_BLACKBOX_EXPORTER_PATH
+
+# 准备 snmp_exporter 编译目录
+mc cp nz/depends/exporter/$SNMP_EXPORTER_FILENAME ./
+tar -xzf ./$SNMP_EXPORTER_FILENAME --strip-components 1 -C ${RPM_SNMP_EXPORTER_PATH}
+
+
+fpm -s dir -t rpm -n ${PACKAGE_NAME} -v ${PACKAGE_VERSION} --iteration ${ITERATION} --rpm-os 'linux'  \
+	--pre-install ./tools/beforeinstall.sh  \
+	--post-install ./tools/afterinstall.sh  \
+	--post-uninstall ./tools/afterremove.sh  \
+	-C $BUILD_PATH -f  \
+	-p $BUILD_PATH 
+# 计算md5值
+cd $BUILD_PATH && md5sum $RPM_FULL_NAME > $RPM_FULL_NAME.md5sum.txt
+cd $CUR_PWD 
+
+# 上传编译的rpm 文件
+mc cp $BUILD_PATH/$RPM_FULL_NAME nz/release/$PACKAGE_NAME/$RPM_FULL_NAME
+mc cp $BUILD_PATH/$RPM_FULL_NAME.md5sum.txt nz/release/$PACKAGE_NAME/$RPM_FULL_NAME.md5sum.txt
+
+echo 'package  finished'