增加安装脚本

7 files changed, 712 insertions, 0 deletions

diff --git a/TSG/centos7.tar b/TSG/centos7.tar
new file mode 100644
index 0000000..b0117ed
--- /dev/null
+++ b/TSG/centos7.tar
diff --git a/TSG/clickhouse-tsg.txt b/TSG/clickhouse-tsg.txt
new file mode 100644
index 0000000..8d8661c
--- /dev/null
+++ b/TSG/clickhouse-tsg.txt
@@ -0,0 +1,486 @@
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy.session_record_log_local on cluster tsg_galaxy(
+    policy_id Int64,
+    user_tags String,
+    action Int64,
+    start_time Int64,
+    end_time Int64,
+    recv_time Int64,
+    trans_proto String,
+    app_proto String,
+    addr_type Int64,
+    server_ip String,
+    d_ip Int64,
+    s_ip Int64,
+    client_ip String,
+    server_port Int64,
+    client_port Int64,
+    service Int64,
+    entrance_id Int64,
+    device_id Int64,
+    link_id Int64,
+    isp String,
+    encap_type Int64,
+    direction Int64,
+    stream_dir Int64,
+    cap_ip String,
+    addr_list String,
+    server_location String,
+    client_location String,
+    client_asn String,
+    server_asn String,
+    subscribe_id String,
+    user_region String,
+    con_duration_ms Int64,
+    url String,
+    host String,
+    domain String,
+    category String,
+    req_line String,
+    res_line String,
+    cookie String,
+    referer String,
+    user_agent String,
+    content_len String,
+    content_type String,
+    set_cookie String,
+    req_header String,
+    resp_header String,
+    req_body_key String,
+    req_body String,
+    res_body_key String,
+    resp_body String,
+    version String,
+    sni String,
+    san String,
+    cn String,
+    app_id Int64,
+    protocol_id Int64,
+    con_latency_ms Int64,
+    pinningst Int64,
+    intercept_state Int64,
+    ssl_server_side_latency Int64,
+    ssl_client_side_latency Int64,
+    ssl_server_side_version String,
+    ssl_client_side_version String,
+    ssl_cert_verify Int64,
+    stream_trace_id String,
+    ssl_error String,
+    c2s_pkt_num Int64,
+    s2c_pkt_num Int64,
+    c2s_byte_num Int64,
+    s2c_byte_num Int64,
+    nas_ip String,
+    framed_ip String,
+    account String,
+    packet_type Int64
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY recv_time;
+
+
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy.policy_event_log_local on cluster tsg_galaxy(
+    policy_id Int64,
+    user_tags String,
+    action Int64,
+    start_time Int64,
+    end_time Int64,
+    recv_time Int64,
+    trans_proto String,
+    app_proto String,
+    addr_type Int64,
+    server_ip String,
+    d_ip Int64,
+    s_ip Int64,
+    client_ip String,
+    server_port Int64,
+    client_port Int64,
+    service Int64,
+    entrance_id Int64,
+    device_id Int64,
+    link_id Int64,
+    isp String,
+    encap_type Int64,
+    direction Int64,
+    stream_dir Int64,
+    cap_ip String,
+    addr_list String,
+    server_location String,
+    client_location String,
+    client_asn String,
+    server_asn String,
+    subscribe_id String,
+    user_region String,
+    con_duration_ms Int64,
+    url String,
+    host String,
+    domain String,
+    category String,
+    req_line String,
+    res_line String,
+    cookie String,
+    referer String,
+    user_agent String,
+    content_len String,
+    content_type String,
+    set_cookie String,
+    req_header String,
+    resp_header String,
+    req_body_key String,
+    req_body String,
+    res_body_key String,
+    resp_body String,
+    version String,
+    sni String,
+    san String,
+    cn String,
+    app_id Int64,
+    protocol_id Int64,
+    con_latency_ms Int64,
+    pinningst Int64,
+    intercept_state Int64,
+    ssl_server_side_latency Int64,
+    ssl_client_side_latency Int64,
+    ssl_server_side_version String,
+    ssl_client_side_version String,
+    ssl_cert_verify Int64,
+    stream_trace_id String,
+    ssl_error String,
+    c2s_pkt_num Int64,
+    s2c_pkt_num Int64,
+    c2s_byte_num Int64,
+    s2c_byte_num Int64,
+    nas_ip String,
+    framed_ip String,
+    account String,
+    packet_type Int64
+) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (recv_time);
+
+CREATE TABLE IF NOT EXISTS tsg_galaxy.radius_log_local on cluster tsg_galaxy(
+    policy_id Int64,
+    user_tags String,
+    action Int64,
+    start_time Int64,
+    end_time Int64,
+    recv_time Int64,
+    trans_proto String,
+    app_proto String,
+    addr_type Int64,
+    server_ip String,
+    d_ip Int64,
+    s_ip Int64,
+    client_ip String,
+    server_port Int64,
+    client_port Int64,
+    service Int64,
+    entrance_id Int64,
+    device_id Int64,
+    link_id Int64,
+    isp String,
+    encap_type Int64,
+    direction Int64,
+    stream_dir Int64,
+    cap_ip String,
+    addr_list String,
+    server_location String,
+    client_location String,
+    client_asn String,
+    server_asn String,
+    subscribe_id String,
+    user_region String,
+    con_duration_ms Int64,
+    url String,
+    host String,
+    domain String,
+    category String,
+    req_line String,
+    res_line String,
+    cookie String,
+    referer String,
+    user_agent String,
+    content_len String,
+    content_type String,
+    set_cookie String,
+    req_header String,
+    resp_header String,
+    req_body_key String,
+    req_body String,
+    res_body_key String,
+    resp_body String,
+    version String,
+    sni String,
+    san String,
+    cn String,
+    app_id Int64,
+    protocol_id Int64,
+    con_latency_ms Int64,
+    pinningst Int64,
+    intercept_state Int64,
+    ssl_server_side_latency Int64,
+    ssl_client_side_latency Int64,
+    ssl_server_side_version String,
+    ssl_client_side_version String,
+    ssl_cert_verify Int64,
+    stream_trace_id String,
+    ssl_error String,
+    c2s_pkt_num Int64,
+    s2c_pkt_num Int64,
+    c2s_byte_num Int64,
+    s2c_byte_num Int64,
+    nas_ip String,
+    framed_ip String,
+    account String,
+    packet_type Int64
+) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (recv_time);
+
+
+CREATE TABLE IF NOT EXISTS ntc_galaxy.session_record_log_local on cluster tsg_galaxy(
+    policy_id Int64,
+    user_tags String,
+    action Int64,
+    start_time Int64,
+    end_time Int64,
+    recv_time Int64,
+    trans_proto String,
+    app_proto String,
+    addr_type Int64,
+    server_ip String,
+    d_ip Int64,
+    s_ip Int64,
+    client_ip String,
+    server_port Int64,
+    client_port Int64,
+    service Int64,
+    entrance_id Int64,
+    device_id Int64,
+    link_id Int64,
+    isp String,
+    encap_type Int64,
+    direction Int64,
+    stream_dir Int64,
+    cap_ip String,
+    addr_list String,
+    server_location String,
+    client_location String,
+    client_asn String,
+    server_asn String,
+    subscribe_id String,
+    user_region String,
+    con_duration_ms Int64,
+    url String,
+    host String,
+    domain String,
+    category String,
+    req_line String,
+    res_line String,
+    cookie String,
+    referer String,
+    user_agent String,
+    content_len String,
+    content_type String,
+    set_cookie String,
+    req_header String,
+    resp_header String,
+    req_body_key String,
+    req_body String,
+    res_body_key String,
+    resp_body String,
+    version String,
+    sni String,
+    san String,
+    cn String,
+    app_id Int64,
+    protocol_id Int64,
+    con_latency_ms Int64,
+    pinningst Int64,
+    intercept_state Int64,
+    ssl_server_side_latency Int64,
+    ssl_client_side_latency Int64,
+    ssl_server_side_version String,
+    ssl_client_side_version String,
+    ssl_cert_verify Int64,
+    stream_trace_id String,
+    ssl_error String,
+    c2s_pkt_num Int64,
+    s2c_pkt_num Int64,
+    c2s_byte_num Int64,
+    s2c_byte_num Int64,
+    nas_ip String,
+    framed_ip String,
+    account String,
+    packet_type Int64
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY recv_time;
+
+
+
+CREATE TABLE IF NOT EXISTS ntc_galaxy.policy_event_log_local on cluster tsg_galaxy(
+    policy_id Int64,
+    user_tags String,
+    action Int64,
+    start_time Int64,
+    end_time Int64,
+    recv_time Int64,
+    trans_proto String,
+    app_proto String,
+    addr_type Int64,
+    server_ip String,
+    d_ip Int64,
+    s_ip Int64,
+    client_ip String,
+    server_port Int64,
+    client_port Int64,
+    service Int64,
+    entrance_id Int64,
+    device_id Int64,
+    link_id Int64,
+    isp String,
+    encap_type Int64,
+    direction Int64,
+    stream_dir Int64,
+    cap_ip String,
+    addr_list String,
+    server_location String,
+    client_location String,
+    client_asn String,
+    server_asn String,
+    subscribe_id String,
+    user_region String,
+    con_duration_ms Int64,
+    url String,
+    host String,
+    domain String,
+    category String,
+    req_line String,
+    res_line String,
+    cookie String,
+    referer String,
+    user_agent String,
+    content_len String,
+    content_type String,
+    set_cookie String,
+    req_header String,
+    resp_header String,
+    req_body_key String,
+    req_body String,
+    res_body_key String,
+    resp_body String,
+    version String,
+    sni String,
+    san String,
+    cn String,
+    app_id Int64,
+    protocol_id Int64,
+    con_latency_ms Int64,
+    pinningst Int64,
+    intercept_state Int64,
+    ssl_server_side_latency Int64,
+    ssl_client_side_latency Int64,
+    ssl_server_side_version String,
+    ssl_client_side_version String,
+    ssl_cert_verify Int64,
+    stream_trace_id String,
+    ssl_error String,
+    c2s_pkt_num Int64,
+    s2c_pkt_num Int64,
+    c2s_byte_num Int64,
+    s2c_byte_num Int64,
+    nas_ip String,
+    framed_ip String,
+    account String,
+    packet_type Int64
+) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (recv_time);
+
+CREATE TABLE IF NOT EXISTS ntc_galaxy.radius_log_local on cluster tsg_galaxy(
+    policy_id Int64,
+    user_tags String,
+    action Int64,
+    start_time Int64,
+    end_time Int64,
+    recv_time Int64,
+    trans_proto String,
+    app_proto String,
+    addr_type Int64,
+    server_ip String,
+    d_ip Int64,
+    s_ip Int64,
+    client_ip String,
+    server_port Int64,
+    client_port Int64,
+    service Int64,
+    entrance_id Int64,
+    device_id Int64,
+    link_id Int64,
+    isp String,
+    encap_type Int64,
+    direction Int64,
+    stream_dir Int64,
+    cap_ip String,
+    addr_list String,
+    server_location String,
+    client_location String,
+    client_asn String,
+    server_asn String,
+    subscribe_id String,
+    user_region String,
+    con_duration_ms Int64,
+    url String,
+    host String,
+    domain String,
+    category String,
+    req_line String,
+    res_line String,
+    cookie String,
+    referer String,
+    user_agent String,
+    content_len String,
+    content_type String,
+    set_cookie String,
+    req_header String,
+    resp_header String,
+    req_body_key String,
+    req_body String,
+    res_body_key String,
+    resp_body String,
+    version String,
+    sni String,
+    san String,
+    cn String,
+    app_id Int64,
+    protocol_id Int64,
+    con_latency_ms Int64,
+    pinningst Int64,
+    intercept_state Int64,
+    ssl_server_side_latency Int64,
+    ssl_client_side_latency Int64,
+    ssl_server_side_version String,
+    ssl_client_side_version String,
+    ssl_cert_verify Int64,
+    stream_trace_id String,
+    ssl_error String,
+    c2s_pkt_num Int64,
+    s2c_pkt_num Int64,
+    c2s_byte_num Int64,
+    s2c_byte_num Int64,
+    nas_ip String,
+    framed_ip String,
+    account String,
+    packet_type Int64
+) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time))
+ORDER BY (recv_time);
+create TABLE IF NOT EXISTS tsg_galaxy.radius_log AS tsg_galaxy.radius_log_local ENGINE =Distributed(tsg_galaxy,tsg_galaxy,radius_log_local,rand());
+create TABLE IF NOT EXISTS tsg_galaxy.policy_event_log AS tsg_galaxy.policy_event_log_local ENGINE =Distributed(tsg_galaxy,tsg_galaxy,policy_event_log_local,rand());
+create TABLE IF NOT EXISTS tsg_galaxy.session_record_log AS tsg_galaxy.session_record_log_local ENGINE =Distributed(tsg_galaxy,tsg_galaxy,session_record_log_local,rand());
+
+create TABLE IF NOT EXISTS ntc_galaxy.radius_log AS ntc_galaxy.radius_log_local ENGINE =Distributed(tsg_galaxy,ntc_galaxy,radius_log_local,rand());
+create TABLE IF NOT EXISTS ntc_galaxy.policy_event_log AS ntc_galaxy.policy_event_log_local ENGINE =Distributed(tsg_galaxy,ntc_galaxy,policy_event_log_local,rand());
+create TABLE IF NOT EXISTS ntc_galaxy.session_record_log AS ntc_galaxy.session_record_log_local ENGINE =Distributed(tsg_galaxy,ntc_galaxy,session_record_log_local,rand());
+
+
diff --git a/TSG/docker.tar.gz b/TSG/docker.tar.gz
new file mode 100644
index 0000000..452d5ed
--- /dev/null
+++ b/TSG/docker.tar.gz
diff --git a/TSG/galaxy-qgw-service-deploy.tar.gz b/TSG/galaxy-qgw-service-deploy.tar.gz
new file mode 100644
index 0000000..d86b4a0
--- /dev/null
+++ b/TSG/galaxy-qgw-service-deploy.tar.gz
diff --git a/TSG/home.md b/TSG/home.md
new file mode 100644
index 0000000..b956db5
--- /dev/null
+++ b/TSG/home.md
@@ -0,0 +1,208 @@
+Galaxy TSG 集成文档
+
+## 1.安装列表
+
+| 应用安装     | 进程名称                    | 依赖                                                         | 备注                        |
+| ------------ | --------------------------- | ------------------------------------------------------------ | --------------------------- |
+| SQL 查询网关 | galaxy-qgw-         service | 1.clickhouse 原始日志检索<br> 2.Druid 时序库，预定义统计<br> 3.Hbase 离线报告结果 | 统一SQL查询代理网关平台     |
+| 报表统计服务 | galaxy-reprot-service       | 1.mariadb 执行任务列表 <br> 2.Hbase 统计结果<br> 3.clickhouse OLAP 统计 | 离线报告统计应用            |
+| Storm应用    | jar                         | 数据加载、实时统计                                           | 拓扑程序，部署到storm集群中 |
+|              |                             |                                                              |                             |
+
+## 2.初始化脚本
+
+| 数据类型         | 所属组件   |      |
+| ---------------- | ---------- | ---- |
+| kafka topic列表  | kafka      |      |
+| clickhouse表语句 | clickhouse |      |
+|                  |            |      |
+
+## 3.数据初始化
+
+### kafka 初始化
+
+1. 登录任意一台所属kafka机器，进入如下目录
+
+   cd /home/ceiec/kafka_2.11-1.0.0/bin
+
+2. 初始化topic
+
+​       拷贝附件 kafka-topic.txt 内容执行即可
+
+### clickhouse初始化
+
+1. 登录clickhouse连接终端 dbeave 建库 
+
+   create database tsg_galaxy on cluster tsg_galaxy ;
+   create database ntc_galaxy on cluster tsg_galaxy ;
+
+2. 终端工具下执行如下语句
+
+   Clickhouse-tsg.txt
+
+   
+
+## 4.应用组件安装
+
+### 构建Docker 环境
+
+> 依赖操作系统为centos7.4
+
+1. docker 安装
+
+   拷贝docker.tar.gz 任意目录，执行 docker-deploy.sh
+
+2. 导入centos 镜像
+   docker load -i centos7.tar
+
+3. 构建jdk1.8 镜像
+
+   解压jdk1.8-docker.tar.gz 执行构建jdk1.8_docker.sh
+
+4. 验证镜像安装
+
+   docker images 
+
+   
+
+### SQL查询网关基于docker安装
+
+
+
+1. 将安装galaxy-traffic-service.ta 包拷贝任意目录下
+
+   执行./installGalaxyTraffic.sh 
+
+2. 修改配置文件  application.properties 
+
+   - Kafka 配置
+
+     ```
+     kafka.consumer.servers=10.4.34.10:9092,10.4.34.11:9092,10.4.34.12:9092,10.4.34.13:9092,10.4.34.14:9092,10.4.34.15:9092,10.4.34.16:9092,10.4.34.17:9092,10.4.34.18:9092,10.4.34.19:9092
+     kafka.consumer.enable.auto.commit=true
+     kafka.consumer.session.timeout=6000
+     kafka.consumer.auto.commit.interval=100
+     kafka.consumer.auto.offset.reset=latest
+     kafka.consumer.topic=NTC-NETWORK-TRAFFIC-STAT
+     kafka.consumer.group.id=CONSUMER-GALAXY-TRAFFIC-SERVICE-2019-07-01
+     
+     ```
+
+     
+
+   - mariadb配置 
+
+   ​     
+
+   ```
+   spring.datasource.druid.mariaDB.url=jdbc:mariadb://10.0.4.9:3306/galaxy-service
+   spring.datasource.druid.mariaDB.username=galaxy
+   spring.datasource.druid.mariaDB.password=ENC(N80MNMCGSZ9wD5UGRxUb2EzOhlMSZueT)
+   
+   ```
+
+   > 注意：password为加密，可使用
+   >
+   > java -cp jasypt-1.9.2.jar org.jasypt.intf.cli.JasyptPBEStringEncryptionCLI input="数据库的明文密码" password=f8car algorithm=PBEWithMD5AndDES
+
+   - clickhouse 配置
+
+     ```
+     spring.datasource.druid.clickhouseDB.url=jdbc:clickhouse://10.0.8.241:8010/k18_galaxy_service?socket_timeout=60000
+     spring.datasource.druid.clickhouseDB.username=default
+     spring.datasource.druid.clickhouseDB.password=ceiec2018
+     
+     ```
+
+     
+
+   - 存储库切换配置
+
+     
+
+     选择将流量统计结果存储到 mariadb 或 clickhouse 
+
+```
+db.switch=clickhouseDB
+
+```
+
+3. 重启应用
+
+   ./galaxy-traffic.sh stop 
+
+    ./galaxy-traffic.sh start 
+
+4. 查看应用日志
+
+   /home/ceiec/galaxy-traffic-service/logs/galaxy-traffic-service.log
+
+   
+
+### galaxy-stat-service安装
+
+1. 将安装galaxy-stat-service.tar.gz 包拷贝任意目录下
+
+   执行./installGalaxyStat.sh
+
+2. 修改配置文件  application.properties 
+
+   - mariadb配置 
+
+     ```
+     multiple.datasource.galaxy-mariaDB.url=jdbc:mariadb://10.0.4.9/galaxy-service
+     multiple.datasource.galaxy-mariaDB.username=galaxy
+     multiple.datasource.galaxy-mariaDB.password=ENC(6Bkc8oDrrZ1lxYtGLYOgWBQu0XMtA5Za)
+     multiple.datasource.galaxy-mariaDB.type=com.alibaba.druid.pool.DruidDataSource
+     multiple.datasource.galaxy-mariaDB.driver-class-name=org.mariadb.jdbc.Driver
+     
+     ```
+
+     > 注意：password为加密，可使用
+     >
+     > java -cp jasypt-1.9.2.jar org.jasypt.intf.cli.JasyptPBEStringEncryptionCLI input="数据库明文密码" password=EbfYkitulv73I2p0mXI50JMXoaxZTKJ7 algorithm=PBEWithMD5AndDES
+
+   - Clickhouse-1 环境
+
+     ```
+     multiple.datasource.galaxy-clickhouse.url=jdbc:clickhouse://clickhouse-read-ys.com:80/k18_ods?socket_timeout=60000
+     multiple.datasource.galaxy-clickhouse.username=wall
+     multiple.datasource.galaxy-clickhouse.password=k18
+     
+     ```
+
+     
+
+   - clickhouse-2 环境
+
+     ```
+     multiple.datasource.galaxy-clickhouse2.url=jdbc:clickhouse://10.0.8.241:8010/k18_galaxy_service?socket_timeout=60000
+     multiple.datasource.galaxy-clickhouse2.username=default
+     multiple.datasource.galaxy-clickhouse2.password=ceiec2018
+     
+     ```
+
+   - influxdb 环境
+
+     > 分两地，若部署一地，地址可相同
+
+     ```
+     http.astana.url=10.0.5.19:8086/query?db=ntc_stat
+     http.alamty.url=10.0.5.15:8086/query?db=ntc_stat
+     ```
+
+   - 重启应用
+
+     ./galaxy-stat.sh stop 
+
+      ./galaxy-stat.sh start 
+
+   - 查看应用日志
+
+     /home/ceiec/galaxy-stat-service/logs/galaxy-stat-service.log
+
+
+
+### 实时统计topology部署
+
+// todo
\ No newline at end of file
diff --git a/TSG/jdk1.8-docker.tar.gz b/TSG/jdk1.8-docker.tar.gz
new file mode 100644
index 0000000..493fb7b
--- /dev/null
+++ b/TSG/jdk1.8-docker.tar.gz
diff --git a/TSG/kafka-topic.txt b/TSG/kafka-topic.txt
new file mode 100644
index 0000000..f76c612
--- /dev/null
+++ b/TSG/kafka-topic.txt
@@ -0,0 +1,18 @@
+
+
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-EXTERNAL-HOST-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-INTERNAL-HOST-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-URLS-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-USER-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-WEBSITE-DOMAIN-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TRAFFIC-METRICS-LOG
+
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  SESSION-RECORD-COMPLETED-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  POLICY-EVENT-COMPLETED-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  RADIUS-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  SESSION-RECORD-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  POLICY-EVENT-LOG
+
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  NTC-POLICY-EVENT-COMPLETED-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  NTC-POLICY-EVENT-LOG
+kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  NTC-TOP-URLS-LOG
\ No newline at end of file