梳理galaxy应用部署版本规范

6 files changed, 64 insertions, 531 deletions

diff --git a/TSG/centos7.tar b/TSG/centos7.tar
deleted file mode 100644
index b0117ed..0000000
--- a/TSG/centos7.tar
+++ /dev/null
diff --git a/TSG/clickhouse-tsg.txt b/TSG/clickhouse-tsg.txt
deleted file mode 100644
index 7a24693..0000000
--- a/TSG/clickhouse-tsg.txt
+++ /dev/null
@@ -1,487 +0,0 @@
-
-
-CREATE TABLE IF NOT EXISTS tsg_galaxy.session_record_log_local on cluster ck_cluster(
-    policy_id Int64,
-    user_tags String,
-    action Int64,
-    start_time Int64,
-    end_time Int64,
-    recv_time Int64,
-    trans_proto String,
-    app_proto String,
-    addr_type Int64,
-    server_ip String,
-    d_ip Int64,
-    s_ip Int64,
-    client_ip String,
-    server_port Int64,
-    client_port Int64,
-    service Int64,
-    entrance_id Int64,
-    device_id Int64,
-    link_id Int64,
-    isp String,
-    encap_type Int64,
-    direction Int64,
-    stream_dir Int64,
-    cap_ip String,
-    addr_list String,
-    server_location String,
-    client_location String,
-    client_asn String,
-    server_asn String,
-    subscribe_id String,
-    user_region String,
-    con_duration_ms Int64,
-    url String,
-    host String,
-    domain String,
-    category String,
-    req_line String,
-    res_line String,
-    cookie String,
-    referer String,
-    user_agent String,
-    content_len String,
-    content_type String,
-    set_cookie String,
-    req_header String,
-    resp_header String,
-    req_body_key String,
-    req_body String,
-    res_body_key String,
-    resp_body String,
-    version String,
-    sni String,
-    san String,
-    cn String,
-    app_id Int64,
-    protocol_id Int64,
-    con_latency_ms Int64,
-    pinningst Int64,
-    intercept_state Int64,
-    ssl_server_side_latency Int64,
-    ssl_client_side_latency Int64,
-    ssl_server_side_version String,
-    ssl_client_side_version String,
-    ssl_cert_verify Int64,
-    stream_trace_id String,
-    ssl_error String,
-    c2s_pkt_num Int64,
-    s2c_pkt_num Int64,
-    c2s_byte_num Int64,
-    s2c_byte_num Int64,
-    nas_ip String,
-    framed_ip String,
-    account String,
-    packet_type Int64
-)
-ENGINE = MergeTree
-PARTITION BY toYYYYMMDD(toDate(recv_time))
-ORDER BY recv_time;
-
-
-
-CREATE TABLE IF NOT EXISTS tsg_galaxy.policy_event_log_local on cluster ck_cluster(
-    policy_id Int64,
-    user_tags String,
-    action Int64,
-    start_time Int64,
-    end_time Int64,
-    recv_time Int64,
-    trans_proto String,
-    app_proto String,
-    addr_type Int64,
-    server_ip String,
-    d_ip Int64,
-    s_ip Int64,
-    client_ip String,
-    server_port Int64,
-    client_port Int64,
-    service Int64,
-    entrance_id Int64,
-    device_id Int64,
-    link_id Int64,
-    isp String,
-    encap_type Int64,
-    direction Int64,
-    stream_dir Int64,
-    cap_ip String,
-    addr_list String,
-    server_location String,
-    client_location String,
-    client_asn String,
-    server_asn String,
-    subscribe_id String,
-    user_region String,
-    con_duration_ms Int64,
-    url String,
-    host String,
-    domain String,
-    category String,
-    req_line String,
-    res_line String,
-    cookie String,
-    referer String,
-    user_agent String,
-    content_len String,
-    content_type String,
-    set_cookie String,
-    req_header String,
-    resp_header String,
-    req_body_key String,
-    req_body String,
-    res_body_key String,
-    resp_body String,
-    version String,
-    sni String,
-    san String,
-    cn String,
-    app_id Int64,
-    protocol_id Int64,
-    con_latency_ms Int64,
-    pinningst Int64,
-    intercept_state Int64,
-    ssl_server_side_latency Int64,
-    ssl_client_side_latency Int64,
-    ssl_server_side_version String,
-    ssl_client_side_version String,
-    ssl_cert_verify Int64,
-    stream_trace_id String,
-    ssl_error String,
-    c2s_pkt_num Int64,
-    s2c_pkt_num Int64,
-    c2s_byte_num Int64,
-    s2c_byte_num Int64,
-    nas_ip String,
-    framed_ip String,
-    account String,
-    packet_type Int64
-) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time))
-ORDER BY (recv_time);
-
-CREATE TABLE IF NOT EXISTS tsg_galaxy.radius_log_local on cluster ck_cluster(
-    policy_id Int64,
-    user_tags String,
-    action Int64,
-    start_time Int64,
-    end_time Int64,
-    recv_time Int64,
-    trans_proto String,
-    app_proto String,
-    addr_type Int64,
-    server_ip String,
-    d_ip Int64,
-    s_ip Int64,
-    client_ip String,
-    server_port Int64,
-    client_port Int64,
-    service Int64,
-    entrance_id Int64,
-    device_id Int64,
-    link_id Int64,
-    isp String,
-    encap_type Int64,
-    direction Int64,
-    stream_dir Int64,
-    cap_ip String,
-    addr_list String,
-    server_location String,
-    client_location String,
-    client_asn String,
-    server_asn String,
-    subscribe_id String,
-    user_region String,
-    con_duration_ms Int64,
-    url String,
-    host String,
-    domain String,
-    category String,
-    req_line String,
-    res_line String,
-    cookie String,
-    referer String,
-    user_agent String,
-    content_len String,
-    content_type String,
-    set_cookie String,
-    req_header String,
-    resp_header String,
-    req_body_key String,
-    req_body String,
-    res_body_key String,
-    resp_body String,
-    version String,
-    sni String,
-    san String,
-    cn String,
-    app_id Int64,
-    protocol_id Int64,
-    con_latency_ms Int64,
-    pinningst Int64,
-    intercept_state Int64,
-    ssl_server_side_latency Int64,
-    ssl_client_side_latency Int64,
-    ssl_server_side_version String,
-    ssl_client_side_version String,
-    ssl_cert_verify Int64,
-    stream_trace_id String,
-    ssl_error String,
-    c2s_pkt_num Int64,
-    s2c_pkt_num Int64,
-    c2s_byte_num Int64,
-    s2c_byte_num Int64,
-    nas_ip String,
-    framed_ip String,
-    account String,
-    packet_type Int64
-) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time))
-ORDER BY (recv_time);
-
-
-CREATE TABLE IF NOT EXISTS ntc_galaxy.session_record_log_local on cluster ck_cluster(
-    policy_id Int64,
-    user_tags String,
-    action Int64,
-    start_time Int64,
-    end_time Int64,
-    recv_time Int64,
-    trans_proto String,
-    app_proto String,
-    addr_type Int64,
-    server_ip String,
-    d_ip Int64,
-    s_ip Int64,
-    client_ip String,
-    server_port Int64,
-    client_port Int64,
-    service Int64,
-    entrance_id Int64,
-    device_id Int64,
-    link_id Int64,
-    isp String,
-    encap_type Int64,
-    direction Int64,
-    stream_dir Int64,
-    cap_ip String,
-    addr_list String,
-    server_location String,
-    client_location String,
-    client_asn String,
-    server_asn String,
-    subscribe_id String,
-    user_region String,
-    con_duration_ms Int64,
-    url String,
-    host String,
-    domain String,
-    category String,
-    req_line String,
-    res_line String,
-    cookie String,
-    referer String,
-    user_agent String,
-    content_len String,
-    content_type String,
-    set_cookie String,
-    req_header String,
-    resp_header String,
-    req_body_key String,
-    req_body String,
-    res_body_key String,
-    resp_body String,
-    version String,
-    sni String,
-    san String,
-    cn String,
-    app_id Int64,
-    protocol_id Int64,
-    con_latency_ms Int64,
-    pinningst Int64,
-    intercept_state Int64,
-    ssl_server_side_latency Int64,
-    ssl_client_side_latency Int64,
-    ssl_server_side_version String,
-    ssl_client_side_version String,
-    ssl_cert_verify Int64,
-    stream_trace_id String,
-    ssl_error String,
-    c2s_pkt_num Int64,
-    s2c_pkt_num Int64,
-    c2s_byte_num Int64,
-    s2c_byte_num Int64,
-    nas_ip String,
-    framed_ip String,
-    account String,
-    packet_type Int64
-)
-ENGINE = MergeTree
-PARTITION BY toYYYYMMDD(toDate(recv_time))
-ORDER BY recv_time;
-
-
-
-CREATE TABLE IF NOT EXISTS ntc_galaxy.policy_event_log_local on cluster ck_cluster(
-    policy_id Int64,
-    user_tags String,
-    action Int64,
-    start_time Int64,
-    end_time Int64,
-    recv_time Int64,
-    trans_proto String,
-    app_proto String,
-    addr_type Int64,
-    server_ip String,
-    d_ip Int64,
-    s_ip Int64,
-    client_ip String,
-    server_port Int64,
-    client_port Int64,
-    service Int64,
-    entrance_id Int64,
-    device_id Int64,
-    link_id Int64,
-    isp String,
-    encap_type Int64,
-    direction Int64,
-    stream_dir Int64,
-    cap_ip String,
-    addr_list String,
-    server_location String,
-    client_location String,
-    client_asn String,
-    server_asn String,
-    subscribe_id String,
-    user_region String,
-    con_duration_ms Int64,
-    url String,
-    host String,
-    domain String,
-    category String,
-    req_line String,
-    res_line String,
-    cookie String,
-    referer String,
-    user_agent String,
-    content_len String,
-    content_type String,
-    set_cookie String,
-    req_header String,
-    resp_header String,
-    req_body_key String,
-    req_body String,
-    res_body_key String,
-    resp_body String,
-    version String,
-    sni String,
-    san String,
-    cn String,
-    app_id Int64,
-    protocol_id Int64,
-    con_latency_ms Int64,
-    pinningst Int64,
-    intercept_state Int64,
-    ssl_server_side_latency Int64,
-    ssl_client_side_latency Int64,
-    ssl_server_side_version String,
-    ssl_client_side_version String,
-    ssl_cert_verify Int64,
-    stream_trace_id String,
-    ssl_error String,
-    c2s_pkt_num Int64,
-    s2c_pkt_num Int64,
-    c2s_byte_num Int64,
-    s2c_byte_num Int64,
-    nas_ip String,
-    framed_ip String,
-    account String,
-    packet_type Int64
-) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time))
-ORDER BY (recv_time);
-
-CREATE TABLE IF NOT EXISTS ntc_galaxy.radius_log_local on cluster ck_cluster(
-    policy_id Int64,
-    user_tags String,
-    action Int64,
-    start_time Int64,
-    end_time Int64,
-    recv_time Int64,
-    trans_proto String,
-    app_proto String,
-    addr_type Int64,
-    server_ip String,
-    d_ip Int64,
-    s_ip Int64,
-    client_ip String,
-    server_port Int64,
-    client_port Int64,
-    service Int64,
-    entrance_id Int64,
-    device_id Int64,
-    link_id Int64,
-    isp String,
-    encap_type Int64,
-    direction Int64,
-    stream_dir Int64,
-    cap_ip String,
-    addr_list String,
-    server_location String,
-    client_location String,
-    client_asn String,
-    server_asn String,
-    subscribe_id String,
-    user_region String,
-    con_duration_ms Int64,
-    url String,
-    host String,
-    domain String,
-    category String,
-    req_line String,
-    res_line String,
-    cookie String,
-    referer String,
-    user_agent String,
-    content_len String,
-    content_type String,
-    set_cookie String,
-    req_header String,
-    resp_header String,
-    req_body_key String,
-    req_body String,
-    res_body_key String,
-    resp_body String,
-    version String,
-    sni String,
-    san String,
-    cn String,
-    app_id Int64,
-    protocol_id Int64,
-    con_latency_ms Int64,
-    pinningst Int64,
-    intercept_state Int64,
-    ssl_server_side_latency Int64,
-    ssl_client_side_latency Int64,
-    ssl_server_side_version String,
-    ssl_client_side_version String,
-    ssl_cert_verify Int64,
-    stream_trace_id String,
-    ssl_error String,
-    c2s_pkt_num Int64,
-    s2c_pkt_num Int64,
-    c2s_byte_num Int64,
-    s2c_byte_num Int64,
-    nas_ip String,
-    framed_ip String,
-    account String,
-    packet_type Int64
-) ENGINE=MergeTree PARTITION BY toYYYYMMDD(toDate(recv_time))
-ORDER BY (recv_time);
-create TABLE IF NOT EXISTS tsg_galaxy.radius_log AS tsg_galaxy.radius_log_local ENGINE =Distributed(ck_cluster,tsg_galaxy,radius_log_local,rand());
-create TABLE IF NOT EXISTS tsg_galaxy.policy_event_log AS tsg_galaxy.policy_event_log_local ENGINE =Distributed(ck_cluster,tsg_galaxy,policy_event_log_local,rand());
-create TABLE IF NOT EXISTS tsg_galaxy.session_record_log AS tsg_galaxy.session_record_log_local ENGINE =Distributed(ck_cluster,tsg_galaxy,session_record_log_local,rand());
-
-create TABLE IF NOT EXISTS ntc_galaxy.radius_log AS ntc_galaxy.radius_log_local ENGINE =Distributed(ck_cluster,ntc_galaxy,radius_log_local,rand());
-create TABLE IF NOT EXISTS ntc_galaxy.policy_event_log AS ntc_galaxy.policy_event_log_local ENGINE =Distributed(ck_cluster,ntc_galaxy,policy_event_log_local,rand());
-create TABLE IF NOT EXISTS ntc_galaxy.session_record_log AS ntc_galaxy.session_record_log_local ENGINE =Distributed(ck_cluster,ntc_galaxy,session_record_log_local,rand());
-
-
diff --git a/TSG/docker.tar.gz b/TSG/docker.tar.gz
deleted file mode 100644
index 452d5ed..0000000
--- a/TSG/docker.tar.gz
+++ /dev/null
diff --git a/TSG/home.md b/TSG/home.md
index 56e1eae..eec95e5 100644
--- a/TSG/home.md
+++ b/TSG/home.md
@@ -8,40 +8,39 @@
 
 # 1.集成所需清单
 
-## 1.1 公共组件
-
-| 组件名称           | 版本      | 约束                                                         | 备注                                                         |
-| ------------------ | --------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
-| 系统要求           | centos7.4 | JDK1.8 <br>时钟同步 <br/>文件句柄 65535<br/> 防火墙关闭(开放公共组件端口) |                                                              |
-| Kafka              | 1.0.0     | 端口：9092                                                   | 需安装(复用) zookeeper                                       |
-| apache-storm-1.0.2 | 1.0.2     | 端口：8080                                                   | 需安装(复用) zookeeper                                       |
-| zookeeper-3.4.9    | 3.4.9     | 端口：2181                                                   |                                                              |
-| ClickHouse         | 19.5.4.1  | 端口：8123                                                   | 需安装(复用) zookeeper                                       |
-| Hbase              |           | 端口：8084                                                   | 单独zookeeper、hdfs(集群)                                    |
-| Druid              | 0.14      | 端口：8082                                                   | 依赖Mysql(mariadb)、zookeeper、hdfs(集群)                    |
-| Redis              |           |                                                              | 补全缓存库                                                   |
-| Mariadb(mysql)     |           | 3309                                                         |                                                              |
-| Minio              |           | 9090                                                         | minio安装目录：https://git.mesalab.cn/zhangchengwei/MinioRelated |
+## 1.1 开源公共组件
+
+| 组件名称           | 版本     | 约束       | 备注                                                         |
+| ------------------ | -------- | ---------- | ------------------------------------------------------------ |
+| Kafka              | 1.0.0    | 端口：9092 | 需安装(复用) zookeeper                                       |
+| apache-storm-1.0.2 | 1.0.2    | 端口：8080 | 需安装(复用) zookeeper                                       |
+| zookeeper-3.4.9    | 3.4.9    | 端口：2181 | 也作为日志ID唯一标识存储                                     |
+| ClickHouse         | 19.5.4.1 | 端口：8123 | 需安装(复用) zookeeper                                       |
+| Hbase              |          | 端口：8084 | 单独zookeeper、hdfs(集群)<br/> subscribe_id 补全应用         |
+| Druid              | 0.14     | 端口：8082 | 依赖Mysql(mariadb)、zookeeper、hdfs(集群)                    |
+| Mariadb(mysql)     |          | 3309       | Druid metadata存储，依赖API库，无需单独构建                  |
+| Minio              |          | 9090       | minio安装目录：https://git.mesalab.cn/zhangchengwei/MinioRelated |
 
 组件链接地址：[大数据平台-公共组件库](https://git.mesalab.cn/bigdata/bigdata-deploy)
 ## 1.2 应用清单
 
-| 应用安装     | 进程名称                 | 依赖                                                         | 备注                           |
-| ------------ | ------------------------ | ------------------------------------------------------------ | ------------------------------ |
-| SQL 查询网关 | galaxy-qgw-service(9999) | 1.clickhouse 日志检索<br> 2.Druid 时序库，预定义统计<br> 3.Hbase 离线报告结果 | 统一SQL查询代理网关平台        |
-| 报表统计服务 | galaxy-reprot-service    | 1.mariadb 执行任务列表 <br> 2.Hbase 统计结果<br> 3.clickhouse OLAP 统计 | 离线报告统计应用               |
-| Storm应用    | jar                      | 数据加载、实时统计                                           | 拓扑程序，部署到storm集群中    |
-| gohangout    | gohangout                | 数据源-kafka;数据输出-ck                                     | 数据预处理，数据加载到CK的应用 |
+| 应用安装      | 进程名称                 | 依赖                                                         | 备注                           |
+| ------------- | ------------------------ | ------------------------------------------------------------ | ------------------------------ |
+| SQL 查询网关  | galaxy-qgw-service(9999) | 1.clickhouse 日志检索<br> 2.Druid 时序库，预定义统计<br> 3.Hbase 离线报告结果 | 统一SQL查询代理网关平台        |
+| 报表统计服务  | galaxy-reprot-service    | 1.mariadb 执行任务列表 <br> 2.Hbase 统计结果<br> 3.clickhouse OLAP 统计 | 离线报告统计应用               |
+| Storm统计应用 | jar包                    | 补全加载、实时统计                                           | 拓扑程序，部署到storm集群中    |
+| gohangout     | gohangout                | 数据源-kafka;数据输出-ck                                     | 数据预处理，数据加载到CK的应用 |
 
-## 1.3 初始化脚本
+## 1.3 数据初始化列表
 
 | 数据类型             | 所属组件     |                                |
 | -------------------- | ------------ | ------------------------------ |
-| kafka topic列表      | kafka        | 初始化topic                    |
+| kafka topic列表      | kafka        | 初始化topic列表                |
 | clickhouse建表初始化 | clickhouse   | 初始化库、授权用户、创建表结构 |
 | Druid 构建统计Schema | Apache Druid | json结构                       |
+| Storm 提交job        | Storm        | jar包                          |
 
-# 2.配置初始化
+# 2.数据初始化步骤
 
 ### kafka 初始化
 
@@ -71,7 +70,46 @@
    用户名 tsg_report 密码 ceiec2019    报表服务使用
    用户名 tsg_insert  密码 ceiec2019    gohangout使用
 
-# 3. 应用安装
+### Druid统计任务初始化
+
+  //todo
+
+### Storm提交topology任务
+
+//todo
+
+
+
+# 3. 版本列表
+
+#### Tsg_Galaxy_v2.0.20190630
+
+| 初始化脚本 | data/clickhouse-db/<br/>data/druid-job/v2.0/--.json<br/>data/Kafka-topics/tsg-topic_2.0.txt<br/>data/storm-job/v2.0/--.jar |
+| ---------- | ------------------------------------------------------------ |
+| 公共组件   | zookeeper:3.4.9<br/>kafka:1.0.0<br/>hbase:1.4.9<br/>clickhouse:19.54.1<br/>druid:0.14.0<br/>storm:1.0.2<br/> |
+| 应用镜像   | galaxy/galaxy-qgw-service:1.1<br/>galaxy/galaxy-report-service:1.1<br/>galaxy/gohangout:1.2.16 |
+
+**包含功能：**
+
+- 对外提供SQL调用接口：通联、事件、radius日志查询、时序统计查询、报表结果查询
+- 接收流量处理端产生的代理通联日志、策略日志
+- 接收总控端解析的radius日志数据，并构建IP与subscriber_id 知识库
+
+
+
+//todo link 单机版目录
+
+//集群 --todo
+
+
+
+
+
+
+
+
+
+# 4. 其它(待删除)
 
 ### 构建Docker 环境
 
@@ -94,7 +132,7 @@
 
    
 
-### SQL查询网关基于docker安装
+### SQL查询网关docker安装
 
 
 
@@ -145,7 +183,7 @@ hbase:
 
 
 
-### galaxy-report-service基于docker安装
+### report服务docker安装
 
 1. 将安装galaxy-report-service-deploy.tar.gz 包拷贝任意目录下
 
diff --git a/TSG/jdk1.8-docker.tar.gz b/TSG/jdk1.8-docker.tar.gz
deleted file mode 100644
index 493fb7b..0000000
--- a/TSG/jdk1.8-docker.tar.gz
+++ /dev/null
diff --git a/TSG/kafka-topic.txt b/TSG/kafka-topic.txt
deleted file mode 100644
index f76c612..0000000
--- a/TSG/kafka-topic.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-EXTERNAL-HOST-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-INTERNAL-HOST-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-URLS-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-USER-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TOP-WEBSITE-DOMAIN-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  TRAFFIC-METRICS-LOG
-
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  SESSION-RECORD-COMPLETED-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  POLICY-EVENT-COMPLETED-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  RADIUS-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  SESSION-RECORD-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  POLICY-EVENT-LOG
-
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  NTC-POLICY-EVENT-COMPLETED-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  NTC-POLICY-EVENT-LOG
-kafka-topics.sh --create --zookeeper 192.168.10.121:2181 --replication-factor 1 --partitions 3 --topic  NTC-TOP-URLS-LOG
\ No newline at end of file