20.07.rc1tsg-version20.07.rc1-deploy

6 files changed, 190 insertions, 135 deletions

diff --git a/roles/tfe/files/tfe-4.3.5.0db794c-1.el7.x86_64.rpm b/roles/tfe/files/tfe-4.3.5.0db794c-1.el7.x86_64.rpm
deleted file mode 100644
index 28234cf..0000000
--- a/roles/tfe/files/tfe-4.3.5.0db794c-1.el7.x86_64.rpm
+++ /dev/null
diff --git a/roles/tfe/files/tfe-4.3.7.39bff00-1.el7.x86_64.rpm b/roles/tfe/files/tfe-4.3.7.39bff00-1.el7.x86_64.rpm
new file mode 100644
index 0000000..9aecf93
--- /dev/null
+++ b/roles/tfe/files/tfe-4.3.7.39bff00-1.el7.x86_64.rpm
diff --git a/roles/tfe/tasks/main.yml b/roles/tfe/tasks/main.yml
index 5356aa2..64af311 100644
--- a/roles/tfe/tasks/main.yml
+++ b/roles/tfe/tasks/main.yml
@@ -14,7 +14,7 @@
   yum:
     name:
       - /tmp/ansible_deploy/tfe-kmod-v1.0.5.20200408-1dkms.noarch.rpm
-      - /tmp/ansible_deploy/tfe-4.3.5.0db794c-1.el7.x86_64.rpm
+      - /tmp/ansible_deploy/tfe-4.3.7.39bff00-1.el7.x86_64.rpm
     state: present
 
 - name: "template tfe-env config"
@@ -37,6 +37,16 @@
     src: "{{ role_path }}/templates/pangu_pxy.conf.j2"
     dest: /opt/tsg/tfe/conf/pangu/pangu_pxy.conf
 
+- name: "create conf/doh/"
+  file:
+    path: /opt/tsg/tfe/conf/doh/ 
+    state: directory
+
+- name: "template the doh.conf"
+  template:
+    src: "{{ role_path }}/templates/doh.conf.j2"
+    dest: /opt/tsg/tfe/conf/doh/doh.conf
+
 - name: "create a override conf - first step, create dir"
   file:
     path: /etc/systemd/system/tfe.service.d/
diff --git a/roles/tfe/templates/doh.conf.j2 b/roles/tfe/templates/doh.conf.j2
new file mode 100644
index 0000000..300e584
--- /dev/null
+++ b/roles/tfe/templates/doh.conf.j2
@@ -0,0 +1,26 @@
+[doh]
+# default 1
+enable=1
+
+[log]
+# default 10
+# RLOG_LV_DEBUG : 10
+# RLOG_LV_INFO  : 20
+# RLOG_LV_FATAL : 30
+log_level=10
+
+[maat]
+# default TSG_OBJ_APP_ID
+table_appid=TSG_OBJ_APP_ID
+# default TSG_SECURITY_ADDR
+table_addr=TSG_SECURITY_ADDR
+# default TSG_FIELD_DOH_QNAME
+table_qname=TSG_FIELD_DOH_QNAME
+# default TSG_FIELD_HTTP_HOST
+table_host=TSG_FIELD_DOH_HOST
+
+[kafka]
+# default 0
+ENTRANCE_ID=0
+# default 1
+en_sendlog=1
diff --git a/roles/tfe/templates/pangu_pxy.conf.j2 b/roles/tfe/templates/pangu_pxy.conf.j2
index 8790677..26d8d15 100644
--- a/roles/tfe/templates/pangu_pxy.conf.j2
+++ b/roles/tfe/templates/pangu_pxy.conf.j2
@@ -1,129 +1,107 @@
-[debug]

-log_level=30

-

-[log]

-{% if tsg_running_type == 0 or 1 %}

-nic_name={{ server.ethname }}

-{% else %}

-nic_name={{ nic_mgr.name }}

-{% endif %}

-entrance_id=0

-device_id_filepath=/opt/tsg/etc/tsg_sn.json

-kafka_brokerlist= {{ log_kafkabrokers.address }}

-kafka_topic=PROXY-EVENT-LOG

-

-#Addresses of minio. Format is defined by WiredLB.

-#minio_ip_list=192.168.10.61-64;

-minio_ip_list= {{ log_minio.address }}

-minio_listen_port= {{ log_minio.port }}

-#Maximum number of connections opened by per host.

-#MAX_CONNECTION_PER_HOST=1

-#Maximum number of requests in a pipeline.

-#MAX_CNNT_PIPELINE_NUM=20

-#Maximum parellel sessions(http and redis) is allowed to open.

-#MAX_CURL_SESSION_NUM=100

-#Maximum time the request is allowed to take(seconds).

-#MAX_CURL_TRANSFER_TIMEOUT_S=0

-

-#Bucket name in minio.

-cache_bucket_name=proxybucket

-#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value.

-max_used_memroy_size_mb=5120

-#Default TTL of objects, i.e. the time after which the object will expire(minumun 60s, i.e. 1 minute).

-cache_default_ttl_second=3600

-#Whether to hash the object key before cache actions. GET/PUT may be faster if you open it.

-cache_object_key_hash_switch=1

-

-#Store way: 0-MINIO; 1-META in REDIS, object in minio; 2-META and small object in Redis, large object in minio;

-cache_store_object_way=0

-#If CACHE_STORE_OBJECT_WAY is 2 and the size of a object is not bigger than this value, object will be stored in redis.

-redis_cache_object_size=1024000

-#Configs of WiredLB for Minios load balancer.

-#WIREDLB_OVERRIDE=1

-wiredlb_health_port=42310

-#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.

-redis_cluster_ip_list=192.168.10.62-63;

-redis_cluster_port_range=6379

-#wired load balancer configuration

-

-wiredlb_override=1

-wiredlb_topic=MinioFileLog

-wiredlb_datacenter=k18consul-tse

-wiredlb_health_port=52102

-wiredlb_group=FileLog

-

-log_fsstat_appname=tango_log_file

-log_fsstat_filepath=./tango_log_file.fs

-log_fsstat_interval=10

-log_fsstat_trig=1

-log_fsstat_dst_ip=10.4.20.202

-log_fsstat_dst_port=8125

-[maat]

-# 0:json 1: redis 2: iris

-maat_input_mode=1

-table_info=resource/pangu/table_info.conf

-json_cfg_file=resource/pangu/pangu_http.json

-stat_file=log/pangu_scan.status

-full_cfg_dir=pangu_policy/full/index/

-inc_cfg_dir=pangu_policy/inc/index/

-

-maat_redis_server={{ maat_redis_server.address }}

-maat_redis_port_range={{ maat_redis_server.port }}

-maat_redis_db_index={{ maat_redis_server.db }}

-effect_interval_s=1

-#accept_tags={"tags":[{"tag":"location","value":"Astana"}]}

-

-[dynamic_maat]

-maat_input_mode=1

-table_info=resource/pangu/dynamic_maat_table_info.conf

-maat_redis_server={{ dynamic_maat_redis_server.address }}

-maat_redis_port_range={{ dynamic_maat_redis_server.port }}

-maat_redis_db_index={{ dynamic_maat_redis_server.db }}

-effect_interval_s=1

-

-[tango_cache]

-enable_cache=0

-minio_ip_list=192.168.10.61-64;

-minio_listen_port=9000

-

-#max_connection_per_host=1

-max_cnnt_pipeline_num=20

-#max_curl_session_num=100

-

-cache_bucket_name=proxybucket

-max_used_memory_size_mb=10240

-cache_default_ttl_second=3600

-cache_object_key_hash_switch=1

-

-#1-minio，2-redis

-#Store way: 0-MINIO; 1-META in REDIS, object in minio; 2-META and small object in Redis, large object in minio;

-cache_store_object_way=0

-#If CACHE_STORE_OBJECT_WAY is 2 and the size of a object is not bigger than this value, object will be stored in redis.

-redis_cache_object_size=102400

-#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.

-redis_cluster_ip_list=192.168.10.62-63;

-redis_cluster_port_range=6379

-#wired load balancer configuration

-wiredlb_override=1

-wiredlb_topic=MinioCache

-wiredlb_datacenter=k18consul-tse

-wiredlb_health_port=52101

-wiredlb_group=TangoCache

-

-cache_undefined_obj=1

-query_undefined_obj=0

-statsd_server={{fs_remote.address}}

-statsd_port={{fs_remote.port}}

-histogram_bins=0.20,0.40,0.6,0.8

-

-log_fsstat_appname=tango_cache

-log_fsstat_filepath=./tango_cache_client.fs

-log_fsstat_interval=10

-log_fsstat_trig=1

-log_fsstat_dst_ip=10.4.20.201

-log_fsstat_dst_port=8125

-

-

-[traffic_mirror]

-table_info=resource/pangu/table_info_traffic_mirror.conf

-stat_file=log/traffic_mirror.status

+[debug]
+log_level=10
+
+[log]
+entrance_id=0
+
+#Addresses of minio. Format is defined by WiredLB.
+#minio_ip_list=192.168.10.61-64;
+minio_ip_list= {{ log_minio.address }}
+minio_listen_port= {{ log_minio.port }}
+#Maximum number of connections opened by per host.
+#MAX_CONNECTION_PER_HOST=1
+#Maximum number of requests in a pipeline.
+#MAX_CNNT_PIPELINE_NUM=20
+#Maximum parellel sessions(http and redis) is allowed to open.
+#MAX_CURL_SESSION_NUM=100
+#Maximum time the request is allowed to take(seconds).
+#MAX_CURL_TRANSFER_TIMEOUT_S=0
+
+#Bucket name in minio.
+cache_bucket_name=proxybucket
+#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value.
+max_used_memroy_size_mb=5120
+#Default TTL of objects, i.e. the time after which the object will expire(minumun 60s, i.e. 1 minute).
+cache_default_ttl_second=3600
+#Whether to hash the object key before cache actions. GET/PUT may be faster if you open it.
+cache_object_key_hash_switch=1
+
+#Store way: 0-MINIO; 1-META in REDIS, object in minio; 2-META and small object in Redis, large object in minio;
+cache_store_object_way=0
+#If CACHE_STORE_OBJECT_WAY is 2 and the size of a object is not bigger than this value, object will be stored in redis.
+redis_cache_object_size=1024000
+#Configs of WiredLB for Minios load balancer.
+#WIREDLB_OVERRIDE=1
+wiredlb_health_port=42310
+#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.
+redis_cluster_ip_list=192.168.10.62-63;
+redis_cluster_port_range=6379
+#wired load balancer configuration
+
+wiredlb_override=1
+wiredlb_topic=MinioFileLog
+wiredlb_datacenter=k18consul-tse
+wiredlb_health_port=52102
+wiredlb_group=FileLog
+
+log_fsstat_appname=tango_log_file
+log_fsstat_filepath=./tango_log_file.fs
+log_fsstat_interval=10
+log_fsstat_trig=1
+log_fsstat_dst_ip=10.4.20.202
+log_fsstat_dst_port=8125
+
+[ratelimit]
+enable=0
+token_name=ratelimit
+redis_server={{ maat_redis_server.address }}
+redis_port={{ maat_redis_server.port }}
+redis_db_index=6
+
+[tango_cache]
+enable_cache=0
+minio_ip_list=192.168.10.61-64;
+minio_listen_port=9000
+
+#max_connection_per_host=1
+max_cnnt_pipeline_num=20
+#max_curl_session_num=100
+
+cache_bucket_name=proxybucket
+max_used_memory_size_mb=10240
+cache_default_ttl_second=3600
+cache_object_key_hash_switch=1
+
+#1-minio，2-redis
+#Store way: 0-MINIO; 1-META in REDIS, object in minio; 2-META and small object in Redis, large object in minio;
+cache_store_object_way=0
+#If CACHE_STORE_OBJECT_WAY is 2 and the size of a object is not bigger than this value, object will be stored in redis.
+redis_cache_object_size=102400
+#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.
+redis_cluster_ip_list=192.168.10.62-63;
+redis_cluster_port_range=6379
+#wired load balancer configuration
+wiredlb_override=1
+wiredlb_topic=MinioCache
+wiredlb_datacenter=k18consul-tse
+wiredlb_health_port=52101
+wiredlb_group=TangoCache
+
+cache_undefined_obj=1
+query_undefined_obj=0
+statsd_server=192.168.10.72
+statsd_port=8126
+histogram_bins=0.20,0.40,0.6,0.8
+
+log_fsstat_appname=tango_cache
+log_fsstat_filepath=./tango_cache_client.fs
+log_fsstat_interval=10
+log_fsstat_trig=1
+log_fsstat_dst_ip=10.4.20.201
+log_fsstat_dst_port=8125
+
+
+[traffic_mirror]
+table_info=resource/pangu/table_info_traffic_mirror.conf
+stat_file=log/traffic_mirror.status
+
diff --git a/roles/tfe/templates/tfe.conf.j2 b/roles/tfe/templates/tfe.conf.j2
index 02beb08..cafdcc8 100644
--- a/roles/tfe/templates/tfe.conf.j2
+++ b/roles/tfe/templates/tfe.conf.j2
@@ -1,14 +1,15 @@
 [system]
 nr_worker_threads={{ tfe.nr_threads }}
-enable_breakpad=1
+enable_breakpad=0
 enable_breakpad_upload=0
 breakpad_minidump_dir=/run/tfe/crashreport/
 breakpad_upload_url=http://127.0.0.1:9000/
 disable_coredump=0
 
+
 [kni]
 ip=192.168.100.1
-scm_port=2475
+cmsg_port=2475
 watchdog_switch=1
 watchdog_port=2476
 
@@ -44,15 +45,17 @@ mc_cache_topic=PXY-EXCH-INTERMEDIA-CERT
 [key_keeper]
 #Mode: debug - generate cert with ca_path, normal - generate cert with cert store
 #0 on cache 1 off cache
-mode= {{ tfe.keykeeper.mode }}
+mode= normal
 no_cache=0
 cert_store_host= {{ cert_store_server.address }}
 cert_store_port= {{ cert_store_server.port }}
 ca_path=resource/tfe/tango-ca-v3-trust-ca.pem
 untrusted_ca_path=resource/tfe/tango-ca-v3-untrust-ca.pem
-enable_health_check=0
+# health_check only for "mode=normal"
+# default 1
+enable_health_check=1
 
-[debug]	
+[debug]
 passthrough_all_tcp=0
 
 [traffic_mirror]
@@ -84,6 +87,44 @@ level=10
 [stat]
 statsd_server={{ fs_remote.address }}
 statsd_port={{ fs_remote.port }}
+statsd_cycle=5
+# FS_OUTPUT_STATSD=1, FS_OUTPUT_INFLUX_LINE=2
+statsd_format=2
 
 [http]
 loglevel=10
+
+[kafka]
+enable=1
+{% if tsg_running_type == 0 or 1 %}
+nic_name={{ server.ethname }}
+{% else %}
+nic_name={{ nic_mgr.name }}
+{% endif %}
+kafka_brokerlist={{ log_kafkabrokers.address }}
+kafka_topic=PROXY-EVENT-LOG
+device_id_filepath=/opt/tsg/etc/tsg_sn.json
+
+[maat]
+# 0:json 1: redis 2: iris
+maat_input_mode=1
+table_info=resource/pangu/table_info.conf
+json_cfg_file=resource/pangu/pangu_http.json
+stat_file=log/pangu_scan.status
+full_cfg_dir=pangu_policy/full/index/
+inc_cfg_dir=pangu_policy/inc/index/
+
+maat_redis_server={{ maat_redis_server.address }}
+maat_redis_port_range={{ maat_redis_server.port }}
+maat_redis_db_index={{ maat_redis_server.db }}
+effect_interval_s=1
+#accept_tags={"tags":[{"tag":"location","value":"Astana"}]}
+
+[dynamic_maat]
+maat_input_mode=1
+table_info=resource/pangu/dynamic_maat_table_info.conf
+maat_redis_server={{ dynamic_maat_redis_server.address }}
+maat_redis_port_range={{ dynamic_maat_redis_server.port }}
+maat_redis_db_index={{ dynamic_maat_redis_server.db }}
+effect_interval_s=1
+