P19 23.07 online-configP19

5 files changed, 111 insertions, 0 deletions

diff --git a/PCAP-PIC/packet_dump/rtp/config/packet_dump.yml b/PCAP-PIC/packet_dump/rtp/config/packet_dump.yml
new file mode 100644
index 0000000..e8e65e6
--- /dev/null
+++ b/PCAP-PIC/packet_dump/rtp/config/packet_dump.yml
@@ -0,0 +1,37 @@
+project:
+  - project_name: rtp
+    storage_file_type: pcap              # [pcap, pcapng]
+    consumer_topic: INTERNAL-RTP-RECORD
+    producer_topic: VOIP-RECORD
+    cache_queue_size: 100000
+    storage_mode: remote   #remote: aws or local
+    aws_bucket: rtp_hos_bucket
+    append_mode: append   #append, appendV2
+    append_max_num: 100000
+    upload_goroutine_num: 15
+    check_file_string_key: [common_server_ip, common_client_ip]
+    check_file_intval_key: [common_server_port, common_client_port, common_start_time]
+    write_back_status_key: [raw_log_status]
+    delete_json_key: [raw_packet, raw_packet_len]
+    write_back_filepath_key: rtp_pcap_path
+    origin_packet_time_ms_key: raw_packet_time_ms
+    origin_packet_key: raw_packet
+    filename_prefix: rtp
+    storage_directory: /var/www/html/rtp/
+    file_timeout: 120
+kafka:
+  broker: [192.168.10.193:9094,192.168.10.194:9094,192.168.10.195:9094]
+  sasl_user: admin
+  sasl_password: galaxy2019
+  offset: -1   # Newest: -1 or Oldest: -2
+aws:
+  endpoint: http://192.168.10.251:9098/hos/
+  token: f5c5186ba4874182b33b9b2b2b6e3f77
+  access_key: default
+  access_key_id: default
+log:
+  path: /data/tsg/olap/galaxy/packet_dump/logs/log
+  level: info                 # painc, fatal, warnning, info, debug, trace
+  RotationSize:  1000000000
+  RotationCount: 14
+  RotationTime:  1 
diff --git a/PCAP-PIC/packet_dump/rtp/logs/log b/PCAP-PIC/packet_dump/rtp/logs/log
new file mode 100644
index 0000000..42f0d7b
--- /dev/null
+++ b/PCAP-PIC/packet_dump/rtp/logs/log
@@ -0,0 +1,8 @@
+time="2023-07-08 15:32:02" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-08 15:32:02" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-08 15:32:02" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-08 15:32:02" level=info msg=partitionList Topic: =INTERNAL-RTP-RECORD partitionLists ="[0 1 2]"
+time="2023-07-08 15:32:02" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-08 15:32:02" level=info Offset: =-1 Topic: =INTERNAL-RTP-RECORD partition: =0
+time="2023-07-08 15:32:02" level=info Offset: =-1 Topic: =INTERNAL-RTP-RECORD partition: =1
+time="2023-07-08 15:32:02" level=info Offset: =-1 Topic: =INTERNAL-RTP-RECORD partition: =2
diff --git a/PCAP-PIC/packet_dump/rtp/logs/log.202307070000 b/PCAP-PIC/packet_dump/rtp/logs/log.202307070000
new file mode 100644
index 0000000..411904e
--- /dev/null
+++ b/PCAP-PIC/packet_dump/rtp/logs/log.202307070000
@@ -0,0 +1,58 @@
+time="2023-07-07 18:17:00" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:17:00" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:17:00" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:00" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:00" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:17:02" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:17:02" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:17:02" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:02" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:02" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:17:03" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:17:03" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:17:03" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:03" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:04" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:17:06" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:17:06" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:17:06" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:06" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:06" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:17:08" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:17:08" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:17:08" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:08" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:09" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:17:11" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:17:11" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:17:11" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:11" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:12" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:17:16" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:17:16" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:17:16" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:16" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:17" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:17:24" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:17:24" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:17:24" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:24" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:25" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:17:38" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:17:38" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:17:38" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:38" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:17:39" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:18:06" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:18:06" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:18:06" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:18:06" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:18:06" level=panic msg="kafka server: Request was for a topic or partition that does not exist on this broker." Topic: =INTERNAL-RTP-RECORD
+time="2023-07-07 18:18:58" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-07 18:18:58" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-07 18:18:58" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:18:58" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-07 18:18:58" level=info msg=partitionList Topic: =INTERNAL-RTP-RECORD partitionLists ="[0 1 2]"
+time="2023-07-07 18:18:58" level=info Offset: =-1 Topic: =INTERNAL-RTP-RECORD partition: =0
+time="2023-07-07 18:18:58" level=info Offset: =-1 Topic: =INTERNAL-RTP-RECORD partition: =1
+time="2023-07-07 18:18:58" level=info Offset: =-1 Topic: =INTERNAL-RTP-RECORD partition: =2
diff --git a/PCAP-PIC/packet_dump/rtp/logs/log.202307080000 b/PCAP-PIC/packet_dump/rtp/logs/log.202307080000
new file mode 100644
index 0000000..42f0d7b
--- /dev/null
+++ b/PCAP-PIC/packet_dump/rtp/logs/log.202307080000
@@ -0,0 +1,8 @@
+time="2023-07-08 15:32:02" level=info msg="BuildAwsS3Session sucess" endpoint: ="http://192.168.10.251:9098/hos/"
+time="2023-07-08 15:32:02" level=info PacketDump ="start time main" Para ="{[{<nil> <nil> map[] <nil> <nil> <nil> rtp pcap INTERNAL-RTP-RECORD VOIP-RECORD 100000 remote rtp_hos_bucket append 100000 15 [common_server_ip common_client_ip] [common_server_port common_client_port common_start_time] [raw_log_status] rtp_pcap_path  rtp /var/www/html/rtp/ [raw_packet raw_packet_len] raw_packet_time_ms raw_packet 120}] {[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094] admin galaxy2019 -1} {http://192.168.10.251:9098/hos/ f5c5186ba4874182b33b9b2b2b6e3f77 default default} {/data/tsg/olap/galaxy/packet_dump/logs/log info 1000000000 14 1}}"
+time="2023-07-08 15:32:02" level=info msg="NewConsumer Successes" Init kafka consumer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-08 15:32:02" level=info msg=partitionList Topic: =INTERNAL-RTP-RECORD partitionLists ="[0 1 2]"
+time="2023-07-08 15:32:02" level=info msg="NewProducer Successes" Init kafka producer handle success, Broker: ="[192.168.10.193:9094 192.168.10.194:9094 192.168.10.195:9094]"
+time="2023-07-08 15:32:02" level=info Offset: =-1 Topic: =INTERNAL-RTP-RECORD partition: =0
+time="2023-07-08 15:32:02" level=info Offset: =-1 Topic: =INTERNAL-RTP-RECORD partition: =1
+time="2023-07-08 15:32:02" level=info Offset: =-1 Topic: =INTERNAL-RTP-RECORD partition: =2
diff --git a/PCAP-PIC/packet_dump/rtp/status b/PCAP-PIC/packet_dump/rtp/status
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/PCAP-PIC/packet_dump/rtp/status