修改删除历史数据逻辑

5 files changed, 154 insertions, 42 deletions

diff --git a/control/business.py b/control/business.py
index 3747589..aa9b832 100644
--- a/control/business.py
+++ b/control/business.py
@@ -70,32 +70,55 @@ def kill_all(instructNo_setup_json_abs_path, tsglogger):
 
     subprocess.Popen(['kill','-STOP', str(ppid) ], stdout=subprocess.PIPE,universal_newlines=True)   #暂停进程
     #subprocess.Popen(['kill','-CONT', str(ppid) ], stdout=subprocess.PIPE,universal_newlines=True)   #继续启动进程
-    print("父进程:{}".format(ppid))
-    def kill_child(ppid):
+    tsglogger.info("父进程:{}".format(ppid))
+    def kill_child(ppid, tsglogger):
         #找到子进程
         child_process = subprocess.Popen("pgrep -P {}".format(ppid), stdout=subprocess.PIPE, shell=True, universal_newlines=True)
         child_pid_list = child_process.communicate()[0].split()
-        print("子进程：{}".format(child_pid_list))
-        #找到子进程的子进程
+        # for i in child_pid_list:
+        #     subprocess.Popen(['kill','-STOP', str(i) ], stdout=subprocess.PIPE,universal_newlines=True)   #暂停进程
+        #print("子进程：{}".format(child_pid_list))
+        tsglogger.info("子进程：{}".format(child_pid_list))
+        #找子子进程
         child_child_list = []
         if len(child_pid_list) > 0:
             for i in child_pid_list:
                 child_child_process = subprocess.Popen("pgrep -P {}".format(i), stdout=subprocess.PIPE, shell=True, universal_newlines=True)
                 child_child_list0 = child_child_process.communicate()[0].split()
-                child_child_list.extend(child_child_list0)
-            print("子子进程:{}".format(child_child_list))
+                if len(child_child_list0) == 0:
+                    continue
+                child_child_list.append([i, child_child_list0[0]])
+            #print("子子进程:{}".format(child_child_list))
+            tsglogger.info("子子进程:{}".format(child_child_list))
 
         #kill掉子子的进程
         if len(child_child_list) > 0:
             for child_pid in child_child_list:
-                kill_child_process = subprocess.Popen(['sudo','kill','-2', child_pid ], stdout=subprocess.PIPE,universal_newlines=True)
-                output = kill_child_process.communicate()
+                if len(child_pid) != 2:
+                    continue
+                child_child_pid_info = subprocess.Popen("sudo ps -ef |grep {}".format(child_pid[-1]), stdout=subprocess.PIPE,universal_newlines=True, shell=True)
+                child_child_pid_info_str = child_child_pid_info.stdout.read()
+                tsglogger.info(child_child_pid_info_str)
+                #tsglogger.info(child_child_list)
+
+                kill_child_child_process = subprocess.Popen(['sudo','kill','-2', child_pid[-1] ], stdout=subprocess.PIPE,universal_newlines=True)
+                time.sleep(0.8)
+                output = kill_child_child_process.communicate()
+                kill_child_child_process = subprocess.Popen(['sudo','kill','-2', child_pid[-1] ], stdout=subprocess.PIPE,universal_newlines=True)
+                time.sleep(0.8)
+                kill_child_process = subprocess.Popen(['sudo','kill','-2', child_pid[0] ], stdout=subprocess.PIPE,universal_newlines=True)
+                time.sleep(0.8)
+                kill_child_process = subprocess.Popen(['sudo','kill','-2', child_pid[0] ], stdout=subprocess.PIPE,universal_newlines=True)
 
-            for child_pid in child_child_list:
-                kill_child_process = subprocess.Popen(['sudo','kill','-2', child_pid ], stdout=subprocess.PIPE,universal_newlines=True)
-                #output = kill_child_process.communicate()
 
-    kill_child(ppid)
+
+            # for child_pid in child_child_list:
+            #     kill_child_process = subprocess.Popen(['sudo','kill','-2', child_pid ], stdout=subprocess.PIPE,universal_newlines=True)
+            #output = kill_child_process.communicate()
+    tsglogger.info("第一次寻找并删除子子进程...")
+    kill_child(ppid, tsglogger)
+    tsglogger.info("第二次寻找并删除子子进程...")
+    kill_child(ppid, tsglogger)
 
 
     """
@@ -131,13 +154,15 @@ def kill_all(instructNo_setup_json_abs_path, tsglogger):
 
     exit(0)
 
-def clear_historical_reort(instructNo_setup_json_abs_path):
+def clear_historical_reort(instructNo_setup_json_abs_path, tsglogger):
     """
     清楚控制层中report的历史文件，清楚agent server层中report的历史文件
     :param instructNo_setup_json_abs_path:   #/opt/test/automation/tasks/2022052717210703000103027569/setup_2022052717210703000103027569.json
     :return:
     """
-    print("清楚历史数据...：{}".format(instructNo_setup_json_abs_path))
+    tsglogger.info("开始函数:{}".format(clear_historical_reort.__name__))
+    #print("清楚历史数据...：{}".format(instructNo_setup_json_abs_path))
+    tsglogger.info("清楚历史数据...：{}".format(instructNo_setup_json_abs_path))
     control_config = configparser.ConfigParser()
     global config_abs_path
     control_config.read(config_abs_path)
@@ -151,20 +176,28 @@ def clear_historical_reort(instructNo_setup_json_abs_path):
 
     #删除控制层中report的历史文件
     clear_path = os.path.dirname(os.path.dirname(instructNo_setup_json_abs_path))
-    print(clear_path)
+    #print(clear_path)
+    tsglogger.info(clear_path)
     clear_dir_list = os.listdir(clear_path)
     for path in clear_dir_list:
         dir_abs_path = os.path.join(clear_path, path)
-        dir_ctime = os.path.getctime(dir_abs_path)
-        print(dir_abs_path)
+        try:
+            dir_ctime = os.path.getctime(dir_abs_path)
+        except Exception as e:
+            tsglogger.error(e)
+            continue
+        #print(dir_abs_path)
+        #tsglogger.info(dir_abs_path)
         if dir_ctime <= last_control_layer_retention_time:
-            if os.path.isdir(dir_abs_path):
-                shutil.rmtree(dir_abs_path)
-            else:
-                os.remove(dir_abs_path)
-
+            try:
+                if os.path.isdir(dir_abs_path):
+                    shutil.rmtree(dir_abs_path)
+                else:
+                    os.remove(dir_abs_path)
+            except Exception as e:
+                tsglogger.error(e)
     #删除agent server上的report历史文件：
-    agent_server_host = Linux(ip="192.168.42.4", username="root", password="Xxg-LAB<>90")
+    agent_server_host = Linux(ip="192.168.42.4", username="root", password="Xxg-LAB<>90", tsglogger=tsglogger)
     agent_server_host.connect()
     clear_cmd = 'find /root/work/oss -ctime +{} -type d -name "20*" -exec rm -rf {{}} \\;'.format(agent_server_layer_retention_time)
     clear_result = agent_server_host.send(clear_cmd)
@@ -180,7 +213,7 @@ def clear_historical_reort(instructNo_setup_json_abs_path):
     time.sleep(0.5)
 
     agent_server_host.close()
-
+    tsglogger.info("结束函数:{}".format(clear_historical_reort.__name__))
 
 def get_setup_abs_path(instructNo, tsglogger):
     tsglogger.info("开始函数:{}".format(get_setup_abs_path.__name__))
@@ -288,27 +321,47 @@ def execute_combine_capture_packet(robot_command, tsglogger):
     eth=eth_info
     ip_list = []    #从执行命令中取出抓包需要的ip
     b = re.findall("operationIp.*?\}", robot_command)
+    tsglogger.debug(f"解析到到的ip信息：{b}")
     for i in range(len(b)):
         ip1 =  re.search("\d.*\d", b[i]).group(0)
-        ip_list.append(ip1)
-    ip = ip_list[0]      #目前只是抓取一个ip的流量，如果是voip的抓包，没有添加抓取两个ip的流量逻辑，
+        ip1 = ip1.encode("utf8").decode("utf8")
+        ip_list.append(str(ip1))
+    ip_list_copy = copy.deepcopy(ip_list)
+    tsglogger.debug(f"可以选择的ip列表:{ip_list_copy}")
+    ip = ip_list_copy[0]      #目前只是抓取一个ip的流量，如果是voip的抓包，没有添加抓取两个ip的流量逻辑，
     #拼抓包保存路径
     d1 = re.search("-d\s.*?\s", robot_command).group(0)
+    tsglogger.debug(f"解析到到的d1信息：{d1}")
     d11 = d1.lstrip("-d ").rstrip("log ") + "capturepacket"    #捕包文件存放路径
     d2 = re.search("--output\s.*?xml", robot_command).group(0)
+    tsglogger.debug(f"解析到到的d2信息：{d2}")
     d22 = d2.lstrip("--output ").rstrip("xml")
     source_file_info = d11 + "/" + d22 + "pcap"       # 捕包文件绝对路径+名称
     if not os.path.exists(d11):     #判断捕包路径是否存在
         os.makedirs(d11)
     #ip = "192.168.50.29"    #ip如果没有流量 则不会捕包
-    dpkt = AsyncSniffer(iface=eth, filter="host {}".format(ip), prn=lambda x:wrpcap(source_file_info, x, append=True))
-    dpkt.start()
-    time.sleep(0.3)
+    #ip_list0 = ['192.168.50.9','192.168.50.19','192.168.50.29','192.168.50.39']
+    #ip = random.choice(ip_list0)
+    #ip = "192.168.42.97"
+
+    #eth = "eth0"
+    tsglogger.debug(f"捕包网卡信息:{eth}")
+    tsglogger.debug(f"捕包的ip信息:{ip}")
+    #tsglogger.error("测试信息。。。")
+    try:
+        dpkt = AsyncSniffer(iface=eth, filter="host {}".format(ip), prn=lambda x:wrpcap(source_file_info, x, append=True))
+    except Exception as e:
+        tsglogger.error(f"捕包错误信息:{e}")
+
+    else:
+        dpkt.start()
+        time.sleep(0.3)
     # capture_command = "tcpdump  -i eth1 host 192.168.50.57 -w ss.pcap "
     # capture_command1 = capture_command.replace("eth1", eth).replace("192.168.50.57", ip).replace("ss.pcap", source_file_info)
     # print(capture_command1)
     # cap = subprocess.Popen(capture_command1, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     # cap_result = str(cap.stdout.read())
+    tsglogger.debug("捕包")
     tsglogger.info(f"线程:{threading.current_thread().name},开启捕包......")
     tsglogger.info(f"数据包信息:{eth}--{ip}--{source_file_info}")
     tsglogger.info(f"结束函数:{execute_combine_capture_packet.__name__}")
@@ -329,7 +382,7 @@ def execute_commond_of_robot(robot_command, f_dict, tsglogger):
     """
     tsglogger.info(f"开始函数:{execute_commond_of_robot.__name__}")
     tsglogger.debug(f"传入参数:robot_command:{robot_command}")
-    tsglogger.debug(f"传入参数:f_dict:{f_dict}")
+    #tsglogger.debug(f"传入参数:f_dict:{f_dict}")
     message = f"执行robot命令:{robot_command}"
     green_message = "\033[0;32;40m%s\033[0m"% message
     tsglogger.info(green_message)
diff --git a/control/control.py b/control/control.py
index 4b47b43..3a45d12 100644
--- a/control/control.py
+++ b/control/control.py
@@ -79,7 +79,7 @@ def executeAutomation(instructNo, runMode, onlyDryrun, kill):
         f.write(f"{ppid}")
 
     #清空历史垃圾数据
-    business.clear_historical_reort(instructNo_setup_json_abs_path)
+    business.clear_historical_reort(instructNo_setup_json_abs_path, tsglogger)
 
     #2、重组保存dryrun、report的目录和使用部分文件名
     instructNo_setup_json_abs_path_list = instructNo_setup_json_abs_path.split("/")
diff --git a/control/control_config.ini b/control/control_config.ini
index 4df6318..4dc04f2 100644
--- a/control/control_config.ini
+++ b/control/control_config.ini
@@ -21,9 +21,9 @@ capture_eth = ens16
 [mobile_info]
 mobile_info_dict = {
                    "vivoV2031A":["facebook", "freegate", "gmail", "instagram", "instagram", "psiphon3", "qq", "skype", "tachyon",
-                   "taobao", "telegram", "tor", "twitter", "wechat", "whatsapp", "wireguard", "youtube"],
+                                "taobao", "telegram", "tor", "twitter", "wechat", "whatsapp", "wireguard", "youtube"],
                    "iphone12":["facebook", "gmail", "instagram", "instagram", "psiphon3", "qq", "skype", "tachyon",
-                   "taobao", "telegram", "tor", "twitter", "wechat", "whatsapp", "wireguard", "youtube"],
+                                "taobao", "telegram", "tor", "twitter", "wechat", "whatsapp", "wireguard", "youtube"],
                    "xiaomi":["freegate", "psiphon3", "tachyon", "tor"],
                    "iphoneXR":["freegate", "psiphon3", "tachyon", "youtube"]
                    }
diff --git a/control/remote_command.py b/control/remote_command.py
index 5745f23..51657d0 100644
--- a/control/remote_command.py
+++ b/control/remote_command.py
@@ -5,7 +5,7 @@ import paramiko
 
 
 class Linux():
-    def __init__(self, ip, username, password, timeout=30):
+    def __init__(self, ip, username, password, timeout=30, tsglogger=True):
         self.ip = ip
         self.username = username
         self.password = password
@@ -13,6 +13,7 @@ class Linux():
         self.t = ""
         self.chan = ""
         self.try_times = 3
+        self.tsglogger = tsglogger
 
     def connect(self):
         while True:
@@ -22,17 +23,22 @@ class Linux():
                 self.chan = self.t.open_session()
                 self.chan.get_pty()
                 self.chan.invoke_shell()
-                print("连接{}成功".format(self.ip))
+                #print("连接{}成功".format(self.ip))
+                self.tsglogger.info("连接{}成功".format(self.ip))
                 self.revc_data = self.chan.recv(65535).decode("utf-8")
-                print("接收数据:{}".format(self.revc_data))
+                #print("接收数据:{}".format(self.revc_data))
+                self.tsglogger.info("接收数据:{}".format(self.revc_data))
                 return
             except Exception as e:
                 if self.try_times != 0:
-                    print(e)
-                    print("重试")
+                    #print(e)
+                    #print("重试")
+                    self.tsglogger.info(e)
+                    self.tsglogger.info("重试...")
                     self.try_times -= 1
                 else:
-                    print("{}连接失败".format(self.ip))
+                    #print("{}连接失败".format(self.ip))
+                    self.tsglogger.info("{}连接失败".format(self.ip))
                     exit(1)
 
     def close(self):
@@ -50,7 +56,8 @@ class Linux():
             ret = sign_1.decode("utf-8")
             result += ret
             result1 += ret
-            print(result1)
+            #print(result1)
+            self.tsglogger.info(result1)
             # print(self.chan.recv_ready())
             # print(2)
             time.sleep(0.5)
diff --git a/control/tsg_log.py b/control/tsg_log.py
index fc70fd5..b15e235 100644
--- a/control/tsg_log.py
+++ b/control/tsg_log.py
@@ -1,10 +1,62 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
+import fcntl
 import os
 import sys
 import logging
+import time
 from logging.handlers import TimedRotatingFileHandler
 
+class MultiCompatibleTimedRotatingFileHandler(TimedRotatingFileHandler):
+
+    def doRollover(self):
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+        # get the time that this sequence started at and make it a TimeTuple
+        currentTime = int(time.time())
+        dstNow = time.localtime(currentTime)[-1]
+        t = self.rolloverAt - self.interval
+        if self.utc:
+            timeTuple = time.gmtime(t)
+        else:
+            timeTuple = time.localtime(t)
+            dstThen = timeTuple[-1]
+            if dstNow != dstThen:
+                if dstNow:
+                    addend = 3600
+                else:
+                    addend = -3600
+                timeTuple = time.localtime(t + addend)
+        dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple)
+        # 兼容多进程并发 LOG_ROTATE
+        if not os.path.exists(dfn):
+            f = open(self.baseFilename, 'a')
+            fcntl.lockf(f.fileno(), fcntl.LOCK_EX)
+            if not os.path.exists(dfn):
+                os.rename(self.baseFilename, dfn)
+            # 释放锁 释放老 log 句柄
+            f.close()
+        if self.backupCount > 0:
+            for s in self.getFilesToDelete():
+                os.remove(s)
+        if not self.delay:
+            self.stream = self._open()
+        newRolloverAt = self.computeRollover(currentTime)
+        while newRolloverAt <= currentTime:
+            newRolloverAt = newRolloverAt + self.interval
+        # If DST changes and midnight or weekly rollover, adjust for this.
+        if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc:
+            dstAtRollover = time.localtime(newRolloverAt)[-1]
+            if dstNow != dstAtRollover:
+                if not dstNow:  # DST kicks in before next rollover, so we need to deduct an hour
+                    addend = -3600
+                else:  # DST bows out before next rollover, so we need to add an hour
+                    addend = 3600
+                newRolloverAt += addend
+        self.rolloverAt = newRolloverAt
+
+
 class TsgLogger():
     def __init__(self, file_name, file_level="DEBUG", console_level="DEBUG", instructNo=111):
         if not os.path.exists(os.path.dirname(file_name)):os.mkdir(os.path.dirname(file_name))
@@ -15,11 +67,11 @@ class TsgLogger():
         stream_handler = logging.StreamHandler()
         stream_handler.setLevel(console_level)
 
-        file_handler = TimedRotatingFileHandler(file_name, when="midnight", backupCount=10)
+        file_handler = MultiCompatibleTimedRotatingFileHandler(file_name, when="midnight", backupCount=10)
         file_handler.setLevel(file_level)
 
         formatter1 = "%(asctime)s | %(levelname)-8s | %(process)-10s | %(threadName)-25s |" \
-                     " %(funcName)-25s | %(lineno)-5d |  {}  |%(message)s |".format(instructNo)
+                     " %(funcName)-25s | %(lineno)-5d |  {}  |%(message)s |".format(self.instrucNO)
         datefmt = "%Y-%m-%d %H:%M:%S"
         tsg_formatter = logging.Formatter(fmt=formatter1, datefmt=datefmt)