适配 CMv23.10版本

6 files changed, 284 insertions, 12 deletions

diff --git a/config/application.yml b/config/application.yml
index 96b0dcb..dda11b5 100644
--- a/config/application.yml
+++ b/config/application.yml
@@ -12,9 +12,11 @@ logging:
   config: ./config/logback-spring.xml
 tsg:
   system:
-    url: http://192.168.44.3:80
-    token: aa2bdec5518ad131f71944b13ce5c298&1&
+    url: http://192.168.44.29:8080
+    ##token: aa2bdec5518ad131f71944b13ce5c298&1&
+    token: 1ca0350bc499a208a0edadb74d76a0ee&1&
     httpTimeout: 3600000
+    isLatestVersion: false
   object:
     ip:
       #指定排除的IP. 1.如果没有可不写值，2.若有多个用英文逗号,分隔如: 127.0.0.1,127.0.0.2，3.可适用IP段匹配，如：127.*
diff --git a/src/main/java/com/geedge/common/constant/TsgObject.java b/src/main/java/com/geedge/common/constant/TsgObject.java
index 9a01eda..a672d29 100644
--- a/src/main/java/com/geedge/common/constant/TsgObject.java
+++ b/src/main/java/com/geedge/common/constant/TsgObject.java
@@ -38,5 +38,27 @@ public class TsgObject {
     public static final Integer VALUE_OBJECT_MEMBER_ITEMS_STRING_EXPR_TYPE_0 = 0;
     public static final Integer VALUE_OBJECT_MEMBER_ITEMS_STRING_IS_HEXBIN_0 = 0;
 
+    public static String TSG_OBJECT_ID_V2310 = "objectId";
+    public static String TSG_OBJECT_TYPE_V2310 = "objectType";
+    public static String TSG_IP_ADDR_OBJECT_V2310 = "ip";
+    public static String TSG_FQDN_OBJECT_V2310 = "fqdn";
+    public static String TSG_OBJECT_NAME_V2310 = "objectName";
+    public static String TSG_IS_BUILTIN_V2310 = "isBuiltin";
+    public static String TSG_IS_EXCLUSION_V2310 = "isExclusion";
+    public static String TSG_IS_VALID_V2310 = "isValid";
+    public static String TSG_ITEM_IP_V2310 = "ip";
+    public static String TSG_ITEM_PORT_V2310 = "port";
+    public static String TSG_VALUE_OBJECT_MEMBER_ITEMS_IP_PORT_RANGE_V2310 = "0-65535";
+    public static String TSG_ITEM_ISSESSION_V2310 = "isSession";
+    public static String TSG_ITEM_ENDPOINT_V2310 = "endpoint";
+    public static String TSG_OP_ACTION_V2310 = "opAction";
+    public static String TSG_UPDATE_V2310 = "update";
+    public static String TSG_OBJECT_LIST_V2310 = "objectList";
+    public static String TSG_VSYS_ID_V2310 = "vsysId";
+    public static String TSG_TSG_OBJECT_IDS_V2310 = "objectIds";
+    public static String TSG_ITEM_TYPE_V2310 = "itemType";
+    public static String TSG_KEYWORDARRAY_V2310 = "keywordArray";
+    public static String TSG_DELETE_ITEMS_BYL_TTIME_V2310 = "deleteItemsByLtTime";
+
 }
 
diff --git a/src/main/java/com/geedge/common/PrometheusMetrics.java b/src/main/java/com/geedge/common/monitor/PrometheusMetrics.java
index 57beb83..9c6d079 100644
--- a/src/main/java/com/geedge/common/PrometheusMetrics.java
+++ b/src/main/java/com/geedge/common/monitor/PrometheusMetrics.java
@@ -1,4 +1,4 @@
-package com.geedge.common;
+package com.geedge.common.monitor;
 
 import io.prometheus.client.exporter.MetricsServlet;
 import org.springframework.web.bind.annotation.GetMapping;
diff --git a/src/main/java/com/geedge/common/util/TsgUtil.java b/src/main/java/com/geedge/common/util/TsgUtil.java
index ac0d11b..3e5b995 100644
--- a/src/main/java/com/geedge/common/util/TsgUtil.java
+++ b/src/main/java/com/geedge/common/util/TsgUtil.java
@@ -6,7 +6,6 @@ import cn.hutool.http.HttpRequest;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
 import com.google.common.base.Stopwatch;
-import com.google.common.collect.Maps;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
@@ -26,8 +25,14 @@ import java.util.concurrent.TimeUnit;
 public class TsgUtil {
 
     public static String TSG_URL;
-    private static Integer httpTimeOut;
     private static String TSG_TOKEN;
+    private static Integer httpTimeOut;
+    public static Boolean isLatestVersion;
+
+    @Value("${tsg.system.isLatestVersion}")
+    public void isLatestVersion(Boolean latestVersion) {
+        isLatestVersion = latestVersion;
+    }
 
     @Value("${tsg.system.httpTimeout}")
     public void setTsgUrl(Integer timeOut) {
@@ -71,6 +76,25 @@ public class TsgUtil {
         }
     }
 
+    public static void updateObjectOld(Map<String, Object> body) {
+        Stopwatch watch = Stopwatch.createStarted();
+        String response = HttpRequest.put(TSG_URL + "/v1/policy/object")
+                .header(Header.AUTHORIZATION, getToken())
+                .body(JSONUtil.toJsonStr(body))
+                .timeout(httpTimeOut)
+                .execute().body();
+        log.info("update tsg-api, cost {} seconds", watch.elapsed(TimeUnit.SECONDS));
+        if (StrUtil.isBlank(response)) {
+            log.error("update {} Object error, response: {}", body, response);
+            throw new RuntimeException("update " + body + " Object error, response: " + response);
+        }
+        JSONObject jsonObject = JSONUtil.parseObj(response);
+        if (!"200".equals(jsonObject.get("code").toString())) {
+            log.error("update {} Object error, response: {}", body, response);
+            throw new RuntimeException("update " + body + " Object error, response: " + response);
+        }
+    }
+
     public static void deleteItemOfObjectById(Integer id, Map<String, Object> form) {
         Stopwatch watch = Stopwatch.createStarted();
         String response = HttpRequest.delete(TSG_URL + "/v1/policy/object/" + id + "/item")
@@ -89,4 +113,23 @@ public class TsgUtil {
             throw new RuntimeException("update " + id + " Object error, response: " + response);
         }
     }
+
+    public static void deleteItemOfObjectOld(Map<String, Object> body) {
+        Stopwatch watch = Stopwatch.createStarted();
+        String response = HttpRequest.delete(TSG_URL + "/v1/policy/items")
+                .header(Header.AUTHORIZATION, getToken())
+                .body(JSONUtil.toJsonStr(body))
+                .timeout(httpTimeOut)
+                .execute().body();
+        log.info("delete tsg-api, cost {} seconds", watch.elapsed(TimeUnit.SECONDS));
+        if (StrUtil.isBlank(response)) {
+            log.error("update {} Object error, response: {}", body, response);
+            throw new RuntimeException("update " + body + " Object error, response: " + response);
+        }
+        JSONObject jsonObject = JSONUtil.parseObj(response);
+        if (!"200".equals(jsonObject.get("code").toString())) {
+            log.error("update {} Object error, response: {}", body, response);
+            throw new RuntimeException("update " + body + " Object error, response: " + response);
+        }
+    }
 }
diff --git a/src/main/java/com/geedge/scheduler/FqdnScheduler.java b/src/main/java/com/geedge/scheduler/FqdnScheduler.java
index ec34a6e..b92c6a1 100644
--- a/src/main/java/com/geedge/scheduler/FqdnScheduler.java
+++ b/src/main/java/com/geedge/scheduler/FqdnScheduler.java
@@ -4,6 +4,8 @@ import cn.hutool.core.date.DatePattern;
 import cn.hutool.core.date.DateTime;
 import cn.hutool.core.date.DateUtil;
 import cn.hutool.core.util.BooleanUtil;
+import cn.hutool.json.JSONArray;
+import cn.hutool.json.JSONObject;
 import com.geedge.common.constant.TsgObject;
 import com.geedge.common.util.TsgUtil;
 import com.google.common.base.Stopwatch;
@@ -18,10 +20,7 @@ import org.springframework.beans.factory.annotation.Value;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -157,6 +156,22 @@ public class FqdnScheduler {
     }
 
     private static void executeUpdate(Integer id, String name, Boolean enable, String sql, Counter counter) {
+        if (TsgUtil.isLatestVersion) {
+            executeUpdateLatest(id, name, enable, sql, counter);
+        } else {
+            executeUpdateOld(id, name, enable, sql, counter);
+        }
+    }
+
+    private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) {
+        if (TsgUtil.isLatestVersion) {
+            executeDeleteLatest(id, enable, offsetSecond);
+        } else {
+            executeDeleteOld(id, enable, offsetSecond);
+        }
+    }
+
+    private static void executeUpdateLatest(Integer id, String name, Boolean enable, String sql, Counter counter) {
         log.info("{}: started update task.", id);
         if (BooleanUtil.isFalse(enable)) {
             log.warn("{}: interrupted update task. enable: {}", id, enable);
@@ -212,7 +227,58 @@ public class FqdnScheduler {
         }
     }
 
-    private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) {
+
+    private static void executeUpdateOld(Integer id, String name, Boolean enable, String sql, Counter counter) {
+        log.info("{}: started update task.", id);
+        if (BooleanUtil.isFalse(enable)) {
+            log.warn("{}: interrupted update task. enable: {}", id, enable);
+            return;
+        }
+        try {
+            Stopwatch watch = Stopwatch.createStarted();
+            List<Record> data = Db.find(sql);
+            log.info("{}: query knowledge base content, cost {} seconds", id, watch.elapsed(TimeUnit.SECONDS));
+            watch.reset().start();
+
+            List<Map<String, Object>> items = Lists.newArrayList();
+            for (Record record : data) {
+                String domain = record.get("domain");
+                List<String> keywordArray = new ArrayList<>();
+                keywordArray.add(domain);
+                JSONObject obj = new JSONObject();
+                obj.set("isHexbin", 0);
+                obj.set(TsgObject.TSG_IS_BUILTIN_V2310, 0);
+                obj.set(TsgObject.TSG_KEYWORDARRAY_V2310, keywordArray);
+                items.add(obj);
+            }
+
+            JSONArray array = new JSONArray();
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.set(TsgObject.TSG_OBJECT_ID_V2310, id);
+            jsonObject.set(TsgObject.TSG_OBJECT_TYPE_V2310, TsgObject.TSG_FQDN_OBJECT_V2310);
+            jsonObject.set(TsgObject.TSG_OBJECT_NAME_V2310, name);
+            jsonObject.set(TsgObject.TSG_IS_BUILTIN_V2310, 0);
+            jsonObject.set(TsgObject.TSG_IS_EXCLUSION_V2310, 0);
+            jsonObject.set(TsgObject.TSG_IS_VALID_V2310, 1);
+            jsonObject.set("addItemList", items);
+            log.info("{}: build api params, items size: {}, cost {} seconds", id, items.size(), watch.elapsed(TimeUnit.SECONDS));
+            if (items.isEmpty()) {
+                return;
+            }
+            array.add(jsonObject);
+            JSONObject body = new JSONObject();
+            body.set(TsgObject.TSG_OP_ACTION_V2310, TsgObject.TSG_UPDATE_V2310);
+            body.set(TsgObject.TSG_OBJECT_LIST_V2310, array);
+            TsgUtil.updateObjectOld(body);
+            counter.inc(items.size());
+        } catch (Exception e) {
+            log.error("{}: failed to execute update task. message: {}", id, e.getMessage());
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    private static void executeDeleteLatest(Integer id, Boolean enable, Integer offsetSecond) {
         log.info("{}: started delete task.", id);
         if (BooleanUtil.isFalse(enable)) {
             log.warn("{}: interrupted delete task. enable: {}", id, enable);
@@ -231,4 +297,28 @@ public class FqdnScheduler {
             throw new RuntimeException(e);
         }
     }
+
+    private static void executeDeleteOld(Integer id, Boolean enable, Integer offsetSecond) {
+        log.info("{}: started delete task.", id);
+        if (BooleanUtil.isFalse(enable)) {
+            log.warn("{}: interrupted delete task. enable: {}", id, enable);
+            return;
+        }
+        try {
+            DateTime dateTime = DateUtil.offsetSecond(new Date(), offsetSecond).setTimeZone(TimeZone.getTimeZone("UTC"));
+            String datetimeZ = DateUtil.format(dateTime, DatePattern.UTC_PATTERN);
+            Map<String, Object> body = new HashMap<>(16);
+            body.put(TsgObject.TSG_DELETE_ITEMS_BYL_TTIME_V2310, datetimeZ);
+            body.put(TsgObject.TSG_ITEM_TYPE_V2310, TsgObject.TSG_FQDN_OBJECT_V2310);
+            body.put(TsgObject.TSG_VSYS_ID_V2310, 1);
+            List<Object> objectIds = new ArrayList<>();
+            objectIds.add(id);
+            body.put(TsgObject.TSG_TSG_OBJECT_IDS_V2310, objectIds);
+            TsgUtil.deleteItemOfObjectOld(body);
+        } catch (Exception e) {
+            log.error("{}: failed to execute delete task. message: {}", id, e.getMessage());
+            throw new RuntimeException(e);
+        }
+    }
+
 }
diff --git a/src/main/java/com/geedge/scheduler/IpScheduler.java b/src/main/java/com/geedge/scheduler/IpScheduler.java
index dcf7fef..bd41728 100644
--- a/src/main/java/com/geedge/scheduler/IpScheduler.java
+++ b/src/main/java/com/geedge/scheduler/IpScheduler.java
@@ -7,6 +7,8 @@ import cn.hutool.core.net.Ipv4Util;
 import cn.hutool.core.net.MaskBit;
 import cn.hutool.core.util.BooleanUtil;
 import cn.hutool.core.util.StrUtil;
+import cn.hutool.json.JSONArray;
+import cn.hutool.json.JSONObject;
 import com.geedge.common.constant.TsgObject;
 import com.geedge.common.enums.AddressFormat;
 import com.geedge.common.util.TsgUtil;
@@ -337,8 +339,24 @@ public class IpScheduler {
         executeDelete(psiphon3vpnServeripId, psiphon3vpnServeripDeleteEnable, psiphon3vpnServeripOffsetSecond);
     }
 
-
     private static void executeUpdate(Integer id, String name, Boolean enable, String sql, Counter counter) {
+        if (TsgUtil.isLatestVersion) {
+            executeUpdateLatest(id, name, enable, sql, counter);
+        } else {
+            executeUpdateOld(id, name, enable, sql, counter);
+        }
+    }
+
+    private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) {
+        if (TsgUtil.isLatestVersion) {
+            executeDeleteLatest(id, enable, offsetSecond);
+        } else {
+            executeDeleteOld(id, enable, offsetSecond);
+        }
+    }
+
+
+    private static void executeUpdateLatest(Integer id, String name, Boolean enable, String sql, Counter counter) {
         log.info("{}: started update task.", id);
         if (BooleanUtil.isFalse(enable)) {
             log.warn("{}: interrupted update task. enable: {}", id, enable);
@@ -413,7 +431,81 @@ public class IpScheduler {
         }
     }
 
-    private static void executeDelete(Integer id, Boolean enable, Integer offsetSecond) {
+    private static void executeUpdateOld(Integer id, String name, Boolean enable, String sql, Counter counter) {
+        log.info("{}: started update task.", id);
+        if (BooleanUtil.isFalse(enable)) {
+            log.warn("{}: interrupted update task. enable: {}", id, enable);
+            return;
+        }
+        try {
+            Stopwatch watch = Stopwatch.createStarted();
+            List<Record> data = Db.find(sql);
+            log.info("{}: query knowledge base content, cost {} seconds", id, watch.elapsed(TimeUnit.SECONDS));
+            watch.reset().start();
+            List<Map<String, Object>> items = Lists.newArrayList();
+            row:
+            for (Record record : data) {
+                String addressFormat = record.get("addrFormat");
+                String ip1 = record.get("ip1");
+                if (StrUtil.isBlank(ip1)) {
+                    continue;
+                }
+                for (String excludeItem : excludeList) {
+                    if (excludeItem.equals(ip1)) {
+                        continue row;
+                    }
+                    if (excludeItem.endsWith("*") && ip1.startsWith(excludeItem.replace("*", ""))) {
+                        continue row;
+                    }
+                }
+                String ip2 = record.get("ip2");
+                Map<String, Object> ip = Maps.newHashMap();
+                if (AddressFormat.CIDR.getValue().equalsIgnoreCase(addressFormat)) {
+                    int maskBit = Integer.parseInt(ip2);
+                    String mask = MaskBit.get(Integer.parseInt(ip2));
+                    String beginIpStr = Ipv4Util.getBeginIpStr(ip1, maskBit);
+                    String ipMask = Ipv4Util.formatIpBlock(beginIpStr, mask);
+                    ip.put(TsgObject.TSG_ITEM_IP_V2310, ipMask);
+                } else if (AddressFormat.RANGE.getValue().equalsIgnoreCase(addressFormat)) {
+                    ip.put(TsgObject.TSG_ITEM_IP_V2310, ip1 + "-" + ip2);
+                } else if (AddressFormat.SINGLE.getValue().equalsIgnoreCase(addressFormat)) {
+                    ip.put(TsgObject.TSG_ITEM_IP_V2310, ip1);
+                } else {
+                    log.warn("address format parse error: {}", record);
+                    continue;
+                }
+                ip.put(TsgObject.TSG_ITEM_PORT_V2310, TsgObject.TSG_VALUE_OBJECT_MEMBER_ITEMS_IP_PORT_RANGE_V2310);
+                ip.put(TsgObject.TSG_ITEM_ISSESSION_V2310, TsgObject.TSG_ITEM_ENDPOINT_V2310);
+                items.add(ip);
+            }
+            JSONArray array = new JSONArray();
+
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.set(TsgObject.TSG_OBJECT_ID_V2310, id);
+            jsonObject.set(TsgObject.TSG_OBJECT_TYPE_V2310, TsgObject.TSG_IP_ADDR_OBJECT_V2310);
+            jsonObject.set(TsgObject.TSG_OBJECT_NAME_V2310, name);
+            jsonObject.set(TsgObject.TSG_IS_BUILTIN_V2310, 0);
+            jsonObject.set(TsgObject.TSG_IS_EXCLUSION_V2310, 0);
+            jsonObject.set(TsgObject.TSG_IS_VALID_V2310, 1);
+            jsonObject.set("addItemList", items);
+            log.info("{}: build api params, items size: {}, cost {} seconds", id, items.size(), watch.elapsed(TimeUnit.SECONDS));
+            if (items.isEmpty()) {
+                return;
+            }
+            array.add(jsonObject);
+            JSONObject body = new JSONObject();
+            body.set(TsgObject.TSG_OP_ACTION_V2310, TsgObject.TSG_UPDATE_V2310);
+            body.set(TsgObject.TSG_OBJECT_LIST_V2310, array);
+
+            TsgUtil.updateObjectOld(body);
+            counter.inc(items.size());
+        } catch (Exception e) {
+            log.error("{}: failed to execute update task. message: {}", id, e.getMessage());
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static void executeDeleteLatest(Integer id, Boolean enable, Integer offsetSecond) {
         log.info("{}: started delete task.", id);
         if (BooleanUtil.isFalse(enable)) {
             log.warn("{}: interrupted delete task. enable: {}", id, enable);
@@ -432,4 +524,27 @@ public class IpScheduler {
             throw new RuntimeException(e);
         }
     }
+
+    private static void executeDeleteOld(Integer id, Boolean enable, Integer offsetSecond) {
+        log.info("{}: started delete task.", id);
+        if (BooleanUtil.isFalse(enable)) {
+            log.warn("{}: interrupted delete task. enable: {}", id, enable);
+            return;
+        }
+        try {
+            DateTime dateTime = DateUtil.offsetSecond(new Date(), offsetSecond).setTimeZone(TimeZone.getTimeZone("UTC"));
+            String datetimeZ = DateUtil.format(dateTime, DatePattern.UTC_PATTERN);
+            Map<String, Object> body = new HashMap<>(16);
+            body.put(TsgObject.TSG_DELETE_ITEMS_BYL_TTIME_V2310, datetimeZ);
+            body.put(TsgObject.TSG_ITEM_TYPE_V2310, TsgObject.TSG_ITEM_IP_V2310);
+            body.put(TsgObject.TSG_VSYS_ID_V2310, 1);
+            List<Object> objectIds = new ArrayList<>();
+            objectIds.add(id);
+            body.put(TsgObject.TSG_TSG_OBJECT_IDS_V2310, objectIds);
+            TsgUtil.deleteItemOfObjectOld(body);
+        } catch (Exception e) {
+            log.error("{}: failed to execute delete task. message: {}", id, e.getMessage());
+            throw new RuntimeException(e);
+        }
+    }
 }