TSG-12524 新增动态更新Psiphon3的IP Object功能

5 files changed, 110 insertions, 1 deletions

diff --git a/src/main/java/com/example/nis/FjTransformApiApplication.java b/src/main/java/com/example/nis/FjTransformApiApplication.java
index b82450e..fd15f13 100644
--- a/src/main/java/com/example/nis/FjTransformApiApplication.java
+++ b/src/main/java/com/example/nis/FjTransformApiApplication.java
@@ -2,8 +2,10 @@ package com.example.nis;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 @SpringBootApplication
+@EnableScheduling
 public class FjTransformApiApplication {
 
     public static void main(String[] args) {
diff --git a/src/main/java/com/example/nis/common/TsgServiceImpl.java b/src/main/java/com/example/nis/common/TsgServiceImpl.java
index 5985539..bef463c 100644
--- a/src/main/java/com/example/nis/common/TsgServiceImpl.java
+++ b/src/main/java/com/example/nis/common/TsgServiceImpl.java
@@ -39,6 +39,7 @@ public class TsgServiceImpl {
         paramMap.put(Constant.TSG_PAGE_NO, 1);
         paramMap.put(Constant.TSG_PAGE_SIZE, -1);
         paramMap.put(Constant.TSG_IS_VALID, 1);
+        paramMap.put(Constant.TSG_VSYS_ID, 1);
         if (StringUtils.isNotEmpty(itemType)) {
             paramMap.put(Constant.TSG_ITEM_TYPE, itemType);
         } else {
diff --git a/src/main/java/com/example/nis/controller/IpController.java b/src/main/java/com/example/nis/controller/IpController.java
index 93d86b9..afd7d07 100644
--- a/src/main/java/com/example/nis/controller/IpController.java
+++ b/src/main/java/com/example/nis/controller/IpController.java
@@ -1,8 +1,11 @@
 package com.example.nis.controller;
 
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONArray;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
+import com.example.nis.common.Code;
 import com.example.nis.common.ResponseData;
 import com.example.nis.common.TsgServiceImpl;
 import com.example.nis.domain.IpObject;
@@ -13,9 +16,13 @@ import com.example.nis.util.ToTsgSystemUtil;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.web.bind.annotation.*;
+import sun.net.util.IPAddressUtil;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 
@@ -26,6 +33,67 @@ import java.util.List;
 public class IpController {
 
     private final TsgServiceImpl tsgService;
+    private List<String> excludeList = new ArrayList<>();
+    private Integer upperLimit = 100000;
+    @Value("${ip.filePath}")
+    private String ipFilePath;
+
+    @Value("${ip.excludeValue}")
+    public void setExcludeList(String values) {
+        if (StrUtil.isBlank(values)) {
+            return;
+        }
+        excludeList = Arrays.asList(values.split(","));
+    }
+
+    @Value("${ip.upperLimit}")
+    public void setUpperLimit(Integer upperLimit) {
+        if (StrUtil.isEmptyIfStr(upperLimit)) {
+            return;
+        }
+        this.upperLimit = upperLimit;
+    }
+
+    @Scheduled(cron = "${ip.cron}")
+    @PutMapping("/dynamic-learning")
+    public ResponseData scheduledExecutor() {
+        ResponseData responseData = getAll();
+        if (!Code.SUCCESS.getCode().equals(responseData.get("code"))) {
+            String msg = String.valueOf(responseData.get("msg"));
+            log.error("get all itemList error: {}", msg);
+            return ResponseData.error(msg);
+        }
+        IpSource ipSource = new IpSource();
+
+        List<IpObject> deleteList = new ArrayList<>();
+        JSONArray jsonArray = JSONUtil.parseArray(responseData.get("data"));
+        for (int i = 0; i < jsonArray.size(); i++) {
+            JSONObject obj = jsonArray.getJSONObject(i);
+            deleteList.add(JSONUtil.toBean(obj, IpObject.class));
+        }
+        ipSource.setDeleteItemList(deleteList);
+
+        List<IpObject> addList = new ArrayList<>();
+        List<String> rawLines = FileUtil.readUtf8Lines(ipFilePath);
+        for (String line : rawLines) {
+            if (addList.size() >= upperLimit) {
+                break;
+            }
+            if (StrUtil.isBlank(line) || excludeList.contains(line)) {
+                continue;
+            }
+            if (!IPAddressUtil.isIPv6LiteralAddress(line) && !IPAddressUtil.isIPv4LiteralAddress(line)) {
+                log.warn("Not IPv4 or IPv6: {}", line);
+                continue;
+            }
+            IpObject ipObject = new IpObject();
+            ipObject.setIp(line);
+            ipObject.setPort("0-65535");
+            addList.add(ipObject);
+        }
+        ipSource.setAddItemList(addList);
+        return put(ipSource);
+    }
 
     @GetMapping
     public ResponseData get(IpObject object) {
@@ -61,6 +129,28 @@ public class IpController {
         }
     }
 
+    public ResponseData getAll() {
+        try {
+            JSONArray itemList = tsgService.getItemList(ToTsgSystemUtil.ipObjectId, Constant.TSG_ITEM_IP, null, null, null, null, null);
+            if (itemList == null || itemList.isEmpty()) {
+                return ResponseData.ok(new ArrayList<>());
+            }
+            List<IpObject> result = new ArrayList<>(itemList.size());
+            for (int i = 0; i < itemList.size(); i++) {
+                JSONObject jsonObject = itemList.getJSONObject(i);
+                IpObject ipObject = new IpObject();
+                ipObject.setItemId(Integer.parseInt(jsonObject.get(Constant.TSG_ITEM_ID).toString()));
+                ipObject.setIp(jsonObject.get(Constant.TSG_ITEM_IP).toString());
+                ipObject.setPort(jsonObject.get(Constant.TSG_ITEM_PORT).toString());
+                result.add(ipObject);
+            }
+            return ResponseData.ok(result);
+        } catch (Exception e) {
+            log.warn("get All itemList error: {}", e.getMessage());
+            return ResponseData.error();
+        }
+    }
+
     @PutMapping
     public ResponseData put(@RequestBody IpSource source) {
         try {
diff --git a/src/main/java/com/example/nis/util/ToTsgSystemUtil.java b/src/main/java/com/example/nis/util/ToTsgSystemUtil.java
index ef464d1..8b4707f 100644
--- a/src/main/java/com/example/nis/util/ToTsgSystemUtil.java
+++ b/src/main/java/com/example/nis/util/ToTsgSystemUtil.java
@@ -1,5 +1,6 @@
 package com.example.nis.util;
 
+import cn.hutool.core.util.StrUtil;
 import cn.hutool.http.HttpRequest;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
@@ -51,6 +52,7 @@ public class ToTsgSystemUtil {
     private static String TSG_USERNAME;
     private static String TSG_PASSWORD;
     private static Integer httpTimeOut;
+    private static String TSG_TOKEN;
     @Value("${tsg.system.httpTimeOut}")
     public void setTsgUrl(Integer timeOut) {
         httpTimeOut = timeOut;
@@ -67,6 +69,10 @@ public class ToTsgSystemUtil {
     private void setTsgPassword(String password) {
         TSG_PASSWORD = password;
     }
+    @Value("${tsg.system.token}")
+    private void setTsgToken(String token) {
+        TSG_TOKEN = token;
+    }
 
 
 
@@ -77,6 +83,9 @@ public class ToTsgSystemUtil {
      * @throws Exception
      */
     public static String sendLoginRequest() throws Exception {
+        if (StrUtil.isNotEmpty(TSG_TOKEN)) {
+            return TSG_TOKEN;
+        }
         String token = "";
         Map<String, Object> map = new HashMap<>();
         map.put("username",TSG_USERNAME);
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index e96a0e3..94dc74b 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -5,13 +5,20 @@ spring:
   application:
     name: FJ-Transform-Project
 
-
+# source object
+ip:
+  filePath: /data/tsg/olap/galaxy/volumes/PsiphonSlokIP.txt
+  upperLimit: 100000
+  #指定排除的IP. 1.如果没有可不写值，2.若有多个用英文逗号,分隔如: 127.0.0.1,127.0.0.2
+  excludeValue: 0.0.0.0
+  cron: 0 0 0/1 * * ?
 # TSG 账号
 tsg:
   system:
     url: http://192.168.45.51:8080/
     username: admin
     password: aSdvVT7Fg81kJBuT7T7T7g==
+    token: 4628529e-f349-4634-99ed-03efd83e0e5a&1339&
     # http 超时时间
     httpTimeOut: 3600000
     ipObjectId: 157