fix(Psiphon3): CN-824 Psiphon3客户端IP Object定期更新

4 files changed, 262 insertions, 1 deletions

diff --git a/src/main/java/com/example/nis/Scheduled/Entity.java b/src/main/java/com/example/nis/Scheduled/Entity.java
new file mode 100644
index 0000000..a1a008d
--- /dev/null
+++ b/src/main/java/com/example/nis/Scheduled/Entity.java
@@ -0,0 +1,181 @@
+package com.example.nis.Scheduled;
+
+import cn.hutool.core.util.BooleanUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.http.HttpResponse;
+import cn.hutool.json.JSONArray;
+import cn.hutool.json.JSONObject;
+import cn.hutool.json.JSONUtil;
+import com.example.nis.common.Code;
+import com.example.nis.common.ResponseData;
+import com.example.nis.common.TsgServiceImpl;
+import com.example.nis.controller.IpController;
+import com.example.nis.domain.IpObject;
+import com.example.nis.domain.IpSource;
+import com.example.nis.util.Constant;
+import com.example.nis.util.ToCKDBUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import org.springframework.web.bind.annotation.RequestBody;
+import sun.net.util.IPAddressUtil;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * TODO
+ *
+ * @Classname EntityIP
+ * @Date 2022/12/6 15:23
+ * @Author wWei
+ */
+@Slf4j
+@Component
+public class Entity {
+    @Autowired
+    TsgServiceImpl tsgService;
+
+    @Value("${entity.psiphon.ipObjectId}")
+    private Integer psiphonIpObjectId;
+    @Value("${entity.psiphon.ipObjectName}")
+    private String psiphonIpObjectName;
+    @Value("${entity.psiphon.enable}")
+    private Boolean psiphonEnable;
+
+    @Scheduled(cron = "${entity.psiphon.cron}")
+    public ResponseData scheduledExecutorOfPsiphon() {
+        if (BooleanUtil.isFalse(psiphonEnable)) {
+            log.warn("psiphon schedule disable");
+            return ResponseData.ok();
+        }
+        ResponseData responseData = getAllByIpObjectId(psiphonIpObjectId);
+        if (!Code.SUCCESS.getCode().equals(responseData.get("code"))) {
+            String msg = String.valueOf(responseData.get("msg"));
+            log.error("get all itemList error: {}", msg);
+            return ResponseData.error(msg);
+        }
+
+        String sql = "select distinct client_ip from psiphon_client_detection_event where event_time >= now()-600 AND event_time < now() AND notEmpty(client_ip)";
+        HttpResponse httpResponse = ToCKDBUtil.exeQuery(sql);
+        if (httpResponse.getStatus() != Code.SUCCESS.getCode()) {
+            String msg = String.valueOf(httpResponse);
+            log.error("get client ip error: {}", msg);
+            return ResponseData.error(msg);
+        }
+
+        Map responseBody = JSONUtil.toBean(httpResponse.body(), Map.class);
+        JSONArray dataJson = JSONUtil.parseArray(responseBody.get("data"));
+        List<Map> data = JSONUtil.toList(dataJson, Map.class);
+        if (data.isEmpty()) {
+            log.warn("ips is empty, No update");
+            return ResponseData.ok();
+        }
+
+        IpSource ipSource = new IpSource();
+        List<IpObject> deleteList = new ArrayList<>();
+        JSONArray jsonArray = JSONUtil.parseArray(responseData.get("data"));
+        for (int i = 0; i < jsonArray.size(); i++) {
+            JSONObject obj = jsonArray.getJSONObject(i);
+            deleteList.add(JSONUtil.toBean(obj, IpObject.class));
+        }
+        ipSource.setDeleteItemList(deleteList);
+        List<IpObject> addList = new ArrayList<>();
+        for (Map map : data) {
+            if (StrUtil.isBlankIfStr(map) || IpController.excludeList.contains(map)) {
+                continue;
+            }
+            String clientIp = String.valueOf(map.get("client_ip"));
+            if (!IPAddressUtil.isIPv6LiteralAddress(clientIp) && !IPAddressUtil.isIPv4LiteralAddress(clientIp)) {
+                log.warn("Not IPv4 or IPv6: {}", map);
+                continue;
+            }
+            IpObject ipObject = new IpObject();
+            ipObject.setIp(clientIp);
+            ipObject.setPort("0-65535");
+            addList.add(ipObject);
+        }
+        ipSource.setAddItemList(addList);
+        return putPsiphon(ipSource);
+    }
+
+
+    public ResponseData getAllByIpObjectId(Integer ipObjectId) {
+        try {
+            JSONArray itemList = tsgService.getItemList(ipObjectId, Constant.TSG_ITEM_IP, null, null, null, null, null);
+            if (itemList == null || itemList.isEmpty()) {
+                return ResponseData.ok(new ArrayList<>());
+            }
+            List<IpObject> result = new ArrayList<>(itemList.size());
+            for (int i = 0; i < itemList.size(); i++) {
+                JSONObject jsonObject = itemList.getJSONObject(i);
+                IpObject ipObject = new IpObject();
+                ipObject.setItemId(Integer.parseInt(jsonObject.get(Constant.TSG_ITEM_ID).toString()));
+                ipObject.setIp(jsonObject.get(Constant.TSG_ITEM_IP).toString());
+                ipObject.setPort(jsonObject.get(Constant.TSG_ITEM_PORT).toString());
+                result.add(ipObject);
+            }
+            return ResponseData.ok(result);
+        } catch (Exception e) {
+            log.warn("get All itemList error: {}", e.getMessage());
+            return ResponseData.error();
+        }
+    }
+
+    public ResponseData putPsiphon(@RequestBody IpSource source) {
+        try {
+            JSONArray array = new JSONArray();
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.set(Constant.TSG_OBJECT_ID, psiphonIpObjectId);
+            jsonObject.set(Constant.TSG_OBJECT_TYPE, Constant.TSG_IP_ADDR_OBJECT);
+            jsonObject.set(Constant.TSG_OBJECT_NAME, psiphonIpObjectName);
+            jsonObject.set(Constant.TSG_IS_BUILTIN, 0);
+            jsonObject.set(Constant.TSG_IS_EXCLUSION, 0);
+            jsonObject.set(Constant.TSG_IS_VALID, 1);
+            if (source.getAddItemList() != null && source.getAddItemList().size() > 0) {
+                JSONArray jsonArray = JSONUtil.parseArray(source.getAddItemList());
+                for (int i = 0; i < jsonArray.size(); i++) {
+                    JSONObject obj = jsonArray.getJSONObject(i);
+                    obj.set(Constant.TSG_ITEM_ISSESSION, Constant.TSG_ITEM_ENDPOINT);
+                }
+                jsonObject.set("addItemList", jsonArray);
+            } else {
+                jsonObject.set("addItemList", new JSONArray());
+            }
+            if (source.getUpdateItemList() != null && source.getUpdateItemList().size() > 0) {
+                JSONArray jsonArray = JSONUtil.parseArray(source.getUpdateItemList());
+                for (int i = 0; i < jsonArray.size(); i++) {
+                    JSONObject obj = jsonArray.getJSONObject(i);
+                    obj.set(Constant.TSG_ITEM_ISSESSION, Constant.TSG_ITEM_ENDPOINT);
+                }
+                jsonObject.set("updateItemList", jsonArray);
+            } else {
+                jsonObject.set("updateItemList", new JSONArray());
+            }
+            if (source.getDeleteItemList() != null && source.getDeleteItemList().size() > 0) {
+                JSONArray jsonArray = JSONUtil.parseArray(source.getDeleteItemList());
+                Integer[] deleteItemIds = new Integer[source.getDeleteItemList().size()];
+                for (int i = 0; i < jsonArray.size(); i++) {
+                    JSONObject obj = jsonArray.getJSONObject(i);
+                    deleteItemIds[i] = Integer.parseInt(obj.get("itemId").toString());
+                }
+                jsonObject.set("deleteItemIds", deleteItemIds);
+            } else {
+                jsonObject.set("deleteItemIds", new JSONArray());
+            }
+            array.add(jsonObject);
+            boolean b = tsgService.updateObject(array);
+            if (b) {
+                return ResponseData.ok();
+            } else {
+                return ResponseData.error();
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            return ResponseData.error();
+        }
+    }
+}
diff --git a/src/main/java/com/example/nis/controller/IpController.java b/src/main/java/com/example/nis/controller/IpController.java
index afd7d07..6b2c304 100644
--- a/src/main/java/com/example/nis/controller/IpController.java
+++ b/src/main/java/com/example/nis/controller/IpController.java
@@ -33,7 +33,7 @@ import java.util.List;
 public class IpController {
 
     private final TsgServiceImpl tsgService;
-    private List<String> excludeList = new ArrayList<>();
+    public static List<String> excludeList = new ArrayList<>();
     private Integer upperLimit = 100000;
     @Value("${ip.filePath}")
     private String ipFilePath;
diff --git a/src/main/java/com/example/nis/util/ToCKDBUtil.java b/src/main/java/com/example/nis/util/ToCKDBUtil.java
new file mode 100644
index 0000000..e988b3e
--- /dev/null
+++ b/src/main/java/com/example/nis/util/ToCKDBUtil.java
@@ -0,0 +1,68 @@
+package com.example.nis.util;
+
+import cn.hutool.core.net.URLEncoder;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.core.util.URLUtil;
+import cn.hutool.http.HttpRequest;
+import cn.hutool.http.HttpResponse;
+import cn.hutool.http.HttpUtil;
+import cn.hutool.json.JSONObject;
+import cn.hutool.json.JSONUtil;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.util.EntityUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Slf4j
+@Component
+public class ToCKDBUtil {
+    public static String CK_URL;
+    private static String CK_DBNAME;
+    private static String CK_USERNAME;
+    private static String CK_PASSWORD;
+
+    @Value("${clickhouse.url}")
+    public void setCKUrl(String url) {
+        CK_URL = url;
+    }
+
+    @Value("${clickhouse.dbname}")
+    public void setCkDbname(String dbname) {
+        CK_DBNAME = dbname;
+    }
+
+    @Value("${clickhouse.username}")
+    private void setCkUsername(String username) {
+        CK_USERNAME = username;
+    }
+
+    @Value("${clickhouse.password}")
+    private void setCkPassword(String password) {
+        CK_PASSWORD = password;
+    }
+
+    /**
+     * 发送登录请求 获取 token
+     *
+     * @return
+     * @throws Exception
+     */
+    public static HttpResponse exeQuery(String sql) {
+        HttpResponse response;
+        String urlParam = "/?" +
+                "user=" + CK_USERNAME +
+                "&password=" + CK_PASSWORD +
+                "&database=" + CK_DBNAME +
+                "&query=" + URLUtil.encode(sql) +
+                " FORMAT JSON";
+        response = HttpRequest.get(CK_URL + urlParam)
+                .timeout(60000)
+                .execute();
+        return response;
+    }
+}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 94dc74b..429e4bb 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -28,5 +28,17 @@ tsg:
     fqdnObjectId: 159
     fqdnObjectName: sg-test
 
+entity:
+  psiphon:
+    cron: 0 0/1 * * * ?
+    enable: true
+    ipObjectId: 8533
+    ipObjectName: Psiphon_Client_IP
+## ClickhHouse configuration
+clickhouse:
+  url: http://192.168.44.12:8123
+  dbname: tsg_galaxy_v3
+  username: tsg_query
+  password: galaxy2018