Merge branch 'feature/cn' into 'develop'

[improve][core] Refactor to use class KnowLedgeBaseFileMeta instead of class KnowledgeMetadata.

See merge request galaxy/platform/groot-stream!17

7 files changed, 19 insertions, 54 deletions

diff --git a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AbstractMultipleKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AbstractMultipleKnowledgeBaseHandler.java
index 1d308a8..d65ec49 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AbstractMultipleKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AbstractMultipleKnowledgeBaseHandler.java
@@ -2,8 +2,8 @@ package com.geedgenetworks.core.utils.KnowlegdeBase.Handler.cn;
 
 import com.alibaba.fastjson2.JSON;
 import com.geedgenetworks.common.config.KnowledgeBaseConfig;
+import com.geedgenetworks.core.pojo.KnowLedgeBaseFileMeta;
 import com.geedgenetworks.core.utils.KnowlegdeBase.Handler.AbstractKnowledgeBaseHandler;
-import com.geedgenetworks.core.utils.cn.common.KnowledgeMetadata;
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import org.apache.http.HttpEntity;
@@ -33,7 +33,7 @@ public abstract class AbstractMultipleKnowledgeBaseHandler extends AbstractKnowl
 
     protected KnowledgeBaseConfig knowledgeBaseConfig;
 
-    protected Map<String, KnowledgeMetadata> knowledgeMetedataCacheMap;
+    protected Map<String, KnowLedgeBaseFileMeta> knowledgeMetedataCacheMap;
     private static final CloseableHttpClient HTTP_CLIENT = HttpClients.createMinimal();
 
     @Override
@@ -53,7 +53,7 @@ public abstract class AbstractMultipleKnowledgeBaseHandler extends AbstractKnowl
     protected abstract Boolean buildKnowledgeBase();
 
     protected Boolean ifNeedUpdate() {
-        Map<String, KnowledgeMetadata> knowledgeMetedataMap = getMetadata(knowledgeBaseConfig.getFsPath());
+        Map<String, KnowLedgeBaseFileMeta> knowledgeMetedataMap = getMetadata(knowledgeBaseConfig.getFsPath());
         if (knowledgeMetedataMap.size() != knowledgeMetedataCacheMap.size()) {
             return true;
         }
@@ -66,7 +66,7 @@ public abstract class AbstractMultipleKnowledgeBaseHandler extends AbstractKnowl
         return false;
     }
 
-    public static Map<String, KnowledgeMetadata> getMetadata(String url) {
+    public static Map<String, KnowLedgeBaseFileMeta> getMetadata(String url) {
         final HttpGet httpGet = new HttpGet(url);
         httpGet.addHeader("Accept", "application/json");
         try {
@@ -75,10 +75,10 @@ public abstract class AbstractMultipleKnowledgeBaseHandler extends AbstractKnowl
             if (entity != null) {
                 String content = EntityUtils.toString(entity, "UTF-8");
                 KnowledgeResponse knowledgeResponse = JSON.parseObject(content, KnowledgeResponse.class);
-                List<KnowledgeMetadata> knowledgeMetedataList = JSON.parseArray(knowledgeResponse.data, KnowledgeMetadata.class);
+                List<KnowLedgeBaseFileMeta> knowledgeMetedataList = JSON.parseArray(knowledgeResponse.data, KnowLedgeBaseFileMeta.class);
                 return knowledgeMetedataList.stream()
                         .filter(metadata -> "latest".equals(metadata.getVersion()) && metadata.getIsValid() == 1 && metadata.getSha256() != null)
-                        .collect(Collectors.toMap(KnowledgeMetadata::getId, Function.identity(), (existing, replacement) -> existing, HashMap::new));
+                        .collect(Collectors.toMap(KnowLedgeBaseFileMeta::getKb_id, Function.identity(), (existing, replacement) -> existing, HashMap::new));
             }
         } catch (IOException e) {
             logger.error("fetch knowledge metadata error", e);
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AbstractSingleKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AbstractSingleKnowledgeBaseHandler.java
index 98f1ce9..031de40 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AbstractSingleKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AbstractSingleKnowledgeBaseHandler.java
@@ -4,7 +4,6 @@ import com.alibaba.fastjson2.JSON;
 import com.geedgenetworks.common.config.KnowledgeBaseConfig;
 import com.geedgenetworks.core.pojo.KnowLedgeBaseFileMeta;
 import com.geedgenetworks.core.utils.KnowlegdeBase.Handler.AbstractKnowledgeBaseHandler;
-import com.geedgenetworks.core.utils.cn.common.KnowledgeMetadata;
 import lombok.Data;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.CloseableHttpResponse;
@@ -67,7 +66,7 @@ public abstract class AbstractSingleKnowledgeBaseHandler extends AbstractKnowled
         }
     }
 
-    public List<KnowledgeMetadata> getMetadata(String url) {
+    public List<KnowLedgeBaseFileMeta> getMetadata(String url) {
         final HttpGet httpGet = new HttpGet(url);
         httpGet.addHeader("Accept", "application/json");
         try {
@@ -76,7 +75,7 @@ public abstract class AbstractSingleKnowledgeBaseHandler extends AbstractKnowled
             if (entity != null) {
                 String content = EntityUtils.toString(entity, "UTF-8");
                 KnowledgeResponse knowledgeResponse = JSON.parseObject(content, KnowledgeResponse.class);
-                return JSON.parseArray(knowledgeResponse.data, KnowledgeMetadata.class).stream().filter(metadata -> "latest".equals(metadata.getVersion()) && metadata.getIsValid() == 1).collect(Collectors.toList());
+                return JSON.parseArray(knowledgeResponse.data, KnowLedgeBaseFileMeta.class).stream().filter(metadata -> "latest".equals(metadata.getVersion()) && metadata.getIsValid() == 1).collect(Collectors.toList());
             }
         } catch (IOException e) {
             logger.error("fetch knowledge metadata error", e);
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AppTagUserDefineKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AppTagUserDefineKnowledgeBaseHandler.java
index f83596c..075d768 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AppTagUserDefineKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/AppTagUserDefineKnowledgeBaseHandler.java
@@ -1,6 +1,6 @@
 package com.geedgenetworks.core.utils.KnowlegdeBase.Handler.cn;
 
-import com.geedgenetworks.core.utils.cn.common.KnowledgeMetadata;
+import com.geedgenetworks.core.pojo.KnowLedgeBaseFileMeta;
 import com.geedgenetworks.core.utils.cn.csv.HighCsvReader;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -44,7 +44,7 @@ public class AppTagUserDefineKnowledgeBaseHandler extends AbstractMultipleKnowle
         return true;
     }
 
-    private void buildSingleKnowledgeBase(Map<String, List<Node>> appTags, Long id, KnowledgeMetadata metadata) {
+    private void buildSingleKnowledgeBase(Map<String, List<Node>> appTags, Long id, KnowLedgeBaseFileMeta metadata) {
         try {
             List<String> needColumns = new ArrayList<>();
             needColumns.add("tag_value");
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/DomainTagUserDefineKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/DomainTagUserDefineKnowledgeBaseHandler.java
index a9417e9..fdd0c3b 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/DomainTagUserDefineKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/DomainTagUserDefineKnowledgeBaseHandler.java
@@ -1,6 +1,6 @@
 package com.geedgenetworks.core.utils.KnowlegdeBase.Handler.cn;
 
-import com.geedgenetworks.core.utils.cn.common.KnowledgeMetadata;
+import com.geedgenetworks.core.pojo.KnowLedgeBaseFileMeta;
 import com.geedgenetworks.core.utils.cn.common.Trie;
 import com.geedgenetworks.core.utils.cn.csv.HighCsvReader;
 import org.apache.commons.lang3.StringUtils;
@@ -49,7 +49,7 @@ public class DomainTagUserDefineKnowledgeBaseHandler extends AbstractMultipleKno
         return true;
     }
 
-    private void buildSingleKnowledgeBase(Trie<Node> fqdnTagsFuzzy, Map<String, List<Node>> fqdnTagsFull, Long id, KnowledgeMetadata metadata) {
+    private void buildSingleKnowledgeBase(Trie<Node> fqdnTagsFuzzy, Map<String, List<Node>> fqdnTagsFull, Long id, KnowLedgeBaseFileMeta metadata) {
         try {
             List<String> needColumns = new ArrayList<>();
             needColumns.add("tag_value");
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/IpTagUserDefineKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/IpTagUserDefineKnowledgeBaseHandler.java
index c6b4ac8..7ff4bfd 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/IpTagUserDefineKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/utils/KnowlegdeBase/Handler/cn/IpTagUserDefineKnowledgeBaseHandler.java
@@ -1,6 +1,6 @@
 package com.geedgenetworks.core.utils.KnowlegdeBase.Handler.cn;
 
-import com.geedgenetworks.core.utils.cn.common.KnowledgeMetadata;
+import com.geedgenetworks.core.pojo.KnowLedgeBaseFileMeta;
 import com.geedgenetworks.core.utils.cn.csv.HighCsvReader;
 import inet.ipaddr.IPAddress;
 import inet.ipaddr.IPAddressString;
@@ -47,7 +47,7 @@ public class IpTagUserDefineKnowledgeBaseHandler extends AbstractMultipleKnowled
         return true;
     }
 
-    private void buildSingleKnowledgeBase(TreeRangeMap<IPAddress, List<Node>> treeRangeMap, Long id, KnowledgeMetadata metadata) {
+    private void buildSingleKnowledgeBase(TreeRangeMap<IPAddress, List<Node>> treeRangeMap, Long id, KnowLedgeBaseFileMeta metadata) {
         try {
             List<String> needColumns = new ArrayList<>();
             needColumns.add("tag_value");
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/utils/cn/common/KnowledgeMetadata.java b/groot-core/src/main/java/com/geedgenetworks/core/utils/cn/common/KnowledgeMetadata.java
deleted file mode 100644
index 6926852..0000000
--- a/groot-core/src/main/java/com/geedgenetworks/core/utils/cn/common/KnowledgeMetadata.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package com.geedgenetworks.core.utils.cn.common;
-
-import com.alibaba.fastjson2.annotation.JSONField;
-import lombok.Data;
-import lombok.ToString;
-
-/**
- * @author gujinkai
- * @version 1.0
- * @date 2024/1/18 17:35
- */
-@Data
-@ToString
-public class KnowledgeMetadata {
-
-    @JSONField(name = "kb_id")
-    private String id;
-
-    private String name;
-
-    private String sha256;
-
-    private String format;
-
-    private String path;
-
-    private String category;
-
-    @JSONField(name = "is_valid")
-    private int isValid = 1;
-
-    private String version;
-}
diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LookupTestUtils.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LookupTestUtils.java
index 6678e2e..2dc0701 100644
--- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LookupTestUtils.java
+++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LookupTestUtils.java
@@ -8,7 +8,6 @@ import com.geedgenetworks.core.pojo.KnowLedgeBaseFileMeta;
 import com.geedgenetworks.core.utils.KnowlegdeBase.Handler.AbstractKnowledgeBaseHandler;
 import com.geedgenetworks.core.utils.KnowlegdeBase.Handler.cn.AbstractMultipleKnowledgeBaseHandler;
 import com.geedgenetworks.core.utils.KnowlegdeBase.KnowledgeBaseUpdateJob;
-import com.geedgenetworks.core.utils.cn.common.KnowledgeMetadata;
 import org.apache.flink.api.common.ExecutionConfig;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.configuration.Configuration;
@@ -75,11 +74,11 @@ public class LookupTestUtils {
 
     static void mockMultipleKnowledgeBaseHandler(String downloadContent) {
         checkStaticMock();
-        KnowledgeMetadata knowledgeMetadata = new KnowledgeMetadata();
-        knowledgeMetadata.setPath(downloadPath);
-        Map<String, KnowledgeMetadata> knowledgeMetadataMap = new HashMap<>();
-        knowledgeMetadataMap.put("1", knowledgeMetadata);
-        abstractMultipleKnowledgeBaseHandlerMockedStatic.when(() -> AbstractMultipleKnowledgeBaseHandler.getMetadata(fsPath)).thenReturn(knowledgeMetadataMap);
+        KnowLedgeBaseFileMeta KnowLedgeBaseFileMeta = new KnowLedgeBaseFileMeta();
+        KnowLedgeBaseFileMeta.setPath(downloadPath);
+        Map<String, KnowLedgeBaseFileMeta> KnowLedgeBaseFileMetaMap = new HashMap<>();
+        KnowLedgeBaseFileMetaMap.put("1", KnowLedgeBaseFileMeta);
+        abstractMultipleKnowledgeBaseHandlerMockedStatic.when(() -> AbstractMultipleKnowledgeBaseHandler.getMetadata(fsPath)).thenReturn(KnowLedgeBaseFileMetaMap);
         abstractKnowledgeBaseHandlerMockedStatic.when(() -> AbstractKnowledgeBaseHandler.downloadFile(downloadPath, 1)).thenReturn(downloadContent.getBytes());
     }