Merge branch 'bugs/knowledge_update' into 'develop'

[Fix][core] fix knowledge metadata isValid not use and improve the download of...

See merge request galaxy/platform/groot-stream!35

3 files changed, 21 insertions, 6 deletions

diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractMultipleKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractMultipleKnowledgeBaseHandler.java
index 4bbafd5..716a480 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractMultipleKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractMultipleKnowledgeBaseHandler.java
@@ -35,14 +35,17 @@ public abstract class AbstractMultipleKnowledgeBaseHandler extends AbstractKnowl
 
     protected KnowledgeBaseConfig knowledgeBaseConfig;
 
-    protected Map<String, KnowLedgeBaseFileMeta> knowledgeMetedataCacheMap;
+    protected Map<String, KnowLedgeBaseFileMeta> knowledgeMetedataCacheMap = new HashMap<>();
     private static final CloseableHttpClient HTTP_CLIENT = HttpClients.createMinimal();
 
     @Override
     public boolean initKnowledgeBase(KnowledgeBaseConfig knowledgeBaseConfig) {
         this.knowledgeBaseConfig = knowledgeBaseConfig;
         if ("http".equals(knowledgeBaseConfig.getFsType())) {
-            this.knowledgeMetedataCacheMap = getMetadata(knowledgeBaseConfig.getFsPath());
+            Map<String, KnowLedgeBaseFileMeta> metadata = getMetadata(knowledgeBaseConfig.getFsPath());
+            if (metadata != null) {
+                this.knowledgeMetedataCacheMap = metadata;
+            }
         }
         return buildKnowledgeBase();
     }
@@ -58,7 +61,7 @@ public abstract class AbstractMultipleKnowledgeBaseHandler extends AbstractKnowl
 
     protected byte[] downloadFile(Long id) {
         if ("http".equals(knowledgeBaseConfig.getFsType())) {
-            return downloadFile(knowledgeMetedataCacheMap.get(encodeId(id)).getPath(), 1);
+            return downloadFile(knowledgeMetedataCacheMap.get(encodeId(id)).getPath(), knowledgeMetedataCacheMap.get(encodeId(id)).getIsValid());
         }
         if ("local".equals(knowledgeBaseConfig.getFsType())) {
             return getFileFromLocal(knowledgeBaseConfig.getFsPath() + id);
@@ -78,6 +81,9 @@ public abstract class AbstractMultipleKnowledgeBaseHandler extends AbstractKnowl
 
     protected Boolean ifNeedUpdate() {
         Map<String, KnowLedgeBaseFileMeta> knowledgeMetedataMap = getMetadata(knowledgeBaseConfig.getFsPath());
+        if (knowledgeMetedataMap == null) {
+            return false;
+        }
         if (knowledgeMetedataMap.size() != knowledgeMetedataCacheMap.size()) {
             this.knowledgeMetedataCacheMap = knowledgeMetedataMap;
             return true;
@@ -110,7 +116,7 @@ public abstract class AbstractMultipleKnowledgeBaseHandler extends AbstractKnowl
         } catch (IOException e) {
             logger.error("fetch knowledge metadata error", e);
         }
-        return new HashMap<>();
+        return null;
     }
 
     public static boolean checkId(String id) {
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractSingleKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractSingleKnowledgeBaseHandler.java
index f44df1f..c460961 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractSingleKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractSingleKnowledgeBaseHandler.java
@@ -45,7 +45,12 @@ public abstract class AbstractSingleKnowledgeBaseHandler extends AbstractKnowled
     @Override
     public void updateKnowledgeBase() {
         if (ifNeedUpdate()) {
-            buildKnowledgeBase();
+            Boolean result = buildKnowledgeBase();
+            if (result) {
+                logger.warn("KnowledgeBase " + knowledgeBaseConfig.getName() + " update success!");
+            } else {
+                logger.error("KnowledgeBase " + knowledgeBaseConfig.getName() + " update failed!");
+            }
         }
     }
 
@@ -53,7 +58,7 @@ public abstract class AbstractSingleKnowledgeBaseHandler extends AbstractKnowled
 
     public byte[] downloadFile() {
         if ("http".equals(knowledgeBaseConfig.getFsType())) {
-            return downloadFile(knowledgeMetedataCache.getPath(), 1);
+            return downloadFile(knowledgeMetedataCache.getPath(), knowledgeMetedataCache.getIsValid());
         }
         if ("local".equals(knowledgeBaseConfig.getFsType())) {
             return getFileFromLocal(knowledgeBaseConfig.getFsPath() + knowledgeBaseConfig.getFiles().get(0));
diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LookupTestUtils.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LookupTestUtils.java
index 20defff..b70edcc 100644
--- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LookupTestUtils.java
+++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LookupTestUtils.java
@@ -35,6 +35,8 @@ public class LookupTestUtils {
 
     private static String fsPath = "testPath";
     private static String fsType = "http";
+
+    private static int isValid = 1;
     private static List<String> fsFiles = Arrays.asList("testFile");
     public static String kbName = "testKbName";
     private static String downloadPath = "testDownloadPath";
@@ -77,6 +79,7 @@ public class LookupTestUtils {
         checkStaticMock();
         KnowLedgeBaseFileMeta knowLedgeBaseFileMeta = new KnowLedgeBaseFileMeta();
         knowLedgeBaseFileMeta.setPath(downloadPath);
+        knowLedgeBaseFileMeta.setIsValid(isValid);
         abstractKnowledgeBaseHandlerMockedStatic.when(() -> AbstractKnowledgeBaseHandler.getMetadata(fsType, fsPath, fsFiles.get(0))).thenReturn(knowLedgeBaseFileMeta);
         abstractKnowledgeBaseHandlerMockedStatic.when(() -> AbstractKnowledgeBaseHandler.downloadFile(downloadPath, 1)).thenReturn(downloadContent.getBytes());
     }
@@ -86,6 +89,7 @@ public class LookupTestUtils {
         KnowLedgeBaseFileMeta KnowLedgeBaseFileMeta = new KnowLedgeBaseFileMeta();
         KnowLedgeBaseFileMeta.setKb_id("1");
         KnowLedgeBaseFileMeta.setPath(downloadPath);
+        KnowLedgeBaseFileMeta.setIsValid(isValid);
         Map<String, KnowLedgeBaseFileMeta> KnowLedgeBaseFileMetaMap = new HashMap<>();
         KnowLedgeBaseFileMetaMap.put("1", KnowLedgeBaseFileMeta);
         abstractMultipleKnowledgeBaseHandlerMockedStatic.when(() -> AbstractMultipleKnowledgeBaseHandler.getMetadata(fsPath)).thenReturn(KnowLedgeBaseFileMetaMap);