[fix][core]修复GAL-532，解决知识库初始化加载失败任务不能启动问题。

5 files changed, 89 insertions, 51 deletions

diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java
index 33c0653..34ee2f9 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java
@@ -35,9 +35,7 @@ public class AsnLookup implements UDF {
         CommonConfig commonConfig = JSON.parseObject(configuration.toMap().get(Constants.SYSPROP_GROOTSTREAM_CONFIG), CommonConfig.class);
         for (KnowledgeBaseConfig knowledgeBaseConfig : commonConfig.getKnowledgeBaseConfig()) {
             if (kbName.equals(knowledgeBaseConfig.getName())) {
-                if (!KnowledgeBaseUpdateJob.registerKnowledgeBase(AsnKnowledgeBaseHandler.getInstance(), knowledgeBaseConfig)) {
-                    throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Register AsnKnowledgeBase error ");
-                }
+                KnowledgeBaseUpdateJob.registerKnowledgeBase(AsnKnowledgeBaseHandler.getInstance(), knowledgeBaseConfig);
                 break;
             }
         }
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java
index 442d499..46885c4 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java
@@ -41,9 +41,7 @@ public class GeoIpLookup implements UDF {
         CommonConfig commonConfig = JSON.parseObject(configuration.toMap().get(Constants.SYSPROP_GROOTSTREAM_CONFIG), CommonConfig.class);
         for (KnowledgeBaseConfig knowledgeBaseConfig : commonConfig.getKnowledgeBaseConfig()) {
             if (kbName.equals(knowledgeBaseConfig.getName())) {
-                if (!KnowledgeBaseUpdateJob.registerKnowledgeBase(GeoIpKnowledgeBaseHandler.getInstance(), knowledgeBaseConfig)) {
-                    throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Register GeoIpKnowledgeBase error ");
-                }
+                KnowledgeBaseUpdateJob.registerKnowledgeBase(GeoIpKnowledgeBaseHandler.getInstance(), knowledgeBaseConfig);
                 break;
             }
         }
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractKnowledgeBaseHandler.java
index dd95198..7daa74e 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AbstractKnowledgeBaseHandler.java
@@ -106,7 +106,12 @@ public abstract class AbstractKnowledgeBaseHandler {
 
         } catch (Exception e) {
             log.error("get file Metadata error! path = " + path + " " + e.getMessage());
-            return null;
+            KnowLedgeBaseFileMeta knowLedgeBaseFileMeta = new KnowLedgeBaseFileMeta();
+            knowLedgeBaseFileMeta.setKb_id(kbId);
+            knowLedgeBaseFileMeta.setPath(path);
+            knowLedgeBaseFileMeta.setSha256("");
+            knowLedgeBaseFileMetaList.add(knowLedgeBaseFileMeta);
+            return knowLedgeBaseFileMeta;
         }
 
         if (!knowLedgeBaseFileMetaList.isEmpty()) {
@@ -114,7 +119,12 @@ public abstract class AbstractKnowledgeBaseHandler {
             return knowLedgeBaseFileMetaList.get(0);
         } else {
             log.error("file Metadata is null ! path = " + path + "?kb_id=" + kbId);
-            return null;
+            KnowLedgeBaseFileMeta knowLedgeBaseFileMeta = new KnowLedgeBaseFileMeta();
+            knowLedgeBaseFileMeta.setKb_id(kbId);
+            knowLedgeBaseFileMeta.setPath(path);
+            knowLedgeBaseFileMeta.setSha256("");
+            knowLedgeBaseFileMetaList.add(knowLedgeBaseFileMeta);
+            return knowLedgeBaseFileMeta;
         }
     }
 }
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AsnKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AsnKnowledgeBaseHandler.java
index 381b0aa..7db67d2 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AsnKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/AsnKnowledgeBaseHandler.java
@@ -7,6 +7,7 @@ import com.geedgenetworks.core.pojo.KnowledgeBaseInfoForIpLookUp;
 import com.geedgenetworks.utils.IpLookupV2;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+
 import java.io.ByteArrayInputStream;
 import java.util.*;
 import java.util.concurrent.ConcurrentHashMap;
@@ -26,14 +27,17 @@ public class AsnKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
     }
 
     public static synchronized void increment() {
-           count++;
+        count++;
     }
+
     public static synchronized void decrement() {
         count--;
     }
+
     public static synchronized int getCount() {
         return count;
     }
+
     public static synchronized AsnKnowledgeBaseHandler getInstance() {
         if (instance == null) {
             instance = new AsnKnowledgeBaseHandler();
@@ -44,11 +48,11 @@ public class AsnKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
     @Override
     public boolean initKnowledgeBase(KnowledgeBaseConfig knowledgeBaseConfig) {
 
-        if("http".equals(knowledgeBaseConfig.getFsType()))
+        if ("http".equals(knowledgeBaseConfig.getFsType()))
             return buildKnowledgeBaseWithMeta(knowledgeBaseConfig);
-         else
+        else
             return buildKnowledgeBaseWithoutMeta(knowledgeBaseConfig);
-        }
+    }
 
     @Override
     public void updateKnowledgeBase() {
@@ -57,10 +61,9 @@ public class AsnKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
             try {
                 if (ifNeedUpdate(entry.getValue())) {
                     boolean result = buildKnowledgeBaseWithMeta(entry.getValue().getKnowledgeConfig());
-                    if(result) {
+                    if (result) {
                         log.warn("AsnKnowledgeBase " + entry.getKey() + " update success!");
-                    }
-                    else {
+                    } else {
                         log.warn("AsnKnowledgeBase " + entry.getKey() + " update error!");
                     }
                 }
@@ -71,14 +74,14 @@ public class AsnKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
 
     }
 
-    public static String lookUp(String kbName,String option, String ip) {
+    public static String lookUp(String kbName, String option, String ip) {
         String result = "";
         try {
-            if("IP_TO_ASN".equals(option)) {
+            if ("IP_TO_ASN".equals(option)) {
                 result = kbNameWithKbConfigAndFileMetas.get(kbName).getIpLookupV2().asnLookup(ip);
             }
         } catch (Exception e) {
-            log.error("AsnKnowledgeBaseHandler lookUp Error ip = " + ip +" kbName =" + kbName + " error " + e.getMessage());
+            log.error("AsnKnowledgeBaseHandler lookUp Error ip = " + ip + " kbName =" + kbName + " error " + e.getMessage());
         }
         return result;
     }
@@ -87,26 +90,29 @@ public class AsnKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
 
         for (KnowLedgeBaseFileMeta knowLedgeEntity : kbInfoForIpLookUp.getKnowLedgeBaseFilesMetas()) {
             KnowLedgeBaseFileMeta metadata = getMetadata(kbInfoForIpLookUp.getKnowledgeConfig().getFsType(), kbInfoForIpLookUp.getKnowledgeConfig().getFsPath(), knowLedgeEntity.getKb_id());
-            if (!knowLedgeEntity.getSha256().equals(Objects.requireNonNull(metadata).getSha256())) {
-                return true;
+            if (metadata.getName() != null) {
+                if (!knowLedgeEntity.getSha256().equals(Objects.requireNonNull(metadata).getSha256())) {
+                    return true;
+                }
             }
         }
         return false;
     }
+
     private boolean buildKnowledgeBaseWithoutMeta(KnowledgeBaseConfig knowledgeBaseConfig) {
 
         IpLookupV2.Builder asnLookupBuilder = new IpLookupV2.Builder(false);
         for (int i = 0; i < knowledgeBaseConfig.getFiles().size(); i++) {
             try {
                 byte[] fileByte = getFileFromLocal(knowledgeBaseConfig.getFsPath() + knowledgeBaseConfig.getFiles().get(i));
-                    switch (i) {
-                        case 0:
-                            asnLookupBuilder.loadAsnDataFile(new ByteArrayInputStream(fileByte));
-                            break;
-                        case 1:
-                            asnLookupBuilder.loadDataFilePrivate(new ByteArrayInputStream(fileByte));
-                            break;
-                    }
+                switch (i) {
+                    case 0:
+                        asnLookupBuilder.loadAsnDataFile(new ByteArrayInputStream(fileByte));
+                        break;
+                    case 1:
+                        asnLookupBuilder.loadDataFilePrivate(new ByteArrayInputStream(fileByte));
+                        break;
+                }
 
             } catch (Exception e) {
                 log.error("buildKnowledgeBase error " + e.getMessage());
@@ -121,6 +127,7 @@ public class AsnKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
         kbNameWithKbConfigAndFileMetas.put(knowledgeBaseConfig.getName(), kbInfoForIpLookUp);
         return true;
     }
+
     private static boolean buildKnowledgeBaseWithMeta(KnowledgeBaseConfig knowledgeBaseConfig) {
 
         IpLookupV2.Builder asnLookupBuilder = new IpLookupV2.Builder(false);
@@ -136,18 +143,29 @@ public class AsnKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
         }
         for (int i = 0; i < knowLedgeBaseFileMetas.size(); i++) {
             try {
-                byte[] fileByte = downloadFile(knowLedgeBaseFileMetas.get(i).getPath(), knowLedgeBaseFileMetas.get(i).getIsValid());
-                if (calculateSHA256(fileByte).equals(knowLedgeBaseFileMetas.get(i).getSha256())) {
-                    switch (i) {
-                        case 0:
-                            asnLookupBuilder.loadAsnDataFile(new ByteArrayInputStream(fileByte));
-                            break;
-                        case 1:
-                            asnLookupBuilder.loadDataFilePrivate(new ByteArrayInputStream(fileByte));
-                            break;
+                if (knowLedgeBaseFileMetas.get(i).getName() != null) {
+                    byte[] fileByte = downloadFile(knowLedgeBaseFileMetas.get(i).getPath(), knowLedgeBaseFileMetas.get(i).getIsValid());
+                    if (calculateSHA256(fileByte).equals(knowLedgeBaseFileMetas.get(i).getSha256())) {
+                        switch (i) {
+                            case 0:
+                                asnLookupBuilder.loadAsnDataFile(new ByteArrayInputStream(fileByte));
+                                break;
+                            case 1:
+                                asnLookupBuilder.loadDataFilePrivate(new ByteArrayInputStream(fileByte));
+                                break;
+                        }
+                    } else {
+                        log.error("check file sha256 error ");
+                        return false;
                     }
                 } else {
-                    log.error("check file sha256 error ");
+                    log.error("buildKnowledgeBase error File is null");
+                    IpLookupV2 ipLookup = asnLookupBuilder.build();
+                    KnowledgeBaseInfoForIpLookUp kbInfoForIpLookUp = new KnowledgeBaseInfoForIpLookUp();
+                    kbInfoForIpLookUp.setKnowLedgeBaseFilesMetas(knowLedgeBaseFileMetas);
+                    kbInfoForIpLookUp.setKnowledgeConfig(knowledgeBaseConfig);
+                    kbInfoForIpLookUp.setIpLookupV2(ipLookup);
+                    kbNameWithKbConfigAndFileMetas.put(knowledgeBaseConfig.getName(), kbInfoForIpLookUp);
                     return false;
                 }
             } catch (Exception e) {
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/GeoIpKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/GeoIpKnowledgeBaseHandler.java
index 4d207a2..c84a0b6 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/GeoIpKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/GeoIpKnowledgeBaseHandler.java
@@ -73,8 +73,10 @@ public class GeoIpKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
 
         for (KnowLedgeBaseFileMeta knowLedgeEntity : kbInfoForIpLookUp.getKnowLedgeBaseFilesMetas()) {
             KnowLedgeBaseFileMeta metadata = getMetadata(kbInfoForIpLookUp.getKnowledgeConfig().getFsType(), kbInfoForIpLookUp.getKnowledgeConfig().getFsPath(), knowLedgeEntity.getKb_id());
-            if (!knowLedgeEntity.getSha256().equals(Objects.requireNonNull(metadata).getSha256())) {
-                return true;
+            if (metadata.getName() != null) {
+                if (!knowLedgeEntity.getSha256().equals(Objects.requireNonNull(metadata).getSha256())) {
+                    return true;
+                }
             }
         }
         return false;
@@ -122,20 +124,32 @@ public class GeoIpKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
         }
         for (int i = 0; i < knowLedgeBaseFileMetas.size(); i++) {
             try {
-                byte[] fileByte = downloadFile(knowLedgeBaseFileMetas.get(i).getPath(), knowLedgeBaseFileMetas.get(i).getIsValid());
-                if (calculateSHA256(fileByte).equals(knowLedgeBaseFileMetas.get(i).getSha256())) {
-                    switch (i) {
-                        case 0:
-                            ipLookupBuilder.loadDataFile(new ByteArrayInputStream(fileByte));
-                            break;
-                        case 1:
-                            ipLookupBuilder.loadDataFilePrivate(new ByteArrayInputStream(fileByte));
-                            break;
+                if (knowLedgeBaseFileMetas.get(i).getName() != null) {
+                    byte[] fileByte = downloadFile(knowLedgeBaseFileMetas.get(i).getPath(), knowLedgeBaseFileMetas.get(i).getIsValid());
+                    if (calculateSHA256(fileByte).equals(knowLedgeBaseFileMetas.get(i).getSha256())) {
+                        switch (i) {
+                            case 0:
+                                ipLookupBuilder.loadDataFile(new ByteArrayInputStream(fileByte));
+                                break;
+                            case 1:
+                                ipLookupBuilder.loadDataFilePrivate(new ByteArrayInputStream(fileByte));
+                                break;
+                        }
+                    } else {
+                        log.error("check file sha256 error ");
+                        return false;
                     }
-                } else {
-                    log.error("check file sha256 error ");
+                }else {
+                    log.error("buildKnowledgeBase error File is null");
+                    IpLookupV2 ipLookup = ipLookupBuilder.build();
+                    KnowledgeBaseInfoForIpLookUp kbInfoForIpLookUp = new KnowledgeBaseInfoForIpLookUp();
+                    kbInfoForIpLookUp.setKnowLedgeBaseFilesMetas(knowLedgeBaseFileMetas);
+                    kbInfoForIpLookUp.setKnowledgeConfig(knowledgeBaseConfig);
+                    kbInfoForIpLookUp.setIpLookupV2(ipLookup);
+                    kbNameWithKbConfigAndFileMetas.put(knowledgeBaseConfig.getName(), kbInfoForIpLookUp);
                     return false;
                 }
+
             } catch (Exception e) {
                 log.error("updateKnowledgeBase error " + e.getMessage());
                 return false;