summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--config/grootstream.yaml2
-rw-r--r--groot-bootstrap/pom.xml17
-rw-r--r--groot-common/src/main/resources/grootstream.yaml2
-rw-r--r--groot-common/src/test/resources/grootstream.yaml2
-rw-r--r--groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java4
-rw-r--r--groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java8
-rw-r--r--groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/GeoIpKnowledgeBaseHandler.java11
-rw-r--r--groot-core/src/test/java/com/geedgenetworks/core/udf/test/AsnLookupFunctionTest.java10
-rw-r--r--groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java31
-rw-r--r--groot-core/src/test/resources/asn.mmdbbin0 -> 2316 bytes
-rw-r--r--groot-core/src/test/resources/ip_builtin.mmdbbin0 -> 4738 bytes
11 files changed, 34 insertions, 53 deletions
diff --git a/config/grootstream.yaml b/config/grootstream.yaml
index fdd240a..acb19b1 100644
--- a/config/grootstream.yaml
+++ b/config/grootstream.yaml
@@ -12,7 +12,7 @@ grootstream:
- 64af7077-eb9b-4b8f-80cf-2ceebc89bea9
- 004390bc-3135-4a6f-a492-3662ecb9e289
properties:
- hos.path: http://192.168.44.12:8089
+ hos.path: http://192.168.44.12:9098/hos
hos.bucket.name.traffic_file: traffic_file_bucket
hos.bucket.name.troubleshooting_file: troubleshooting_file_bucket
scheduler.knowledge_base.update.interval.minutes: 5
diff --git a/groot-bootstrap/pom.xml b/groot-bootstrap/pom.xml
index 62c2dc8..0a1fd9c 100644
--- a/groot-bootstrap/pom.xml
+++ b/groot-bootstrap/pom.xml
@@ -121,23 +121,6 @@
<scope>${scope}</scope>
</dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <scope>${scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>log4j-over-slf4j</artifactId>
- </exclusion>
-
- </exclusions>
- </dependency>
-
<dependency>
<groupId>org.apache.flink</groupId>
diff --git a/groot-common/src/main/resources/grootstream.yaml b/groot-common/src/main/resources/grootstream.yaml
index efc5b52..1a9a974 100644
--- a/groot-common/src/main/resources/grootstream.yaml
+++ b/groot-common/src/main/resources/grootstream.yaml
@@ -12,6 +12,6 @@ grootstream:
- 64af7077-eb9b-4b8f-80cf-2ceebc89bea9
- 004390bc-3135-4a6f-a492-3662ecb9e289
properties:
- hos.path: http://192.168.44.12:8089
+ hos.path: http://192.168.44.12:9098/hos
hos.bucket.name.traffic_file: traffic_file_bucket
hos.bucket.name.troubleshooting_file: troubleshooting_file_bucket \ No newline at end of file
diff --git a/groot-common/src/test/resources/grootstream.yaml b/groot-common/src/test/resources/grootstream.yaml
index efc5b52..1a9a974 100644
--- a/groot-common/src/test/resources/grootstream.yaml
+++ b/groot-common/src/test/resources/grootstream.yaml
@@ -12,6 +12,6 @@ grootstream:
- 64af7077-eb9b-4b8f-80cf-2ceebc89bea9
- 004390bc-3135-4a6f-a492-3662ecb9e289
properties:
- hos.path: http://192.168.44.12:8089
+ hos.path: http://192.168.44.12:9098/hos
hos.bucket.name.traffic_file: traffic_file_bucket
hos.bucket.name.troubleshooting_file: troubleshooting_file_bucket \ No newline at end of file
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java
index 422251f..33c0653 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java
@@ -49,7 +49,7 @@ public class AsnLookup implements UDF {
this.lookupFieldName = udfContext.getLookup_fields().get(0);
this.outputFieldName = udfContext.getOutput_fields().get(0);
AsnKnowledgeBaseHandler.increment();
- log.warn("AsnKnowledgeBaseHandlerCountIn"+AsnKnowledgeBaseHandler.getCount());
+ log.warn("AsnKnowledgeBaseHandlerCount "+AsnKnowledgeBaseHandler.getCount());
}
@@ -74,9 +74,9 @@ public class AsnLookup implements UDF {
@Override
public void close() {
AsnKnowledgeBaseHandler.decrement();
- log.warn("+++++++++++++++++++++++++++++++++++AsnKnowledgeBaseHandlerCountDe"+AsnKnowledgeBaseHandler.getCount());
if (AsnKnowledgeBaseHandler.getCount()<=0) {
KnowledgeBaseUpdateJob.removeKnowledgeBase(kbName);
+ log.warn("AsnKnowledgeBaseHandler removeKnowledgeBase "+kbName);
}
}
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java
index 9763758..442d499 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java
@@ -56,6 +56,8 @@ public class GeoIpLookup implements UDF {
if(udfContext.getOutput_fields()!=null && !udfContext.getOutput_fields().isEmpty()){
this.outputFieldName = udfContext.getOutput_fields().get(0);
}
+ GeoIpKnowledgeBaseHandler.increment();
+ log.warn("GeoIpKnowledgeBaseHandler "+GeoIpKnowledgeBaseHandler.getCount());
}
@@ -163,6 +165,10 @@ public class GeoIpLookup implements UDF {
@Override
public void close() {
- KnowledgeBaseUpdateJob.removeKnowledgeBase(kbName);
+ GeoIpKnowledgeBaseHandler.decrement();
+ if (GeoIpKnowledgeBaseHandler.getCount()<=0) {
+ KnowledgeBaseUpdateJob.removeKnowledgeBase(kbName);
+ log.warn("GeoIpKnowledgeBaseHandler removeKnowledgeBase "+kbName);
+ }
}
}
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/GeoIpKnowledgeBaseHandler.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/GeoIpKnowledgeBaseHandler.java
index 5272ada..4d207a2 100644
--- a/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/GeoIpKnowledgeBaseHandler.java
+++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/knowlegdebase/handler/GeoIpKnowledgeBaseHandler.java
@@ -20,10 +20,19 @@ public class GeoIpKnowledgeBaseHandler extends AbstractKnowledgeBaseHandler {
private static Map<String, KnowledgeBaseInfoForIpLookUp> kbNameWithKbConfigAndFileMetas = new ConcurrentHashMap<>();
private static GeoIpKnowledgeBaseHandler instance;
+ private static int count = 0;
private GeoIpKnowledgeBaseHandler() {
}
-
+ public static synchronized void increment() {
+ count++;
+ }
+ public static synchronized void decrement() {
+ count--;
+ }
+ public static synchronized int getCount() {
+ return count;
+ }
public static synchronized GeoIpKnowledgeBaseHandler getInstance() {
if (instance == null) {
instance = new GeoIpKnowledgeBaseHandler();
diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/AsnLookupFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/AsnLookupFunctionTest.java
index 69a074b..5bcf89e 100644
--- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/AsnLookupFunctionTest.java
+++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/AsnLookupFunctionTest.java
@@ -70,14 +70,14 @@ public class AsnLookupFunctionTest {
knowledgeBaseConfig.setName("tsg_ip_asn");
Map<String, String> parameters = new HashMap<>();
knowledgeBaseConfig.setProperties(parameters);
- knowledgeBaseConfig.setFsType("http");
- knowledgeBaseConfig.setFsPath("http://192.168.44.67:9999/v1/knowledge_base");
- knowledgeBaseConfig.setFiles(Collections.singletonList("f9f6bc91-2142-4673-8249-e097c00fe1ea"));
+ knowledgeBaseConfig.setFsType("local");
+ knowledgeBaseConfig.setFsPath("");
+ knowledgeBaseConfig.setFiles(Collections.singletonList("src/test/resources/asn.mmdb"));
knowledgeBaseConfigList.add(knowledgeBaseConfig);
commonConfig.setKnowledgeBaseConfig(knowledgeBaseConfigList);
KnowledgeBaseUpdateJob.registerKnowledgeBase(AsnKnowledgeBaseHandler.getInstance(), knowledgeBaseConfig);
- String asn = AsnKnowledgeBaseHandler.lookUp("tsg_ip_asn","IP_TO_ASN","2600:1015:b002::");
- assertEquals("6167", asn);
+ String asn = AsnKnowledgeBaseHandler.lookUp("tsg_ip_asn","IP_TO_ASN","12.16.98.1");
+ assertEquals("31978", asn);
}
diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java
index faa65f4..f487500 100644
--- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java
+++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java
@@ -29,7 +29,6 @@ public class GeoIpLookupFunctionTest {
udfContext.setOutput_fields(Collections.singletonList("iplocation"));
}
-//com.geedgenetworks.core.udf.GeoIpLookup单元测试
@Test
public void testInit(){
GeoIpLookup geoIpLookup = new GeoIpLookup();
@@ -69,33 +68,17 @@ public class GeoIpLookupFunctionTest {
knowledgeBaseConfig.setName("tsg_ip_location");
Map<String, String> parameters = new HashMap<>();
knowledgeBaseConfig.setProperties(parameters);
- knowledgeBaseConfig.setFsType("http");
- knowledgeBaseConfig.setFsPath("http://192.168.44.67:9999/v1/knowledge_base");
- knowledgeBaseConfig.setFiles(Arrays.asList("64af7077-eb9b-4b8f-80cf-2ceebc89bea9","004390bc-3135-4a6f-a492-3662ecb9e289"));
+ knowledgeBaseConfig.setFsType("local");
+ knowledgeBaseConfig.setFsPath("");
+ knowledgeBaseConfig.setFiles(Collections.singletonList("src/test/resources/ip_builtin.mmdb"));
knowledgeBaseConfigList.add(knowledgeBaseConfig);
commonConfig.setKnowledgeBaseConfig(knowledgeBaseConfigList);
KnowledgeBaseUpdateJob.registerKnowledgeBase(GeoIpKnowledgeBaseHandler.getInstance(), knowledgeBaseConfig);
- String countryLookup = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_COUNTRY","2600:1015:b002::");
- assertEquals("美国", countryLookup);
- String provinceLookup = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_PROVINCE","2600:1015:b002::");
- assertEquals("印第安纳州", provinceLookup);
- String cityLookup = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_CITY","2600:1015:b002::");
- assertEquals("Unknown", cityLookup);
- String cityLookupDetail = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_SUBDIVISION_ADDR","2600:1015:b002::");
- assertEquals("美国.印第安纳州.Unknown", cityLookupDetail);
- String locationLookupDetail = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_DETAIL","2600:1015:b002::");
- assertEquals("美国.印第安纳州.Unknown.", locationLookupDetail.trim());
- String latLngLookup = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_LATLNG","2600:1015:b002::");
- assertEquals("39.9388,-86.1137", latLngLookup);
- String ispLookup = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_PROVIDER","2600:1015:b002::");
- assertEquals("", ispLookup);
- String infoLookupToJSONString = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_JSON","2600:1015:b002::");
- assertEquals(JSONObject.parseObject(infoLookupToJSONString).getString("country"), "美国");
- String country = GeoIpKnowledgeBaseHandler.lookUpObject("tsg_ip_location","2600:1015:b002::").getCountry();
- assertEquals("美国", country);
+ String countryLookup = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_COUNTRY","86.111.175.128");
+ assertEquals("United Kingdom", countryLookup);
+ String country = GeoIpKnowledgeBaseHandler.lookUpObject("tsg_ip_location","86.111.175.128").getCountry();
+ assertEquals("United Kingdom", country);
assertNull(GeoIpKnowledgeBaseHandler.lookUpObject("tsg_ip_location", "255.255.255.255"));
}
-
-
}
diff --git a/groot-core/src/test/resources/asn.mmdb b/groot-core/src/test/resources/asn.mmdb
new file mode 100644
index 0000000..a20e238
--- /dev/null
+++ b/groot-core/src/test/resources/asn.mmdb
Binary files differ
diff --git a/groot-core/src/test/resources/ip_builtin.mmdb b/groot-core/src/test/resources/ip_builtin.mmdb
new file mode 100644
index 0000000..b333c64
--- /dev/null
+++ b/groot-core/src/test/resources/ip_builtin.mmdb
Binary files differ
generated by cgit v1.2.3 (git 2.39.1) at 2025-12-30 15:07:35 +0000