diff options
15 files changed, 114 insertions, 43 deletions
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeUDF.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeUDF.java index bf21b62..309ac81 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeUDF.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeUDF.java @@ -7,6 +7,8 @@ import com.geedgenetworks.common.udf.UDF; import com.geedgenetworks.common.udf.UDFContext; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; import java.util.List; import java.util.stream.Collectors; @@ -26,8 +28,18 @@ public abstract class AbstractKnowledgeUDF implements UDF { protected List<KnowledgeBaseConfig> knowledgeBaseConfigs; + protected MetricGroup metrics; + + protected Counter lookUpExistCounter; + protected Counter hitCounter; + @Override public void open(RuntimeContext runtimeContext, UDFContext udfContext) { + String metricPrefix = buildMetricPrefix(udfContext); + metrics = runtimeContext.getMetricGroup().addGroup(metricPrefix + "_udf_metrics"); + lookUpExistCounter = metrics.counter("look_up_exist"); + hitCounter = metrics.counter("knowledge_hit"); + String kbName = udfContext.getParameters().get("kb_name").toString(); Configuration configuration = (Configuration) runtimeContext @@ -44,5 +56,9 @@ public abstract class AbstractKnowledgeUDF implements UDF { } } + private String buildMetricPrefix(UDFContext udfContext) { + return functionName().toLowerCase() + "_" + udfContext.getLookup_fields().get(0); + } + protected abstract void registerKnowledges(); } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeWithRuleUDF.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeWithRuleUDF.java index 0d63ad2..f72f8e1 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeWithRuleUDF.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeWithRuleUDF.java @@ -6,6 +6,7 @@ import com.geedgenetworks.common.config.KnowledgeBaseConfig; import com.geedgenetworks.common.udf.UDFContext; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; +import org.apache.flink.metrics.Counter; import java.util.ArrayList; import java.util.Collections; @@ -25,6 +26,8 @@ public abstract class AbstractKnowledgeWithRuleUDF extends AbstractKnowledgeUDF protected String internalIocTypeListFieldName = cnInternalFieldNamePrefix + "ioc_type_list"; + protected Counter ruleHitCounter; + @Override public void open(RuntimeContext runtimeContext, UDFContext udfContext) { Configuration configuration = (Configuration) runtimeContext @@ -35,6 +38,7 @@ public abstract class AbstractKnowledgeWithRuleUDF extends AbstractKnowledgeUDF ruleConfigs = commonConfig.getKnowledgeBaseConfig().stream().filter(knowledgeBaseConfig -> knowledgeBaseConfig.getName().equals("cn_rule")).collect(Collectors.toList()); super.open(runtimeContext, udfContext); + ruleHitCounter = metrics.counter("rule_hit"); } protected enum IocType { diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AnonymityLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AnonymityLookup.java index fc02244..12817af 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AnonymityLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AnonymityLookup.java @@ -36,15 +36,18 @@ public class AnonymityLookup extends AbstractKnowledgeWithRuleUDF { @SuppressWarnings("unchecked") public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { + lookUpExistCounter.inc(); String lookupValue = event.getExtractedFields().get(lookupFieldName).toString(); RuleMetadata ruleMetadata = new RuleMetadata(); switch (option) { case "IP_TO_NODE_TYPE": String ipNodeType = knowledgeBaseHandler.lookupByIp(lookupValue); if (ipNodeType != null) { + hitCounter.inc(); event.getExtractedFields().put(outputFieldName, ipNodeType); RuleKnowledgeBaseHandler.Rule ipRule = ruleKnowledgeBaseHandler.lookupByName(ipNodeType); if (ipRule != null) { + ruleHitCounter.inc(); ruleMetadata.addRule(ipRule.getRuleId(), IocType.IP.getType()); } } @@ -53,9 +56,11 @@ public class AnonymityLookup extends AbstractKnowledgeWithRuleUDF { case "DOMAIN_TO_NODE_TYPE": String domainNodeType = knowledgeBaseHandler.lookupByDomain(lookupValue); if (domainNodeType != null) { + hitCounter.inc(); event.getExtractedFields().put(outputFieldName, domainNodeType); RuleKnowledgeBaseHandler.Rule domainRule = ruleKnowledgeBaseHandler.lookupByName(domainNodeType); if (domainRule != null) { + ruleHitCounter.inc(); ruleMetadata.addRule(domainRule.getRuleId(), IocType.DOMAIN.getType()); } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AppCategoryLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AppCategoryLookup.java index 208fd34..5b3371a 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AppCategoryLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AppCategoryLookup.java @@ -2,8 +2,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.Event; import com.geedgenetworks.common.udf.UDFContext; -import com.geedgenetworks.core.udf.knowlegdebase.handler.AppCategoryKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; +import com.geedgenetworks.core.udf.knowlegdebase.handler.AppCategoryKnowledgeBaseHandler; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,8 +32,10 @@ public class AppCategoryLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { + lookUpExistCounter.inc(); AppCategoryKnowledgeBaseHandler.AppCategory appCategory = knowledgeBaseHandler.lookup(event.getExtractedFields().get(lookupFieldName).toString()); if (appCategory != null) { + hitCounter.inc(); fieldMapping.forEach((key, value) -> { switch (key) { case "CATEGORY": diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookup.java index 21a4ed9..efe13b3 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookup.java @@ -1,8 +1,10 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.Event; -import com.geedgenetworks.core.udf.knowlegdebase.handler.DnsServerInfoKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; +import com.geedgenetworks.core.udf.knowlegdebase.handler.DnsServerInfoKnowledgeBaseHandler; + +import java.util.List; /** * @author gujinkai @@ -16,12 +18,14 @@ public class DnsServerInfoLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { - event.getExtractedFields().put( - outputFieldName, - knowledgeBaseHandler.lookup( - event.getExtractedFields().get(lookupFieldName).toString() - ) + lookUpExistCounter.inc(); + List<String> dnsServerRoleList = knowledgeBaseHandler.lookup( + event.getExtractedFields().get(lookupFieldName).toString() ); + if (dnsServerRoleList != null) { + hitCounter.inc(); + event.getExtractedFields().put(outputFieldName, dnsServerRoleList); + } } return event; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookup.java index 0a485c0..d499d00 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookup.java @@ -2,8 +2,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.Event; import com.geedgenetworks.common.udf.UDFContext; -import com.geedgenetworks.core.udf.knowlegdebase.handler.FqdnCategoryKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; +import com.geedgenetworks.core.udf.knowlegdebase.handler.FqdnCategoryKnowledgeBaseHandler; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,8 +32,10 @@ public class FqdnCategoryLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { + lookUpExistCounter.inc(); FqdnCategoryKnowledgeBaseHandler.FqdnCategory fqdnCategory = knowledgeBaseHandler.lookup(event.getExtractedFields().get(lookupFieldName).toString()); if (fqdnCategory != null) { + hitCounter.inc(); fieldMapping.forEach((key, value) -> { switch (key) { case "NAME": diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookup.java index 6afc541..e228e6a 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookup.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.Event; -import com.geedgenetworks.core.udf.knowlegdebase.handler.FqdnWhoisKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; +import com.geedgenetworks.core.udf.knowlegdebase.handler.FqdnWhoisKnowledgeBaseHandler; /** * @author gujinkai @@ -16,12 +16,14 @@ public class FqdnWhoisLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { - event.getExtractedFields().put( - outputFieldName, - knowledgeBaseHandler.lookup( - event.getExtractedFields().get(lookupFieldName).toString() - ) + lookUpExistCounter.inc(); + String whoisRegistrantOrg = knowledgeBaseHandler.lookup( + event.getExtractedFields().get(lookupFieldName).toString() ); + if (whoisRegistrantOrg != null) { + hitCounter.inc(); + event.getExtractedFields().put(outputFieldName, whoisRegistrantOrg); + } } return event; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IcpLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IcpLookup.java index ccaf2c1..0bd4045 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IcpLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IcpLookup.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.Event; -import com.geedgenetworks.core.udf.knowlegdebase.handler.FqdnIcpKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; +import com.geedgenetworks.core.udf.knowlegdebase.handler.FqdnIcpKnowledgeBaseHandler; /** * @author gujinkai @@ -16,12 +16,14 @@ public class IcpLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { - event.getExtractedFields().put( - outputFieldName, - knowledgeBaseHandler.lookup( - event.getExtractedFields().get(lookupFieldName).toString() - ) + lookUpExistCounter.inc(); + String icpCompanyName = knowledgeBaseHandler.lookup( + event.getExtractedFields().get(lookupFieldName).toString() ); + if (icpCompanyName != null) { + hitCounter.inc(); + event.getExtractedFields().put(outputFieldName, icpCompanyName); + } } return event; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IdcRenterLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IdcRenterLookup.java index b749bd1..7f9be21 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IdcRenterLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IdcRenterLookup.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.Event; -import com.geedgenetworks.core.udf.knowlegdebase.handler.IdcRenterKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; +import com.geedgenetworks.core.udf.knowlegdebase.handler.IdcRenterKnowledgeBaseHandler; /** * @author gujinkai @@ -16,12 +16,14 @@ public class IdcRenterLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { - event.getExtractedFields().put( - outputFieldName, - knowledgeBaseHandler.lookup( - event.getExtractedFields().get(lookupFieldName).toString() - ) + lookUpExistCounter.inc(); + String idcRenter = knowledgeBaseHandler.lookup( + event.getExtractedFields().get(lookupFieldName).toString() ); + if (idcRenter != null) { + hitCounter.inc(); + event.getExtractedFields().put(outputFieldName, idcRenter); + } } return event; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookup.java index e04df2e..e386437 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookup.java @@ -32,22 +32,29 @@ public class IntelligenceIndicatorLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { + lookUpExistCounter.inc(); String lookupValue = event.getExtractedFields().get(lookupFieldName).toString(); switch (option) { case "IP_TO_TAG": List<String> ipTags = knowledgeBaseHandler.lookupByIp(lookupValue); - if (event.getExtractedFields().get(outputFieldName) != null && event.getExtractedFields().get(outputFieldName) instanceof List) { - ((List<String>) event.getExtractedFields().get(outputFieldName)).addAll(ipTags); - } else { - event.getExtractedFields().put(outputFieldName, ipTags); + if (ipTags != null) { + hitCounter.inc(); + if (event.getExtractedFields().get(outputFieldName) != null && event.getExtractedFields().get(outputFieldName) instanceof List) { + ((List<String>) event.getExtractedFields().get(outputFieldName)).addAll(ipTags); + } else { + event.getExtractedFields().put(outputFieldName, ipTags); + } } break; case "DOMAIN_TO_TAG": List<String> domainTags = knowledgeBaseHandler.lookupByDomain(lookupValue); - if (event.getExtractedFields().get(outputFieldName) != null && event.getExtractedFields().get(outputFieldName) instanceof List) { - ((List<String>) event.getExtractedFields().get(outputFieldName)).addAll(domainTags); - } else { - event.getExtractedFields().put(outputFieldName, domainTags); + if (domainTags != null) { + hitCounter.inc(); + if (event.getExtractedFields().get(outputFieldName) != null && event.getExtractedFields().get(outputFieldName) instanceof List) { + ((List<String>) event.getExtractedFields().get(outputFieldName)).addAll(domainTags); + } else { + event.getExtractedFields().put(outputFieldName, domainTags); + } } break; default: diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IocLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IocLookup.java index 4386bde..30383af 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IocLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IocLookup.java @@ -36,15 +36,18 @@ public class IocLookup extends AbstractKnowledgeWithRuleUDF { @SuppressWarnings("unchecked") public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { + lookUpExistCounter.inc(); String lookupValue = event.getExtractedFields().get(lookupFieldName).toString(); RuleMetadata ruleMetadata = new RuleMetadata(); switch (option) { case "IP_TO_MALWARE": String ipMalware = knowledgeBaseHandler.lookupByIp(lookupValue); if (ipMalware != null) { + hitCounter.inc(); event.getExtractedFields().put(outputFieldName, ipMalware); RuleKnowledgeBaseHandler.Rule ipRule = ruleKnowledgeBaseHandler.lookupByName(ipMalware); if (ipRule != null) { + ruleHitCounter.inc(); ruleMetadata.addRule(ipRule.getRuleId(), IocType.IP.getType()); } } @@ -52,9 +55,11 @@ public class IocLookup extends AbstractKnowledgeWithRuleUDF { case "DOMAIN_TO_MALWARE": String domainMalware = knowledgeBaseHandler.lookupByDomain(lookupValue); if (domainMalware != null) { + hitCounter.inc(); event.getExtractedFields().put(outputFieldName, domainMalware); RuleKnowledgeBaseHandler.Rule domainRule = ruleKnowledgeBaseHandler.lookupByName(domainMalware); if (domainRule != null) { + ruleHitCounter.inc(); ruleMetadata.addRule(domainRule.getRuleId(), IocType.DOMAIN.getType()); } } @@ -62,9 +67,11 @@ public class IocLookup extends AbstractKnowledgeWithRuleUDF { case "HTTP_URL_TO_MALWARE": String urlMalware = knowledgeBaseHandler.lookupByUrl(lookupValue); if (urlMalware != null) { + hitCounter.inc(); event.getExtractedFields().put(outputFieldName, urlMalware); RuleKnowledgeBaseHandler.Rule urlRule = ruleKnowledgeBaseHandler.lookupByName(urlMalware); if (urlRule != null) { + ruleHitCounter.inc(); ruleMetadata.addRule(urlRule.getRuleId(), IocType.URL.getType()); } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IpZoneLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IpZoneLookup.java index 6453225..9a7df14 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IpZoneLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IpZoneLookup.java @@ -19,6 +19,7 @@ public class IpZoneLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { + lookUpExistCounter.inc(); String ip = event.getExtractedFields().get(lookupFieldName).toString(); if (knowledgeBaseHandler.isInternal(ip)) { event.getExtractedFields().put(outputFieldName, "internal"); diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookup.java index 12b86d2..71964f0 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookup.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.Event; -import com.geedgenetworks.core.udf.knowlegdebase.handler.LinkDirectionKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; +import com.geedgenetworks.core.udf.knowlegdebase.handler.LinkDirectionKnowledgeBaseHandler; /** * @author gujinkai @@ -16,12 +16,14 @@ public class LinkDirectionLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { - event.getExtractedFields().put( - outputFieldName, - knowledgeBaseHandler.lookup( - event.getExtractedFields().get(lookupFieldName).toString() - ) + lookUpExistCounter.inc(); + String peerCity = knowledgeBaseHandler.lookup( + event.getExtractedFields().get(lookupFieldName).toString() ); + if (peerCity != null) { + hitCounter.inc(); + event.getExtractedFields().put(outputFieldName, peerCity); + } } return event; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookup.java index 78b2b98..3e924ab 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookup.java @@ -2,9 +2,10 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.Event; import com.geedgenetworks.common.udf.UDFContext; -import com.geedgenetworks.core.udf.knowlegdebase.handler.*; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; +import com.geedgenetworks.core.udf.knowlegdebase.handler.*; import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.metrics.Counter; import java.util.ArrayList; import java.util.List; @@ -23,16 +24,20 @@ public class UserDefineTagLookup extends AbstractKnowledgeWithRuleUDF { private AppTagUserDefineKnowledgeBaseHandler appKnowledgeBaseHandler; private RuleKnowledgeBaseHandler ruleKnowledgeBaseHandler; + private Counter lookupTagsCounter; + @Override public void open(RuntimeContext runtimeContext, UDFContext udfContext) { option = udfContext.getParameters().get("option").toString(); super.open(runtimeContext, udfContext); + lookupTagsCounter = metrics.counter("lookup_tags"); } @Override @SuppressWarnings("unchecked") public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { + lookUpExistCounter.inc(); String lookupValue = event.getExtractedFields().get(lookupFieldName).toString(); List<String> tags = new ArrayList<>(); RuleMetadata ruleMetadata = new RuleMetadata(); @@ -40,9 +45,11 @@ public class UserDefineTagLookup extends AbstractKnowledgeWithRuleUDF { case "IP_TO_TAG": List<AbstractMultipleKnowledgeBaseHandler.Node> ipNodes = ipKnowledgeBaseHandler.lookup(lookupValue); ipNodes.forEach(node -> { + lookupTagsCounter.inc(); tags.add(node.getTag()); List<RuleKnowledgeBaseHandler.Rule> rules = ruleKnowledgeBaseHandler.lookupByKbId(node.getKbId()); if (rules != null) { + ruleHitCounter.inc(); rules.forEach(rule -> ruleMetadata.addRule(rule.getRuleId(), IocType.IP.getType())); } }); @@ -50,9 +57,11 @@ public class UserDefineTagLookup extends AbstractKnowledgeWithRuleUDF { case "DOMAIN_TO_TAG": List<AbstractMultipleKnowledgeBaseHandler.Node> domainNodes = domainKnowledgeBaseHandler.lookup(lookupValue); domainNodes.forEach(node -> { + lookupTagsCounter.inc(); tags.add(node.getTag()); List<RuleKnowledgeBaseHandler.Rule> rules = ruleKnowledgeBaseHandler.lookupByKbId(node.getKbId()); if (rules != null) { + ruleHitCounter.inc(); rules.forEach(rule -> ruleMetadata.addRule(rule.getRuleId(), IocType.DOMAIN.getType())); } }); @@ -60,9 +69,11 @@ public class UserDefineTagLookup extends AbstractKnowledgeWithRuleUDF { case "APP_TO_TAG": List<AbstractMultipleKnowledgeBaseHandler.Node> appNodes = appKnowledgeBaseHandler.lookup(lookupValue); appNodes.forEach(node -> { + lookupTagsCounter.inc(); tags.add(node.getTag()); List<RuleKnowledgeBaseHandler.Rule> rules = ruleKnowledgeBaseHandler.lookupByKbId(node.getKbId()); if (rules != null) { + ruleHitCounter.inc(); rules.forEach(rule -> ruleMetadata.addRule(rule.getRuleId(), IocType.APP.getType())); } }); diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/VpnLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/VpnLookup.java index 661a220..2c78ab9 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/VpnLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/VpnLookup.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.Event; import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.DomainVpnKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.handler.IpVpnKnowledgeBaseHandler; -import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +33,7 @@ public class VpnLookup extends AbstractKnowledgeUDF { @Override public Event evaluate(Event event) { if (event.getExtractedFields().get(lookupFieldName) != null && event.getExtractedFields().get(lookupFieldName).toString().length() != 0) { + lookUpExistCounter.inc(); String lookup = event.getExtractedFields().get(lookupFieldName).toString(); String result = null; switch (option) { @@ -46,7 +47,10 @@ public class VpnLookup extends AbstractKnowledgeUDF { logger.error("unknown option: " + option); break; } - event.getExtractedFields().put(outputFieldName, result); + if (result != null) { + hitCounter.inc(); + event.getExtractedFields().put(outputFieldName, result); + } } return event; } |