diff options
| author | tanghao <[email protected]> | 2024-11-07 09:36:11 +0800 |
|---|---|---|
| committer | tanghao <[email protected]> | 2024-11-07 09:36:11 +0800 |
| commit | c65254d7f0abe36ef98ef8653c55aad5ce3b35fe (patch) | |
| tree | 61655b744b95d0b89ce8a2308ec981e39df5ff3d | |
| parent | ac93e1aed3a567af9ec72c3a030dd3907b4bb05d (diff) | |
fix: 修复location接口返回类型错误问题
| -rw-r--r-- | cn-admin/src/main/resources/db/R__AZ_magic_api_file.sql | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/cn-admin/src/main/resources/db/R__AZ_magic_api_file.sql b/cn-admin/src/main/resources/db/R__AZ_magic_api_file.sql index 065e707..22f38c2 100644 --- a/cn-admin/src/main/resources/db/R__AZ_magic_api_file.sql +++ b/cn-admin/src/main/resources/db/R__AZ_magic_api_file.sql @@ -349,7 +349,7 @@ INSERT INTO `magic_api_file` VALUES ('/api/v1/entity/explorer/overview/group.jso INSERT INTO `magic_api_file` VALUES ('/api/v1/entity/explorer/overview/数据概览.ms', '{\r\n \"properties\" : { },\r\n \"id\" : \"e857c6ece26c4e479ee60df7e70f6fa2\",\r\n \"script\" : null,\r\n \"groupId\" : \"605d302dc4f847b596819f332c7f5605\",\r\n \"name\" : \"数据概览\",\r\n \"createTime\" : null,\r\n \"updateTime\" : 1711102678729,\r\n \"lock\" : null,\r\n \"createBy\" : null,\r\n \"updateBy\" : \"admin\",\r\n \"path\" : \"/{metricType}\",\r\n \"method\" : \"GET\",\r\n \"parameters\" : [ {\r\n \"name\" : \"resource\",\r\n \"value\" : \"\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"startTime\",\r\n \"value\" : \"\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Long\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"endTime\",\r\n \"value\" : \"\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Long\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n } ],\r\n \"options\" : [ ],\r\n \"requestBody\" : \"\",\r\n \"headers\" : [ ],\r\n \"paths\" : [ {\r\n \"name\" : \"metricType\",\r\n \"value\" : \"total\",\r\n \"description\" : null,\r\n \"required\" : true,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n } ],\r\n \"responseBody\" : \"{\\n \\\"code\\\": 200,\\n \\\"data\\\": {\\n \\\"appCount\\\": 837,\\n \\\"domainCount\\\": 73310,\\n \\\"ipCount\\\": 466740,\\n \\\"totalCount\\\": 540887,\\n \\\"subscriberCount\\\": 0\\n },\\n \\\"time\\\": \\\"2024-03-22 09:45:27\\\",\\n \\\"originalUrl\\\": \\\"SELECT COUNTIF(DISTINCT(app_name),notEmpty(app_name)) as appCount, COUNTIF(DISTINCT(domain),notEmpty(domain)) as domainCount, COUNTIF(DISTINCT(ip),notEmpty(ip)) as ipCount ,plus(ipCount,plus(appCount,domainCount)) AS totalCount FROM( SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time FROM session_relation_domain_ip_app e WHERE 1=1 )\\\",\\n \\\"message\\\": \\\"success\\\",\\n \\\"status\\\": 200\\n}\",\r\n \"description\" : null,\r\n \"requestBodyDefinition\" : null,\r\n \"responseBodyDefinition\" : {\r\n \"name\" : \"\",\r\n \"value\" : \"\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Object\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ {\r\n \"name\" : \"code\",\r\n \"value\" : \"200\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"data\",\r\n \"value\" : \"\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Object\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ {\r\n \"name\" : \"appCount\",\r\n \"value\" : \"120\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"domainCount\",\r\n \"value\" : \"559\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"ipCount\",\r\n \"value\" : \"841\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"totalCount\",\r\n \"value\" : \"1520\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n } ]\r\n }, {\r\n \"name\" : \"time\",\r\n \"value\" : \"2023-08-31 03:27:28\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"originalUrl\",\r\n \"value\" : \"http://192.168.44.55:9999/?query=SELECT COUNTIF(DISTINCT(app_name),notEmpty(app_name)) as appCount, COUNTIF(DISTINCT(domain),notEmpty(domain)) as domainCount, COUNTIF(DISTINCT(ip),notEmpty(ip)) as ipCount ,plus(ipCount,plus(appCount,domainCount)) AS totalCount FROM( SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time FROM session_relation_domain_ip_app e WHERE 1=1 )&format=json&option=real_time\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"message\",\r\n \"value\" : \"success\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"status\",\r\n \"value\" : \"200\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n } ]\r\n }\r\n}\r\n================================\r\nimport cn.hutool.log.Log;\r\nimport cn.hutool.core.util.ObjectUtil;\r\nimport net.geedge.common.utils.FreeMarkerUtil as freeMarkerUtil;\r\nimport net.geedge.common.utils.JSONUtil as jsonUtil;\r\nimport net.geedge.common.utils.R as R;\r\nimport net.geedge.modules.entity.service.EntityExplorerService as EntityExplorerService;\r\nimport \'@/pre/timeHandleNone\' as timeHandle;\r\nimport \'@/entity/entityAggHandle\' as entityAggHandle;\r\nimport \'@/public/postGalaxyResult\' as postGalaxyResult;\r\n/**\r\n * 入参整理 包括调用公共函数\r\n */\r\nvar parameter={\r\n \"metricType\":metricType,\r\n \"resource\":resource,\r\n \"startTime\":startTime,\r\n \"endTime\":endTime,\r\n \"dr\":false,\r\n \"fqdnCategory\":false,\r\n \"whois\":false,\r\n \"appCategory\":false\r\n}\r\ntimeHandle(parameter);\r\nentityAggHandle(parameter);\r\n\r\nvar isAssociationQuery = EntityExplorerService.isAssociationQuery(parameter.resource,\"entityMetadata\");\r\nif(isAssociationQuery){\r\n var sql = \"{\\\"sql\\\":\\\"SELECT COUNT(DISTINCT(app_name)) FILTER (WHERE app_name != \'\') AS appCount, COUNT(DISTINCT(domain)) FILTER (WHERE domain != \'\') AS domainCount, COUNT(DISTINCT(ip)) FILTER (WHERE ip != \'\') AS ipCount FROM( SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time FROM clickhouse.cyber_narrator_galaxy.session_relation_domain_ip_app e <#if fqdnCategory> JOIN mysql.cn_api.cn_fqdn_category_built_in fqdnCategory ON (e.domain = fqdnCategory.fqdn <#if fqdnCategory\\.category_name?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_name} </#if> <#if fqdnCategory\\.category_group?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_group} </#if> <#if fqdnCategory\\.reputation_level?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.reputation_level}</#if>) </#if> <#if whois> JOIN mysql.cn_api.cn_fqdn_who_is_built_in whois ON (e.domain = whois.fqdn <#if whois\\.whois_domain?default(\'\')?trim? length gt 0> AND ${whois\\.whois_domain} </#if> <#if whois\\.whois_registrant_name?default(\'\')?trim? length gt 0> AND ${whois\\.whois_registrant_name}</#if>) </#if> <#if appCategory> JOIN mysql.cn_api.cn_app_category_built_in appCategory ON (e.app_name = appCategory.app_name <#if appCategory\\.app_category?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_category} </#if> <#if appCategory\\.app_subcategory?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_subcategory} </#if> <#if appCategory\\.app_risk?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_risk}</#if>) </#if> <#if dr\\.port?default(\'\')?trim? length gt 0> JOIN clickhouse.cyber_narrator_galaxy.cn_ip_dynamic_attribute dr ON (e.ip = dr.ip AND ${dr\\.port}) </#if> WHERE 1=1 <#if metricType==\\\"active\\\"> AND e.update_time >= ${startTime} AND e.update_time < ${endTime} </#if> <#if filter?default(\'\')?trim? length gt 0> ${filter} </#if> )\\\"}\"\r\n var aTemplate = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerTopAggCountry\",sql,parameter);\r\n var ares = jsonUtil.toBean(aTemplate,HashMap.class);\r\n var log = Log.get();\r\n log.info(ares.sql);\r\n var aresult = db[\'trino\'].select(ares.sql);\r\n var data = aresult[0];\r\n var url = db.select(\"select cvalue from sys_config where ckey = \'galaxy_url\'\");\r\n return R.ok(data).put(\"originalUrl\",url.cvalue+ares.sql).put(\"status\",200);\r\n}\r\n/**\r\n * 接口sql相关内容整理 通过freemarker模板生成最后执行语句\r\n */\r\n//ip,domain,app\r\nvar q = \"{\\r\\n \\\"query\\\": \\\"SELECT COUNTIF(DISTINCT(app_name),notEmpty(app_name)) as appCount, COUNTIF(DISTINCT(domain),notEmpty(domain)) as domainCount, COUNTIF(DISTINCT(ip),notEmpty(ip)) as ipCount <#if metricType==\\\"total\\\"> ,plus(ipCount,plus(appCount,domainCount)) AS totalCount </#if> FROM( SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time FROM session_relation_domain_ip_app e <#if dr> JOIN cn_ip_dynamic_attribute dr ON (e.ip = dr.ip) </#if> WHERE 1=1 <#if metricType==\\\"active\\\"> AND e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if dr> AND dr.update_time >= ${startTime} AND dr.update_time < ${endTime} </#if> </#if><#if filter?default(\\\'\\\')?trim? length gt 0> AND (${filter}) </#if> )\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\nvar temp = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerTopAggCountry\",q,parameter);\r\n\r\n//subscriber\r\nvar subscriber = \"{\\r\\n \\\"query\\\": \\\"SELECT COUNT(DISTINCT(subscriber_id)) as subscriberCount FROM session_relation_subscriber_app WHERE notEmpty(subscriber_id) <#if metricType==\\\"active\\\"> AND update_time >= ${startTime} AND update_time < ${endTime} </#if> <#if filter?default(\\\'\\\')?trim? length gt 0> AND (${filter}) </#if>\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\nvar subscriberTemp = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerOverview\",subscriber,parameter);\r\n/**\r\n * 调用galaxy接口获取结果信息\r\n */\r\nvar res = jsonUtil.toBean(temp,HashMap.class);\r\nvar result = postGalaxyResult(res);\r\n\r\nvar subscriberRes = jsonUtil.toBean(subscriberTemp,HashMap.class);\r\nvar subscriberResult = postGalaxyResult(subscriberRes);\r\n\r\nvar data = {};\r\nif(result.success){\r\n for(k,v in result.data[0]){\r\n data.put(k,v.asInt());\r\n }\r\n}\r\nif(subscriberResult.success){\r\n for(k,v in subscriberResult.data[0]){\r\n data.put(k,v.asInt());\r\n }\r\n}\r\nif(ObjectUtil.isNotEmpty(data.get(\"totalCount\")) && ObjectUtil.isNotEmpty(data.get(\"subscriberCount\"))){\r\n data.put(\"totalCount\",data.get(\"totalCount\")+data.get(\"subscriberCount\"))\r\n}\r\nreturn R.ok(data).put(\"originalUrl\",result.originalUrl).put(\"status\",200);'); INSERT INTO `magic_api_file` VALUES ('/api/v1/entity/explorer/query/', 'this is directory'); INSERT INTO `magic_api_file` VALUES ('/api/v1/entity/explorer/query/group.json', '{\n \"properties\" : { },\n \"id\" : \"9e64aeec148c46259a28506f2f1e4cac\",\n \"name\" : \"query\",\n \"type\" : \"api\",\n \"parentId\" : \"877b51b4c5184132b246ca6f050d5776\",\n \"path\" : \"/query\",\n \"createTime\" : 1688465971314,\n \"updateTime\" : null,\n \"createBy\" : \"ljy\",\n \"updateBy\" : null,\n \"paths\" : [ ],\n \"options\" : [ ]\n}'); -INSERT INTO `magic_api_file` VALUES ('/api/v1/entity/explorer/query/实体列表.ms', '{\r\n \"properties\" : { },\r\n \"id\" : \"efe394db28ce4eef89cc7a3089936d6c\",\r\n \"script\" : null,\r\n \"groupId\" : \"9e64aeec148c46259a28506f2f1e4cac\",\r\n \"name\" : \"实体列表\",\r\n \"createTime\" : null,\r\n \"updateTime\" : 1730196735135,\r\n \"lock\" : null,\r\n \"createBy\" : null,\r\n \"updateBy\" : null,\r\n \"path\" : \"/list\",\r\n \"method\" : \"GET\",\r\n \"parameters\" : [ {\r\n \"name\" : \"resource\",\r\n \"value\" : \"has(tag,\'FWDNS\')\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"startTime\",\r\n \"value\" : \"1730190809\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Long\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"endTime\",\r\n \"value\" : \"1730194409\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Long\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"pageNo\",\r\n \"value\" : \"1\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"pageSize\",\r\n \"value\" : \"20\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"hideRelated\",\r\n \"value\" : \"true\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Boolean\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n } ],\r\n \"options\" : [ ],\r\n \"requestBody\" : \"\",\r\n \"headers\" : [ {\r\n \"name\" : \"cn-authorization\",\r\n \"value\" : \"adacf41c-8e3f-412e-987e-f31592d2e0ed\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n } ],\r\n \"paths\" : [ ],\r\n \"responseBody\" : \"{\\n \\\"code\\\": 200,\\n \\\"data\\\": {\\n \\\"list\\\": [\\n {\\n \\\"entityType\\\": \\\"ip\\\",\\n \\\"entityValue\\\": \\\"125.35.213.106\\\",\\n \\\"isRelated\\\": false\\n },\\n {\\n \\\"entityType\\\": \\\"ip\\\",\\n \\\"entityValue\\\": \\\"124.167.218.157\\\",\\n \\\"isRelated\\\": false\\n },\\n {\\n \\\"entityType\\\": \\\"ip\\\",\\n \\\"entityValue\\\": \\\"114.114.114.114\\\",\\n \\\"isRelated\\\": false\\n },\\n {\\n \\\"entityType\\\": \\\"ip\\\",\\n \\\"entityValue\\\": \\\"223.6.6.6\\\",\\n \\\"isRelated\\\": false\\n },\\n {\\n \\\"entityType\\\": \\\"ip\\\",\\n \\\"entityValue\\\": \\\"162.14.21.178\\\",\\n \\\"isRelated\\\": false\\n },\\n {\\n \\\"entityType\\\": \\\"ip\\\",\\n \\\"entityValue\\\": \\\"183.239.114.142\\\",\\n \\\"isRelated\\\": false\\n }\\n ],\\n \\\"pageNo\\\": 1,\\n \\\"pageSize\\\": 20\\n },\\n \\\"time\\\": \\\"2024-10-29 10:07:48\\\",\\n \\\"originalUrl\\\": \\\"SELECT arrayJoin(entity_values) AS entity,MAX(IF(is_ip_tag_primary, 1, 0)) AS ip_is_primary,MAX(IF(is_app_tag_primary, 1, 0)) AS app_is_primary,MAX(IF(is_domain_tag_primary, 1, 0)) AS domain_is_primary,MAX(IF(is_subscriber_tag_primary, 1, 0)) AS subscriber_is_primary FROM (SELECT e.update_time AS update_time,e.app_name AS app_name,e.domain AS domain,e.ip AS ip,\'\' AS subscriber_id, if( has(e.entity_tags, concat(\'ip.\', \'FWDNS\')) , 1, 0) AS is_ip_tag_primary ,if( has(e.entity_tags, concat(\'app.\', \'FWDNS\')) , 1, 0) AS is_app_tag_primary ,if( has(e.entity_tags, concat(\'domain.\', \'FWDNS\')) , 1, 0) AS is_domain_tag_primary, 0 AS is_subscriber_tag_primary,( (splitByChar(\'$\',concat(\'$ip=\',IF(is_ip_tag_primary, ip, \'\'),\'$app=\',IF(is_app_tag_primary, app_name, \'\'),\'$domain=\',IF(is_domain_tag_primary, domain, \'\')))) ) AS entity_values FROM session_relation_domain_ip_app e WHERE e.update_time >= 1730190809 AND e.update_time < 1730194409 AND ((has(e.entity_tags, concat(\'ip.\', \'FWDNS\')) OR has(e.entity_tags, concat(\'domain.\', \'FWDNS\')) OR has(e.entity_tags, concat(\'app.\', \'FWDNS\')))) UNION ALL SELECT e.update_time AS update_time,e.app_name AS app_name,\'\' AS domain,\'\' AS ip,e.subscriber_id AS subscriber_id,0 AS is_ip_tag_primary,0 AS is_domain_tag_primary, if( has(e.entity_tags, concat(\'app.\', \'FWDNS\')) , 1, 0) AS is_app_tag_primary ,if( has(e.entity_tags, concat(\'subscriber.\', \'FWDNS\')) , 1, 0) AS is_subscriber_tag_primary , ( (splitByChar(\'$\',concat(\'$app=\',IF(is_app_tag_primary, app_name, \'\'),\'$subscriber=\',IF(is_subscriber_tag_primary, subscriber_id, \'\')))) ) AS entity_values FROM session_relation_subscriber_app e WHERE e.update_time >= 1730190809 AND e.update_time < 1730194409 AND ((has(e.entity_tags, concat(\'ip.\', \'FWDNS\')) OR has(e.entity_tags, concat(\'domain.\', \'FWDNS\')) OR has(e.entity_tags, concat(\'app.\', \'FWDNS\')))) ) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC LIMIT 0,20\\\",\\n \\\"message\\\": \\\"success\\\",\\n \\\"status\\\": 200\\n}\",\r\n \"description\" : null,\r\n \"requestBodyDefinition\" : null,\r\n \"responseBodyDefinition\" : {\r\n \"name\" : \"\",\r\n \"value\" : \"\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Object\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ {\r\n \"name\" : \"code\",\r\n \"value\" : \"999\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"time\",\r\n \"value\" : \"2023-08-31 03:44:05\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"message\",\r\n \"value\" : \"对象为空\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n } ]\r\n }\r\n}\r\n================================\r\nimport cn.hutool.log.Log;\r\nimport cn.hutool.core.util.ObjectUtil;\r\nimport net.geedge.common.utils.FreeMarkerUtil as freeMarkerUtil;\r\nimport net.geedge.common.utils.JSONUtil as jsonUtil;\r\nimport net.geedge.common.utils.R as R;\r\nimport net.geedge.modules.entity.service.EntityExplorerService as EntityExplorerService;\r\nimport \'@/pre/timeHandleHalf\' as timeHandle;\r\nimport \'@/pre/pageFieldsHandle\' as pageHandle;\r\nimport \'@/entity/entityAggHandle\' as entityAggHandle;\r\nimport \'@/entity/crossEntityFilterHandle\' as crossEntityFilterHandle;\r\nimport \'@/entity/isExistEntity\' as isExistEntity;\r\nimport \'@/entity/isSubscriberEntity\' as isSubscriberEntity;\r\nimport \'@/entity/notQueryHandle\' as notQueryHandle;\r\nimport \'@/public/postGalaxyResult\' as postGalaxyResult;\r\n\r\n/**\r\n * 入参整理 包括调用公共函数\r\n */\r\nvar parameter={\r\n \"resource\":resource,\r\n \"startTime\":startTime,\r\n \"endTime\":endTime,\r\n \"pageNo\":pageNo,\r\n \"pageSize\":pageSize,\r\n \"dr\":false,\r\n \"fqdnCategory\":false,\r\n \"whois\":false,\r\n \"appCategory\":false,\r\n \"is_ip_attribut_primary\":1,\r\n \"is_app_attribut_primary\":1,\r\n \"is_domain_attribut_primary\":1,\r\n \"hideRelated\":ObjectUtil.isEmpty(hideRelated)?false:hideRelated\r\n}\r\ntimeHandle(parameter);\r\npageHandle(parameter);\r\n\r\nif(ObjectUtil.isNotEmpty(resource) && (resource.contains(\"ip.\") || resource.contains(\"app.\") || resource.contains(\"domain.\") || resource.contains(\"has(\")) && resource.contains(\"subscriber.\")){\r\n return notQueryHandle();\r\n}\r\n\r\nvar q = \"\";\r\n\r\nif(ObjectUtil.isEmpty(resource)){\r\n//默认无筛选\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT arrayJoin(entity_values) AS entity, 1 AS ip_is_primary, 1 AS app_is_primary, 1 AS domain_is_primary, 1 AS subscriber_id_is_primary FROM ( SELECT e.update_time AS update_time, e.app_name AS app_name, e.domain AS domain, e.ip AS ip, \'\' AS subscriber_id, (splitByChar(\'$\',concat(\'$ip=\',ip,\'$app=\', app_name,\'$domain=\', domain)) ) AS entity_values FROM session_relation_domain_ip_app e WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} UNION ALL SELECT update_time AS update_time, app_name AS app_name, \'\' AS domain, \'\' AS ip, subscriber_id AS subscriber_id, (splitByChar(\'$\',concat(\'$app=\', app_name,\'$subscriber_id=\', subscriber_id)) ) AS entity_values FROM session_relation_subscriber_app WHERE update_time >= ${startTime} AND update_time < ${endTime} ) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC <#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\n} else if(isSubscriberEntity(resource)){\r\n//检索属性属于Subscriber\r\n parameter.filter = EntityExplorerService.parseExpression(parameter.resource,\"entityMetadata\");\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT arrayJoin(entity_values) AS entity, MAX(IF(1 AND startsWith(entity,\'subscriber_id=\'), 1, 0)) AS subscriber_id_is_primary FROM ( SELECT app_name AS app_name, subscriber_id AS subscriber_id, update_time AS update_time, (<#if hideRelated> splitByChar(\'$\',concat( \'$subscriber_id=\',IF(1, subscriber_id, \'\'))) <#else> splitByChar(\'$\',concat(\'$app=\', app_name,\'$subscriber_id=\', subscriber_id)) </#if>) AS entity_values FROM session_relation_subscriber_app su WHERE update_time >= ${startTime} AND update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if> ) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC <#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\n} else if(!resource.contains(\"ip.\") && !resource.contains(\"app.\") && !resource.contains(\"domain.\") && resource.contains(\"has(\") && !resource.contains(\"subscriber.\")){\r\n // 仅tag\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && parameter.resource.contains(\"has(\")) {\r\n parameter.tags = EntityExplorerService.getTagsForFilter(parameter.resource);\r\n }\r\n entityAggHandle(parameter);\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT arrayJoin(entity_values) AS entity,MAX(IF(is_ip_tag_primary, 1, 0)) AS ip_is_primary,MAX(IF(is_app_tag_primary, 1, 0)) AS app_is_primary,MAX(IF(is_domain_tag_primary, 1, 0)) AS domain_is_primary,MAX(IF(is_subscriber_tag_primary, 1, 0)) AS subscriber_is_primary FROM (SELECT e.update_time AS update_time,e.app_name AS app_name,e.domain AS domain,e.ip AS ip,\'\' AS subscriber_id,<#if tags?exists> if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'ip.\', ${tag})) <#else> has(e.entity_tags, concat(\'ip.\', ${tag})) AND </#if></#list>, 1, 0) AS is_ip_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) AND </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'domain.\', ${tag})) <#else> has(e.entity_tags, concat(\'domain.\', ${tag})) AND </#if></#list>, 1, 0) AS is_domain_tag_primary, </#if>0 AS is_subscriber_tag_primary,(<#if hideRelated> (splitByChar(\'$\',concat(\'$ip=\',IF(is_ip_tag_primary, ip, \'\'),\'$app=\',IF(is_app_tag_primary, app_name, \'\'),\'$domain=\',IF(is_domain_tag_primary, domain, \'\')))) <#else> splitByChar(\'$\',concat(\'$ip=\',ip,\'$app=\', app_name,\'$domain=\', domain)) </#if>) AS entity_values FROM session_relation_domain_ip_app e WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if> UNION ALL SELECT e.update_time AS update_time,e.app_name AS app_name,\'\' AS domain,\'\' AS ip,e.subscriber_id AS subscriber_id,0 AS is_ip_tag_primary,0 AS is_domain_tag_primary,<#if tags?exists> if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) AND </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'subscriber.\', ${tag})) <#else> has(e.entity_tags, concat(\'subscriber.\', ${tag})) AND </#if></#list>, 1, 0) AS is_subscriber_tag_primary ,</#if> (<#if hideRelated> (splitByChar(\'$\',concat(\'$app=\',IF(is_app_tag_primary, app_name, \'\'),\'$subscriber=\',IF(is_subscriber_tag_primary, subscriber_id, \'\')))) <#else> splitByChar(\'$\',concat(\'$app=\', app_name,\'$subscriber=\', subscriber_id)) </#if>) AS entity_values FROM session_relation_subscriber_app e WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if>) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC <#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n} else {\r\n//检索属性属于APP/IP/DOMAIN\r\n //过滤标签时处理\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && parameter.resource.contains(\"has(\")) {\r\n parameter.tags = EntityExplorerService.getTagsForFilter(parameter.resource);\r\n }\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && parameter.resource.contains(\"has(\") && isExistEntity(parameter)) {\r\n parameter.is_ip_attribut_primary = 0;\r\n parameter.is_app_attribut_primary = 0;\r\n parameter.is_domain_attribut_primary = 0;\r\n }\r\n //有过滤时\r\n var primay = EntityExplorerService.getPrimayForFilter(parameter.resource,\"entityMetadata\");\r\n if(ObjectUtil.isNotEmpty(primay)) {\r\n parameter.is_ip_attribut_primary = 0;\r\n parameter.is_app_attribut_primary = 0;\r\n parameter.is_domain_attribut_primary = 0;\r\n for(k,v in primay) {\r\n if(v.asString().contains(\"ip\")) {\r\n parameter.is_ip_attribut_primary = 1;\r\n } else if(v.asString().contains(\"app\")) {\r\n parameter.is_app_attribut_primary = 1;\r\n } else if(v.asString().contains(\"domain\")) {\r\n parameter.is_domain_attribut_primary = 1;\r\n }\r\n }\r\n }\r\n entityAggHandle(parameter);\r\n //如果跨实体不查询返回空;如果单一实体和tag同时过滤只过滤该实体的tag\r\n if(crossEntityFilterHandle(parameter) != 0) {\r\n return crossEntityFilterHandle(parameter);\r\n }\r\n\r\n var isAssociationQuery = EntityExplorerService.isAssociationQuery(parameter.resource,\"entityMetadata\");\r\n if(isAssociationQuery){\r\n var sql = \"{\\\"sql\\\":\\\"SELECT a.entityValue AS entityValue, (case when ip = entityValue then \'ip\' when domain= entityValue then \'domain\' when app_name = entityValue then \'app\' else \'\' end) AS entityType FROM (SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time, concat(e.app_name,\'$\',e.domain,\'$\',e.ip) AS entity_value FROM clickhouse.cyber_narrator_galaxy.session_relation_domain_ip_app e <#if fqdnCategory> JOIN mysql.cn_api.cn_fqdn_category_built_in fqdnCategory ON (e.domain = fqdnCategory.fqdn <#if fqdnCategory\\.category_name?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_name} </#if> <#if fqdnCategory\\.category_group?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_group} </#if> <#if fqdnCategory\\.reputation_level?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.reputation_level}</#if>) </#if> <#if whois> JOIN mysql.cn_api.cn_fqdn_who_is_built_in whois ON (e.domain = whois.fqdn <#if whois\\.whois_domain?default(\'\')?trim? length gt 0> AND ${whois\\.whois_domain} </#if> <#if whois\\.whois_registrant_name?default(\'\')?trim? length gt 0> AND ${whois\\.whois_registrant_name}</#if>) </#if> <#if appCategory> JOIN mysql.cn_api.cn_app_category_built_in appCategory ON (e.app_name = appCategory.app_name <#if appCategory\\.app_category?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_category} </#if> <#if appCategory\\.app_subcategory?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_subcategory} </#if> <#if appCategory\\.app_risk?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_risk}</#if>) </#if> <#if dr\\.port?default(\'\')?trim? length gt 0> JOIN clickhouse.cyber_narrator_galaxy.cn_ip_dynamic_attribute dr ON (e.ip = dr.ip AND ${dr\\.port}) </#if> WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> ${filter} </#if> ) CROSS JOIN unnest(split(entity_value,\'$\')) AS a(entityValue) WHERE entityValue != \'\' GROUP BY entityValue,ip,domain,app_name ORDER BY MAX(update_time) DESC <#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\"}\"\r\n var aTemplate = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerTopAggCountry\",sql,parameter);\r\n var ares = jsonUtil.toBean(aTemplate,HashMap.class);\r\n var log = Log.get();\r\n log.info(ares.sql);\r\n var aresult = db[\'trino\'].select(ares.sql);\r\n var data = {};\r\n data.list = aresult;\r\n data.total = total;\r\n data.pageNo = ObjectUtil.isNotEmpty(pageNo) ? pageNo : 1;\r\n data.pageSize = ObjectUtil.isNotEmpty(pageSize) ? pageSize : parameter.pageSize;\r\n return R.ok(data).put(\"originalUrl\",ares.sql).put(\"status\",200);\r\n }\r\n /**\r\n * 接口sql相关内容整理 通过freemarker模板生成最后执行语句\r\n */\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT arrayJoin(entity_values) AS entity, MAX(IF(${is_ip_attribut_primary} <#if tags?exists> AND is_ip_tag_primary </#if>, 1, 0)) AS ip_is_primary, MAX(IF(${is_app_attribut_primary} <#if tags?exists> AND is_app_tag_primary </#if>, 1, 0)) AS app_is_primary, MAX(IF(${is_domain_attribut_primary} <#if tags?exists> AND is_domain_tag_primary </#if>, 1, 0)) AS domain_is_primary FROM ( SELECT e.update_time AS update_time, e.app_name AS app_name, e.domain AS domain, e.ip AS ip, <#if tags?exists> if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'ip.\', ${tag})) <#else> has(e.entity_tags, concat(\'ip.\', ${tag})) AND </#if></#list>, 1, 0) AS is_ip_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) AND </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'domain.\', ${tag})) <#else> has(e.entity_tags, concat(\'domain.\', ${tag})) AND </#if></#list>, 1, 0) AS is_domain_tag_primary, </#if> (<#if hideRelated> splitByChar(\'$\',concat( \'$ip=\',IF(${is_ip_attribut_primary} <#if tags?exists> AND is_ip_tag_primary </#if>, ip, \'\'), \'$app=\',IF(${is_app_attribut_primary} <#if tags?exists> AND is_app_tag_primary </#if>, app_name, \'\'), \'$domain=\',IF(${is_domain_attribut_primary} <#if tags?exists> AND is_domain_tag_primary </#if>, domain, \'\'))) <#else> splitByChar(\'$\',concat(\'$ip=\',ip,\'$app=\', app_name,\'$domain=\', domain)) </#if>) AS entity_values FROM session_relation_domain_ip_app e <#if dr> JOIN cn_ip_dynamic_attribute dr ON (e.ip = dr.ip) </#if> WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if dr> AND dr.update_time >= ${startTime} AND dr.update_time < ${endTime} </#if> <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if> ) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC<#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\n}\r\n\r\nvar result = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerQueryList\",q,parameter);\r\n\r\n/**\r\n * 调用galaxy接口获取结果信息\r\n */\r\nvar res = jsonUtil.toBean(result,HashMap.class);\r\nvar result = postGalaxyResult(res);\r\n/**\r\n * 处理galaxy返回的数据结果 将新结果返回前端界面\r\n */\r\nvar data = {};\r\nif(result.success){\r\n var list = new ArrayList();\r\n for(k,v in result.data) {\r\n var map = {};\r\n var entity = Arrays.asList(v.entity.asString().split(\"=\"));\r\n map.entityType = entity.get(0);\r\n map.entityValue = entity.get(1);\r\n map.isRelated = true;\r\n if(entity.get(0).equals(\"ip\") && v.ip_is_primary==1) {\r\n map.isRelated = false;\r\n } else if(entity.get(0).equals(\"app\") && v.app_is_primary==1) {\r\n map.isRelated = false;\r\n } else if(entity.get(0).equals(\"domain\") && v.domain_is_primary==1) {\r\n map.isRelated = false;\r\n } else if(entity.get(0).equals(\"subscriber_id\") && v.subscriber_id_is_primary==1) {\r\n map.isRelated = false;\r\n }\r\n list.add(map);\r\n }\r\n data.list = list;\r\n data.pageNo = ObjectUtil.isNotEmpty(pageNo) ? pageNo : 1;\r\n data.pageSize = ObjectUtil.isNotEmpty(pageSize) ? pageSize : parameter.pageSize;\r\n}\r\n\r\nreturn R.ok(data).put(\"originalUrl\",result.originalUrl).put(\"status\",200);'); +INSERT INTO `magic_api_file` VALUES ('/api/v1/entity/explorer/query/实体列表.ms', '{\n \"properties\" : { },\n \"id\" : \"efe394db28ce4eef89cc7a3089936d6c\",\n \"script\" : null,\n \"groupId\" : \"9e64aeec148c46259a28506f2f1e4cac\",\n \"name\" : \"实体列表\",\n \"createTime\" : null,\n \"updateTime\" : 1730702048112,\n \"lock\" : null,\n \"createBy\" : null,\n \"updateBy\" : null,\n \"path\" : \"/list\",\n \"method\" : \"GET\",\n \"parameters\" : [ {\n \"name\" : \"resource\",\n \"value\" : \"subscriber.id like \'%4%\'\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"String\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"startTime\",\n \"value\" : \"1730697118\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Long\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"endTime\",\n \"value\" : \"1730700718\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Long\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"pageNo\",\n \"value\" : \"1\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Integer\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"pageSize\",\n \"value\" : \"20\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Integer\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"hideRelated\",\n \"value\" : \"false\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Boolean\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n } ],\n \"options\" : [ ],\n \"requestBody\" : \"\",\n \"headers\" : [ {\n \"name\" : \"cn-authorization\",\n \"value\" : \"5cd6af79-3cb8-48f0-bd47-dad0ebf8c7f5\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"String\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n } ],\n \"paths\" : [ ],\n \"responseBody\" : \"{\\n \\\"code\\\": 200,\\n \\\"data\\\": {\\n \\\"list\\\": [\\n {\\n \\\"entityType\\\": \\\"app\\\",\\n \\\"entityValue\\\": \\\"demo8\\\",\\n \\\"isRelated\\\": true\\n },\\n {\\n \\\"entityType\\\": \\\"subscriber_id\\\",\\n \\\"entityValue\\\": \\\"415678605002892\\\",\\n \\\"isRelated\\\": false\\n },\\n {\\n \\\"entityType\\\": \\\"app\\\",\\n \\\"entityValue\\\": \\\"surfshark vpn\\\",\\n \\\"isRelated\\\": true\\n },\\n {\\n \\\"entityType\\\": \\\"app\\\",\\n \\\"entityValue\\\": \\\"lantern_domain\\\",\\n \\\"isRelated\\\": true\\n }\\n ],\\n \\\"pageNo\\\": 1,\\n \\\"pageSize\\\": 20\\n },\\n \\\"time\\\": \\\"2024-11-04 06:21:38\\\",\\n \\\"originalUrl\\\": \\\"SELECT arrayJoin(entity_values) AS entity, MAX(IF(1 AND startsWith(entity,\'subscriber_id=\'), 1, 0)) AS subscriber_id_is_primary FROM ( SELECT app_name AS app_name, subscriber_id AS subscriber_id, update_time AS update_time, ( splitByChar(\'$\',concat(\'$app=\', app_name,\'$subscriber_id=\', subscriber_id)) ) AS entity_values FROM session_relation_subscriber_app su WHERE update_time >= 1730697118 AND update_time < 1730700718 AND (su.subscriber_id LIKE \'%4%\') ) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC LIMIT 0,20\\\",\\n \\\"message\\\": \\\"success\\\",\\n \\\"status\\\": 200\\n}\",\n \"description\" : null,\n \"requestBodyDefinition\" : null,\n \"responseBodyDefinition\" : {\n \"name\" : \"\",\n \"value\" : \"\",\n \"description\" : \"\",\n \"required\" : false,\n \"dataType\" : \"Object\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : \"\",\n \"error\" : \"\",\n \"expression\" : \"\",\n \"children\" : [ {\n \"name\" : \"code\",\n \"value\" : \"999\",\n \"description\" : \"\",\n \"required\" : false,\n \"dataType\" : \"Integer\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : \"\",\n \"error\" : \"\",\n \"expression\" : \"\",\n \"children\" : [ ]\n }, {\n \"name\" : \"time\",\n \"value\" : \"2023-08-31 03:44:05\",\n \"description\" : \"\",\n \"required\" : false,\n \"dataType\" : \"String\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : \"\",\n \"error\" : \"\",\n \"expression\" : \"\",\n \"children\" : [ ]\n }, {\n \"name\" : \"message\",\n \"value\" : \"对象为空\",\n \"description\" : \"\",\n \"required\" : false,\n \"dataType\" : \"String\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : \"\",\n \"error\" : \"\",\n \"expression\" : \"\",\n \"children\" : [ ]\n } ]\n }\n}\r\n================================\r\nimport cn.hutool.log.Log;\r\nimport cn.hutool.core.util.ObjectUtil;\r\nimport net.geedge.common.utils.FreeMarkerUtil as freeMarkerUtil;\r\nimport net.geedge.common.utils.JSONUtil as jsonUtil;\r\nimport net.geedge.common.utils.R as R;\r\nimport net.geedge.modules.entity.service.EntityExplorerService as EntityExplorerService;\r\nimport \'@/pre/timeHandleHalf\' as timeHandle;\r\nimport \'@/pre/pageFieldsHandle\' as pageHandle;\r\nimport \'@/entity/entityAggHandle\' as entityAggHandle;\r\nimport \'@/entity/crossEntityFilterHandle\' as crossEntityFilterHandle;\r\nimport \'@/entity/isExistEntity\' as isExistEntity;\r\nimport \'@/entity/isSubscriberEntity\' as isSubscriberEntity;\r\nimport \'@/entity/notQueryHandle\' as notQueryHandle;\r\nimport \'@/public/postGalaxyResult\' as postGalaxyResult;\r\n\r\n/**\r\n * 入参整理 包括调用公共函数\r\n */\r\nvar parameter={\r\n \"resource\":resource,\r\n \"startTime\":startTime,\r\n \"endTime\":endTime,\r\n \"pageNo\":pageNo,\r\n \"pageSize\":pageSize,\r\n \"dr\":false,\r\n \"fqdnCategory\":false,\r\n \"whois\":false,\r\n \"appCategory\":false,\r\n \"is_ip_attribut_primary\":1,\r\n \"is_app_attribut_primary\":1,\r\n \"is_domain_attribut_primary\":1,\r\n \"hideRelated\":ObjectUtil.isEmpty(hideRelated)?false:hideRelated\r\n}\r\ntimeHandle(parameter);\r\npageHandle(parameter);\r\n\r\nif(ObjectUtil.isNotEmpty(resource) && (resource.contains(\"ip.\") || resource.contains(\"app.\") || resource.contains(\"domain.\") || resource.contains(\"has(\")) && resource.contains(\"subscriber.\")){\r\n return notQueryHandle();\r\n}\r\n\r\nvar q = \"\";\r\n\r\nif(ObjectUtil.isEmpty(resource)){\r\n//默认无筛选\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT arrayJoin(entity_values) AS entity, 1 AS ip_is_primary, 1 AS app_is_primary, 1 AS domain_is_primary, 1 AS subscriber_id_is_primary FROM ( SELECT e.update_time AS update_time, e.app_name AS app_name, e.domain AS domain, e.ip AS ip, \'\' AS subscriber_id, (splitByChar(\'$\',concat(\'$ip=\',ip,\'$app=\', app_name,\'$domain=\', domain)) ) AS entity_values FROM session_relation_domain_ip_app e WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} UNION ALL SELECT update_time AS update_time, app_name AS app_name, \'\' AS domain, \'\' AS ip, subscriber_id AS subscriber_id, (splitByChar(\'$\',concat(\'$app=\', app_name,\'$subscriber_id=\', subscriber_id)) ) AS entity_values FROM session_relation_subscriber_app WHERE update_time >= ${startTime} AND update_time < ${endTime} ) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC <#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\n} else if(isSubscriberEntity(resource)){\r\n//检索属性属于Subscriber\r\n parameter.filter = EntityExplorerService.parseExpression(parameter.resource,\"entityMetadata\");\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT arrayJoin(entity_values) AS entity, MAX(IF(1 AND startsWith(entity,\'subscriber_id=\'), 1, 0)) AS subscriber_id_is_primary FROM ( SELECT app_name AS app_name, subscriber_id AS subscriber_id, update_time AS update_time, (<#if hideRelated> splitByChar(\'$\',concat( \'$subscriber_id=\',IF(1, subscriber_id, \'\'))) <#else> splitByChar(\'$\',concat(\'$app=\', app_name,\'$subscriber_id=\', subscriber_id)) </#if>) AS entity_values FROM session_relation_subscriber_app su WHERE update_time >= ${startTime} AND update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if> ) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC <#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\n} else if(!resource.contains(\"ip.\") && !resource.contains(\"app.\") && !resource.contains(\"domain.\") && resource.contains(\"has(\") && !resource.contains(\"subscriber.\")){\r\n // 仅tag\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && parameter.resource.contains(\"has(\")) {\r\n parameter.tags = EntityExplorerService.getTagsForFilter(parameter.resource);\r\n }\r\n entityAggHandle(parameter);\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT arrayJoin(entity_values) AS entity,MAX(IF(is_ip_tag_primary, 1, 0)) AS ip_is_primary,MAX(IF(is_app_tag_primary, 1, 0)) AS app_is_primary,MAX(IF(is_domain_tag_primary, 1, 0)) AS domain_is_primary,MAX(IF(is_subscriber_tag_primary, 1, 0)) AS subscriber_is_primary FROM (SELECT e.update_time AS update_time,e.app_name AS app_name,e.domain AS domain,e.ip AS ip,\'\' AS subscriber_id,<#if tags?exists> if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'ip.\', ${tag})) <#else> has(e.entity_tags, concat(\'ip.\', ${tag})) AND </#if></#list>, 1, 0) AS is_ip_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) AND </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'domain.\', ${tag})) <#else> has(e.entity_tags, concat(\'domain.\', ${tag})) AND </#if></#list>, 1, 0) AS is_domain_tag_primary, </#if>0 AS is_subscriber_tag_primary,(<#if hideRelated> (splitByChar(\'$\',concat(\'$ip=\',IF(is_ip_tag_primary, ip, \'\'),\'$app=\',IF(is_app_tag_primary, app_name, \'\'),\'$domain=\',IF(is_domain_tag_primary, domain, \'\')))) <#else> splitByChar(\'$\',concat(\'$ip=\',ip,\'$app=\', app_name,\'$domain=\', domain)) </#if>) AS entity_values FROM session_relation_domain_ip_app e WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if> UNION ALL SELECT e.update_time AS update_time,e.app_name AS app_name,\'\' AS domain,\'\' AS ip,e.subscriber_id AS subscriber_id,0 AS is_ip_tag_primary,0 AS is_domain_tag_primary,<#if tags?exists> if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) AND </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'subscriber.\', ${tag})) <#else> has(e.entity_tags, concat(\'subscriber.\', ${tag})) AND </#if></#list>, 1, 0) AS is_subscriber_tag_primary ,</#if> (<#if hideRelated> (splitByChar(\'$\',concat(\'$app=\',IF(is_app_tag_primary, app_name, \'\'),\'$subscriber_id=\',IF(is_subscriber_tag_primary, subscriber_id, \'\')))) <#else> splitByChar(\'$\',concat(\'$app=\', app_name,\'$subscriber_id=\', subscriber_id)) </#if>) AS entity_values FROM session_relation_subscriber_app e WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if>) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC <#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n} else {\r\n//检索属性属于APP/IP/DOMAIN\r\n //过滤标签时处理\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && parameter.resource.contains(\"has(\")) {\r\n parameter.tags = EntityExplorerService.getTagsForFilter(parameter.resource);\r\n }\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && parameter.resource.contains(\"has(\") && isExistEntity(parameter)) {\r\n parameter.is_ip_attribut_primary = 0;\r\n parameter.is_app_attribut_primary = 0;\r\n parameter.is_domain_attribut_primary = 0;\r\n }\r\n //有过滤时\r\n var primay = EntityExplorerService.getPrimayForFilter(parameter.resource,\"entityMetadata\");\r\n if(ObjectUtil.isNotEmpty(primay)) {\r\n parameter.is_ip_attribut_primary = 0;\r\n parameter.is_app_attribut_primary = 0;\r\n parameter.is_domain_attribut_primary = 0;\r\n for(k,v in primay) {\r\n if(v.asString().contains(\"ip\")) {\r\n parameter.is_ip_attribut_primary = 1;\r\n } else if(v.asString().contains(\"app\")) {\r\n parameter.is_app_attribut_primary = 1;\r\n } else if(v.asString().contains(\"domain\")) {\r\n parameter.is_domain_attribut_primary = 1;\r\n }\r\n }\r\n }\r\n entityAggHandle(parameter);\r\n //如果跨实体不查询返回空;如果单一实体和tag同时过滤只过滤该实体的tag\r\n if(crossEntityFilterHandle(parameter) != 0) {\r\n return crossEntityFilterHandle(parameter);\r\n }\r\n\r\n var isAssociationQuery = EntityExplorerService.isAssociationQuery(parameter.resource,\"entityMetadata\");\r\n if(isAssociationQuery){\r\n var sql = \"{\\\"sql\\\":\\\"SELECT a.entityValue AS entityValue, (case when ip = entityValue then \'ip\' when domain= entityValue then \'domain\' when app_name = entityValue then \'app\' else \'\' end) AS entityType FROM (SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time, concat(e.app_name,\'$\',e.domain,\'$\',e.ip) AS entity_value FROM clickhouse.cyber_narrator_galaxy.session_relation_domain_ip_app e <#if fqdnCategory> JOIN mysql.cn_api.cn_fqdn_category_built_in fqdnCategory ON (e.domain = fqdnCategory.fqdn <#if fqdnCategory\\.category_name?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_name} </#if> <#if fqdnCategory\\.category_group?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_group} </#if> <#if fqdnCategory\\.reputation_level?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.reputation_level}</#if>) </#if> <#if whois> JOIN mysql.cn_api.cn_fqdn_who_is_built_in whois ON (e.domain = whois.fqdn <#if whois\\.whois_domain?default(\'\')?trim? length gt 0> AND ${whois\\.whois_domain} </#if> <#if whois\\.whois_registrant_name?default(\'\')?trim? length gt 0> AND ${whois\\.whois_registrant_name}</#if>) </#if> <#if appCategory> JOIN mysql.cn_api.cn_app_category_built_in appCategory ON (e.app_name = appCategory.app_name <#if appCategory\\.app_category?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_category} </#if> <#if appCategory\\.app_subcategory?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_subcategory} </#if> <#if appCategory\\.app_risk?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_risk}</#if>) </#if> <#if dr\\.port?default(\'\')?trim? length gt 0> JOIN clickhouse.cyber_narrator_galaxy.cn_ip_dynamic_attribute dr ON (e.ip = dr.ip AND ${dr\\.port}) </#if> WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> ${filter} </#if> ) CROSS JOIN unnest(split(entity_value,\'$\')) AS a(entityValue) WHERE entityValue != \'\' GROUP BY entityValue,ip,domain,app_name ORDER BY MAX(update_time) DESC <#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\"}\"\r\n var aTemplate = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerTopAggCountry\",sql,parameter);\r\n var ares = jsonUtil.toBean(aTemplate,HashMap.class);\r\n var log = Log.get();\r\n log.info(ares.sql);\r\n var aresult = db[\'trino\'].select(ares.sql);\r\n var data = {};\r\n data.list = aresult;\r\n data.total = total;\r\n data.pageNo = ObjectUtil.isNotEmpty(pageNo) ? pageNo : 1;\r\n data.pageSize = ObjectUtil.isNotEmpty(pageSize) ? pageSize : parameter.pageSize;\r\n return R.ok(data).put(\"originalUrl\",ares.sql).put(\"status\",200);\r\n }\r\n /**\r\n * 接口sql相关内容整理 通过freemarker模板生成最后执行语句\r\n */\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT arrayJoin(entity_values) AS entity, MAX(IF(${is_ip_attribut_primary} <#if tags?exists> AND is_ip_tag_primary </#if>, 1, 0)) AS ip_is_primary, MAX(IF(${is_app_attribut_primary} <#if tags?exists> AND is_app_tag_primary </#if>, 1, 0)) AS app_is_primary, MAX(IF(${is_domain_attribut_primary} <#if tags?exists> AND is_domain_tag_primary </#if>, 1, 0)) AS domain_is_primary FROM ( SELECT e.update_time AS update_time, e.app_name AS app_name, e.domain AS domain, e.ip AS ip, <#if tags?exists> if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'ip.\', ${tag})) <#else> has(e.entity_tags, concat(\'ip.\', ${tag})) AND </#if></#list>, 1, 0) AS is_ip_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) AND </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'domain.\', ${tag})) <#else> has(e.entity_tags, concat(\'domain.\', ${tag})) AND </#if></#list>, 1, 0) AS is_domain_tag_primary, </#if> (<#if hideRelated> splitByChar(\'$\',concat( \'$ip=\',IF(${is_ip_attribut_primary} <#if tags?exists> AND is_ip_tag_primary </#if>, ip, \'\'), \'$app=\',IF(${is_app_attribut_primary} <#if tags?exists> AND is_app_tag_primary </#if>, app_name, \'\'), \'$domain=\',IF(${is_domain_attribut_primary} <#if tags?exists> AND is_domain_tag_primary </#if>, domain, \'\'))) <#else> splitByChar(\'$\',concat(\'$ip=\',ip,\'$app=\', app_name,\'$domain=\', domain)) </#if>) AS entity_values FROM session_relation_domain_ip_app e <#if dr> JOIN cn_ip_dynamic_attribute dr ON (e.ip = dr.ip) </#if> WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if dr> AND dr.update_time >= ${startTime} AND dr.update_time < ${endTime} </#if> <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if> ) WHERE notEmpty(SUBSTRING(entity, position(entity, \'=\') + 1)) GROUP BY entity ORDER BY MAX(update_time) DESC<#if pageSize?default(\\\'\\\')?trim? length gt 0 && pageNo?default(\\\'\\\')?trim? length gt 0> LIMIT ${pageNo},${pageSize}</#if>\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\n}\r\n\r\nvar result = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerQueryList\",q,parameter);\r\n\r\n/**\r\n * 调用galaxy接口获取结果信息\r\n */\r\nvar res = jsonUtil.toBean(result,HashMap.class);\r\nvar result = postGalaxyResult(res);\r\n/**\r\n * 处理galaxy返回的数据结果 将新结果返回前端界面\r\n */\r\nvar data = {};\r\nif(result.success){\r\n var list = new ArrayList();\r\n for(k,v in result.data) {\r\n var map = {};\r\n var entity = Arrays.asList(v.entity.asString().split(\"=\"));\r\n map.entityType = entity.get(0);\r\n map.entityValue = entity.get(1);\r\n map.isRelated = true;\r\n if(entity.get(0).equals(\"ip\") && v.ip_is_primary==1) {\r\n map.isRelated = false;\r\n } else if(entity.get(0).equals(\"app\") && v.app_is_primary==1) {\r\n map.isRelated = false;\r\n } else if(entity.get(0).equals(\"domain\") && v.domain_is_primary==1) {\r\n map.isRelated = false;\r\n } else if(entity.get(0).equals(\"subscriber_id\") && v.subscriber_id_is_primary==1) {\r\n map.isRelated = false;\r\n }\r\n list.add(map);\r\n }\r\n data.list = list;\r\n data.pageNo = ObjectUtil.isNotEmpty(pageNo) ? pageNo : 1;\r\n data.pageSize = ObjectUtil.isNotEmpty(pageSize) ? pageSize : parameter.pageSize;\r\n}\r\n\r\nreturn R.ok(data).put(\"originalUrl\",result.originalUrl).put(\"status\",200);'); INSERT INTO `magic_api_file` VALUES ('/api/v1/entity/explorer/query/实体基数统计.ms', '{\n \"properties\" : { },\n \"id\" : \"c5f77524ffab4c02a0cdc05b44b95ad0\",\n \"script\" : null,\n \"groupId\" : \"9e64aeec148c46259a28506f2f1e4cac\",\n \"name\" : \"实体基数统计\",\n \"createTime\" : null,\n \"updateTime\" : 1730194824861,\n \"lock\" : null,\n \"createBy\" : null,\n \"updateBy\" : null,\n \"path\" : \"/summaryCount\",\n \"method\" : \"GET\",\n \"parameters\" : [ {\n \"name\" : \"startTime\",\n \"value\" : \"1730184879\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Long\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"endTime\",\n \"value\" : \"1730188479\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Long\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"hideRelated\",\n \"value\" : \"true\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Boolean\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"pageNo\",\n \"value\" : \"\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Integer\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"pageSize\",\n \"value\" : \"\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"Integer\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n }, {\n \"name\" : \"resource\",\n \"value\" : \"has(tag,\'FWDNS\')\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"String\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n } ],\n \"options\" : [ ],\n \"requestBody\" : \"\",\n \"headers\" : [ {\n \"name\" : \"cn-authorization\",\n \"value\" : \"adacf41c-8e3f-412e-987e-f31592d2e0ed\",\n \"description\" : null,\n \"required\" : false,\n \"dataType\" : \"String\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : null,\n \"error\" : null,\n \"expression\" : null,\n \"children\" : null\n } ],\n \"paths\" : [ ],\n \"responseBody\" : \"{\\n \\\"code\\\": 999,\\n \\\"time\\\": \\\"2024-10-29 09:39:56\\\",\\n \\\"message\\\": \\\"找不到函数toHump(String)\\\"\\n}\",\n \"description\" : null,\n \"requestBodyDefinition\" : null,\n \"responseBodyDefinition\" : {\n \"name\" : \"\",\n \"value\" : \"\",\n \"description\" : \"\",\n \"required\" : false,\n \"dataType\" : \"Object\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : \"\",\n \"error\" : \"\",\n \"expression\" : \"\",\n \"children\" : [ {\n \"name\" : \"code\",\n \"value\" : \"999\",\n \"description\" : \"\",\n \"required\" : false,\n \"dataType\" : \"Integer\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : \"\",\n \"error\" : \"\",\n \"expression\" : \"\",\n \"children\" : [ ]\n }, {\n \"name\" : \"time\",\n \"value\" : \"2023-11-07 09:03:06\",\n \"description\" : \"\",\n \"required\" : false,\n \"dataType\" : \"String\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : \"\",\n \"error\" : \"\",\n \"expression\" : \"\",\n \"children\" : [ ]\n }, {\n \"name\" : \"message\",\n \"value\" : \"404 null at Row:10~10,Col:32~36\\n\\nvar galaxyResult = httpRequest.get().getBody();\\r\\n ^^^^^ \",\n \"description\" : \"\",\n \"required\" : false,\n \"dataType\" : \"String\",\n \"type\" : null,\n \"defaultValue\" : null,\n \"validateType\" : \"\",\n \"error\" : \"\",\n \"expression\" : \"\",\n \"children\" : [ ]\n } ]\n }\n}\r\n================================\r\nimport cn.hutool.log.Log;\r\nimport cn.hutool.core.util.ObjectUtil;\r\nimport net.geedge.common.utils.FreeMarkerUtil as freeMarkerUtil;\r\nimport net.geedge.common.utils.JSONUtil as jsonUtil;\r\nimport net.geedge.common.utils.R as R;\r\nimport net.geedge.modules.entity.service.EntityExplorerService as EntityExplorerService;\r\nimport \'@/pre/timeHandleHalf\' as timeHandle;\r\nimport \'@/pre/pageFieldsHandle\' as pageFieldsHandle;\r\nimport \'@/entity/entityAggHandle\' as entityAggHandle;\r\nimport \'@/entity/isExistEntity\' as isExistEntity;\r\nimport \'@/entity/isSubscriberEntity\' as isSubscriberEntity;\r\nimport \'@/entity/notQueryHandle\' as notQueryHandle;\r\nimport \'@/entity/crossEntityFilterHandle\' as crossEntityFilterHandle;\r\nimport \'@/public/postGalaxyResult\' as postGalaxyResult;\r\nimport \'@/public/toHump\' as toHump\r\n/**\r\n * 入参整理 包括调用公共函数\r\n */\r\nvar parameter={\r\n \"resource\":resource,\r\n \"startTime\":startTime,\r\n \"endTime\":endTime,\r\n \"pageNo\":pageNo,\r\n \"pageSize\":pageSize,\r\n \"dr\":false,\r\n \"fqdnCategory\":false,\r\n \"whois\":false,\r\n \"appCategory\":false,\r\n \"is_ip_attribut_primary\":1,\r\n \"is_app_attribut_primary\":1,\r\n \"is_domain_attribut_primary\":1,\r\n \"hideRelated\":ObjectUtil.isEmpty(hideRelated)?false:hideRelated\r\n}\r\ntimeHandle(parameter);\r\npageFieldsHandle(parameter);\r\n\r\nif(ObjectUtil.isNotEmpty(resource) && (resource.contains(\"ip.\") || resource.contains(\"app.\") || resource.contains(\"domain.\") || resource.contains(\"has(\")) && resource.contains(\"subscriber.\")){\r\n return notQueryHandle();\r\n}\r\n\r\nvar q = \"\";\r\n\r\nif(ObjectUtil.isEmpty(resource)){\r\n//默认无筛选\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT IFNULL(COUNTIF(DISTINCT(IF(1,app_name, NULL)),notEmpty(app_name)),0) as appCount, IFNULL(COUNTIF(DISTINCT(IF(1,domain, NULL)),notEmpty(domain)),0) as domainCount, IFNULL(COUNTIF(DISTINCT(IF(1,ip, NULL)),notEmpty(ip)),0) as ipCount, IFNULL(COUNTIF(DISTINCT(IF(1,subscriber_id, NULL)),notEmpty(subscriber_id)),0) as subscriberCount, plus(subscriberCount,plus(ipCount,plus(appCount,domainCount))) AS totalCount FROM ( SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time, NULL AS subscriber_id FROM session_relation_domain_ip_app e WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} UNION ALL SELECT app_name AS app_name, NULL AS domain, NULL AS ip, update_time AS update_time, subscriber_id AS subscriber_id FROM session_relation_subscriber_app WHERE update_time >= ${startTime} AND update_time < ${endTime} )\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\n} else if(isSubscriberEntity(resource)){\r\n//检索属性属于Subscriber\r\n if(hideRelated){\r\n parameter.is_app_attribut_primary = 0;\r\n }\r\n parameter.filter = EntityExplorerService.parseExpression(parameter.resource,\"entityMetadata\");\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT IFNULL(COUNTIF(DISTINCT(IF(${is_app_attribut_primary},app_name, NULL)),notEmpty(app_name)),0) as appCount, IFNULL(COUNTIF(DISTINCT(IF(1,subscriber_id, NULL)),notEmpty(subscriber_id)),0) as subscriberCount, plus(subscriberCount,appCount) AS totalCount FROM ( SELECT app_name AS app_name, subscriber_id AS subscriber_id FROM session_relation_subscriber_app su WHERE update_time >= ${startTime} AND update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> AND ${filter} </#if> )\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\n}else if (!resource.contains(\"ip.\") && !resource.contains(\"app.\") && !resource.contains(\"domain.\") && resource.contains(\"has(\") && !resource.contains(\"subscriber.\")){\r\n // 仅tag\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && parameter.resource.contains(\"has(\")) {\r\n parameter.tags = EntityExplorerService.getTagsForFilter(parameter.resource);\r\n }\r\n entityAggHandle(parameter);\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT IFNULL(COUNTIF(DISTINCT(IF(is_app_tag_primary,app_name, NULL)),notEmpty(app_name)),0) as app_count,IFNULL(COUNTIF(DISTINCT(IF(is_domain_tag_primary,domain, NULL)),notEmpty(domain)),0) as domain_count,IFNULL(COUNTIF(DISTINCT(IF(is_ip_tag_primary,ip, NULL)),notEmpty(ip)),0) as ip_count,IFNULL(COUNTIF(DISTINCT(IF(is_subscriber_tag_primary,subscriber_id, NULL)),notEmpty(subscriber_id)),0) as subscriber_count,plus(plus(ip_count, app_count),plus(subscriber_count, domain_count)) AS total_count FROM ( SELECT e.app_name AS app_name,e.domain AS domain,e.ip AS ip,NULL AS subscriber_id,e.update_time AS update_time <#if hideRelated> ,0 AS is_subscriber_tag_primary <#if tags?exists>, if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'ip.\', ${tag})) <#else> has(e.entity_tags, concat(\'ip.\', ${tag})) AND </#if></#list>, 1, 0) AS is_ip_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) AND </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'domain.\', ${tag})) <#else> has(e.entity_tags, concat(\'domain.\', ${tag})) AND </#if></#list>, 1, 0) AS is_domain_tag_primary</#if> <#else> ,1 AS is_ip_tag_primary,1 AS is_app_tag_primary,1 AS is_domain_tag_primary,0 AS is_subscriber_tag_primary </#if> FROM session_relation_domain_ip_app e WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if> UNION ALL SELECT e.app_name AS app_name,NULL AS domain,NULL AS ip,e.subscriber_id AS subscriber_id,e.update_time AS update_time <#if hideRelated> ,0 AS is_ip_tag_primary,0 AS is_domain_tag_primary <#if tags?exists>, if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) AND </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'subscriber.\', ${tag})) <#else> has(e.entity_tags, concat(\'subscriber.\', ${tag})) AND </#if></#list>, 1, 0) AS is_subscriber_tag_primary</#if><#else>,0 AS is_ip_tag_primary,1 AS is_app_tag_primary,0 AS is_domain_tag_primary,1 AS is_subscriber_tag_primary </#if> FROM session_relation_subscriber_app e WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if>)\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n}else {\r\n//检索属性属于APP/IP/DOMAIN\r\n //过滤标签时处理\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && parameter.resource.contains(\"has(\")) {\r\n parameter.tags = EntityExplorerService.getTagsForFilter(parameter.resource);\r\n }\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && hideRelated && parameter.resource.contains(\"has(\")){\r\n parameter.is_ip_attribut_primary = 0;\r\n parameter.is_app_attribut_primary = 0;\r\n parameter.is_domain_attribut_primary = 0;\r\n }\r\n if(ObjectUtil.isNotEmpty(parameter.resource) && hideRelated && parameter.resource.contains(\"has(\") && isExistEntity(parameter)) {\r\n parameter.is_ip_attribut_primary = 0;\r\n parameter.is_app_attribut_primary = 0;\r\n parameter.is_domain_attribut_primary = 0;\r\n }\r\n //隐藏相关时处理\r\n if(hideRelated){\r\n var primay = EntityExplorerService.getPrimayForFilter(parameter.resource,\"entityMetadata\");\r\n if(ObjectUtil.isNotEmpty(primay)) {\r\n parameter.is_ip_attribut_primary = 0;\r\n parameter.is_app_attribut_primary = 0;\r\n parameter.is_domain_attribut_primary = 0;\r\n for(k,v in primay) {\r\n if(v.asString().contains(\"ip\")) {\r\n parameter.is_ip_attribut_primary = 1;\r\n } else if(v.asString().contains(\"app\")) {\r\n parameter.is_app_attribut_primary = 1;\r\n } else if(v.asString().contains(\"domain\")) {\r\n parameter.is_domain_attribut_primary = 1;\r\n }\r\n }\r\n }\r\n }\r\n\r\n entityAggHandle(parameter);\r\n //如果跨实体不查询返回空;如果单一实体和tag同时过滤只过滤该实体的tag\r\n if(crossEntityFilterHandle(parameter) != 0) {\r\n return crossEntityFilterHandle(parameter);\r\n }\r\n\r\n var isAssociationQuery = EntityExplorerService.isAssociationQuery(parameter.resource,\"entityMetadata\");\r\n if(isAssociationQuery){\r\n var sql = \"{\\\"sql\\\":\\\"SELECT COUNT(DISTINCT(app_name)) FILTER (WHERE app_name != \'\') AS appCount, COUNT(DISTINCT(domain)) FILTER (WHERE domain != \'\') AS domainCount, COUNT(DISTINCT(ip)) FILTER (WHERE ip != \'\') AS ipCount, COUNT(DISTINCT(app_name)) FILTER (WHERE app_name != \'\') + COUNT(DISTINCT(domain)) FILTER (WHERE domain != \'\') + COUNT(DISTINCT(ip)) FILTER (WHERE ip != \'\') AS total FROM( SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time FROM clickhouse.cyber_narrator_galaxy.session_relation_domain_ip_app e <#if fqdnCategory> JOIN mysql.cn_api.cn_fqdn_category_built_in fqdnCategory ON (e.domain = fqdnCategory.fqdn <#if fqdnCategory\\.category_name?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_name} </#if> <#if fqdnCategory\\.category_group?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_group} </#if> <#if fqdnCategory\\.reputation_level?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.reputation_level}</#if>) </#if> <#if whois> JOIN mysql.cn_api.cn_fqdn_who_is_built_in whois ON (e.domain = whois.fqdn <#if whois\\.whois_domain?default(\'\')?trim? length gt 0> AND ${whois\\.whois_domain} </#if> <#if whois\\.whois_registrant_name?default(\'\')?trim? length gt 0> AND ${whois\\.whois_registrant_name}</#if>) </#if> <#if appCategory> JOIN mysql.cn_api.cn_app_category_built_in appCategory ON (e.app_name = appCategory.app_name <#if appCategory\\.app_category?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_category} </#if> <#if appCategory\\.app_subcategory?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_subcategory} </#if> <#if appCategory\\.app_risk?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_risk}</#if>) </#if> <#if dr\\.port?default(\'\')?trim? length gt 0> JOIN clickhouse.cyber_narrator_galaxy.cn_ip_dynamic_attribute dr ON (e.ip = dr.ip AND ${dr\\.port}) </#if> WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if filter?default(\'\')?trim? length gt 0> ${filter} </#if> )\\\"}\"\r\n var aTemplate = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerQuerySummaryCount\",sql,parameter);\r\n var ares = jsonUtil.toBean(aTemplate,HashMap.class);\r\n var log = Log.get();\r\n log.info(ares.sql);\r\n var aresult = db[\'trino\'].select(ares.sql);\r\n var data = aresult[0];\r\n var url = db.select(\"select cvalue from sys_config where ckey = \'galaxy_url\'\");\r\n return R.ok(data).put(\"originalUrl\",url.cvalue+ares.sql).put(\"status\",200);\r\n }\r\n /**\r\n * 接口sql相关内容整理 通过freemarker模板生成最后执行语句\r\n */\r\n q = \"{\\r\\n \\\"query\\\": \\\"SELECT IFNULL(COUNTIF(DISTINCT(IF(${is_app_attribut_primary} <#if tags?exists> OR is_app_tag_primary </#if>,app_name, NULL)),notEmpty(app_name)),0) as appCount, IFNULL(COUNTIF(DISTINCT(IF(${is_domain_attribut_primary} <#if tags?exists> OR is_domain_tag_primary </#if>,domain, NULL)),notEmpty(domain)),0) as domainCount, IFNULL(COUNTIF(DISTINCT(IF(${is_ip_attribut_primary} <#if tags?exists> OR is_ip_tag_primary </#if>,ip, NULL)),notEmpty(ip)),0) as ipCount, plus(ipCount,plus(appCount,domainCount)) AS totalCount FROM( SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time <#if tags?exists>, if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'ip.\', ${tag})) <#else> has(e.entity_tags, concat(\'ip.\', ${tag})) AND </#if></#list>, 1, 0) AS is_ip_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) AND </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'domain.\', ${tag})) <#else> has(e.entity_tags, concat(\'domain.\', ${tag})) AND </#if></#list>, 1, 0) AS is_domain_tag_primary </#if> FROM session_relation_domain_ip_app e <#if dr> JOIN cn_ip_dynamic_attribute dr ON (e.ip = dr.ip) </#if> WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if dr> AND dr.update_time >= ${startTime} AND dr.update_time < ${endTime} </#if> <#if filter?default(\'\')?trim? length gt 0> AND (${filter}) </#if> )\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n}\r\n\r\n\r\nvar result = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerQuerySummaryCount\",q,parameter);\r\n\r\n/**\r\n * 调用galaxy接口获取结果信息\r\n */\r\nvar res = jsonUtil.toBean(result,HashMap.class);\r\nvar result = postGalaxyResult(res);\r\n/**\r\n * 处理galaxy返回的数据结果 将新结果返回前端界面\r\n */\r\nvar data = {appCount:0,domainCount:0,ipCount:0,subscriberCount:0,totalCount:0};\r\nif(result.success){\r\n for(k,v in result.data[0]){\r\n data.put(toHump(k),v.asInt());\r\n }\r\n}\r\n\r\nreturn R.ok(data).put(\"originalUrl\",result.originalUrl).put(\"status\",200);'); INSERT INTO `magic_api_file` VALUES ('/api/v1/entity/explorer/top/', 'this is directory'); INSERT INTO `magic_api_file` VALUES ('/api/v1/entity/explorer/top/App类别-实体基数统计.ms', '{\r\n \"properties\" : { },\r\n \"id\" : \"fd06a1755cbb415ab4eefa767aaba193\",\r\n \"script\" : null,\r\n \"groupId\" : \"80392cde8e944a6e9c947b4ad8661c29\",\r\n \"name\" : \"App类别-实体基数统计\",\r\n \"createTime\" : null,\r\n \"updateTime\" : 1710924249614,\r\n \"lock\" : null,\r\n \"createBy\" : null,\r\n \"updateBy\" : \"admin\",\r\n \"path\" : \"/aggAppCategory\",\r\n \"method\" : \"GET\",\r\n \"parameters\" : [ {\r\n \"name\" : \"resource\",\r\n \"value\" : \"has(tag,\'FWDNS\') AND has(tag,\'mtproxy\')\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"startTime\",\r\n \"value\" : \"\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Long\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"endTime\",\r\n \"value\" : \"\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Long\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n }, {\r\n \"name\" : \"hideRelated\",\r\n \"value\" : \"\",\r\n \"description\" : null,\r\n \"required\" : false,\r\n \"dataType\" : \"Boolean\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : null,\r\n \"error\" : null,\r\n \"expression\" : null,\r\n \"children\" : null\r\n } ],\r\n \"options\" : [ ],\r\n \"requestBody\" : \"\",\r\n \"headers\" : [ ],\r\n \"paths\" : [ ],\r\n \"responseBody\" : \"{\\n \\\"code\\\": 200,\\n \\\"data\\\": {\\n \\\"list\\\": [\\n {\\n \\\"value\\\": \\\"general-internet\\\",\\n \\\"uniqueEntities\\\": \\\"3\\\"\\n }\\n ]\\n },\\n \\\"time\\\": \\\"2023-12-29 06:30:48\\\",\\n \\\"originalUrl\\\": \\\"http://192.168.44.55:9999/v1/query/sql\\\",\\n \\\"message\\\": \\\"success\\\",\\n \\\"status\\\": 200\\n}\",\r\n \"description\" : null,\r\n \"requestBodyDefinition\" : null,\r\n \"responseBodyDefinition\" : {\r\n \"name\" : \"\",\r\n \"value\" : \"\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Object\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ {\r\n \"name\" : \"code\",\r\n \"value\" : \"999\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"Integer\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"time\",\r\n \"value\" : \"2023-08-29 07:57:31\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n }, {\r\n \"name\" : \"message\",\r\n \"value\" : \"404 null at Row:10~10,Col:32~36\\\\n\\\\nvar galaxyResult = httpRequest.get().getBody();\\\\r\\\\n ^^^^^\",\r\n \"description\" : \"\",\r\n \"required\" : false,\r\n \"dataType\" : \"String\",\r\n \"type\" : null,\r\n \"defaultValue\" : null,\r\n \"validateType\" : \"\",\r\n \"error\" : \"\",\r\n \"expression\" : \"\",\r\n \"children\" : [ ]\r\n } ]\r\n }\r\n}\r\n================================\r\nimport cn.hutool.log.Log;\r\nimport cn.hutool.core.util.ObjectUtil;\r\nimport net.geedge.common.utils.FreeMarkerUtil as freeMarkerUtil;\r\nimport net.geedge.common.utils.JSONUtil as jsonUtil;\r\nimport net.geedge.common.utils.R as R;\r\nimport net.geedge.modules.entity.service.EntityExplorerService as EntityExplorerService;\r\nimport \'@/public/toHump\' as toHump;\r\nimport \'@/pre/timeHandleHalf\' as timeHandle;\r\nimport \'@/entity/entityAggHandle\' as entityAggHandle;\r\nimport \'@/entity/entityHideHandle\' as entityHideHandle;\r\nimport \'@/entity/crossEntityFilterHandle\' as crossEntityFilterHandle;\r\nimport \'@/entity/notQueryHandle\' as notQueryHandle;\r\nimport \'@/public/postGalaxyResult\' as postGalaxyResult;\r\n/**\r\n * 入参整理 包括调用公共函数\r\n */\r\nvar parameter={\r\n \"resource\":resource,\r\n \"startTime\":startTime,\r\n \"endTime\":endTime,\r\n \"dr\":false,\r\n \"fqdnCategory\":false,\r\n \"whois\":false,\r\n \"appCategory\":false,\r\n \"hideRelated\":ObjectUtil.isEmpty(hideRelated)?false:hideRelated,\r\n \"is_ip_attribut_primary\":0,\r\n \"is_app_attribut_primary\":0,\r\n \"is_domain_attribut_primary\":0\r\n}\r\ntimeHandle(parameter);\r\n//过滤标签时处理\r\nif(ObjectUtil.isNotEmpty(parameter.resource) && parameter.resource.contains(\"has(\")) {\r\n parameter.tags = EntityExplorerService.getTagsForFilter(parameter.resource);\r\n}\r\nentityAggHandle(parameter);\r\nentityHideHandle(parameter);\r\n//如果跨实体不查询返回空;如果单一实体和tag同时过滤只过滤该实体的tag\r\nif(crossEntityFilterHandle(parameter) != 0) {\r\n return crossEntityFilterHandle(parameter);\r\n}\r\n\r\nif(ObjectUtil.isNotEmpty(parameter.filter) && hideRelated && !parameter.filter.contains(\"has(e.entity_tags\") && parameter.is_app_attribut_primary == 0) {\r\n return notQueryHandle();\r\n}\r\n\r\nvar isAssociationQuery = EntityExplorerService.isAssociationQuery(parameter.resource,\"entityMetadata\");\r\nif(isAssociationQuery){\r\n var sql = \"{\\\"sql\\\":\\\"SELECT app_category AS app_category, COUNT(DISTINCT(app_name)) FILTER (WHERE app_name != \'\') AS count FROM( SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time, e.app_category AS app_category FROM clickhouse.cyber_narrator_galaxy.session_relation_domain_ip_app e <#if fqdnCategory> JOIN mysql.cn_api.cn_fqdn_category_built_in fqdnCategory ON (e.domain = fqdnCategory.fqdn <#if fqdnCategory\\.category_name?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_name} </#if> <#if fqdnCategory\\.category_group?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.category_group} </#if> <#if fqdnCategory\\.reputation_level?default(\'\')?trim? length gt 0> AND ${fqdnCategory\\.reputation_level}</#if>) </#if> <#if whois> JOIN mysql.cn_api.cn_fqdn_who_is_built_in whois ON (e.domain = whois.fqdn <#if whois\\.whois_domain?default(\'\')?trim? length gt 0> AND ${whois\\.whois_domain} </#if> <#if whois\\.whois_registrant_name?default(\'\')?trim? length gt 0> AND ${whois\\.whois_registrant_name}</#if>) </#if> <#if appCategory> JOIN mysql.cn_api.cn_app_category_built_in appCategory ON (e.app_name = appCategory.app_name <#if appCategory\\.app_category?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_category} </#if> <#if appCategory\\.app_subcategory?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_subcategory} </#if> <#if appCategory\\.app_risk?default(\'\')?trim? length gt 0> AND ${appCategory\\.app_risk}</#if>) </#if> <#if dr\\.port?default(\'\')?trim? length gt 0> JOIN clickhouse.cyber_narrator_galaxy.cn_ip_dynamic_attribute dr ON (e.ip = dr.ip AND ${dr\\.port}) </#if> WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} AND app_category != \'\' <#if filter?default(\'\')?trim? length gt 0> ${filter} </#if> ) GROUP BY app_category ORDER BY count DESC\\\"}\"\r\n var aTemplate = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerTopAggTag\",sql,parameter);\r\n var ares = jsonUtil.toBean(aTemplate,HashMap.class);\r\n var log = Log.get();\r\n log.info(ares.sql);\r\n var aresult = db[\'trino\'].select(ares.sql);\r\n var list=new ArrayList();\r\n if(ObjectUtil.isNotEmpty(aresult)){\r\n if(aresult.length > 0){\r\n for(key,value in aresult){\r\n var obj = {};\r\n obj.value = value.app_category;\r\n obj.uniqueEntities = value.count;\r\n list.add(obj);\r\n }\r\n }\r\n }\r\n var data = {};\r\n data.list = list;\r\n var url = db.select(\"select cvalue from sys_config where ckey = \'galaxy_url\'\");\r\n return R.ok(data).put(\"originalUrl\",url.cvalue+ares.sql).put(\"status\",200);\r\n}\r\n/**\r\n * 接口sql相关内容整理 通过freemarker模板生成最后执行语句\r\n */\r\nvar q = \"{\\r\\n \\\"query\\\": \\\"SELECT app_category as app_category, COUNTIF(DISTINCT(app_name),notEmpty(app_name)) as count FROM( SELECT e.app_name AS app_name, e.domain AS domain, e.ip AS ip, e.update_time AS update_time, <#if tags?exists> if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'ip.\', ${tag})) <#else> has(e.entity_tags, concat(\'ip.\', ${tag})) OR </#if></#list>, 1 , 0) AS is_ip_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'app.\', ${tag})) <#else> has(e.entity_tags, concat(\'app.\', ${tag})) OR </#if></#list>, 1, 0) AS is_app_tag_primary ,if(<#list tags as tag><#if !tag_has_next> has(e.entity_tags, concat(\'domain.\', ${tag})) <#else> has(e.entity_tags, concat(\'domain.\', ${tag})) OR </#if></#list>, 1, 0) AS is_domain_tag_primary, </#if> IF(<#if hideRelated>${is_app_attribut_primary} <#else> 1 </#if> <#if tags?exists> OR is_app_tag_primary </#if>,e.app_category,\'\') as app_category FROM session_relation_domain_ip_app e <#if dr> JOIN cn_ip_dynamic_attribute dr ON (e.ip = dr.ip) </#if> WHERE e.update_time >= ${startTime} AND e.update_time < ${endTime} <#if dr> AND dr.update_time >= ${startTime} AND dr.update_time < ${endTime} </#if> AND notEmpty(app_category) <#if filter?default(\\\'\\\')?trim? length gt 0> AND (${filter}) </#if> ) GROUP BY app_category ORDER BY count DESC limit 100\\\",\\r\\n \\\"option\\\": \\\"real_time\\\",\\r\\n \\\"format\\\": \\\"json\\\"\\r\\n}\"\r\n\r\nvar result = freeMarkerUtil.processTemplateIntoString(\"v1EntityExplorerTopAggTag\",q,parameter);\r\n\r\n/**\r\n * 调用galaxy接口获取结果信息\r\n */\r\nvar res = jsonUtil.toBean(result,HashMap.class);\r\nvar result = postGalaxyResult(res);\r\n/**\r\n * 处理galaxy返回的数据结果 将新结果返回前端界面\r\n */\r\nvar data = {};\r\nvar list = new ArrayList();\r\nif(result.success){\r\n for(key,value in result.data){\r\n var obj = {};\r\n obj.value = value.app_category;\r\n obj.uniqueEntities = value.count;\r\n list.add(obj);\r\n }\r\n}\r\ndata.list = list;\r\nreturn R.ok(data).put(\"originalUrl\",result.originalUrl).put(\"status\",200);'); |