diff options
Diffstat (limited to 'src/test')
38 files changed, 8182 insertions, 779 deletions
diff --git a/src/test/java/com/mesalab/GalaxyQGWApplicationTests.java b/src/test/java/com/mesalab/GalaxyQGWApplicationTests.java index 276c3fa3..fd89b068 100644 --- a/src/test/java/com/mesalab/GalaxyQGWApplicationTests.java +++ b/src/test/java/com/mesalab/GalaxyQGWApplicationTests.java @@ -9,6 +9,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.web.server.LocalServerPort; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; @@ -18,12 +19,15 @@ import java.util.Map; import java.util.Objects; @RunWith(SpringRunner.class) -@SpringBootTest(classes = {GalaxyQGWApplication.class}, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT) +@SpringBootTest(classes = {GalaxyQGWApplication.class}, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @EnableAutoConfiguration @ActiveProfiles("test") public class GalaxyQGWApplicationTests { private static final Log log = LogFactory.get(); + @LocalServerPort + protected int testPort; + static { System.setProperty("jasypt.encryptor.password", "galaxy"); System.setProperty("JM.SNAPSHOT.PATH", "config"); diff --git a/src/test/java/com/mesalab/knowledge/JsonSchemaTest.java b/src/test/java/com/mesalab/knowledge/JsonSchemaTest.java deleted file mode 100644 index ec78f262..00000000 --- a/src/test/java/com/mesalab/knowledge/JsonSchemaTest.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.mesalab.knowledge; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.fasterxml.jackson.databind.JsonNode; -import com.github.fge.jackson.JsonLoader; -import com.github.fge.jsonschema.core.exceptions.ProcessingException; -import com.github.fge.jsonschema.core.report.ProcessingMessage; -import com.github.fge.jsonschema.core.report.ProcessingReport; -import com.github.fge.jsonschema.main.JsonSchema; -import com.github.fge.jsonschema.main.JsonSchemaFactory; -import com.mesalab.GalaxyQGWApplicationTests; -import org.apache.commons.lang3.Validate; -import org.junit.Test; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -import java.io.IOException; -import java.util.Iterator; - -/** - * @description: - * @author: zhq - * @create: 2020-07-31 - **/ -@EnableAutoConfiguration -public class JsonSchemaTest extends GalaxyQGWApplicationTests { - private static final Log log = LogFactory.get(); - - @Test - public void jsonTest() { - //创建jsonschema工厂 - String jsonStr = "{\n" + - " \"clientId\": null,\n" + - " \"query\": {\n" + - " \"queryType\": \"iplearning\",\n" + - " \"dataSource\": \"IP_LEARNING_VIEW\",\n" + - " \"parameters\": {\n" + - " \"match\": [\n" + - " {\n" + - " \"type\": \"substring\",\n" + - " \"fieldKey\": \"FQDN_NAME\",\n" + - " \"fieldValues\": \n" + - " [\"360\"]\n" + - " \n" + - " }\n" + - " ],\n" + - " \"range\": [\n" + - " {\n" + - " \"type\": \"ge\",\n" + - " \"fieldKey\": \"PROTOCOL\",\n" + - " \"fieldValues\": [\n" + - " \"HTTP\"\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"type\": \"eq\",\n" + - " \"fieldKey\": \"DEPTH\",\n" + - " \"fieldValues\": [\n" + - " 1\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"type\": \"ge\",\n" + - " \"fieldKey\": \"UNIQ_CIP\",\n" + - " \"fieldValues\": \n" + - " [5]\n" + - " \n" + - " }\n" + - " ],\n" + - " \"intervals\": [\n" + - " \"2020-07-01 00:00:00/2020-08-02 00:00:00\"]\n" + - " ,\n" + - " \"limit\": \"15\"\n" + - " }\n" + - " }\n" + - "}"; - try { - JsonSchemaFactory factory = JsonSchemaFactory.byDefault(); - //通过jsonschemaFactory获取jsonnode对象 - JsonNode schemaNode = JsonLoader.fromResource("/dsl-validation.json"); - //通过jsonstr字符串获取对应的jsonnode对象 - JsonNode dataNode = JsonLoader.fromString(jsonStr); - JsonSchema jsonSchema = factory.getJsonSchema(schemaNode); - //使用json-schema-validator中的jsonschema对象的validate方法对数据进行校验 - //获取处理的报告信息 - ProcessingReport processingReport = jsonSchema.validate(dataNode); - //获取完整的报告信息 - Iterator<ProcessingMessage> iterator = processingReport.iterator(); - StringBuffer sb = new StringBuffer(); - while (iterator.hasNext()) { - ProcessingMessage next = iterator.next(); - JsonNode jsonNode = next.asJson(); - sb.append("pointer on "); - sb.append(jsonNode.get("instance").get("pointer")); - sb.append(", "); - sb.append(next.getMessage()); - sb.append(". "); - } - //判断校验是否成功，如果为true成功 - Validate.isTrue(processingReport.isSuccess(), sb.toString()); - } catch (ProcessingException e) { - log.error("ProcessingException information: ", e.getMessage()); - } catch (IOException e) { - log.error("IOException information: ", e.getMessage()); - } - } -} diff --git a/src/test/java/com/mesalab/qgw/service/ApplicationAndProtocolTest.java b/src/test/java/com/mesalab/qgw/service/ApplicationAndProtocolTest.java new file mode 100644 index 00000000..dffa9959 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/ApplicationAndProtocolTest.java @@ -0,0 +1,113 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.context.request.async.DeferredResult; + +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class ApplicationAndProtocolTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + + @Test + public void testAppAndProtocolSummary() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_summary", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatusById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testAppAndProtocolTreeComposition() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_tree_composition", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResultById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testAppAndProtocolTreeThroughput() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_tree_throughput", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResult(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testAppAndProtocolTopApp() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_top_apps", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatus(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testAppAndProtocolTreeAppSummaryOneshot() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_app_summary", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 200); + } + + @Test + public void testAppAndProtocolAppRelatedInternalIpsOneshot() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_app_related_internal_ips", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 200); + } + + @Test + public void testAppAndProtocolAppThroughputOneshot() { + DSLQueryRequestParam body = jsonToInParameter("parameters/applicationAndProtocolTest.json", "application_and_protocol_app_throughput", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 200); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/AutoPeriodDSLTest.java b/src/test/java/com/mesalab/qgw/service/AutoPeriodDSLTest.java new file mode 100644 index 00000000..cabe8c5a --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/AutoPeriodDSLTest.java @@ -0,0 +1,204 @@ +package com.mesalab.qgw.service; + +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.google.common.collect.Lists; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.utils.sqlparser.AutoPeriodHelper; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +/** + * TODO + * + * @Classname AutoPeriodTest + * @Date 2024/6/20 09:24 + * @Author wWei + */ +@EnableAutoConfiguration +public class AutoPeriodDSLTest extends GalaxyQGWApplicationTests { + private static final Log log = LogFactory.get(); + private static final DateTime now; + + static { + now = new DateTime(); + } + + @Test + public void testDSLLast5Minutes() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -5), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1S", granularity); + } + + @Test + public void testDSLLast5MinutesISO8601() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -5), "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1S", granularity); + } + + @Test + public void testDSLLast30Minutes() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -30), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT30S", granularity); + } + + @Test + public void testDSLLast1Hour() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -1), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT30S", granularity); + } + + @Test + public void testDSLLast6Hour() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -6), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1M", granularity); + } + + @Test + public void testDSLLast12Hour() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -12), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1M", granularity); + } + + @Test + public void testDSLLast1Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -1), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT5M", granularity); + } + + @Test + public void testDSLLast2Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -2), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT5M", granularity); + } + + @Test + public void testDSLLast3Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -3), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT5M", granularity); + } + + @Test + public void testDSLLast1Week() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetWeek(now, -1), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1H", granularity); + } + + @Test + public void testDSLLast30Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -30), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1H", granularity); + } + + @Test + public void testDSLLast365Day() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -365), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "P1D", granularity); + } + + @Test + public void testDSLConst() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_const", DSLQueryRequestParam.class); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1S", granularity); + } + + @Test + public void testDSLAutoRange() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "application_and_protocol_summary_auto_const_range", DSLQueryRequestParam.class); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "P1D", granularity); + } + + @Test + public void testDSLTrafficSpectrumNetworkThroughputTrend() { + DSLQueryRequestParam dslQueryRequestParam = jsonToInParameter("parameters/dslAutoGranularityTest.json", "traffic_spectrum_network_throughput_trend_auto", DSLQueryRequestParam.class); + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -1), "yyyy-MM-dd HH:mm:ss"); + dslQueryRequestParam.setIntervals(Lists.newArrayList(startTime + "/" + endTime)); + AutoPeriodHelper.buildDslGranularity(dslQueryRequestParam); + String granularity = dslQueryRequestParam.getGranularity(); + log.info(granularity); + Assert.assertEquals("ERROR: don't expect period", "PT1M", granularity); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/AutoPeriodSQLTest.java b/src/test/java/com/mesalab/qgw/service/AutoPeriodSQLTest.java new file mode 100644 index 00000000..214d0554 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/AutoPeriodSQLTest.java @@ -0,0 +1,227 @@ +package com.mesalab.qgw.service; + +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.utils.sqlparser.AutoPeriodHelper; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +/** + * TODO + * + * @Classname AutoPeriodTest + * @Date 2024/6/20 09:24 + * @Author wWei + */ +@EnableAutoConfiguration +public class AutoPeriodSQLTest extends GalaxyQGWApplicationTests { + private static final Log log = LogFactory.get(); + private static final DateTime now; + + static { + now = new DateTime(); + } + + private static final String originalSQL = "SELECT\n" + + " FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY('${start_time}', '${end_time}'), 'zero')) AS stat_time,\n" + + " AVG( in_pkts_per_sec ) AS avg_in_pkts_per_sec\n" + + "FROM\n" + + " (\n" + + " SELECT\n" + + " TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY('${start_time}', '${end_time}')) AS stat_time,\n" + + " RATE( in_pkts, SAMPLE_GRANULARITY('${start_time}', '${end_time}'), 1) AS in_pkts_per_sec\n" + + " FROM\n" + + " traffic_general_stat\n" + + " WHERE\n" + + " __time >= '${start_time}'\n" + + " AND __time < '${end_time}'\n" + + " AND vsys_id IN ( 1)\n" + + " AND ( 1 = 1 )\n" + + " GROUP BY\n" + + " TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY('${start_time}', '${end_time}'))\n" + + " HAVING TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY('${start_time}', '${end_time}')) >= '${start_time}'\n" + + " ORDER BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), SAMPLE_GRANULARITY('${start_time}', '${end_time}')) DESC\n" + + " )\n" + + "GROUP BY\n" + + " FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(stat_time, CHART_GRANULARITY('${start_time}', '${end_time}'), 'zero'))\n" + + "ORDER BY\n" + + " stat_time ASC\n" + + "LIMIT 10"; + + @Test + public void testSQLLast5Minutes() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -5), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: not contains PT1S", afterReplaceSQL.contains("PT1S")); + } + + @Test + public void testSQLLast5MinutesISO8601() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -5), "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: not contains PT1S", afterReplaceSQL.contains("PT1S")); + } + + @Test + public void testSQLLast30Minutes() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetMinute(now, -30), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1S") && afterReplaceSQL.contains("PT30S")); + } + + @Test + public void testSQLLast1Hour() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -1), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1S") && afterReplaceSQL.contains("PT30S")); + } + + @Test + public void testSQLLast6Hour() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -6), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT30S") && afterReplaceSQL.contains("PT1M")); + } + + @Test + public void testSQLLast12Hour() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -6), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT30S") && afterReplaceSQL.contains("PT1M")); + } + + @Test + public void testSQLLast1Day() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -1), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1M") && afterReplaceSQL.contains("PT5M")); + } + + @Test + public void testSQLLast2Day() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -2), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1M") && afterReplaceSQL.contains("PT5M")); + } + + @Test + public void testSQLLast3Day() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -3), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1M") && afterReplaceSQL.contains("PT5M")); + } + + @Test + public void testSQLLast1Week() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetWeek(now, -1), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT5M") && afterReplaceSQL.contains("PT1H")); + } + + @Test + public void testSQLLast1Month() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -30), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT5M") && afterReplaceSQL.contains("PT1H")); + } + + @Test + public void testSQLLast365Day() throws JSQLParserException { + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetDay(now, -365), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = originalSQL.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT1H") && afterReplaceSQL.contains("P1D")); + } + + @Test + public void testSQLDrillDown() throws JSQLParserException { + String sql = "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY('${start_time}', '${end_time}'), 'zero')) AS \"Time\", device_group AS \"Device Group\", RATE(sessions, CHART_GRANULARITY('${start_time}', '${end_time}'), 1) AS \"Sessions\" FROM druid.statistics_rule WHERE __time >= '${start_time}' AND __time < '${end_time}' AND rule_id = 787368 AND template_id = 40010 AND chart_id = 44044 AND version = 1 AND device_group IN (SELECT device_group FROM statistics_rule WHERE __time >= '${start_time}' AND __time < '${end_time}' AND rule_id = 787368 AND template_id = 40010 AND chart_id = 44044 AND version = 1 AND statistics_rule.vsys_id IN (1) GROUP BY device_group ORDER BY RATE(sessions, CHART_GRANULARITY('${start_time}', '${end_time}'), 1) DESC LIMIT 10) AND statistics_rule.vsys_id IN (1) GROUP BY device_group, FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time), CHART_GRANULARITY('${start_time}', '${end_time}'), 'zero')) ORDER BY \"Time\" ASC LIMIT 65536"; + String endTime = DateUtil.format(now, "yyyy-MM-dd HH:mm:ss"); + String startTime = DateUtil.format(DateUtil.offsetHour(now, -1), "yyyy-MM-dd HH:mm:ss"); + String afterReplaceSQL = sql.replace("${start_time}", startTime); + afterReplaceSQL = afterReplaceSQL.replace("${end_time}", endTime); + Statement statement = CCJSqlParserUtil.parse(afterReplaceSQL); + afterReplaceSQL = AutoPeriodHelper.buildSqlGranularity(statement).toString(); + log.info(afterReplaceSQL); + Assert.assertFalse("ERROR: contains CHART/SAMPLE_GRANULARITY", afterReplaceSQL.contains("CHART_GRANULARITY") || afterReplaceSQL.contains("SAMPLE_GRANULARITY")); + Assert.assertTrue("ERROR: don't expect period", afterReplaceSQL.contains("PT30S")); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/AvroSchemaFormatTest.java b/src/test/java/com/mesalab/qgw/service/AvroSchemaFormatTest.java deleted file mode 100644 index efffa34d..00000000 --- a/src/test/java/com/mesalab/qgw/service/AvroSchemaFormatTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.mesalab.qgw.service; - -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import org.apache.http.HttpStatus; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -import static org.junit.Assert.assertEquals; - -@EnableAutoConfiguration -public class AvroSchemaFormatTest extends GalaxyQGWApplicationTests { - private static final Log log = LogFactory.get(); - @Autowired - public DiagnosisService diagnosisService; - @Before - public void testBefore() { - log.info("=================================================SCHEMA TEST BEGIN================================================="); - } - - @After - public void testAfter() { - log.info("=================================================SCHEMA TEST END================================================="); - } - - /** - * 测试schema - */ - @Test - public void testCheckSchema() { - BaseResult baseResult = diagnosisService.validateSchema(); - assertEquals("Schema format failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); - } - -} diff --git a/src/test/java/com/mesalab/qgw/service/ClickHouseTest.java b/src/test/java/com/mesalab/qgw/service/ClickHouseTest.java index 4e9dd11d..f6500835 100644 --- a/src/test/java/com/mesalab/qgw/service/ClickHouseTest.java +++ b/src/test/java/com/mesalab/qgw/service/ClickHouseTest.java @@ -8,7 +8,7 @@ import cn.hutool.log.LogFactory; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.enums.DBTypeEnum; +import com.mesalab.common.enums.DBEngineType; import com.mesalab.qgw.benchmark.DialectWriter; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Optional; //clickhouse-benchmark -i 93 --host 127.0.0.1 --port 9001 --user default --password "ceiec2019" --database tsg_galaxy_v3 < /root/session_record_queries_20210113203103.sql @@ -38,7 +37,7 @@ public class ClickHouseTest extends GalaxyQGWApplicationTests { @Test public void testSqlParser() { - String pocFile = dialectWriter.buildPocSQL(Lists.newLinkedList(), DBTypeEnum.CLICKHOUSE.getValue(), null); + String pocFile = dialectWriter.buildPocSQL(Lists.newLinkedList(), DBEngineType.CLICKHOUSE.getValue(), null); Optional<File> ckTestFile = Optional.of(new File(pocFile)); if (ckTestFile.isPresent()) { if (!ckTestFile.get().isFile()) { diff --git a/src/test/java/com/mesalab/qgw/service/DruidTest.java b/src/test/java/com/mesalab/qgw/service/DruidTest.java index a8b4fb51..55ffd62a 100644 --- a/src/test/java/com/mesalab/qgw/service/DruidTest.java +++ b/src/test/java/com/mesalab/qgw/service/DruidTest.java @@ -8,7 +8,7 @@ import cn.hutool.log.LogFactory; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.enums.DBTypeEnum; +import com.mesalab.common.enums.DBEngineType; import com.mesalab.qgw.benchmark.DialectWriter; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.parser.CCJSqlParserUtil; @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Optional; @@ -35,7 +34,7 @@ public class DruidTest extends GalaxyQGWApplicationTests { @Test public void testSqlParser() { - String pocFile = dialectWriter.buildPocSQL(Lists.newLinkedList(), DBTypeEnum.DRUID.getValue(), null); + String pocFile = dialectWriter.buildPocSQL(Lists.newLinkedList(), DBEngineType.DRUID.getValue(), null); Optional<File> ckTestFile = Optional.of(new File(pocFile)); if (ckTestFile.isPresent()) { if (!ckTestFile.get().isFile()) { diff --git a/src/test/java/com/mesalab/qgw/service/EngineTest.java b/src/test/java/com/mesalab/qgw/service/EngineTest.java index 65629173..b69d3e8a 100644 --- a/src/test/java/com/mesalab/qgw/service/EngineTest.java +++ b/src/test/java/com/mesalab/qgw/service/EngineTest.java @@ -4,7 +4,6 @@ import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.mesalab.GalaxyQGWApplicationTests; import com.mesalab.common.entity.BaseResult; -import com.mesalab.common.enums.DiagnosisOptionEnum; import org.apache.http.HttpStatus; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -24,16 +23,5 @@ public class EngineTest extends GalaxyQGWApplicationTests { assertEquals("Schema validate failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); } - @Test - public void testMetadata() { - BaseResult baseResult = diagnosisService.validateMetadata(); - assertEquals("Metadata validate failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); - } - - @Test - public void testPocSql() { - BaseResult baseResult = diagnosisService.runPocSQL(false, DiagnosisOptionEnum.PARSE.getValue(), null); - assertEquals("POC sql parse failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); - } } diff --git a/src/test/java/com/mesalab/qgw/service/EntityTest.java b/src/test/java/com/mesalab/qgw/service/EntityTest.java deleted file mode 100644 index 2c5f498d..00000000 --- a/src/test/java/com/mesalab/qgw/service/EntityTest.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.service.EntityService; -import com.mesalab.services.service.RelationService; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -/** - * @Author wxs - * @Date 2022/9/28 - */ -@EnableAutoConfiguration -public class EntityTest extends GalaxyQGWApplicationTests { - - @Autowired - private EntityService entityService; - - @Autowired - private RelationService relationService; - - @Test - public void activeClientIp() { - - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "activeClientIp", ComDSLObject.class); - BaseResult entityInfo = entityService.getEntityInfo("activeclientip", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - - } - - @Test - public void topServerIp() { - - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "topServerIp", ComDSLObject.class); - BaseResult entityInfo = entityService.getEntityInfo("topserverip", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - - } - - @Test - public void topSni() { - - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "topSni", ComDSLObject.class); - BaseResult entityInfo = entityService.getEntityInfo("topsni", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - - } - - @Test - public void subScriberidPool() { - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "subScriberidPool", ComDSLObject.class); - BaseResult entityInfo = relationService.getRealRelation("subscriberidpool", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - } - - @Test - public void gtpc() { - ComDSLObject dslObject = jsonToInParameter("parameters/entityTest.json", "gtpc", ComDSLObject.class); - BaseResult entityInfo = relationService.getRealRelation("gtpc", dslObject); - Assert.assertTrue(entityInfo.getMessage(), entityInfo.getStatus() == 200); - } -} diff --git a/src/test/java/com/mesalab/qgw/service/ExampleDataTest.java b/src/test/java/com/mesalab/qgw/service/ExampleDataTest.java new file mode 100644 index 00000000..c0f985c3 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/ExampleDataTest.java @@ -0,0 +1,62 @@ +package com.mesalab.qgw.service; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.DataTypeMapping; +import com.mesalab.common.utils.sqlparser.ExampleDataHelper; +import com.mesalab.common.utils.sqlparser.SelectItemHelper; +import com.mesalab.qgw.constant.DataTypeConst; +import com.mesalab.qgw.constant.MetaConst; +import lombok.extern.slf4j.Slf4j; +import net.sf.jsqlparser.JSQLParserException; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +import java.util.List; +import java.util.Map; + + +@Slf4j +@EnableAutoConfiguration +public class ExampleDataTest extends GalaxyQGWApplicationTests { + + @Test + public void testSelectItemHelper() throws JSQLParserException { + String sql = "SELECT filed_1 AS \"Field A\", SUM(field_2) AS \"SUM(field_2)\", MAX(field_3) FROM table GROUP BY filed_1 ORDER BY \"Field A\" DESC LIMIT 10"; + Map<String, SelectItemHelper.AliasObject> selectItem = SelectItemHelper.getSelectItem(sql); + Assert.assertEquals("ok", 3, selectItem.size()); + } + + @Test + public void testExampleDataHelper() { + List<Map<String, String>> meta = initMeta(); + List<Object> objects = ExampleDataHelper.buildExampleData(ExampleDataHelper.QUERY_TYPE_GROUP_BY, meta, 1L, null, true); + Assert.assertEquals("ok", 1, objects.size()); + } + + private static List<Map<String, String>> initMeta() { + List<Map<String, String>> meta = Lists.newArrayList(); + Map<String, String> item1 = Maps.newHashMap(); + item1.put(MetaConst.META_NAME, "Field A"); + item1.put(MetaConst.META_TYPE, DataTypeMapping.STRING); + item1.put(MetaConst.META_CATEGORY, MetaConst.META_CATEGORY_DIMENSION); + item1.put(MetaConst.META_DATA_TYPE, DataTypeConst.IP); + meta.add(item1); + + Map<String, String> item2 = Maps.newHashMap(); + item2.put(MetaConst.META_NAME, "SUM(field_2)"); + item2.put(MetaConst.META_TYPE, DataTypeMapping.LONG); + item2.put(MetaConst.META_CATEGORY, MetaConst.META_CATEGORY_METRIC); + meta.add(item2); + + Map<String, String> item3 = Maps.newHashMap(); + item3.put(MetaConst.META_NAME, "MAX(field_3)"); + item3.put(MetaConst.META_TYPE, DataTypeMapping.LONG); + item3.put(MetaConst.META_CATEGORY, MetaConst.META_CATEGORY_METRIC); + meta.add(item3); + return meta; + } + +} diff --git a/src/test/java/com/mesalab/qgw/service/FieldDiscoveryTest.java b/src/test/java/com/mesalab/qgw/service/FieldDiscoveryTest.java new file mode 100644 index 00000000..db0ba405 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/FieldDiscoveryTest.java @@ -0,0 +1,114 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.context.request.async.DeferredResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class FieldDiscoveryTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + private static final List<String> fields = new ArrayList<>(); + + static { + fields.add("flags"); + fields.add("security_rule_list"); + fields.add("client_port"); + fields.add("server_ip"); + } + + @Test + public void testFieldDiscoveryDefault() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_default", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatusById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testFieldDiscoverySessions() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_sessions", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResultById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testFieldDiscoveryBytes() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_bytes", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResult(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testFieldDiscoveryIncomingBytes() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_incoming_bytes", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatus(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testFieldDiscoveryOutgoingBytes() { + DSLQueryRequestParam body = jsonToInParameter("parameters/fieldDiscoveryTest.json", "field_discovery_outgoing_bytes", DSLQueryRequestParam.class); + body.getCustomRequestParam().put("custom.field_discovery.fields", fields); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/HttpClientServiceTest.java b/src/test/java/com/mesalab/qgw/service/HttpClientServiceTest.java new file mode 100644 index 00000000..d761f4fc --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/HttpClientServiceTest.java @@ -0,0 +1,24 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.enums.HttpStatusCodeEnum; +import com.mesalab.qgw.service.impl.HttpClientServiceV2; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +@EnableAutoConfiguration +public class HttpClientServiceTest extends GalaxyQGWApplicationTests { + + @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") + @Autowired + private HttpClientServiceV2 httpClientServiceV2; + + @Test + public void testHttpGet() { + Assert.assertTrue(httpClientServiceV2.get("https://www.baidu.com", 1000).getStatusCode()==200); + Assert.assertTrue(httpClientServiceV2.get("http://www.baidu.com", 1).getStatusCode() == HttpStatusCodeEnum.GATEWAY_TIMEOUT.getCode()); + Assert.assertTrue(httpClientServiceV2.get("http://fake.www.baidu.com", 1000).getStatusCode() == HttpStatusCodeEnum.BAD_GATEWAY.getCode()); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/JobTest.java b/src/test/java/com/mesalab/qgw/service/JobTest.java deleted file mode 100644 index 4df5e0fd..00000000 --- a/src/test/java/com/mesalab/qgw/service/JobTest.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.controller.JobController; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.web.context.request.async.DeferredResult; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @Author wxs - * @Date 2022/9/28 - */ -@EnableAutoConfiguration -public class JobTest extends GalaxyQGWApplicationTests { - - @Autowired - private JobController jobController; - - @Test - public void fieldDiscoveryDefault() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "field_discovery_default", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - DeferredResult<BaseResult> baseResultDeferredResult = jobController.statisticsResult(id); - if(baseResultDeferredResult.hasResult()){ - BaseResult result = (BaseResult) baseResultDeferredResult.getResult(); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - } - - @Test - public void fieldDiscoveryBytes() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "field_discovery_bytes", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - DeferredResult<BaseResult> baseResultDeferredResult = jobController.statisticsResult(id); - if(baseResultDeferredResult.hasResult()){ - BaseResult result = (BaseResult) baseResultDeferredResult.getResult(); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - } - - @Test - public void longTerm() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "long_term", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - DeferredResult<BaseResult> baseResultDeferredResult = jobController.statisticsResult(id); - if(baseResultDeferredResult.hasResult()){ - BaseResult result = (BaseResult) baseResultDeferredResult.getResult(); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - - } - - @Test - public void report() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "report", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - BaseResult result = jobController.savedQueryStatus(id); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - BaseResult cancelled = jobController.cancelSavedQuery(id); - Assert.assertTrue(cancelled.getMessage(), cancelled.getStatus() == 202); - - } - - @Test - public void statistics() { - HashMap param = jsonToInParameter("parameters/jobTest.json", "statistics_top", HashMap.class); - BaseResult committed = jobController.commit(param); - Assert.assertTrue(committed.getMessage(), committed.getStatus() == 201); - String id = ((List<Map>) committed.getData()).get(0).get("job_id").toString(); - DeferredResult<BaseResult> baseResultDeferredResult = jobController.statisticsResult(id); - if(baseResultDeferredResult.hasResult()){ - BaseResult result = (BaseResult) baseResultDeferredResult.getResult(); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - } - -// @Test -// public void getResultReport() { -// BaseResult result = jobController.getResultReport(jobId); -// Assert.assertTrue(result.getMessage(), result.getStatus() == 200); -// } - -} diff --git a/src/test/java/com/mesalab/qgw/service/JsonSchemaValidatorTest.java b/src/test/java/com/mesalab/qgw/service/JsonSchemaValidatorTest.java new file mode 100644 index 00000000..a4352c20 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/JsonSchemaValidatorTest.java @@ -0,0 +1,86 @@ +package com.mesalab.qgw.service; + + +import cn.hutool.core.io.resource.ClassPathResource; +import cn.hutool.core.io.resource.Resource; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.utils.JsonSchemaValidator; +import com.mesalab.qgw.exception.QGWBusinessException; +import org.apache.commons.io.IOUtils; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.test.context.TestPropertySource; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; + + +@EnableAutoConfiguration +@TestPropertySource("classpath:") +public class JsonSchemaValidatorTest extends GalaxyQGWApplicationTests { + @Autowired + DatabaseService databaseService; + + @Test + public void testJsonSchema() { + try { + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + Map<String, Object> schemaMap = databaseService.getSchemaInfo("fields", "session_record", false); + boolean isValidSuccess = jsonSchemaValidator + .addRule("schema-syntax-validation.json", "schema") + .validateSchema(com.alibaba.fastjson2.JSON.toJSONString(schemaMap)); + Assert.assertTrue(isValidSuccess); + } catch (QGWBusinessException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testInvalidJsonSchema() { + boolean isValidError = true; + try { + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + Resource resource = new ClassPathResource("examples/invalidSessionRecordTest.json"); + String jsonStr = new String(IOUtils.toByteArray(resource.getStream()), StandardCharsets.UTF_8); + jsonSchemaValidator + .addRule("schema-syntax-validation.json", "schema") + .validateSchema(jsonStr); + } catch (Exception e) { + isValidError = false; + } + Assert.assertFalse(isValidError); + } + + @Test + public void testDSLRequest() { + try { + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + Resource resource = new ClassPathResource("examples/validDSLRequestTest.json"); + String jsonStr = new String(IOUtils.toByteArray(resource.getStream()), StandardCharsets.UTF_8); + boolean isValidSuccess = jsonSchemaValidator + .addRule("dsl-validation.json", "dsl") + .validateDSL(jsonStr); + Assert.assertTrue(isValidSuccess); + } catch (QGWBusinessException | IOException e) { + Assert.fail(e.getMessage()); + } + } + + @Test + public void testInvalidDSLRequest() { + boolean isValidError = true; + try { + JsonSchemaValidator jsonSchemaValidator = JsonSchemaValidator.getInstance(); + Resource resource = new ClassPathResource("examples/invalidDSLRequestTest.json"); + String jsonStr = new String(IOUtils.toByteArray(resource.getStream()), StandardCharsets.UTF_8); + jsonSchemaValidator + .addRule("dsl-validation.json", "dsl") + .validateDSL(jsonStr); + } catch (Exception e) { + isValidError = false; + } + Assert.assertFalse(isValidError); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/KnowledgeBaseTest.java b/src/test/java/com/mesalab/qgw/service/KnowledgeBaseTest.java index 583f2e6f..2164e03b 100644 --- a/src/test/java/com/mesalab/qgw/service/KnowledgeBaseTest.java +++ b/src/test/java/com/mesalab/qgw/service/KnowledgeBaseTest.java @@ -2,10 +2,9 @@ package com.mesalab.qgw.service; import com.mesalab.GalaxyQGWApplicationTests; import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.entity.KnowledgeBase; +import com.mesalab.services.common.entity.KnowledgeBaseRequest; import com.mesalab.services.common.entity.KnowledgeConstant; -import com.mesalab.services.service.KnowledgeBaseService; -import org.apache.http.entity.ContentType; +import com.mesalab.services.service.KBService; import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -15,7 +14,6 @@ import org.springframework.web.multipart.MultipartFile; import java.io.File; import java.io.FileInputStream; -import java.util.Map; /** * @Author wxs @@ -23,58 +21,53 @@ import java.util.Map; */ @EnableAutoConfiguration public class KnowledgeBaseTest extends GalaxyQGWApplicationTests { + @Autowired - private KnowledgeBaseService knowledgeService; + private KBService kbService; + - private String id; @Test public void query() { - BaseResult result = knowledgeService.queryKnowledge(); + BaseResult result = kbService.getList(null, null); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); } @Test public void cycle() throws Exception { publish(); - update(); - toggle(); + updateFile(); + updateStatus(); delete(); + } private void publish() throws Exception { - KnowledgeBase knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "publishTest", KnowledgeBase.class); - File file = new File(KnowledgeConstant.DAT + File.separator + KnowledgeConstant.IP_BUILTIN + KnowledgeConstant.MMDB); + KnowledgeBaseRequest knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "publish", KnowledgeBaseRequest.class); + File file = new File(KnowledgeConstant.DAT + File.separator + KnowledgeConstant.ASN_BUILTIN + KnowledgeConstant.MMDB); FileInputStream inputStream = new FileInputStream(file); - MultipartFile multipartFile = new MockMultipartFile(file.getName(), file.getName(), - ContentType.APPLICATION_OCTET_STREAM.toString(), inputStream); - BaseResult result = knowledgeService.publishKnowledge(multipartFile, knowledgeBase); + MultipartFile multipartFile = new MockMultipartFile(file.getName(), inputStream); + BaseResult result = kbService.publishKnowledge(multipartFile, knowledgeBase); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - id = ((Map) result.getData()).get("id").toString(); } - private void update() throws Exception { - File file = new File(KnowledgeConstant.DAT + File.separator + KnowledgeConstant.IP_BUILTIN + KnowledgeConstant.MMDB); + private void updateFile() throws Exception { + KnowledgeBaseRequest knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "update", KnowledgeBaseRequest.class); + File file = new File(KnowledgeConstant.DAT + File.separator + KnowledgeConstant.ASN_BUILTIN + KnowledgeConstant.MMDB); FileInputStream inputStream = new FileInputStream(file); - MultipartFile multipartFile = new MockMultipartFile(file.getName(), file.getName(), - ContentType.APPLICATION_OCTET_STREAM.toString(), inputStream); - KnowledgeBase knowledgeBase = new KnowledgeBase(); - knowledgeBase.setId(id); - BaseResult result = knowledgeService.updateKnowledge(multipartFile, knowledgeBase); + MultipartFile multipartFile = new MockMultipartFile(file.getName(), inputStream); + BaseResult result = kbService.updateKnowledge(multipartFile, knowledgeBase); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); } - private void toggle() throws Exception{ - KnowledgeBase knowledgeBase = new KnowledgeBase(); - knowledgeBase.setId(id); - knowledgeBase.setIsValid(KnowledgeConstant.VALID); - BaseResult result = knowledgeService.toggleKnowledge(knowledgeBase); + private void updateStatus() { + KnowledgeBaseRequest knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "update_status", KnowledgeBaseRequest.class); + BaseResult result = kbService.updateStatus(knowledgeBase); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); } - private void delete() throws Exception { - KnowledgeBase knowledgeBase = new KnowledgeBase(); - knowledgeBase.setId(id); - BaseResult result = knowledgeService.deleteKnowledge(id); + private void delete() { + KnowledgeBaseRequest knowledgeBase = jsonToInParameter("parameters/knowledgeBase.json", "delete", KnowledgeBaseRequest.class); + BaseResult result = kbService.deleteKnowledge(knowledgeBase); Assert.assertTrue(result.getMessage(), result.getStatus() == 200); } } diff --git a/src/test/java/com/mesalab/qgw/service/MergeFunctionsHelperTest.java b/src/test/java/com/mesalab/qgw/service/MergeFunctionsHelperTest.java new file mode 100644 index 00000000..02a6ff6a --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/MergeFunctionsHelperTest.java @@ -0,0 +1,36 @@ +package com.mesalab.qgw.service; + +import cn.hutool.log.Log; +import cn.hutool.log.LogFactory; +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.utils.sqlparser.FunctionsMergeHelper; +import net.sf.jsqlparser.JSQLParserException; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +/** + * TODO + * + * @Classname MergeFunctionsHelperTest + * @Date 2024/2/24 14:22 + * @Author wWei + */ +@EnableAutoConfiguration +public class MergeFunctionsHelperTest extends GalaxyQGWApplicationTests { + private static final Log log = LogFactory.get(); + + @Test + public void testPlainSelectSQL() throws JSQLParserException { + String originalSQL = "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero')) as a ,TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero') as b ,TIME_FLOOR_WITH_FILL(long_column, 'PT5M', 'zero') as c, SUM(bytes) AS bytes FROM table GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero')) ,TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero'),TIME_FLOOR_WITH_FILL(long_column, 'PT5M', 'zero') order by FROM_UNIXTIME(TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero')) asc ,TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(column), 'PT5M', 'zero') desc ,TIME_FLOOR_WITH_FILL(long_column, 'PT5M', 'zero') desc, bytes asc, a desc limit 1"; + String build = FunctionsMergeHelper.build(originalSQL); + Assert.assertTrue(build.equals("SELECT TIME_FORMAT(TIME_FLOOR(column, 'PT5M'), 'yyyy-MM-dd HH:mm:ss') AS a, TIMESTAMP_TO_MILLIS(TIME_FLOOR(column, 'PT5M')) / 1000 AS b, TIMESTAMP_TO_MILLIS(TIME_FLOOR(MILLIS_TO_TIMESTAMP(long_column * 1000), 'PT5M')) / 1000 AS c, SUM(bytes) AS bytes FROM table GROUP BY TIME_FLOOR(column, 'PT5M'), TIME_FLOOR(column, 'PT5M'), TIME_FLOOR(MILLIS_TO_TIMESTAMP(long_column * 1000), 'PT5M') ORDER BY TIME_FLOOR(column, 'PT5M') ASC, TIME_FLOOR(column, 'PT5M') DESC, TIME_FLOOR(MILLIS_TO_TIMESTAMP(long_column * 1000), 'PT5M') DESC, bytes ASC, TIME_FLOOR(column, 'PT5M') DESC LIMIT 1")); + } + + @Test + public void testNestedSQL() throws JSQLParserException { + String originalSQL = "SELECT FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( stat_time, 'PT5S','zero' )) AS stat_time, SUM(bytes) AS bytes FROM ( SELECT TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1S') AS stat_time, SUM(in_bytes + out_bytes) AS bytes FROM statistics_rule WHERE __time >= '2024-02-01 00:00:00' and __time < '2024-02-02 00:00:00' and rule_id = 301 GROUP BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1S') ORDER BY TIME_FLOOR_WITH_FILL(UNIX_TIMESTAMP(__time),'PT1S') ASC ) GROUP BY FROM_UNIXTIME(TIME_FLOOR_WITH_FILL( stat_time,'PT5S','zero' )) ORDER BY stat_time ASC, bytes DESC LIMIT 10"; + String build = FunctionsMergeHelper.build(originalSQL); + Assert.assertTrue(build.equals("SELECT TIME_FORMAT(TIME_FLOOR(MILLIS_TO_TIMESTAMP(stat_time * 1000), 'PT5S'), 'yyyy-MM-dd HH:mm:ss') AS stat_time, SUM(bytes) AS bytes FROM (SELECT TIMESTAMP_TO_MILLIS(TIME_FLOOR(__time, 'PT1S')) / 1000 AS stat_time, SUM(in_bytes + out_bytes) AS bytes FROM statistics_rule WHERE __time >= '2024-02-01 00:00:00' AND __time < '2024-02-02 00:00:00' AND rule_id = 301 GROUP BY TIME_FLOOR(__time, 'PT1S') ORDER BY TIME_FLOOR(__time, 'PT1S') ASC) GROUP BY TIME_FLOOR(MILLIS_TO_TIMESTAMP(stat_time * 1000), 'PT5S') ORDER BY TIME_FLOOR(MILLIS_TO_TIMESTAMP(stat_time * 1000), 'PT5S') ASC, bytes DESC LIMIT 10")); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/NetworkMonitorTest.java b/src/test/java/com/mesalab/qgw/service/NetworkMonitorTest.java deleted file mode 100644 index e74a4f9d..00000000 --- a/src/test/java/com/mesalab/qgw/service/NetworkMonitorTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.network.dsl.DSLObject; -import com.mesalab.network.service.NetworkMonitorService; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -/** - * TODO - * - * @Classname NetworkMonitorTest - * @Date 2022/12/7 09:45 - * @Author wWei - */ -@EnableAutoConfiguration -public class NetworkMonitorTest extends GalaxyQGWApplicationTests { - - @Autowired - NetworkMonitorService networkMonitorService; - - @Test - public void appDataSummary() { - DSLObject dslObject = jsonToInParameter("parameters/networkMonitorTest.json", "appDataSummary", DSLObject.class); - BaseResult result = networkMonitorService.buildAppData(dslObject); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - - @Test - public void internalIPDataSummary() { - DSLObject dslObject = jsonToInParameter("parameters/networkMonitorTest.json", "internalIPDataSummary", DSLObject.class); - BaseResult result = networkMonitorService.buildInternalIPData(dslObject); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - - @Test - public void appDataRateSummary() { - DSLObject dslObject = jsonToInParameter("parameters/networkMonitorTest.json", "appDataRateSummary", DSLObject.class); - BaseResult result = networkMonitorService.buildAppDataRate(dslObject); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } - - @Test - public void appTrafficSummary() { - DSLObject dslObject = jsonToInParameter("parameters/networkMonitorTest.json", "appTrafficSummary", DSLObject.class); - BaseResult result = networkMonitorService.buildAppTraffic(dslObject); - Assert.assertTrue(result.getMessage(), result.getStatus() == 200); - } -} diff --git a/src/test/java/com/mesalab/qgw/service/ProtocolTreeTest.java b/src/test/java/com/mesalab/qgw/service/ProtocolTreeTest.java index 4121e9b5..76ed3ba8 100644 --- a/src/test/java/com/mesalab/qgw/service/ProtocolTreeTest.java +++ b/src/test/java/com/mesalab/qgw/service/ProtocolTreeTest.java @@ -6,7 +6,6 @@ import cn.hutool.log.LogFactory; import com.alibaba.fastjson2.JSON; import com.google.common.base.Stopwatch; import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.network.service.NetworkMonitorService; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.EnableAutoConfiguration; @@ -20,7 +19,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; public class ProtocolTreeTest extends GalaxyQGWApplicationTests { private static final Log log = LogFactory.get(); @Autowired - public NetworkMonitorService networkMonitorService; + public DSLService dslService; private List<Map> getLiveChartsData(String fileName) { File file = new File("benchmark" + File.separator + "live_traffic_chart" + File.separator + fileName); @@ -32,7 +31,7 @@ public class ProtocolTreeTest extends GalaxyQGWApplicationTests { @Test public void test5MinuteTreeOutput() { Stopwatch stopwatch = Stopwatch.createStarted(); - networkMonitorService.buildHierarchicalStructure(networkMonitorService.buildFlatStructure(getLiveChartsData("livecharts-5min.txt"))); + dslService.buildHierarchicalStructure(dslService.buildFlatStructure(getLiveChartsData("livecharts-5min.txt"))); stopwatch.stop(); long millis = stopwatch.elapsed(MILLISECONDS); log.info("time:{} ms", millis); @@ -41,7 +40,7 @@ public class ProtocolTreeTest extends GalaxyQGWApplicationTests { @Test public void test10MinuteTreeOutput() { Stopwatch stopwatch = Stopwatch.createStarted(); - networkMonitorService.buildHierarchicalStructure(networkMonitorService.buildFlatStructure(getLiveChartsData("livecharts-10min.txt"))); + dslService.buildHierarchicalStructure(dslService.buildFlatStructure(getLiveChartsData("livecharts-10min.txt"))); stopwatch.stop(); long millis = stopwatch.elapsed(MILLISECONDS); log.info("time:{} ms", millis); @@ -49,7 +48,7 @@ public class ProtocolTreeTest extends GalaxyQGWApplicationTests { @Test public void test30MinuteTreeOutput() { Stopwatch stopwatch = Stopwatch.createStarted(); - networkMonitorService.buildHierarchicalStructure(networkMonitorService.buildFlatStructure(getLiveChartsData("livecharts-30min.txt"))); + dslService.buildHierarchicalStructure(dslService.buildFlatStructure(getLiveChartsData("livecharts-30min.txt"))); stopwatch.stop(); long millis = stopwatch.elapsed(MILLISECONDS); log.info("time:{} ms", millis); @@ -57,7 +56,7 @@ public class ProtocolTreeTest extends GalaxyQGWApplicationTests { @Test public void test1HourTreeOutput() { Stopwatch stopwatch = Stopwatch.createStarted(); - networkMonitorService.buildHierarchicalStructure(networkMonitorService.buildFlatStructure(getLiveChartsData("livecharts-1h.txt"))); + dslService.buildHierarchicalStructure(dslService.buildFlatStructure(getLiveChartsData("livecharts-1h.txt"))); stopwatch.stop(); long millis = stopwatch.elapsed(MILLISECONDS); log.info("time:{} ms", millis); diff --git a/src/test/java/com/mesalab/qgw/service/RecommendTest.java b/src/test/java/com/mesalab/qgw/service/RecommendTest.java new file mode 100644 index 00000000..497e71e2 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/RecommendTest.java @@ -0,0 +1,48 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.DSLQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.context.request.async.DeferredResult; + +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class RecommendTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + + @Test + public void testIpLearningFqdnRelateIp() { + DSLQueryRequestParam body = jsonToInParameter("parameters/recommendTest.json", "ip_learning_fqdn_relate_ip", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatusById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testIpLearningActiveIpOneshot() { + DSLQueryRequestParam body = jsonToInParameter("parameters/recommendTest.json", "ip_learning_active_ip", DSLQueryRequestParam.class); + BaseResult commitResult = queryController.createDSLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 200); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/SQLAdHocTest.java b/src/test/java/com/mesalab/qgw/service/SQLAdHocTest.java new file mode 100644 index 00000000..5f9083d9 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/SQLAdHocTest.java @@ -0,0 +1,123 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.SqlQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.web.context.request.async.DeferredResult; + +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class SQLAdHocTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + + @Test + public void testQuerySqlDefault() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_default", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatusById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQuerySqlOneshot() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_oneshot", SqlQueryRequestParam.class); + BaseResult baseResult = queryController.createSQLQuery(body); + Assert.assertTrue(baseResult.getMessage(), baseResult.getStatus() == 200); + } + + @Test + public void testQuerySqlNormal() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_normal", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobStatus(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQuerySqlBlocking() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_blocking", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResultById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQuerySqlJson() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_json", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResult(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQuerySqlCsv() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_csv", SqlQueryRequestParam.class); + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + DeferredResult<BaseResult> getResult = new DeferredResult<>(); + while (true) { + if (getResult.isSetOrExpired()) { + break; + } + getResult = queryController.getJobResultById(id, 0); + } + BaseResult result = (BaseResult) getResult.getResult(); + Assert.assertTrue(result.getMessage(), result.getStatus() == 200); + } + + @Test + public void testQueryNoneSubQuery() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlAdHocTest.json", "query_sql_oneshot_error_trigger_sub_query", SqlQueryRequestParam.class); + BaseResult baseResult = queryController.createSQLQuery(body); + Assert.assertTrue(baseResult.getMessage(), baseResult.getStatus() == 200); + } +} diff --git a/src/test/java/com/mesalab/qgw/service/SQLSavedTest.java b/src/test/java/com/mesalab/qgw/service/SQLSavedTest.java new file mode 100644 index 00000000..c6048642 --- /dev/null +++ b/src/test/java/com/mesalab/qgw/service/SQLSavedTest.java @@ -0,0 +1,41 @@ +package com.mesalab.qgw.service; + +import com.mesalab.GalaxyQGWApplicationTests; +import com.mesalab.common.entity.BaseResult; +import com.mesalab.qgw.controller.QueryController; +import com.mesalab.qgw.model.basic.SqlQueryRequestParam; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; + +import java.util.Map; + +/** + * @Author ww + * @Date 2024/1/30 + */ +@EnableAutoConfiguration +public class SQLSavedTest extends GalaxyQGWApplicationTests { + + @Autowired + private QueryController queryController; + @Autowired + private QueryJobService queryJobService; + + @Test + public void testWholeProcess() { + SqlQueryRequestParam body = jsonToInParameter("parameters/sqlSavedTest.json", "default", SqlQueryRequestParam.class); + + BaseResult commitResult = queryController.createSQLQuery(body); + Assert.assertTrue(commitResult.getMessage(), commitResult.getStatus() == 201); + String id = ((Map<String, Object>) commitResult.getJob()).get("job_id").toString(); + + BaseResult statusResult = queryJobService.getSavedQueryStatusById(id); + Assert.assertTrue(statusResult.getMessage(), statusResult.getStatus() == 200); + + BaseResult deletedResult = queryController.deleteJobById(id, 1); + Assert.assertTrue(deletedResult.getMessage(), deletedResult.getStatus() == 202); + + } +}
\ No newline at end of file diff --git a/src/test/java/com/mesalab/qgw/service/AvroSchemaDynamicTest.java b/src/test/java/com/mesalab/qgw/service/SchemaDynamicTest.java index 274e9487..1c7fb876 100644 --- a/src/test/java/com/mesalab/qgw/service/AvroSchemaDynamicTest.java +++ b/src/test/java/com/mesalab/qgw/service/SchemaDynamicTest.java @@ -14,14 +14,14 @@ import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import static org.junit.Assert.assertEquals; @EnableAutoConfiguration -public class AvroSchemaDynamicTest extends GalaxyQGWApplicationTests { +public class SchemaDynamicTest extends GalaxyQGWApplicationTests { private static final Log log = LogFactory.get(); @Autowired public DiagnosisService diagnosisService; @Test public void testValidateMetadata(){ - BaseResult baseResult = diagnosisService.validateMetadata(); - assertEquals("Validate metadata failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); + BaseResult baseResult = diagnosisService.validateSchema(); + assertEquals("Validate schema failure.", String.valueOf(baseResult.getStatus()), String.valueOf(HttpStatus.SC_OK)); } diff --git a/src/test/java/com/mesalab/qgw/service/TopEntityTest.java b/src/test/java/com/mesalab/qgw/service/TopEntityTest.java deleted file mode 100644 index 318cf863..00000000 --- a/src/test/java/com/mesalab/qgw/service/TopEntityTest.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.mesalab.qgw.service; - -import cn.hutool.core.io.FileUtil; -import cn.hutool.core.util.CharsetUtil; -import cn.hutool.core.util.NumberUtil; -import cn.hutool.core.util.StrUtil; -import cn.hutool.log.Log; -import cn.hutool.log.LogFactory; -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.services.service.EntityService; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -import java.io.File; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -@EnableAutoConfiguration -public class TopEntityTest extends GalaxyQGWApplicationTests { - private static final Log log = LogFactory.get(); - - @Autowired - EntityService entityService; - - private List<Map<String, Object>> getEntityData(String fileName) { - File file = new File("benchmark" + File.separator + "entity_dataset" + File.separator + fileName); - List<String> list = FileUtil.readLines(file, CharsetUtil.UTF_8); - List<Map<String, Object>> data = Lists.newArrayList(); - for (String s : list) { - String[] split = StrUtil.split(s, "\t"); - Map<String, Object> map = new HashMap<>(16); - if (split.length >= 2) { - map.put("sessions", NumberUtil.isLong(split[1]) ? split[1] : "0"); - } - if (split.length >= 1) { - map.put("ssl_sni", split[0]); - data.add(map); - } - } - return data; - } - - @Test - public void testCUCCSNIOutput() { - Stopwatch stopwatch = Stopwatch.createStarted(); - entityService.mergeSNI(getEntityData("XJ-CUCC-SNI-Top200w.txt")); - stopwatch.stop(); - long millis = stopwatch.elapsed(MILLISECONDS); - log.info("time:{} ms", millis); - } - - @Test - public void testE21SNIOutput() { - Stopwatch stopwatch = Stopwatch.createStarted(); - entityService.mergeSNI(getEntityData("E21-SNI-Top200w.txt")); - stopwatch.stop(); - long millis = stopwatch.elapsed(MILLISECONDS); - log.info("time:{} ms", millis); - } - - @Test - public void testE21SNI20221011Output() { - Stopwatch stopwatch = Stopwatch.createStarted(); - entityService.mergeSNI(getEntityData("E21-SNI-Top3W-20221011.txt")); - stopwatch.stop(); - long millis = stopwatch.elapsed(MILLISECONDS); - log.info("time:{} ms", millis); - } - - @Test - public void testE21SNI20221020Output() { - Stopwatch stopwatch = Stopwatch.createStarted(); - entityService.mergeSNI(getEntityData("E21-SNI-Top120W-20221020.txt")); - stopwatch.stop(); - long millis = stopwatch.elapsed(MILLISECONDS); - log.info("time:{} ms", millis); - } - - - -} diff --git a/src/test/java/com/mesalab/qgw/service/UnstructuredTest.java b/src/test/java/com/mesalab/qgw/service/UnstructuredTest.java deleted file mode 100644 index 4285bc5a..00000000 --- a/src/test/java/com/mesalab/qgw/service/UnstructuredTest.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.mesalab.qgw.service; - -import com.mesalab.GalaxyQGWApplicationTests; -import com.mesalab.common.entity.BaseResult; -import com.mesalab.services.common.dsl.ComDSLObject; -import com.mesalab.services.service.UnstructuredService; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; - -@EnableAutoConfiguration -public class UnstructuredTest extends GalaxyQGWApplicationTests { - @Autowired - UnstructuredService unstructuredService; - - @Test - public void allFilePath(){ - ComDSLObject dslObject = jsonToInParameter("parameters/unstructuredTest.json", "all", ComDSLObject.class); - BaseResult UnstructuredData = unstructuredService.getUnstructuredData("all", dslObject); - Assert.assertTrue(UnstructuredData.getMessage(), UnstructuredData.getStatus() == 200); - } - - @Test - public void mailFilePath(){ - ComDSLObject dslObject = jsonToInParameter("parameters/unstructuredTest.json", "mail", ComDSLObject.class); - BaseResult UnstructuredData = unstructuredService.getUnstructuredData("mail", dslObject); - Assert.assertTrue(UnstructuredData.getMessage(), UnstructuredData.getStatus() == 200); - } - - @Test - public void pcapFilePath(){ - ComDSLObject dslObject = jsonToInParameter("parameters/unstructuredTest.json", "pcap", ComDSLObject.class); - BaseResult UnstructuredData = unstructuredService.getUnstructuredData("pcap", dslObject); - Assert.assertTrue(UnstructuredData.getMessage(), UnstructuredData.getStatus() == 200); - } - - @Test - public void httpFilePath(){ - ComDSLObject dslObject = jsonToInParameter("parameters/unstructuredTest.json", "http", ComDSLObject.class); - BaseResult UnstructuredData = unstructuredService.getUnstructuredData("http", dslObject); - Assert.assertTrue(UnstructuredData.getMessage(), UnstructuredData.getStatus() == 200); - } -} diff --git a/src/test/resources/examples/invalidDSLRequestTest.json b/src/test/resources/examples/invalidDSLRequestTest.json new file mode 100644 index 00000000..903f8f53 --- /dev/null +++ b/src/test/resources/examples/invalidDSLRequestTest.json @@ -0,0 +1,50 @@ +{ + "query": { + "parameters": { + "intervals": [ + "2024-03-14 00:00:00/2024-03-15 00:00:00" + ], + "limit": "3", + "match": [ + { + "fieldValues": [ + "SSL", + "HTTP", + "DNS" + ], + "type": "exactly" + }, + { + "fieldKey": "FQDN_NAME", + "fieldValues": [ + "itunes.apple", + "itunes.apple.com" + ], + "type": "exactly" + } + ], + "range": [ + { + "fieldKey": "VSYS_ID", + "type": "eq" + }, + { + "fieldKey": "DEPTH", + "fieldValues": [ + 1 + ], + "type": "eq" + }, + { + "fieldKey": "UNIQ_CIP", + "fieldValues": [ + 12 + ], + "type": "gt" + } + ], + "sort": [] + }, + "queryType": "iplearning" + } +}
\ No newline at end of file diff --git a/src/test/resources/examples/invalidSessionRecordTest.json b/src/test/resources/examples/invalidSessionRecordTest.json new file mode 100644 index 00000000..632ab4b0 --- /dev/null +++ b/src/test/resources/examples/invalidSessionRecordTest.json @@ -0,0 +1,6764 @@ +{ + "type": "record", + "name": "session_record", + "namespace": "tsg_galaxy_v3", + "doc": { + "primary_key": "log_id", + "partition_key": "recv_time", + "index_key": [ + "vsys_id", + "security_action", + "proxy_action", + "decoded_as", + "data_center", + "device_group", + "recv_time" + ], + "ttl": 2592000, + "default_ttl": 2592000, + "functions": { + "aggregation": [ + { + "name": "COUNT", + "label": "COUNT", + "function": "count(expr)", + "metric_type": "counter", + "unit": "short" + }, + { + "name": "COUNT_DISTINCT", + "label": "COUNT_DISTINCT", + "function": "COUNT_DISTINCT(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "AVG", + "label": "AVG", + "function": "avg(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "SUM", + "label": "SUM", + "function": "sum(expr)", + "metric_type": "counter", + "unit": "short" + }, + { + "name": "MAX", + "label": "MAX", + "function": "max(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "MIN", + "label": "MIN", + "function": "min(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "MEDIAN", + "label": "MEDIAN", + "function": "MEDIAN(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "MEDIAN_HDR", + "label": "MEDIAN_HDR", + "function": "MEDIAN_HDR(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "QUANTILE", + "label": "QUANTILE", + "function": "QUANTILE(expr,level)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P95_PERCENTILE", + "label": "P95_PERCENTILE", + "function": "QUANTILE(expr,0.95)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P99_PERCENTILE", + "label": "P99_PERCENTILE", + "function": "QUANTILE(expr,0.99)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "QUANTILE_HDR", + "label": "QUANTILE_HDR", + "function": "QUANTILE_HDR(expr,level)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P95_PERCENTILE_HDR", + "label": "P95_PERCENTILE_HDR", + "function": "QUANTILE_HDR(expr,0.95)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "P99_PERCENTILE_HDR", + "label": "P99_PERCENTILE_HDR", + "function": "QUANTILE_HDR(expr,0.99)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "PERCENTILES_HDR", + "label": "PERCENTILES_HDR", + "function": "PERCENTILES_HDR(expr)", + "metric_type": "histogram", + "unit": "short" + }, + { + "name": "APPROX_COUNT_DISTINCT_HLLD", + "label": "COUNT_DISTINCT_HLLD", + "function": "APPROX_COUNT_DISTINCT_HLLD(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "APPROX_COUNT_DISTINCT_DS_HLL", + "label": "COUNT_DISTINCT_DS_HLL", + "function": "APPROX_COUNT_DISTINCT_DS_HLL(expr)", + "metric_type": "gauge", + "unit": "short" + }, + { + "name": "RATE", + "label": "RATE", + "function": "RATE(expr,duration)", + "metric_type": "gauge", + "unit": "sps" + }, + { + "name": "BITRATE", + "label": "BITRATE", + "function": "RATE(expr,duration)*8", + "metric_type": "gauge", + "unit": "bps" + } + ], + "date": [ + { + "name": "UNIX_TIMESTAMP", + "label": "UNIX_TIMESTAMP", + "function": "UNIX_TIMESTAMP(expr)" + }, + { + "name": "UNIX_TIMESTAMP_MILLIS", + "label": "UNIX_TIMESTAMP_MILLIS", + "function": "UNIX_TIMESTAMP_MILLIS(expr)" + }, + { + "name": "FROM_UNIXTIME", + "label": "FROM_UNIXTIME", + "function": "FROM_UNIXTIME(expr)" + }, + { + "name": "FROM_UNIXTIME_MILLIS", + "label": "FROM_UNIXTIME_MILLIS", + "function": "FROM_UNIXTIME_MILLIS(expr)" + }, + { + "name": "DATE_FORMAT", + "label": "DATE_FORMAT", + "function": "DATE_FORMAT(expr,format)" + }, + { + "name": "CONVERT_TZ", + "label": "CONVERT_TZ", + "function": "CONVERT_TZ(expr, from_tz, to_tz)" + }, + { + "name": "TIME_FLOOR_WITH_FILL", + "label": "TIME_FLOOR_WITH_FILL", + "function": "TIME_FLOOR_WITH_FILL(expr,period,fill)" + } + ], + "operator": [ + { + "name": "=", + "label": "=", + "function": "expr = value" + }, + { + "name": "!=", + "label": "!=", + "function": "expr != value" + }, + { + "name": ">", + "label": ">", + "function": "expr > value" + }, + { + "name": "<", + "label": "<", + "function": "expr < value" + }, + { + "name": ">=", + "label": ">=", + "function": "expr >= value" + }, + { + "name": "<=", + "label": "<=", + "function": "expr <= value" + }, + { + "name": "has", + "label": "HAS", + "function": "has(expr, value)" + }, + { + "name": "in", + "label": "IN", + "function": "expr in (values)" + }, + { + "name": "not in", + "label": "NOT IN", + "function": "expr not in (values)" + }, + { + "name": "like", + "label": "LIKE", + "function": "expr like value" + }, + { + "name": "not like", + "label": "NOT LIKE", + "function": "expr not like value" + }, + { + "name": "notEmpty", + "label": "NOT EMPTY", + "function": "notEmpty(expr)" + }, + { + "name": "empty", + "label": "EMPTY", + "function": "empty(expr)" + }, + { + "name": "bitAnd", + "label": "Bitwise AND", + "function": "bitAnd(expr, value)=value" + } + ] + }, + "schema_query": { + "time": [ + "recv_time", + "test", + "start_timestamp_ms", + "end_timestamp_ms", + "processing_time", + "ingestion_time", + "insert_time" + ], + "dimensions": [ + "session_id", + "session_id", + "device_id", + "out_link_id", + "in_link_id", + "data_center", + "sled_ip", + "device_group", + "address_type", + "decoded_as", + "vsys_id", + "flags", + "flags_identify_info", + "security_action", + "security_rule_list", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_action", + "proxy_rule_list", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "in_dest_mac", + "out_dest_mac", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "quic_sni", + "quic_version", + "quic_user_agent", + "ftp_account", + "ftp_url", + "ftp_link_type", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "security_rule_id", + "monitor_rule_id", + "proxy_rule_id", + "statistics_rule_id", + "shaping_rule_id", + "sc_rule_id" + ], + "metrics": [ + "client_ip", + "client_port", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "in_src_mac", + "out_src_mac", + "server_ip", + "server_port", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "in_dest_mac", + "out_dest_mac", + "app", + "decoded_path", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "duration_ms", + "tcp_handshake_latency_ms", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "http_url", + "http_host", + "http_user_agent", + "http_request_content_length", + "http_response_content_length", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_attachment_name", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qtype", + "dns_qclass", + "dns_qname", + "dns_cname", + "dns_response_latency_ms", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "quic_sni", + "quic_user_agent", + "ftp_account", + "ftp_url", + "sip_call_id", + "sip_server", + "ssh_hassh" + ], + "filters": [ + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "processing_time", + "ingestion_time", + "device_id", + "data_center", + "sled_ip", + "device_group", + "address_type", + "decoded_as", + "vsys_id", + "flags", + "flags_identify_info", + "security_action", + "security_rule_list", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "in_src_mac", + "out_src_mac", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "in_dest_mac", + "out_dest_mac", + "app_transition", + "app_debug_info", + "app", + "app_content", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qtype", + "dns_qclass", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "quic_sni", + "quic_version", + "quic_user_agent", + "ftp_account", + "ftp_url", + "ftp_link_type", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_originator_dir", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method", + "internal_ip_list", + "external_ip_list", + "security_rule_id", + "monitor_rule_id", + "proxy_rule_id", + "statistics_rule_id", + "shaping_rule_id", + "sc_rule_id" + ], + "references": { + "aggregation": [ + { + "type": "int", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "long", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "float", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "double", + "functions": "COUNT,COUNT_DISTINCT,AVG,SUM,MAX,MIN,MEDIAN,P95_PERCENTILE,P99_PERCENTILE,RATE" + }, + { + "type": "string", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "date", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "datetime", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "unix_timestamp", + "functions": "COUNT,COUNT_DISTINCT,MAX,MIN" + }, + { + "type": "array", + "functions": "COUNT,COUNT_DISTINCT" + }, + { + "type": "bit", + "functions": "COUNT,COUNT_DISTINCT" + } + ], + "operator": [ + { + "type": "int", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "long", + "functions": "=,!=,>,<,>=,<=,in,not in" + }, + { + "type": "float", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "double", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "string", + "functions": "=,!=,in,not in,like,not like,notEmpty,empty" + }, + { + "type": "date", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "dateTime", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "unix_timestamp", + "functions": "=,!=,>,<,>=,<=" + }, + { + "type": "array", + "functions": "has,notEmpty,empty" + }, + { + "type": "bit", + "functions": "=,!=,bitAnd" + } + ] + }, + "details": { + "general": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info" + ], + "treatment": [ + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes" + ], + "source": [ + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number" + ], + "destination": [ + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain" + ], + "application": [ + "app_transition", + "app", + "app_debug_info", + "app_content", + "fqdn_category_list" + ], + "protocol": [ + "ip_protocol", + "decoded_path", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject", + "ftp_account", + "ftp_url", + "ftp_link_type", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file", + "quic_version", + "quic_sni", + "quic_user_agent", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe" + ], + "transmission": [ + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn" + ], + "other": [ + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc" + ] + } + }, + "data_view": { + "PROXY_INTERCEPT_EVENT_VIEW": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "security_rule_list", + "security_action", + "client_ip", + "client_port", + "server_fqdn", + "app", + "server_ip", + "server_port" + ] + } + }, + "decoded_as": { + "BASE": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "server_ip", + "server_port" + ] + }, + "HTTP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "http_url", + "http_host", + "http_request_line", + "http_response_line", + "http_request_content_length", + "http_request_content_type", + "http_response_content_length", + "http_response_content_type", + "http_request_body", + "http_response_body", + "http_proxy_flag", + "http_sequence", + "http_cookie", + "http_referer", + "http_user_agent", + "http_set_cookie", + "http_version", + "http_status_code", + "http_response_latency_ms", + "http_session_duration_ms", + "http_action_file_size" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "http_url", + "server_ip", + "server_port" + ] + }, + "MAIL": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "mail_protocol_type", + "mail_account", + "mail_from_cmd", + "mail_to_cmd", + "mail_from", + "mail_password", + "mail_to", + "mail_cc", + "mail_bcc", + "mail_subject", + "mail_subject_charset", + "mail_attachment_name", + "mail_attachment_name_charset", + "mail_eml_file" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "mail_from", + "mail_to", + "mail_subject", + "server_ip", + "server_port" + ] + }, + "DNS": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "dns_message_id", + "dns_qr", + "dns_opcode", + "dns_aa", + "dns_tc", + "dns_rd", + "dns_ra", + "dns_rcode", + "dns_qdcount", + "dns_ancount", + "dns_nscount", + "dns_arcount", + "dns_qname", + "dns_qtype", + "dns_qclass", + "dns_cname", + "dns_sub", + "dns_rr", + "dns_response_latency_ms" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "dns_qr", + "dns_qname", + "dns_qtype", + "server_ip", + "server_port" + ] + }, + "SSL": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "ssl_version", + "ssl_sni", + "ssl_san", + "ssl_cn", + "ssl_handshake_latency_ms", + "ssl_ja3_hash", + "ssl_ja3s_hash", + "ssl_cert_issuer", + "ssl_cert_subject", + "ssl_esni_flag", + "ssl_ech_flag" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "ssl_sni", + "server_ip", + "server_port" + ] + }, + "DTLS": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "dtls_cookie", + "dtls_version", + "dtls_sni", + "dtls_san", + "dtls_cn", + "dtls_handshake_latency_ms", + "dtls_ja3_fingerprint", + "dtls_ja3_hash", + "dtls_cert_issuer", + "dtls_cert_subject" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "dtls_sni", + "server_ip", + "server_port" + ] + }, + "QUIC": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "quic_version", + "quic_sni", + "quic_user_agent" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "quic_sni", + "server_ip", + "server_port" + ] + }, + "FTP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "ftp_account", + "ftp_url", + "ftp_link_type" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "ftp_url", + "server_ip", + "server_port" + ] + }, + "SIP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "sip_call_id", + "sip_originator_description", + "sip_responder_description", + "sip_user_agent", + "sip_server", + "sip_originator_sdp_connect_ip", + "sip_originator_sdp_media_port", + "sip_originator_sdp_media_type", + "sip_originator_sdp_content", + "sip_responder_sdp_connect_ip", + "sip_responder_sdp_media_port", + "sip_responder_sdp_media_type", + "sip_responder_sdp_content", + "sip_duration_s", + "sip_bye" + ], + "default_columns": [ + "recv_time", + "client_ip", + "client_port", + "sip_originator_description", + "sip_responder_description", + "sip_call_id", + "server_ip", + "server_port" + ] + }, + "RTP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "rtp_payload_type_c2s", + "rtp_payload_type_s2c", + "rtp_pcap_path", + "rtp_originator_dir" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "rtp_pcap_path", + "rtp_originator_dir" + ] + }, + "RDP": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "rdp_cookie", + "rdp_security_protocol", + "rdp_client_channels", + "rdp_keyboard_layout", + "rdp_client_version", + "rdp_client_name", + "rdp_client_product_id", + "rdp_desktop_width", + "rdp_desktop_height", + "rdp_requested_color_depth", + "rdp_certificate_type", + "rdp_certificate_count", + "rdp_certificate_permanent", + "rdp_encryption_level", + "rdp_encryption_method" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "rdp_client_version", + "rdp_client_name" + ] + }, + "SSH": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "ssh_version", + "ssh_auth_success", + "ssh_client_version", + "ssh_server_version", + "ssh_cipher_alg", + "ssh_mac_alg", + "ssh_compression_alg", + "ssh_kex_alg", + "ssh_host_key_alg", + "ssh_host_key", + "ssh_hassh" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "ssh_auth_success" + ] + }, + "Stratum": { + "columns": [ + "recv_time", + "log_id", + "decoded_as", + "session_id", + "start_timestamp_ms", + "end_timestamp_ms", + "duration_ms", + "tcp_handshake_latency_ms", + "ingestion_time", + "processing_time", + "insert_time", + "device_id", + "out_link_id", + "in_link_id", + "device_tag", + "data_center", + "device_group", + "sled_ip", + "address_type", + "vsys_id", + "t_vsys_id", + "flags", + "flags_identify_info", + "security_rule_list", + "security_action", + "monitor_rule_list", + "shaping_rule_list", + "sc_rule_list", + "statistics_rule_list", + "sc_rsp_raw", + "sc_rsp_decrypted", + "proxy_rule_list", + "proxy_action", + "proxy_pinning_status", + "proxy_intercept_status", + "proxy_passthrough_reason", + "proxy_client_side_latency_ms", + "proxy_server_side_latency_ms", + "proxy_client_side_version", + "proxy_server_side_version", + "proxy_cert_verify", + "proxy_intercept_error", + "monitor_mirrored_pkts", + "monitor_mirrored_bytes", + "client_ip", + "client_port", + "client_os_desc", + "client_geolocation", + "client_asn", + "subscriber_id", + "imei", + "imsi", + "apn", + "phone_number", + "server_ip", + "server_port", + "server_os_desc", + "server_geolocation", + "server_asn", + "server_fqdn", + "server_domain", + "app_transition", + "app", + "app_debug_info", + "app_content", + "ip_protocol", + "decoded_path", + "fqdn_category_list", + "sent_pkts", + "received_pkts", + "sent_bytes", + "received_bytes", + "tcp_c2s_ip_fragments", + "tcp_s2c_ip_fragments", + "tcp_c2s_lost_bytes", + "tcp_s2c_lost_bytes", + "tcp_c2s_o3_pkts", + "tcp_s2c_o3_pkts", + "tcp_c2s_rtx_pkts", + "tcp_s2c_rtx_pkts", + "tcp_c2s_rtx_bytes", + "tcp_s2c_rtx_bytes", + "tcp_rtt_ms", + "tcp_client_isn", + "tcp_server_isn", + "packet_capture_file", + "in_src_mac", + "out_src_mac", + "in_dest_mac", + "out_dest_mac", + "encapsulation", + "dup_traffic_flag", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program", + "stratum_mining_subscribe" + ], + "default_columns": [ + "recv_time", + "subscriber_id", + "client_port", + "client_ip", + "server_ip", + "server_port", + "stratum_cryptocurrency", + "stratum_mining_pools", + "stratum_mining_program" + ] + } + }, + "default_columns": [ + "recv_time", + "subscriber_id", + "client_ip", + "client_port", + "server_ip", + "server_port", + "decoded_as", + "server_fqdn" + ], + "internal_columns": [ + "recv_time", + "log_id", + "flags_identify_info", + "encapsulation", + "app_debug_info", + "app_content", + "packet_capture_file", + "tunnel_endpoint_a_desc", + "tunnel_endpoint_b_desc" + ], + "tunnel_type": { + "GTP": [ + { + "name": "gtp_endpoint_a_ip", + "label": "Endpoint A IP", + "type": "string" + }, + { + "name": "gtp_endpoint_b_ip", + "label": "Endpoint B IP", + "type": "string" + }, + { + "name": "gtp_endpoint_a_port", + "label": "Endpoint A Port", + "type": "int" + }, + { + "name": "gtp_endpoint_b_port", + "label": "Endpoint B Port", + "type": "int" + }, + { + "name": "gtp_endpoint_a2b_teid", + "label": "Endpoint A2B TEID", + "type": "long" + }, + { + "name": "gtp_endpoint_b2a_teid", + "label": "Endpoint B2A TEID", + "type": "long" + } + ], + "MPLS": [ + { + "name": "mpls_c2s_direction_label", + "label": "Multiprotocol Label (c2s)", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + } + }, + { + "name": "mpls_s2c_direction_label", + "label": "Multiprotocol Label (s2c)", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + } + } + ], + "VLAN": [ + { + "name": "vlan_c2s_direction_id", + "label": "VLAN Direction (c2s)", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + } + }, + { + "name": "vlan_s2c_direction_id", + "label": "VLAN Direction (s2c)", + "type": { + "type": "array", + "items": "int", + "logicalType": "array" + } + } + ], + "ETHERNET": [ + { + "name": "source_mac", + "label": "Source MAC", + "type": "string" + }, + { + "name": "destination_mac", + "label": "Destination MAC", + "type": "string" + } + ], + "MULTIPATH_ETHERNET": [ + { + "name": "c2s_source_mac", + "label": "Source MAC (c2s)", + "type": "string" + }, + { + "name": "c2s_destination_mac", + "label": "Destination MAC (c2s)", + "type": "string" + }, + { + "name": "s2c_source_mac", + "label": "Source MAC (s2c)", + "type": "string" + }, + { + "name": "s2c_destination_mac", + "label": "Destination MAC (s2c)", + "type": "string" + } + ], + "L2TP": [ + { + "name": "l2tp_version", + "label": "Version", + "type": "string" + }, + { + "name": "l2tp_lac2lns_tunnel_id", + "label": "LAC2LNS Tunnel ID", + "type": "int" + }, + { + "name": "l2tp_lns2lac_tunnel_id", + "label": "LNS2LAC Tunnel ID", + "type": "int" + }, + { + "name": "l2tp_lac2lns_session_id", + "label": "LAC2LNS Session ID", + "type": "int" + }, + { + "name": "l2tp_lns2lac_session_id", + "label": "LNS2LAC Session ID", + "type": "int" + }, + { + "name": "l2tp_access_concentrator_ip", + "label": "Access Concentrator IP", + "type": "string" + }, + { + "name": "l2tp_access_concentrator_port", + "label": "Access Concentrator Port", + "type": "int" + }, + { + "name": "l2tp_network_server_ip", + "label": "Network Server IP", + "type": "string" + }, + { + "name": "l2tp_network_server_port", + "label": "Network Server Port", + "type": "int" + } + ], + "PPTP": [ + { + "name": "pptp_uplink_tunnel_id", + "label": "UpLink Tunnel ID", + "type": "int" + }, + { + "name": "pptp_downlink_tunnel_id", + "label": "Down Tunnel ID", + "type": "int" + } + ], + "IPv4": [ + { + "name": "client_ip", + "label": "Client IP", + "type": "string" + }, + { + "name": "server_ip", + "label": "Server IP", + "type": "string" + } + ], + "IPv6": [ + { + "name": "client_ip", + "label": "Client IP", + "type": "string" + }, + { + "name": "server_ip", + "label": "Server IP", + "type": "string" + } + ] + }, + "measurements": { + "aggregated_metric_unit": { + "SUM": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes" + }, + "RATE": { + "sent_pkts": "pps", + "received_pkts": "pps", + "sent_bytes": "Bps", + "received_bytes": "Bps", + "tcp_c2s_lost_bytes": "Bps", + "tcp_s2c_lost_bytes": "Bps", + "tcp_c2s_o3_pkts": "pps", + "tcp_s2c_o3_pkts": "pps", + "tcp_c2s_rtx_pkts": "pps", + "tcp_s2c_rtx_pkts": "pps", + "tcp_c2s_rtx_bytes": "Bps", + "tcp_s2c_rtx_bytes": "Bps", + "http_request_content_length": "Bps", + "http_response_content_length": "Bps" + }, + "BITRATE": { + "sent_bytes": "bps", + "received_bytes": "bps", + "tcp_c2s_lost_bytes": "bps", + "tcp_s2c_lost_bytes": "bps", + "tcp_c2s_rtx_bytes": "bps", + "tcp_s2c_rtx_bytes": "bps", + "http_request_content_length": "bps", + "http_response_content_length": "bps" + }, + "MAX": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MIN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "AVG": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "MEDIAN": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P95_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + }, + "P99_PERCENTILE": { + "sent_pkts": "packets", + "received_pkts": "packets", + "sent_bytes": "bytes", + "received_bytes": "bytes", + "duration_ms": "ms", + "tcp_handshake_latency_ms": "ms", + "tcp_c2s_lost_bytes": "bytes", + "tcp_s2c_lost_bytes": "bytes", + "tcp_c2s_o3_pkts": "packets", + "tcp_s2c_o3_pkts": "packets", + "tcp_c2s_rtx_pkts": "packets", + "tcp_s2c_rtx_pkts": "packets", + "tcp_c2s_rtx_bytes": "bytes", + "tcp_s2c_rtx_bytes": "bytes", + "tcp_rtt_ms": "ms", + "http_request_content_length": "bytes", + "http_response_content_length": "bytes", + "http_response_latency_ms": "ms", + "http_session_duration_ms": "ms", + "dtls_handshake_latency_ms": "ms", + "dns_response_latency_ms": "ms", + "ssl_handshake_latency_ms": "ms" + } + }, + "field_discovery_metric": { + "sessions": [ + { + "fn": "count", + "column": "log_id", + "value": "sessions", + "label": "Sessions", + "unit": "sessions" + } + ], + "bytes": [ + { + "fn": "sum", + "column": "sent_bytes + received_bytes", + "value": "bytes", + "label": "Bytes", + "unit": "bytes" + } + ], + "incoming_bytes": [ + { + "fn": "sum", + "column": "if(bitAnd(flags, 8) = 8, received_bytes, sent_bytes)", + "value": "incoming_bytes", + "label": "Incoming Bytes", + "unit": "bytes" + } + ], + "outgoing_bytes": [ + { + "fn": "sum", + "column": "if(bitAnd(flags, 8) = 8, sent_bytes, received_bytes)", + "value": "outgoing_bytes", + "label": "Outgoing Bytes", + "unit": "bytes" + } + ] + } + }, + "expression_fields": [ + { + "name": "internal_ip_list", + "label": "Internal IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)=8, client_ip,''), if(bitAnd(flags, 16)=16, server_ip,''))" + } + }, + { + "name": "external_ip_list", + "label": "External IP List", + "type": { + "type": "array", + "items": "string", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "expression": "array(if(bitAnd(flags, 8)!=8, client_ip,''), if(bitAnd(flags, 16)!=16, server_ip,''))" + } + }, + { + "name": "security_rule_id", + "label": "Security Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(security_rule_list), arrayPushFront(security_rule_list, null),security_rule_list))" + } + }, + { + "name": "proxy_rule_id", + "label": "Proxy Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(proxy_rule_list), arrayPushFront(proxy_rule_list, null),proxy_rule_list))" + } + }, + { + "name": "monitor_rule_id", + "label": "Monitor Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(monitor_rule_list), arrayPushFront(monitor_rule_list, null),monitor_rule_list))" + } + }, + { + "name": "shaping_rule_id", + "label": "Shaping Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(shaping_rule_list), arrayPushFront(shaping_rule_list, null),shaping_rule_list))" + } + }, + { + "name": "sc_rule_id", + "label": "Service Chaining Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(sc_rule_list), arrayPushFront(sc_rule_list, null),sc_rule_list))" + } + }, + { + "name": "statistics_rule_id", + "label": "Statistics Rule ID", + "type": "long", + "doc": { + "expression": "arrayJoin(IF(empty(statistics_rule_list), arrayPushFront(statistics_rule_list, null),statistics_rule_list))" + } + } + ], + "size": 0 + }, + "fields": [ + { + "name": "recv_time", + "type": { + "type": "long", + "logicalType": "unix_timestamp" + }, + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "label": "Receive Time" + }, + { + "name": "log_id", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Log ID" + }, + { + "name": "decoded_as", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "BASE", + "value": "BASE" + }, + { + "code": "MAIL", + "value": "MAIL" + }, + { + "code": "DNS", + "value": "DNS" + }, + { + "code": "HTTP", + "value": "HTTP" + }, + { + "code": "SSL", + "value": "SSL" + }, + { + "code": "DTLS", + "value": "DTLS" + }, + { + "code": "QUIC", + "value": "QUIC" + }, + { + "code": "FTP", + "value": "FTP" + }, + { + "code": "SSH", + "value": "SSH" + }, + { + "code": "Stratum", + "value": "Stratum" + }, + { + "code": "RDP", + "value": "RDP" + }, + { + "code": "SIP", + "value": "SIP" + }, + { + "code": "RTP", + "value": "RTP" + } + ], + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "label": "Decoded AS" + }, + { + "name": "session_id", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Session ID" + }, + { + "name": "start_timestamp_ms", + "type": { + "type": "string", + "logicalType": "datetime64" + }, + "doc": { + "constraints": { + "operator_functions": "=,!=,>,<,>=,<=", + "type": "datetime64" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Start Time" + }, + { + "name": "end_timestamp_ms", + "type": { + "type": "string", + "logicalType": "datetime64" + }, + "doc": { + "constraints": { + "operator_functions": "=,!=,>,<,>=,<=", + "type": "datetime64" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "End Time" + }, + { + "name": "duration_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Duration (ms)" + }, + { + "name": "tcp_handshake_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "TCP Handshake Latency (ms)" + }, + { + "name": "processing_time", + "type": { + "type": "long", + "logicalType": "unix_timestamp" + }, + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Processing Time" + }, + { + "name": "ingestion_time", + "type": { + "type": "long", + "logicalType": "unix_timestamp" + }, + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Ingestion Time" + }, + { + "name": "insert_time", + "type": { + "type": "long", + "logicalType": "unix_timestamp" + }, + "doc": { + "constraints": { + "type": "unix_timestamp" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Insert Time" + }, + { + "name": "device_id", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Device ID" + }, + { + "name": "out_link_id", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Outgoing Link ID" + }, + { + "name": "in_link_id", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Incoming Link ID" + }, + { + "name": "device_tag", + "type": "string", + "doc": { + "visibility": "hidden", + "ttl": 2592000, + "size": 0 + }, + "label": "Device Tag" + }, + { + "name": "data_center", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [], + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "label": "Data Center" + }, + { + "name": "device_group", + "type": "string", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "City A", + "value": "City A" + }, + { + "code": "City B", + "value": "City B" + }, + { + "code": "City C", + "value": "City C" + }, + { + "code": "City D", + "value": "City D" + }, + { + "code": "City E", + "value": "City E" + }, + { + "code": "City F", + "value": "City F" + }, + { + "code": "City G", + "value": "City G" + }, + { + "code": "City H", + "value": "City H" + }, + { + "code": "City I", + "value": "City I" + }, + { + "code": "City J", + "value": "City J" + }, + { + "code": "City K", + "value": "City K" + }, + { + "code": "City L", + "value": "City L" + }, + { + "code": "City M", + "value": "City M" + }, + { + "code": "City N", + "value": "City N" + } + ], + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "label": "Device Group" + }, + { + "name": "sled_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Sled IP" + }, + { + "name": "address_type", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "4", + "value": "ipv4" + }, + { + "code": "6", + "value": "ipv6" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Address Type" + }, + { + "name": "vsys_id", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": null, + "size": 0 + }, + "default": 1, + "label": "Vsys ID" + }, + { + "name": "t_vsys_id", + "type": "int", + "doc": { + "allow_query": "false", + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Traffic Vsys ID" + }, + { + "name": "flags", + "type": { + "type": "long", + "logicalType": "bit" + }, + "doc": { + "constraints": { + "type": "bit", + "operator_functions": "=,!=,bitAnd" + }, + "data": [ + { + "code": "1", + "value": "Asymmetric" + }, + { + "code": "2", + "value": "Bulky" + }, + { + "code": "4", + "value": "CBR Streaming" + }, + { + "code": "8", + "value": "Client is Local" + }, + { + "code": "16", + "value": "Server is Local" + }, + { + "code": "32", + "value": "Download" + }, + { + "code": "64", + "value": "Interactive" + }, + { + "code": "128", + "value": "Inbound" + }, + { + "code": "256", + "value": "Outbound" + }, + { + "code": "512", + "value": "Pseudo Unidirectional" + }, + { + "code": "1024", + "value": "Streaming" + }, + { + "code": "2048", + "value": "Unidirectional" + }, + { + "code": "4096", + "value": "Random looking" + }, + { + "code": "8192", + "value": "C2S" + }, + { + "code": "16384", + "value": "S2C" + }, + { + "code": "32768", + "value": "Bidirectional" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Flags" + }, + { + "name": "flags_identify_info", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Flags Identify Info" + }, + { + "name": "security_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Security Rule List" + }, + { + "name": "security_action", + "type": "string", + "doc": { + "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "Deny", + "value": "Deny" + }, + { + "code": "Allow", + "value": "Allow" + } + ], + "ttl": null, + "size": 0 + }, + "label": "Security Action" + }, + { + "name": "monitor_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Monitor Rule List" + }, + { + "name": "sc_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Service Chaining Rule List" + }, + { + "name": "statistics_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Statistics Rule List" + }, + { + "name": "sc_rsp_raw", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Service Chaining Rendered Service Path (Raw)" + }, + { + "name": "sc_rsp_decrypted", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Service Chaining Rendered Service Path (Decrypted)" + }, + { + "name": "shaping_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Shaping Rule List" + }, + { + "name": "proxy_rule_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Rule List" + }, + { + "name": "proxy_action", + "type": "string", + "doc": { + "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "Intercept", + "value": "Intercept" + }, + { + "code": "No Intercept", + "value": "No Intercept" + } + ], + "ttl": null, + "size": 0 + }, + "label": "Proxy Action" + }, + { + "name": "proxy_pinning_status", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "not pinning" + }, + { + "code": "1", + "value": "pinning" + }, + { + "code": "2", + "value": "maybe pinning" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Pinning Status" + }, + { + "name": "proxy_intercept_status", + "type": "int", + "doc": { + "data": [ + { + "code": "0", + "value": "passthrough" + }, + { + "code": "1", + "value": "intercept" + }, + { + "code": "2", + "value": "shutdown" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Intercept Status" + }, + { + "name": "proxy_passthrough_reason", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Passthrough Reason" + }, + { + "name": "proxy_server_side_latency_ms", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Server-Side Latency (ms)" + }, + { + "name": "proxy_client_side_latency_ms", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Client-Side Latency (ms)" + }, + { + "name": "proxy_client_side_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Client-Side Version" + }, + { + "name": "proxy_server_side_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Server-Side Version" + }, + { + "name": "proxy_cert_verify", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Certificate Verify" + }, + { + "name": "proxy_intercept_error", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Proxy Intercept Error" + }, + { + "name": "monitor_mirrored_pkts", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Monitor Mirrored Packets" + }, + { + "name": "monitor_mirrored_bytes", + "type": "int", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Monitor Mirrored Bytes" + }, + { + "name": "client_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client IP" + }, + { + "name": "client_port", + "type": "int", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client Port" + }, + { + "name": "client_os_desc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client OS Description" + }, + { + "name": "client_geolocation", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client Geolocation" + }, + { + "name": "client_asn", + "type": "long", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client ASN" + }, + { + "name": "subscriber_id", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Subscriber ID" + }, + { + "name": "imei", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "IMEI" + }, + { + "name": "imsi", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "IMSI" + }, + { + "name": "apn", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "APN" + }, + { + "name": "phone_number", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Phone Number" + }, + { + "name": "server_ip", + "type": "string", + "doc": { + "constraints": { + "type": "ip" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server IP" + }, + { + "name": "server_port", + "type": "int", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server Port" + }, + { + "name": "server_os_desc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server OS Description" + }, + { + "name": "server_geolocation", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server Geolocation" + }, + { + "name": "server_asn", + "type": "long", + "doc": { + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server ASN" + }, + { + "name": "server_fqdn", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server FQDN" + }, + { + "name": "server_domain", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server Domain" + }, + { + "name": "app_transition", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Application Transition" + }, + { + "name": "app", + "type": "string", + "doc": { + "ttl": 2592000, + "size": 0 + }, + "label": "Application" + }, + { + "name": "app_debug_info", + "type": "string", + "doc": { + "visibility": "hidden", + "ttl": 2592000, + "size": 0 + }, + "label": "Application Debug Info" + }, + { + "name": "app_content", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Application Content" + }, + { + "name": "ip_protocol", + "type": "string", + "doc": { + "data": [ + { + "code": "tcp", + "value": "tcp" + }, + { + "code": "udp", + "value": "udp" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "IP Protocol" + }, + { + "name": "decoded_path", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Decoded Path" + }, + { + "name": "fqdn_category_list", + "type": { + "type": "array", + "items": "long", + "logicalType": "array" + }, + "doc": { + "constraints": { + "operator_functions": "has,notEmpty,empty" + }, + "dict_location": { + "path": "/v1/policy/object?type=fqdn_category", + "key": "category_id", + "value": "category_name" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "FQDN Category List" + }, + { + "name": "sent_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Packets Sent" + }, + { + "name": "received_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Packets Received" + }, + { + "name": "sent_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Bytes Sent" + }, + { + "name": "received_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Bytes Received" + }, + { + "name": "tcp_c2s_ip_fragments", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server IP Fragments" + }, + { + "name": "tcp_s2c_ip_fragments", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client IP Fragments" + }, + { + "name": "tcp_c2s_lost_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server Lost Bytes" + }, + { + "name": "tcp_s2c_lost_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client Lost Bytes" + }, + { + "name": "tcp_c2s_o3_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server Out-of-Order Packets" + }, + { + "name": "tcp_s2c_o3_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client Out-of-Order Packets" + }, + { + "name": "tcp_c2s_rtx_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server Retransmission Packets" + }, + { + "name": "tcp_s2c_rtx_pkts", + "type": "long", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client Retransmission Packets" + }, + { + "name": "tcp_c2s_rtx_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client-to-Server Retransmission Bytes" + }, + { + "name": "tcp_s2c_rtx_bytes", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server-to-Client Retransmission Bytes" + }, + { + "name": "tcp_rtt_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Round-trip Time (ms)" + }, + { + "name": "tcp_client_isn", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Client ISN" + }, + { + "name": "tcp_server_isn", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Server ISN" + }, + { + "name": "packet_capture_file", + "type": "string", + "doc": { + "allow_query": "false", + "visibility": "enabled", + "constraints": { + "type": "file" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "Packet Capture File" + }, + { + "name": "in_src_mac", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Incoming Source MAC" + }, + { + "name": "out_src_mac", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Outgoing Source MAC" + }, + { + "name": "in_dest_mac", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Incoming Destination MAC" + }, + { + "name": "out_dest_mac", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Outgoing Destination MAC" + }, + { + "name": "encapsulation", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Encapsulation" + }, + { + "name": "dup_traffic_flag", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "No" + }, + { + "code": "1", + "value": "Yes" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Duplicate Traffic Flag" + }, + { + "name": "tunnel_endpoint_a_desc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Tunnel Endpoint A Description" + }, + { + "name": "tunnel_endpoint_b_desc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Tunnel Endpoint B Description" + }, + { + "name": "http_url", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.URL" + }, + { + "name": "http_host", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Host" + }, + { + "name": "http_request_line", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Request Line" + }, + { + "name": "http_response_line", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Line" + }, + { + "name": "http_request_content_length", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Request Content-Length" + }, + { + "name": "http_request_content_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Request Content-Type" + }, + { + "name": "http_response_content_length", + "type": "long", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Content-Length" + }, + { + "name": "http_response_content_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Content Type" + }, + { + "name": "http_request_body", + "type": "string", + "doc": { + "allow_query": "false", + "constraints": { + "type": "file" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Request Body" + }, + { + "name": "http_response_body", + "type": "string", + "doc": { + "allow_query": "false", + "constraints": { + "type": "file" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Body" + }, + { + "name": "http_proxy_flag", + "type": "int", + "doc": { + "visibility": "hidden", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Proxy Flag" + }, + { + "name": "http_sequence", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Sequence" + }, + { + "name": "http_cookie", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Cookie" + }, + { + "name": "http_referer", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Referer" + }, + { + "name": "http_user_agent", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.User-Agent" + }, + { + "name": "http_set_cookie", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Set-Cookie" + }, + { + "name": "http_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Version" + }, + { + "name": "http_status_code", + "type": "int", + "doc": { + "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Status Code" + }, + { + "name": "http_response_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Response Latency (ms)" + }, + { + "name": "http_action_file_size", + "type": "long", + "doc": { + "constraints": { + "type": "bytes" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Action File Size" + }, + { + "name": "http_session_duration_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "HTTP.Session Duration (ms)" + }, + { + "name": "mail_protocol_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Protocol Type" + }, + { + "name": "mail_account", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Account" + }, + { + "name": "mail_from_cmd", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.From CMD" + }, + { + "name": "mail_to_cmd", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.To CMD" + }, + { + "name": "mail_from", + "type": "string", + "doc": { + "constraints": { + "type": "email" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.From" + }, + { + "name": "mail_password", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Password" + }, + { + "name": "mail_to", + "type": "string", + "doc": { + "constraints": { + "type": "email" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.To" + }, + { + "name": "mail_cc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.CC" + }, + { + "name": "mail_bcc", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.BCC" + }, + { + "name": "mail_subject", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Subject" + }, + { + "name": "mail_subject_charset", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Subject Charset" + }, + { + "name": "mail_attachment_name", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Attachment Name" + }, + { + "name": "mail_attachment_name_charset", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.Attachment Name Charset" + }, + { + "name": "mail_eml_file", + "type": "string", + "doc": { + "constraints": { + "type": "file" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "MAIL.EML File" + }, + { + "name": "dns_message_id", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.Message ID" + }, + { + "name": "dns_qr", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "RESPONSE" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QR" + }, + { + "name": "dns_opcode", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": [ + { + "code": "0", + "value": "QUERY" + }, + { + "code": "1", + "value": "IQUERY" + }, + { + "code": "2", + "value": "STATUS" + }, + { + "code": "5", + "value": "UPDATE" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.OPCODE" + }, + { + "name": "dns_aa", + "type": "int", + "doc": { + "visibility": "enabled", + "constraints": { + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.AA" + }, + { + "name": "dns_tc", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.TC" + }, + { + "name": "dns_rd", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.RD" + }, + { + "name": "dns_ra", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.RA" + }, + { + "name": "dns_rcode", + "type": "int", + "doc": { + "data": [ + { + "code": 0, + "value": "NoError" + }, + { + "code": 1, + "value": "FormErr" + }, + { + "code": 2, + "value": "ServFail" + }, + { + "code": 3, + "value": "NXDomain" + }, + { + "code": 4, + "value": "NotImp" + }, + { + "code": 5, + "value": "Refused" + }, + { + "code": 6, + "value": "YXDomain" + }, + { + "code": 7, + "value": "YXRRSet" + }, + { + "code": 8, + "value": "NXRRSet" + }, + { + "code": 9, + "value": "NotAuth" + }, + { + "code": 10, + "value": "NotZone" + }, + { + "code": 16, + "value": "BADSIG" + }, + { + "code": 17, + "value": "BADKEY" + }, + { + "code": 18, + "value": "BADTIME" + }, + { + "code": 19, + "value": "BADMODE" + }, + { + "code": 20, + "value": "BADNAME" + }, + { + "code": 21, + "value": "BADALG" + } + ], + "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.RCODE" + }, + { + "name": "dns_qdcount", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QDCOUNT" + }, + { + "name": "dns_ancount", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.ANCOUNT" + }, + { + "name": "dns_nscount", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.NSCOUNT" + }, + { + "name": "dns_arcount", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.ARCOUNT" + }, + { + "name": "dns_qname", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QNAME" + }, + { + "name": "dns_qtype", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "data": [ + { + "code": "1", + "value": "A" + }, + { + "code": "2", + "value": "NS" + }, + { + "code": "3", + "value": "MD" + }, + { + "code": "4", + "value": "MF" + }, + { + "code": "5", + "value": "CNAME" + }, + { + "code": "6", + "value": "SOA" + }, + { + "code": "7", + "value": "MB" + }, + { + "code": "8", + "value": "MG" + }, + { + "code": "9", + "value": "MR" + }, + { + "code": "10", + "value": "NULL" + }, + { + "code": "11", + "value": "WKS" + }, + { + "code": "12", + "value": "PTR" + }, + { + "code": "13", + "value": "HINFO" + }, + { + "code": "14", + "value": "MINFO" + }, + { + "code": "15", + "value": "MX" + }, + { + "code": "16", + "value": "TXT" + }, + { + "code": "17", + "value": "RP" + }, + { + "code": "18", + "value": "AFSDB" + }, + { + "code": "19", + "value": "X25" + }, + { + "code": "20", + "value": "ISDN" + }, + { + "code": "21", + "value": "RT" + }, + { + "code": "22", + "value": "NSAP" + }, + { + "code": "23", + "value": "NSAP" + }, + { + "code": "24", + "value": "SIG" + }, + { + "code": "25", + "value": "KEY" + }, + { + "code": "26", + "value": "PX" + }, + { + "code": "27", + "value": "GPOS" + }, + { + "code": "28", + "value": "AAAA" + }, + { + "code": "29", + "value": "LOC" + }, + { + "code": "30", + "value": "EID" + }, + { + "code": "31", + "value": "NIMLOC" + }, + { + "code": "32", + "value": "NB" + }, + { + "code": "33", + "value": "SRV" + }, + { + "code": "34", + "value": "ATMA" + }, + { + "code": "35", + "value": "NAPTR" + }, + { + "code": "36", + "value": "KX" + }, + { + "code": "37", + "value": "CERT" + }, + { + "code": "38", + "value": "A6" + }, + { + "code": "39", + "value": "DNAME" + }, + { + "code": "40", + "value": "SINK" + }, + { + "code": "41", + "value": "OPT" + }, + { + "code": "42", + "value": "APL" + }, + { + "code": "43", + "value": "DS" + }, + { + "code": "44", + "value": "SSHFP" + }, + { + "code": "45", + "value": "IPSECKEY" + }, + { + "code": "46", + "value": "RRSIG" + }, + { + "code": "47", + "value": "NSEC" + }, + { + "code": "48", + "value": "DNSKEY" + }, + { + "code": "49", + "value": "DHCID" + }, + { + "code": "50", + "value": "NSEC3" + }, + { + "code": "51", + "value": "NSEC3PARAM" + }, + { + "code": "52", + "value": "TLSA" + }, + { + "code": "53", + "value": "SMIMEA" + }, + { + "code": "55", + "value": "HIP" + }, + { + "code": "59", + "value": "CDS" + }, + { + "code": "60", + "value": "CDNSKEY" + }, + { + "code": "61", + "value": "OPENPGPKEY" + }, + { + "code": "62", + "value": "CSYNC" + }, + { + "code": "63", + "value": "ZONEMD" + }, + { + "code": "64", + "value": "SVCB" + }, + { + "code": "65", + "value": "HTTPS" + }, + { + "code": "99", + "value": "SPF" + }, + { + "code": "100", + "value": "UINFO" + }, + { + "code": "101", + "value": "UID" + }, + { + "code": "102", + "value": "GID" + }, + { + "code": "103", + "value": "UNSPEC" + }, + { + "code": "108", + "value": "EUI48" + }, + { + "code": "109", + "value": "EUI64" + }, + { + "code": "249", + "value": "TKEY" + }, + { + "code": "250", + "value": "TSIG" + }, + { + "code": "251", + "value": "IXFR" + }, + { + "code": "252", + "value": "AXFR" + }, + { + "code": "253", + "value": "MAILB" + }, + { + "code": "254", + "value": "MAILA" + }, + { + "code": "255", + "value": "*" + }, + { + "code": "256", + "value": "URI" + }, + { + "code": "257", + "value": "CAA" + }, + { + "code": "32768", + "value": "TA" + }, + { + "code": "32769", + "value": "DLV" + }, + { + "code": "65521", + "value": "INTEGRITY" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QTYPE" + }, + { + "name": "dns_qclass", + "type": "int", + "doc": { + "visibility": "enabled", + "constraints": { + "operator_functions": "=,!=,in,not in", + "aggregation_functions": "COUNT, COUNT_DISTINCT" + }, + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.QCLASS" + }, + { + "name": "dns_cname", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.CNAME" + }, + { + "name": "dns_sub", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "1", + "value": "DNS" + }, + { + "code": "2", + "value": "DNSSEC" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.SUB" + }, + { + "name": "dns_rr", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.RR" + }, + { + "name": "dns_response_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DNS.Response Latency (ms)" + }, + { + "name": "ssl_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.Version" + }, + { + "name": "ssl_sni", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.SNI" + }, + { + "name": "ssl_san", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.SAN" + }, + { + "name": "ssl_cn", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.CN" + }, + { + "name": "ssl_handshake_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.Handshake Latency (ms)" + }, + { + "name": "ssl_ja3_hash", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.JA3 Hash" + }, + { + "name": "ssl_ja3s_hash", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.JA3S Hash" + }, + { + "name": "ssl_cert_issuer", + "type": "string", + "doc": { + "constraints": { + "type": "items" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.Issuer" + }, + { + "name": "ssl_cert_subject", + "type": "string", + "doc": { + "constraints": { + "type": "items" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.Subject" + }, + { + "name": "ssl_esni_flag", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSL.ESNI Flag" + }, + { + "name": "ssl_ech_flag", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": " SSL.ECH Flag" + }, + { + "name": "dtls_cookie", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Cookie" + }, + { + "name": "dtls_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Version" + }, + { + "name": "dtls_sni", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.SNI" + }, + { + "name": "dtls_san", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.SAN" + }, + { + "name": "dtls_cn", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.CN" + }, + { + "name": "dtls_handshake_latency_ms", + "type": "int", + "doc": { + "constraints": { + "type": "decimal", + "aggregation_functions": "AVG, MAX, MIN, MEDIAN, P95_PERCENTILE, P99_PERCENTILE" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Handshake Latency (ms)" + }, + { + "name": "dtls_ja3_fingerprint", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.JA3 Fingerprint" + }, + { + "name": "dtls_ja3_hash", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.JA3 Hash" + }, + { + "name": "dtls_cert_issuer", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Certificate Issuer" + }, + { + "name": "dtls_cert_subject", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "DTLS.Certificate Subject" + }, + { + "name": "quic_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "QUIC.Version" + }, + { + "name": "quic_sni", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "QUIC.SNI" + }, + { + "name": "quic_user_agent", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "QUIC.User-Agent" + }, + { + "name": "ftp_account", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "FTP.Account" + }, + { + "name": "ftp_url", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "FTP.URL" + }, + { + "name": "ftp_link_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "FTP.Link Type" + }, + { + "name": "sip_call_id", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Call-ID" + }, + { + "name": "sip_originator_description", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator" + }, + { + "name": "sip_responder_description", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder" + }, + { + "name": "sip_user_agent", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.User-Agent" + }, + { + "name": "sip_server", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Server" + }, + { + "name": "sip_originator_sdp_connect_ip", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator IP" + }, + { + "name": "sip_originator_sdp_media_port", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator Port" + }, + { + "name": "sip_originator_sdp_media_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator Media Type" + }, + { + "name": "sip_originator_sdp_content", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Originator Content" + }, + { + "name": "sip_responder_sdp_connect_ip", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder IP" + }, + { + "name": "sip_responder_sdp_media_port", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder Port" + }, + { + "name": "sip_responder_sdp_media_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder Media Type" + }, + { + "name": "sip_responder_sdp_content", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Responder Content" + }, + { + "name": "sip_duration_s", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Duration (s)" + }, + { + "name": "sip_bye", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SIP.Bye" + }, + { + "name": "rtp_payload_type_c2s", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RTP.Payload Type (C2S)" + }, + { + "name": "rtp_payload_type_s2c", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "PCMU" + }, + { + "code": "1", + "value": "1016" + }, + { + "code": "2", + "value": "G721" + }, + { + "code": "3", + "value": "GSM" + }, + { + "code": "4", + "value": "G723" + }, + { + "code": "5", + "value": "DVI4_8000" + }, + { + "code": "6", + "value": "DVI4_16000" + }, + { + "code": "7", + "value": "LPC" + }, + { + "code": "8", + "value": "PCMA" + }, + { + "code": "9", + "value": "G722" + }, + { + "code": "10", + "value": "L16_STEREO" + }, + { + "code": "11", + "value": "L16_MONO" + }, + { + "code": "12", + "value": "QCELP" + }, + { + "code": "13", + "value": "CN" + }, + { + "code": "14", + "value": "MPA" + }, + { + "code": "15", + "value": "G728" + }, + { + "code": "16", + "value": "DVI4_11025" + }, + { + "code": "17", + "value": "DVI4_22050" + }, + { + "code": "18", + "value": "G729" + }, + { + "code": "19", + "value": "CN_OLD" + }, + { + "code": "25", + "value": "CELB" + }, + { + "code": "26", + "value": "JPEG" + }, + { + "code": "28", + "value": "NV" + }, + { + "code": "31", + "value": "H261" + }, + { + "code": "32", + "value": "MPV" + }, + { + "code": "33", + "value": "MP2T" + }, + { + "code": "34", + "value": "H263" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RTP.Payload Type (S2C)" + }, + { + "name": "rtp_pcap_path", + "type": "string", + "doc": { + "allow_query": "false", + "constraints": { + "type": "file" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RTP.PCAP" + }, + { + "name": "rtp_originator_dir", + "type": "int", + "doc": { + "constraints": { + "operator_functions": "=,!=,in,not in" + }, + "data": [ + { + "code": "0", + "value": "unknown" + }, + { + "code": "1", + "value": "c2s" + }, + { + "code": "2", + "value": "s2c" + } + ], + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RTP.Direction" + }, + { + "name": "ssh_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Version" + }, + { + "name": "ssh_auth_success", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Authentication Result" + }, + { + "name": "ssh_client_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Client Version" + }, + { + "name": "ssh_server_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Server Version" + }, + { + "name": "ssh_cipher_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Encryption Algorithm" + }, + { + "name": "ssh_mac_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Signing Algorithm" + }, + { + "name": "ssh_compression_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Compression Algorithm" + }, + { + "name": "ssh_kex_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Key Exchange Algorithm" + }, + { + "name": "ssh_host_key_alg", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Server Host Key Algorithm" + }, + { + "name": "ssh_host_key", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.Server Key Fingerprint" + }, + { + "name": "ssh_hassh", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "SSH.HASSH" + }, + { + "name": "stratum_cryptocurrency", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Stratum.Cryptocurrency" + }, + { + "name": "stratum_mining_pools", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Stratum.Mining Pools" + }, + { + "name": "stratum_mining_program", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Stratum.Mining Program" + }, + { + "name": "stratum_mining_subscribe", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "Stratum.Mining Subscribe" + }, + { + "name": "rdp_cookie", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Cookie" + }, + { + "name": "rdp_security_protocol", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Security Protocol" + }, + { + "name": "rdp_client_channels", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Client Channels" + }, + { + "name": "rdp_keyboard_layout", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Keyboard Layout" + }, + { + "name": "rdp_client_version", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Client Version" + }, + { + "name": "rdp_client_name", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Client Name" + }, + { + "name": "rdp_client_product_id", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Client Product ID" + }, + { + "name": "rdp_desktop_width", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Desktop Width" + }, + { + "name": "rdp_desktop_height", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Desktop Height" + }, + { + "name": "rdp_requested_color_depth", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Requested Color Depth" + }, + { + "name": "rdp_certificate_type", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Certificate Type" + }, + { + "name": "rdp_certificate_count", + "type": "int", + "doc": { + "constraints": { + "type": "decimal" + }, + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Certificate Count" + }, + { + "name": "rdp_certificate_permanent", + "type": "int", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Certificate Permanent" + }, + { + "name": "rdp_encryption_level", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Encryption Level" + }, + { + "name": "rdp_encryption_method", + "type": "string", + "doc": { + "visibility": "enabled", + "ttl": 2592000, + "size": 0 + }, + "label": "RDP.Encryption Method" + } + ] +} diff --git a/src/test/resources/examples/validDSLRequestTest.json b/src/test/resources/examples/validDSLRequestTest.json new file mode 100644 index 00000000..c7e2225c --- /dev/null +++ b/src/test/resources/examples/validDSLRequestTest.json @@ -0,0 +1,55 @@ +{ + "query": { + "dataSource": "IP_LEARNING_VIEW", + "parameters": { + "intervals": [ + "2024-03-14 00:00:00/2024-03-15 00:00:00" + ], + "limit": "3", + "match": [ + { + "fieldKey": "PROTOCOL", + "fieldValues": [ + "SSL", + "HTTP", + "DNS" + ], + "type": "exactly" + }, + { + "fieldKey": "FQDN_NAME", + "fieldValues": [ + "itunes.apple", + "itunes.apple.com" + ], + "type": "exactly" + } + ], + "range": [ + { + "fieldKey": "VSYS_ID", + "fieldValues": [ + 1 + ], + "type": "eq" + }, + { + "fieldKey": "DEPTH", + "fieldValues": [ + 1 + ], + "type": "eq" + }, + { + "fieldKey": "UNIQ_CIP", + "fieldValues": [ + 12 + ], + "type": "gt" + } + ], + "sort": [] + }, + "queryType": "iplearning" + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/applicationAndProtocolTest.json b/src/test/resources/parameters/applicationAndProtocolTest.json new file mode 100644 index 00000000..2a8c043d --- /dev/null +++ b/src/test/resources/parameters/applicationAndProtocolTest.json @@ -0,0 +1,60 @@ +{ + "application_and_protocol_summary": { + "name": "application-and-protocol-summary", + "granularity": "PT5S", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ] + }, + "application_and_protocol_tree_composition": { + "name": "application-and-protocol-tree-composition", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30T00:00:00+08:00/2024-01-31T00:00:00+08:00" + ] + }, + "application_and_protocol_tree_throughput": { + "name": "application-and-protocol-tree-throughput", + "granularity": "PT1H", + "filter": " (vsys_id = 1) AND (protocol_stack_id = 'ETHERNET.IPv4' OR ( protocol_stack_id LIKE 'ETHERNET.IPv4.%' AND NOT CONTAINS_STRING(REPLACE(protocol_stack_id, 'ETHERNET.IPv4.', ''), '.')))", + "intervals": [ + "2024-01-30T00:00:00.000+01:00/2024-01-31T00:00:00.000+01:00" + ] + }, + "application_and_protocol_top_apps": { + "name": "application-and-protocol-top-apps", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "limit": 10 + }, + "application_and_protocol_app_summary": { + "name": "application-and-protocol-app-summary", + "execution_mode":"oneshot", + "filter": " vsys_id = 1 AND app_name IN ('ftp', 'http')", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ] + }, + "application_and_protocol_app_related_internal_ips": { + "name": "application-and-protocol-app-related-internal-ips", + "execution_mode":"oneshot", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "limit": 10 + }, + "application_and_protocol_app_throughput": { + "name": "application-and-protocol-app-throughput", + "execution_mode":"oneshot", + "granularity": "PT15S", + "filter": "vsys_id = 1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "limit": 10 + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/dslAutoGranularityTest.json b/src/test/resources/parameters/dslAutoGranularityTest.json new file mode 100644 index 00000000..2f8a9d4f --- /dev/null +++ b/src/test/resources/parameters/dslAutoGranularityTest.json @@ -0,0 +1,27 @@ +{ + "application_and_protocol_summary_auto": { + "name": "application-and-protocol-summary", + "filter": "vsys_id = 1" + }, + "application_and_protocol_summary_const": { + "name": "application-and-protocol-summary", + "granularity": "PT1S", + "filter": "vsys_id = 1", + "interval": [ + "2019-01-01 00:00:00/2019-10-01 00:00:10" + ] + }, + "application_and_protocol_summary_auto_const_range": { + "name": "application-and-protocol-summary", + "granularity": "CHART_GRANULARITY('2019-01-01 00:00:00', '2019-10-01 00:00:10')", + "filter": "vsys_id = 1", + "interval": [ + "2019-01-01 00:00:00/2019-10-01 00:00:10" + ] + }, + "traffic_spectrum_network_throughput_trend_auto": { + "name": "traffic-spectrum-network-throughput-trend", + "filter": "vsys_id in (1) ", + "execution_mode": "oneshot" + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/entityTest.json b/src/test/resources/parameters/entityTest.json deleted file mode 100644 index 5a94461e..00000000 --- a/src/test/resources/parameters/entityTest.json +++ /dev/null @@ -1,132 +0,0 @@ -{ - "activeClientIp": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "session_record", - "limit": "10000", - "parameters": { - "match": [ - { - "type": "exactly", - "fieldKey": "app", - "fieldValues": [ - "Freegate" - ] - } - ], - "range": [ - { - "type": "eq", - "fieldKey": "vsys_id", - "fieldValues": [ - 1 - ] - } - ], - "intervals": [ - "2020-08-15T00:00:00.865Z/2022-08-15T00:30:00.865Z" - ] - } - } - }, - "topServerIp": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "session_record", - "limit": "10000", - "parameters": { - "range": [ - { - "type": "eq", - "fieldKey": "vsys_id", - "fieldValues": [ - 1 - ] - } - ], - "intervals": [ - "2020-08-15T00:00:00Z/2022-08-16T00:00:00Z" - ] - } - } - }, - "topSni": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "session_record", - "limit": "10000", - "parameters": { - "range": [ - { - "type": "eq", - "fieldKey": "vsys_id", - "fieldValues": [ - 1 - ] - } - ], - "intervals": [ - "2020-08-15T00:00:00.865+08:00/2022-08-16T00:00:00.865+08:00" - ] - } - } - }, - "subScriberidPool": { - "clientId":null, - "query":{ - "dataEngine":"AnalysisEngine", - "dataSource":"SUBSCRIBER_ID_VIEW", - "parameters":{ - "match":[ - { - "type":"exactly", - "fieldKey":"SUBSCRIBER_ID", - "fieldValues":[ - "test01", - "test02" - ] - } - ], - "range":[ - { - "type":"eq", - "fieldKey":"vsys_id", - "fieldValues":[ - 1 - ] - } - ] - } - } - }, - "gtpc": { - "clientId":null, - "query":{ - "dataEngine":"AnalysisEngine", - "dataSource":"gtpc_knowledge_base", - "parameters":{ - "match":[ - { - "type":"prefix", - "fieldKey":"phone_number", - "fieldValues":[ - "1761041" - ] - } - ], - "range":[ - { - "type":"eq", - "fieldKey":"vsys_id", - "fieldValues":[ - 1 - ] - } - ] - } - } - } -} diff --git a/src/test/resources/parameters/fieldDiscoveryTest.json b/src/test/resources/parameters/fieldDiscoveryTest.json new file mode 100644 index 00000000..e6ed275b --- /dev/null +++ b/src/test/resources/parameters/fieldDiscoveryTest.json @@ -0,0 +1,35 @@ +{ + "field_discovery_default": { + "name": "field_discovery", + "data_source": "session_record", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + }, + "field_discovery_sessions": { + "name": "field_discovery", + "data_source": "session_record", + "custom.field_discovery.metric": "sessions", + "custom.field_discovery.metric.fn": "count", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + }, + "field_discovery_bytes": { + "name": "field_discovery", + "data_source": "session_record", + "custom.field_discovery.metric": "bytes", + "custom.field_discovery.metric.fn": "sum", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + }, + "field_discovery_incoming_bytes": { + "name": "field_discovery", + "data_source": "session_record", + "custom.field_discovery.metric": "incoming_bytes", + "custom.field_discovery.metric.fn": "sum", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + }, + "field_discovery_outgoing_bytes": { + "name": "field_discovery", + "data_source": "session_record", + "custom.field_discovery.metric": "outgoing_bytes", + "custom.field_discovery.metric.fn": "sum", + "filter": "recv_time >= UNIX_TIMESTAMP(now()) - 500 AND recv_time <= UNIX_TIMESTAMP(now()) AND vsys_id = 1" + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/jobTest.json b/src/test/resources/parameters/jobTest.json deleted file mode 100644 index 777418ff..00000000 --- a/src/test/resources/parameters/jobTest.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "field_discovery_default": { - "query.type": "field_discovery", - "query.data_source": "session_record", - "custom.field_discovery.fields": [ - "log_id", - "security_action" - ], - "custom.field_discovery.filter": "vsys_id in (1,2) and client_ip='192.168.0.1' AND server_port = 80" - }, - "field_discovery_bytes": { - "query.type": "field_discovery", - "query.data_source": "session_record", - "custom.field_discovery.metric": "bytes", - "custom.field_discovery.metric.fn": "sum", - "custom.field_discovery.fields": [ - "security_action", - "proxy_action" - ], - "custom.field_discovery.filter": "vsys_id in (1,2) and client_ip='192.168.0.1' AND server_port = 80" - }, - "long_term": { - "query.type": "long_term", - "query.data_source": "session_record", - "custom.long_term.sql": "select client_ip, count(*) as count from session_record where vsys_id in (1,2) and client_ip='192.168.0.1' AND server_port = 80 group by client_ip order by count asc limit 10" - }, - "report": { - "query.type": "report", - "query.data_source": "session_record", - "custom.report.sql": "SELECT log_id, recv_time FROM session_record LIMIT 12 " - }, - "statistics_top": { - "query.type": "statistics", - "query.data_source": "session_record", - "custom.statistics.sql": "select client_ip, count(*) as count from session_record where vsys_id in (1,2) and client_ip='192.168.0.1' AND server_port = 80 group by client_ip order by count desc limit 10" - } -}
\ No newline at end of file diff --git a/src/test/resources/parameters/knowledgeBase.json b/src/test/resources/parameters/knowledgeBase.json index c5eadaaa..8c2bd42e 100644 --- a/src/test/resources/parameters/knowledgeBase.json +++ b/src/test/resources/parameters/knowledgeBase.json @@ -1,7 +1,21 @@ { - "publishTest": { + "publish": { + "kb_id": "test", "name": "test", - "format": "test", - "type": "test" + "format": "format", + "category": "category", + "is_valid": 1 + }, + "update": { + "kb_id": "test", + "version": "latest" + }, + "update_status": { + "kb_id": "test", + "version": "latest", + "is_valid": 0 + }, + "delete": { + "kb_id": "test" } }
\ No newline at end of file diff --git a/src/test/resources/parameters/recommendTest.json b/src/test/resources/parameters/recommendTest.json new file mode 100644 index 00000000..34ad5030 --- /dev/null +++ b/src/test/resources/parameters/recommendTest.json @@ -0,0 +1,20 @@ +{ + "ip_learning_fqdn_relate_ip": { + "name": "ip-learning-fqdn-relate-ip", + "filter": "VSYS_ID in (1,2,3,4,5) AND PROTOCOL in ('SSL', 'HTTP', 'DNS') AND DEPTH = 1 and UNIQ_NAME > 12 AND FQDN_NAME in ('google.com', 'itunes.apple.com')", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "limit": 100 + }, + "ip_learning_active_ip": { + "name": "ip-learning-active-ip", + "execution_mode": "oneshot", + "filter": "vsys_id in (1) AND 1=1", + "intervals": [ + "2024-01-30 00:00:00/2024-01-31 00:00:00" + ], + "order_by": "BYTES_TOTAL desc, LAST_FOUND_TIME desc", + "limit": 1 + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/sqlAdHocTest.json b/src/test/resources/parameters/sqlAdHocTest.json new file mode 100644 index 00000000..efc526fd --- /dev/null +++ b/src/test/resources/parameters/sqlAdHocTest.json @@ -0,0 +1,29 @@ +{ + "query_sql_default": { + "statement": "select * from session_record limit 1" + }, + "query_sql_oneshot": { + "statement": "select * from session_record limit 1", + "execution_mode": "oneshot" + }, + "query_sql_normal": { + "statement": "select * from session_record limit 1", + "execution_mode": "normal" + }, + "query_sql_blocking": { + "statement": "select * from session_record limit 1", + "execution_mode": "blocking" + }, + "query_sql_json": { + "statement": "select * from session_record limit 1", + "output_mode": "json" + }, + "query_sql_csv": { + "statement": "select * from session_record limit 1", + "output_mode": "csv" + }, + "query_sql_oneshot_error_trigger_sub_query": { + "statement": "SELECT COUNT_DISTINCT(client_ip) AS \"Client IP\" FROM security_event WHERE ((security_action = 'Deny')) AND recv_time >= UNIX_TIMESTAMP('2024-03-12T00:00:00+08:00') AND recv_time < UNIX_TIMESTAMP('2024-03-12T23:59:59+08:00') AND security_event.vsys_id IN (1) ORDER BY \"Client IP\" DESC LIMIT 20", + "execution_mode": "oneshot" + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/sqlSavedTest.json b/src/test/resources/parameters/sqlSavedTest.json new file mode 100644 index 00000000..dd128a67 --- /dev/null +++ b/src/test/resources/parameters/sqlSavedTest.json @@ -0,0 +1,6 @@ +{ + "default": { + "statement": "select * from session_record limit 1", + "is_saved_query": 1 + } +}
\ No newline at end of file diff --git a/src/test/resources/parameters/unstructuredTest.json b/src/test/resources/parameters/unstructuredTest.json deleted file mode 100644 index 64412dd7..00000000 --- a/src/test/resources/parameters/unstructuredTest.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "all": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "tsg_galaxy_v3", - "limit": "1000", - "parameters": { - "intervals": [ - "2023-03-01T00:00:00+08:00/2023-04-06T00:00:00+08:00" - ] - } - } - }, - "mail": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "session_record", - "limit": "1000", - "parameters": { - "intervals": [ - "2023-03-01T00:00:00+08:00/2023-04-06T00:00:00+08:00" - ] - } - } - }, - "http": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "security_event", - "limit": "1000", - "parameters": { - "intervals": [ - "2023-03-01T00:00:00+08:00/2023-04-06T00:00:00+08:00" - ] - } - } - }, - "pcap": { - "clientId": null, - "query": { - "dataEngine": "BusinessEngine", - "dataSource": "voip_record", - "limit": "1000", - "parameters": { - "intervals": [ - "2023-03-01T00:00:00+08:00/2023-04-06T00:00:00+08:00" - ] - } - } - } -}
\ No newline at end of file |