From d2d809407163b172333f69a9dd28f6eed6f4a44e Mon Sep 17 00:00:00 2001 From: doufenghu Date: Thu, 1 Aug 2024 10:29:46 +0800 Subject: [feature][e2e-test] add e2e-clickhouse module --- groot-tests/pom.xml | 1 + groot-tests/test-e2e-clickhouse/pom.xml | 63 ++++ .../test/e2e/clickhouse/ClickHouseIT.java | 363 ++++++++++++++++++++ .../src/test/resources/init/clickhouse_init.conf | 143 ++++++++ .../test/e2e/kafka/KafkaConnectorIT.java | 375 --------------------- .../com/geedgenetworks/test/e2e/kafka/KafkaIT.java | 372 ++++++++++++++++++++ pom.xml | 46 +-- 7 files changed, 966 insertions(+), 397 deletions(-) create mode 100644 groot-tests/test-e2e-clickhouse/pom.xml create mode 100644 groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java create mode 100644 groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_init.conf delete mode 100644 groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaConnectorIT.java create mode 100644 groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java diff --git a/groot-tests/pom.xml b/groot-tests/pom.xml index 76f533a..b46ad10 100644 --- a/groot-tests/pom.xml +++ b/groot-tests/pom.xml @@ -16,6 +16,7 @@ test-common test-e2e-base test-e2e-kafka + test-e2e-clickhouse diff --git a/groot-tests/test-e2e-clickhouse/pom.xml b/groot-tests/test-e2e-clickhouse/pom.xml new file mode 100644 index 0000000..c576100 --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + com.geedgenetworks + groot-tests + ${revision} + + + test-e2e-clickhouse + Groot : Tests : E2E : ClickHouse + + + 11 + 11 + UTF-8 + 0.3.2-patch9 + + + + + + com.geedgenetworks + test-common + ${project.version} + test-jar + test + + + + org.testcontainers + clickhouse + ${testcontainer.version} + test + + + + com.clickhouse + clickhouse-jdbc + ${clickhouse.jdbc.version} + test + + + + com.geedgenetworks + connector-clickhouse + ${project.version} + test + + + + org.xerial.snappy + snappy-java + ${snappy-java.version} + test + + + + + + \ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java new file mode 100644 index 0000000..f752d57 --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java @@ -0,0 +1,363 @@ +package com.geedgenetworks.test.e2e.clickhouse; + +import com.geedgenetworks.test.common.TestResource; +import com.geedgenetworks.test.common.TestSuiteBase; +import com.geedgenetworks.test.common.container.ContainerUtil; +import com.geedgenetworks.test.common.container.TestContainer; +import com.geedgenetworks.test.common.container.TestContainerId; +import com.geedgenetworks.test.common.junit.DisabledOnContainer; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.shaded.org.apache.commons.io.IOUtils; +import org.testcontainers.utility.DockerLoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.sql.*; +import java.sql.Date; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +@DisabledOnContainer( + value = {TestContainerId.FLINK_1_17}, + disabledReason = "Override TestSuiteBase @DisabledOnContainer") +public class ClickHouseIT extends TestSuiteBase implements TestResource { + private static final String CLICKHOUSE_DOCKER_IMAGE = "clickhouse/clickhouse-server:23.3.19.32"; + private static final String DRIVER_CLASS = "com.clickhouse.jdbc.ClickHouseDriver"; + private static final String INIT_CLICKHOUSE_PATH = "/init/clickhouse_init.conf"; + private static final String DATABASE = "default"; + private static final String SOURCE_TABLE = "source_table"; + private static final String SINK_TABLE = "sink_table"; + private static final String INSERT_SQL = "insert_sql"; + private static final String COMPARE_SQL = "compare_sql"; + private static final String HOST = "clickhouse"; + private static final Config CONFIG = getInitClickhouseConfig(); + private ClickHouseContainer clickHouseContainer; + private Connection connection; + private static final ListTEST_DATASET = + generateTestDataSet(); + private static final String[] default_columns = new String[] { + "id", + "c_map", + "c_array_string", + "c_array_short", + "c_array_int", + "c_array_long", + "c_array_float", + "c_array_double", + "c_string", + "c_boolean", + "c_int8", + "c_int16", + "c_int32", + "c_int64", + "c_float32", + "c_float64", + "c_decimal", + "c_date", + "c_datetime", + "c_nullable", + "c_lowcardinality", + "c_nested.int", + "c_nested.double", + "c_nested.string", + "c_int128", + "c_uint128", + "c_int256", + "c_uint256", + "c_point", + "c_ring" + }; + + @BeforeAll + @Override + public void startUp() throws Exception { + this.clickHouseContainer = + new ClickHouseContainer(CLICKHOUSE_DOCKER_IMAGE) + .withNetwork(NETWORK) + .withNetworkAliases(HOST) + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger(CLICKHOUSE_DOCKER_IMAGE))); + Startables.deepStart(Stream.of(this.clickHouseContainer)).join(); + log.info("Clickhouse container started"); + Awaitility.given() + .ignoreExceptions() + .await() + .atMost(360L, TimeUnit.SECONDS) + .untilAsserted(this::initConnection); + this.initializeClickhouseTable(); + this.batchInsertData(); + + } + + + @TestTemplate + public void testClickhouse(TestContainer container) throws Exception { + assertHasData(SOURCE_TABLE); + + //assertHasData(SINK_TABLE); + //compareResult(); + //clearSinkTable(); + + } + + private void assertHasData(String table) { + try (Statement statement = connection.createStatement()) { + String sql = String.format("select * from %s.%s limit 1", DATABASE, table); + ResultSet source = statement.executeQuery(sql); + Assertions.assertTrue(source.next()); + } catch (SQLException e) { + throw new RuntimeException("test clickhouse server image error", e); + } + } + + private void clearSinkTable() { + try (Statement statement = connection.createStatement()) { + statement.execute(String.format("truncate table %s.%s", DATABASE, SINK_TABLE)); + } catch (SQLException e) { + throw new RuntimeException("Test clickhouse server image error", e); + } + } + + private void compareResult() throws SQLException, IOException { + String sourceSql = "select * from " + SOURCE_TABLE + " order by id"; + String sinkSql = "select * from " + SINK_TABLE + " order by id"; + List columnList = + Arrays.stream(default_columns).collect(Collectors.toList()); + Statement sourceStatement = connection.createStatement(); + Statement sinkStatement = connection.createStatement(); + ResultSet sourceResultSet = sourceStatement.executeQuery(sourceSql); + ResultSet sinkResultSet = sinkStatement.executeQuery(sinkSql); + Assertions.assertEquals( + sourceResultSet.getMetaData().getColumnCount(), + sinkResultSet.getMetaData().getColumnCount()); + while (sourceResultSet.next()) { + if (sinkResultSet.next()) { + for (String column : columnList) { + Object source = sourceResultSet.getObject(column); + Object sink = sinkResultSet.getObject(column); + if (!Objects.deepEquals(source, sink)) { + InputStream sourceAsciiStream = sourceResultSet.getBinaryStream(column); + InputStream sinkAsciiStream = sinkResultSet.getBinaryStream(column); + String sourceValue = + IOUtils.toString(sourceAsciiStream, StandardCharsets.UTF_8); + String sinkValue = + IOUtils.toString(sinkAsciiStream, StandardCharsets.UTF_8); + Assertions.assertEquals(sourceValue, sinkValue); + } + Assertions.assertTrue(true); + } + } + } + String columns = String.join(",", default_columns); + Assertions.assertTrue( + compare(String.format(CONFIG.getString(COMPARE_SQL), columns, columns))); + } + + private Boolean compare(String sql) { + try (Statement statement = connection.createStatement()) { + ResultSet resultSet = statement.executeQuery(sql); + return !resultSet.next(); + } catch (SQLException e) { + throw new RuntimeException("result compare error", e); + } + } + + + private void batchInsertData() { + String sql = CONFIG.getString(INSERT_SQL); + PreparedStatement preparedStatement = null; + try { + this.connection.setAutoCommit(true); + preparedStatement = this.connection.prepareStatement(sql); + for (Object[] row : TEST_DATASET) { + preparedStatement.setLong(1, (Long) row[0]); + preparedStatement.setObject(2, row[1]); + preparedStatement.setArray(3, toSqlArray(row[2])); + preparedStatement.setArray(4, toSqlArray(row[3])); + preparedStatement.setArray(5, toSqlArray(row[4])); + preparedStatement.setArray(6, toSqlArray(row[5])); + preparedStatement.setArray(7, toSqlArray(row[6])); + preparedStatement.setArray(8, toSqlArray(row[7])); + preparedStatement.setString(9, (String) row[8]); + preparedStatement.setBoolean(10, (Boolean) row[9]); + preparedStatement.setByte(11, (Byte) row[10]); + preparedStatement.setShort(12, (Short) row[11]); + preparedStatement.setInt(13, (Integer) row[12]); + preparedStatement.setLong(14, (Long) row[13]); + preparedStatement.setFloat(15, (Float) row[14]); + preparedStatement.setDouble(16, (Double) row[15]); + preparedStatement.setBigDecimal(17, (BigDecimal) row[16]); + preparedStatement.setDate(18, Date.valueOf((LocalDate) row[17])); + preparedStatement.setTimestamp( + 19, Timestamp.valueOf((LocalDateTime) row[18])); + preparedStatement.setInt(20, (Integer) row[19]); + preparedStatement.setString(21, (String) row[20]); + preparedStatement.setArray(22, toSqlArray(row[21])); + preparedStatement.setArray(23, toSqlArray(row[22])); + preparedStatement.setArray(24, toSqlArray(row[23])); + preparedStatement.setObject(25, row[24]); + preparedStatement.setObject(26, row[25]); + preparedStatement.setObject(27, row[26]); + preparedStatement.setObject(28, row[27]); + preparedStatement.setObject(29, row[28]); + preparedStatement.setObject(30, row[29]); + preparedStatement.addBatch(); + } + + preparedStatement.executeBatch(); + preparedStatement.clearBatch(); + + } catch (SQLException e) { + throw new RuntimeException("Batch insert data failed!", e); + } finally { + if (preparedStatement != null) { + try { + preparedStatement.close(); + } catch (SQLException e) { + throw new RuntimeException("PreparedStatement close failed!", e); + } + } + } + + + + } + + + private Array toSqlArray(Object value) throws SQLException { + Object[] elements = null; + String sqlType = null; + if (String[].class.equals(value.getClass())) { + sqlType = "TEXT"; + elements = (String[]) value; + } else if (Boolean[].class.equals(value.getClass())) { + sqlType = "BOOLEAN"; + elements = (Boolean[]) value; + } else if (Byte[].class.equals(value.getClass())) { + sqlType = "TINYINT"; + elements = (Byte[]) value; + } else if (Short[].class.equals(value.getClass())) { + sqlType = "SMALLINT"; + elements = (Short[]) value; + } else if (Integer[].class.equals(value.getClass())) { + sqlType = "INTEGER"; + elements = (Integer[]) value; + } else if (Long[].class.equals(value.getClass())) { + sqlType = "BIGINT"; + elements = (Long[]) value; + } else if (Float[].class.equals(value.getClass())) { + sqlType = "REAL"; + elements = (Float[]) value; + } else if (Double[].class.equals(value.getClass())) { + sqlType = "DOUBLE"; + elements = (Double[]) value; + } + if (sqlType == null) { + throw new IllegalArgumentException( + "array inject error, not supported data type: " + value.getClass()); + } + return connection.createArrayOf(sqlType, elements); + } + + private static List generateTestDataSet() { + List rows = new ArrayList<>(); + for (int i = 0; i < 100; ++i) { + Object[] row = new Object[] { + (long) i, + Collections.singletonMap("key", Integer.parseInt("1")), + new String[] {"string"}, + new Short[] {Short.parseShort("1")}, + new Integer[] {Integer.parseInt("1")}, + new Long[] {Long.parseLong("1")}, + new Float[] {Float.parseFloat("1.1")}, + new Double[] {Double.parseDouble("1.1")}, + "string", + Boolean.FALSE, + Byte.parseByte("1"), + Short.parseShort("1"), + Integer.parseInt("1"), + Long.parseLong("1"), + Float.parseFloat("1.1"), + Double.parseDouble("1.1"), + BigDecimal.valueOf(11L, 1), + LocalDate.now(), + LocalDateTime.now(), + i, + "string", + new Integer[] {Integer.parseInt("1")}, + new Double[] {Double.parseDouble("1.1")}, + new String[] {"1"}, + "170141183460469231731687303715884105727", + "340282366920938463463374607431768211455", + "57896044618658097711785492504343953926634992332820282019728792003956564819967", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + new double[] {1, 2}, + new double[][] {{2, 3}, {4, 5}} + }; + rows.add(row); + } + return rows; + } + + private void initConnection() + throws SQLException, ClassNotFoundException, InstantiationException, + IllegalAccessException { + final Properties info = new Properties(); + info.put("user", this.clickHouseContainer.getUsername()); + info.put("password", this.clickHouseContainer.getPassword()); + this.connection = + ((Driver) Class.forName(DRIVER_CLASS).newInstance()) + .connect(this.clickHouseContainer.getJdbcUrl(), info); + } + + private void initializeClickhouseTable() { + try { + Statement statement = this.connection.createStatement(); + statement.execute(CONFIG.getString(SOURCE_TABLE)); + statement.execute(CONFIG.getString(SINK_TABLE)); + } catch (SQLException e) { + throw new RuntimeException("Initializing Clickhouse table failed!", e); + } + } + + private static Config getInitClickhouseConfig() { + File file = ContainerUtil.getResourcesFile(INIT_CLICKHOUSE_PATH); + Config config = ConfigFactory.parseFile(file); + assert config.hasPath(SOURCE_TABLE) + && config.hasPath(SINK_TABLE) + && config.hasPath(INSERT_SQL) + && config.hasPath(COMPARE_SQL); + return config; + } + + @AfterAll + @Override + public void tearDown() throws Exception { + if (this.connection != null) { + this.connection.close(); + } + if (this.clickHouseContainer != null) { + this.clickHouseContainer.stop(); + } + + } +} diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_init.conf b/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_init.conf new file mode 100644 index 0000000..78f2daa --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_init.conf @@ -0,0 +1,143 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +source_table = """ +set allow_experimental_geo_types = 1; +create table if not exists `default`.source_table( + `id` Int64, + `c_map` Map(String, Int32), + `c_array_string` Array(String), + `c_array_short` Array(Int16), + `c_array_int` Array(Int32), + `c_array_long` Array(Int64), + `c_array_float` Array(Float32), + `c_array_double` Array(Float64), + `c_string` String, + `c_boolean` Boolean, + `c_int8` Int8, + `c_int16` Int16, + `c_int32` Int32, + `c_int64` Int64, + `c_float32` Float32, + `c_float64` Float64, + `c_decimal` Decimal(9,4), + `c_date` Date, + `c_datetime` DateTime64, + `c_nullable` Nullable(Int32), + `c_lowcardinality` LowCardinality(String), + `c_nested` Nested + ( + `int` UInt32, + `double` Float64, + `string` String + ), + `c_int128` Int128, + `c_uint128` UInt128, + `c_int256` Int256, + `c_uint256` UInt256, + `c_point` Point, + `c_ring` Ring +)engine=Memory; +""" + +sink_table = """ +create table if not exists `default`.sink_table( + `id` Int64, + `c_map` Map(String, Int32), + `c_array_string` Array(String), + `c_array_short` Array(Int16), + `c_array_int` Array(Int32), + `c_array_long` Array(Int64), + `c_array_float` Array(Float32), + `c_array_double` Array(Float64), + `c_string` String, + `c_boolean` Boolean, + `c_int8` Int8, + `c_int16` Int16, + `c_int32` Int32, + `c_int64` Int64, + `c_float32` Float32, + `c_float64` Float64, + `c_decimal` Decimal(9,4), + `c_date` Date, + `c_datetime` DateTime64, + `c_nullable` Nullable(Int32), + `c_lowcardinality` LowCardinality(String), + `c_nested` Nested + ( + `int` UInt32, + `double` Float64, + `string` String + ), + `c_int128` Int128, + `c_uint128` UInt128, + `c_int256` Int256, + `c_uint256` UInt256, + `c_point` Point, + `c_ring` Ring +)engine=Memory; +""" + +insert_sql = """ +insert into `default`.source_table +( + `id`, + `c_map`, + `c_array_string`, + `c_array_short`, + `c_array_int`, + `c_array_long`, + `c_array_float`, + `c_array_double`, + `c_string`, + `c_boolean`, + `c_int8`, + `c_int16`, + `c_int32`, + `c_int64`, + `c_float32`, + `c_float64`, + `c_decimal`, + `c_date`, + `c_datetime`, + `c_nullable`, + `c_lowcardinality`, + `c_nested.int`, + `c_nested.double`, + `c_nested.string`, + `c_int128`, + `c_uint128`, + `c_int256`, + `c_uint256`, + `c_point`, + `c_ring` +) +values +(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) +""" + +compare_sql = """ +select + %s + from ( + select * from default.source_table +union all + select * from default.sink_table + ) +group by %s +having count(*) < 2 +""" \ No newline at end of file diff --git a/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaConnectorIT.java b/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaConnectorIT.java deleted file mode 100644 index 56108c5..0000000 --- a/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaConnectorIT.java +++ /dev/null @@ -1,375 +0,0 @@ -package com.geedgenetworks.test.e2e.kafka; - -import com.geedgenetworks.core.types.StructType; -import com.geedgenetworks.core.types.Types; -import com.geedgenetworks.formats.json.JsonSerializer; -import com.geedgenetworks.test.common.TestResource; -import com.geedgenetworks.test.common.TestSuiteBase; -import com.geedgenetworks.test.common.container.TestContainer; -import com.geedgenetworks.test.common.container.TestContainerId; -import com.geedgenetworks.test.common.junit.DisabledOnContainer; -import com.google.common.collect.Lists; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.consumer.*; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.config.SaslConfigs; -import org.apache.kafka.common.security.auth.SecurityProtocol; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.awaitility.Awaitility; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.TestTemplate; -import org.testcontainers.containers.Container; -import org.testcontainers.containers.KafkaContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.lifecycle.Startables;; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.DockerLoggerFactory; -import org.testcontainers.utility.MountableFile; - -import java.io.IOException; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; - -import static org.awaitility.Awaitility.await; - -@Slf4j -@DisabledOnContainer( - value = {TestContainerId.FLINK_1_17}, - disabledReason = "Override TestSuiteBase @DisabledOnContainer") -public class KafkaConnectorIT extends TestSuiteBase implements TestResource { - - private KafkaContainer kafkaContainer; - - private static final String KAFKA_IMAGE_NAME = "confluentinc/cp-kafka:7.4.0"; - private static final String KAFKA_HOST = "kafkaCluster"; - private KafkaProducer producer; - private static final String DEFAULT_TEST_TOPIC_SOURCE = "test_topic_source"; - private static final String DEFAULT_TEST_TOPIC_CONSUME_GROUP = "test-consume-group"; - - @Override - @BeforeAll - public void startUp() { - kafkaContainer = new KafkaContainer(DockerImageName.parse(KAFKA_IMAGE_NAME)) - .withNetwork(NETWORK) - .withNetworkAliases(KAFKA_HOST) - .withEnv("KAFKA_AUTO_CREATE_TOPICS_ENABLE", "true") - .withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1") - .withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "PLAINTEXT:SASL_PLAINTEXT,BROKER:SASL_PLAINTEXT") - .withEnv("KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL", "PLAIN") - .withEnv("KAFKA_LISTENER_NAME_PLAINTEXT_SASL_ENABLED_MECHANISMS", "PLAIN") - .withEnv("KAFKA_LISTENER_NAME_BROKER_SASL_ENABLED_MECHANISMS", "PLAIN") - // .withEnv("KAFKA_AUTHORIZER_CLASS_NAME", "kafka.security.authorizer.AclAuthorizer") - .withEnv("KAFKA_SUPER_USERS", "User:admin") - .withEnv("KAFKA_OPTS", "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf") - .withCopyFileToContainer(MountableFile.forClasspathResource("kafka_server_jaas.conf"), "/etc/kafka/kafka_server_jaas.conf") - .withCopyFileToContainer(MountableFile.forClasspathResource("kafka_client_jass_cli.properties"), "/etc/kafka/kafka_client_jass_cli.properties") - .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(KAFKA_IMAGE_NAME))); - Startables.deepStart(Stream.of(kafkaContainer)).join(); - log.info("Kafka container started successfully"); - Awaitility.given() - .ignoreExceptions() - .atLeast(100, TimeUnit.MILLISECONDS) - .pollInterval(500, TimeUnit.MILLISECONDS) - .atMost(180, TimeUnit.SECONDS) - .untilAsserted(this::initKafkaProducer); - - log.info("Write 100 records to topic test_topic_source"); - generateTestData(DEFAULT_TEST_TOPIC_SOURCE,0, 100); - - - } - - @TestTemplate - public void testKafkaAsSourceConsume(TestContainer container) { - generateTestData("test_topic_json", 0, 10); - CompletableFuture.supplyAsync( - () -> { - try { - return container.executeJob("/kafka_source.yaml"); - } catch (Exception e) { - log.error("Commit task exception :" + e.getMessage()); - throw new RuntimeException(e); - } - }); - - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> { - String logs = container.getServerLogs(); - Assertions.assertEquals(StringUtils.countMatches(logs, "PrintSinkFunction"), 10); - }); - } - - @TestTemplate - public void testKafkaAsSourceConsumeErrorSchema(TestContainer container) { - generateTestData("test_topic_error_json", 0, 10); - CompletableFuture.supplyAsync( - () -> { - try { - Container.ExecResult execResult = container.executeJob("/kafka_source_error_schema.yaml"); - Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); - return execResult; - } catch (Exception e) { - log.error("Commit task exception :" + e.getMessage()); - throw new RuntimeException(e); - } - }); - - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> { - String logs = container.getServerLogs(); - Assertions.assertTrue(StringUtils.contains(logs, "NumberFormatException")); - }); - } - - @TestTemplate - public void testKafkaAsSink(TestContainer container) throws IOException, InterruptedException { - CompletableFuture.supplyAsync( - () -> { - try { - Container.ExecResult execResult = container.executeJob("/kafka_sink.yaml"); - Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); - return execResult; - } catch (Exception e) { - log.error("Commit task exception :" + e.getMessage()); - throw new RuntimeException(e); - } - }); - - - List data = Lists.newArrayList(); - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> { - data.addAll(getKafkaConsumerListData("test_sink_topic")); - Assertions.assertEquals(10, data.size()); // Check if all 10 records are consumed - }); - - } - - @TestTemplate - public void testKafkaAsSinkProducerQuota(TestContainer container) throws IOException, InterruptedException { - //Create topic with 3 partitions - executeShell("kafka-topics --create --topic SESSION-RECORD-QUOTA-TEST --bootstrap-server kafkaCluster:9092 --partitions 3 --replication-factor 1 --command-config /etc/kafka/kafka_client_jass_cli.properties"); - //Set producer quota to 5KB/s - executeShell("kafka-configs --bootstrap-server kafkaCluster:9092 --alter --add-config 'producer_byte_rate=5120' --entity-type users --entity-name admin --entity-type clients --entity-name SESSION-RECORD-QUOTA-TEST --command-config /etc/kafka/kafka_client_jass_cli.properties "); - - CompletableFuture.supplyAsync( - () -> { - try { - Container.ExecResult execResult = container.executeJob("/kafka_producer_quota.yaml"); - Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); - return execResult; - } catch (Exception e) { - log.error("Commit task exception :" + e.getMessage()); - throw new RuntimeException(e); - } - }); - - List data = Lists.newArrayList(); - await().atMost(300000, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> { - data.addAll(getKafkaConsumerListData("SESSION-RECORD-QUOTA-TEST")); - Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "TimeoutException") && data.size()>1000); - }); - - } - - - - @TestTemplate - public void testKafkaAsSinkHandleErrorJsonFormat(TestContainer container) throws IOException, InterruptedException { - CompletableFuture. supplyAsync( - () -> { - try { - Container.ExecResult execResult = container.executeJob("/kafka_sink_handle_error_json_format.yaml"); - Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); - return execResult; - } catch (Exception e) { - log.error("Commit task exception :" + e.getMessage()); - throw new RuntimeException(e); - } - }); - - - List data = Lists.newArrayList(); - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> { - data.addAll(getKafkaConsumerListData("test_handle_error_json_format_topic")); - Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "UnsupportedOperationException")); - Assertions.assertEquals(0, data.size()); - }); - - - - } - - @TestTemplate - public void testKafkaSinkSkipErrorJsonFormat(TestContainer container) throws IOException, InterruptedException { - CompletableFuture.supplyAsync( - () -> { - try { - Container.ExecResult execResult = container.executeJob("/kafka_sink_skip_error_json_format.yaml"); - Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); - return execResult; - } catch (Exception e) { - log.error("Commit task exception :" + e.getMessage()); - throw new RuntimeException(e); - } - }); - - - List data = Lists.newArrayList(); - await().atMost(60000, TimeUnit.MILLISECONDS) - .untilAsserted( - () -> { - data.addAll(getKafkaConsumerListData("test_skip_error_json_format_topic")); - Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "NullPointerException")); - Assertions.assertEquals(0, data.size()); - }); - } - - - - private void generateTestData(String topic, int start, int end) { - StructType dataType = Types.parseStructType("id: int, client_ip: string, server_ip: string, flag: string"); - JsonSerializer serializer = new JsonSerializer(dataType); - for (int i = start; i < end; i++) { - Map row = Map - .of("id", i, - "client_ip", "192.168.40.12", - "server_ip", "8.8.8.8" , - "flag", Boolean.FALSE.booleanValue()); - ProducerRecord record = - new ProducerRecord<>(topic, serializer.serialize(row)); - producer.send(record); - } - - } - - - @AfterAll - @Override - public void tearDown() throws Exception { - if (producer != null) { - producer.close(); - } - if (kafkaContainer != null) { - kafkaContainer.close(); - } - - } - - private void initKafkaProducer() { - Properties properties = new Properties(); - String bootstrapServers = kafkaContainer.getBootstrapServers(); - properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); - properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); - properties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); - properties.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); - properties.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"admin\";"); - producer = new KafkaProducer<>(properties); - } - - private Properties kafkaConsumerConfig(String consumeGroup) { - Properties properties = new Properties(); - properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); - properties.put(ConsumerConfig.GROUP_ID_CONFIG, consumeGroup); - properties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); - properties.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); - properties.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"admin\";"); - properties.put( - ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, - OffsetResetStrategy.EARLIEST.toString().toLowerCase()); - properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - return properties; - } - - private Properties kafkaByteConsumerConfig() { - Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-consume-group"); - props.put( - ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, - OffsetResetStrategy.EARLIEST.toString().toLowerCase()); - props.setProperty( - ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, - ByteArrayDeserializer.class.getName()); - props.setProperty( - ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, - ByteArrayDeserializer.class.getName()); - return props; - } - - private Map getKafkaConsumerData(String topicName) { - Map data = new HashMap<>(); - try (KafkaConsumer consumer = new KafkaConsumer<>(kafkaConsumerConfig(DEFAULT_TEST_TOPIC_CONSUME_GROUP))) { - consumer.subscribe(Arrays.asList(topicName)); - Map offsets = - consumer.endOffsets(Arrays.asList(new TopicPartition(topicName, 0))); - Long endOffset = offsets.entrySet().iterator().next().getValue(); - Long lastProcessedOffset = -1L; - - do { - ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); - for (ConsumerRecord record : records) { - if (lastProcessedOffset < record.offset()) { - data.put(record.key(), record.value()); - } - lastProcessedOffset = record.offset(); - } - } while (lastProcessedOffset < endOffset - 1); - } - return data; - } - - private List getKafkaConsumerListData(String topicName) { - List data = new ArrayList<>(); - try (KafkaConsumer consumer = new KafkaConsumer<>(kafkaConsumerConfig(DEFAULT_TEST_TOPIC_CONSUME_GROUP))) { - consumer.subscribe(Arrays.asList(topicName)); - Map offsets = - consumer.endOffsets(Arrays.asList(new TopicPartition(topicName, 0))); - Long endOffset = offsets.entrySet().iterator().next().getValue(); - Long lastProcessedOffset = -1L; - - do { - ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); - for (ConsumerRecord record : records) { - if (lastProcessedOffset < record.offset()) { - data.add(record.value()); - } - lastProcessedOffset = record.offset(); - } - } while (lastProcessedOffset < endOffset - 1); - } - return data; - } - - private void executeShell(String command) { - try { - Container.ExecResult result = kafkaContainer.execInContainer("/bin/sh", "-c", command); - log.info("Execute shell command result: {},{}", result.getStdout(), result.getStderr()); - - } catch (Exception e) { - log.error("Execute shell command error: {}", e.getMessage()); - } - } - -} diff --git a/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java b/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java new file mode 100644 index 0000000..1c40e01 --- /dev/null +++ b/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java @@ -0,0 +1,372 @@ +package com.geedgenetworks.test.e2e.kafka; + +import com.geedgenetworks.core.types.StructType; +import com.geedgenetworks.core.types.Types; +import com.geedgenetworks.formats.json.JsonSerializer; +import com.geedgenetworks.test.common.TestResource; +import com.geedgenetworks.test.common.TestSuiteBase; +import com.geedgenetworks.test.common.container.TestContainer; +import com.geedgenetworks.test.common.container.TestContainerId; +import com.geedgenetworks.test.common.junit.DisabledOnContainer; +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.consumer.*; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables;; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; + +import java.io.IOException; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import static org.awaitility.Awaitility.await; + +@Slf4j +@DisabledOnContainer( + value = {TestContainerId.FLINK_1_17}, + disabledReason = "Override TestSuiteBase @DisabledOnContainer") +public class KafkaIT extends TestSuiteBase implements TestResource { + + private KafkaContainer kafkaContainer; + + private static final String KAFKA_IMAGE_NAME = "confluentinc/cp-kafka:7.4.0"; + private static final String KAFKA_HOST = "kafkaCluster"; + private KafkaProducer producer; + private static final String DEFAULT_TEST_TOPIC_SOURCE = "test_topic_source"; + private static final String DEFAULT_TEST_TOPIC_CONSUME_GROUP = "test-consume-group"; + + @Override + @BeforeAll + public void startUp() { + kafkaContainer = new KafkaContainer(DockerImageName.parse(KAFKA_IMAGE_NAME)) + .withNetwork(NETWORK) + .withNetworkAliases(KAFKA_HOST) + .withEnv("KAFKA_AUTO_CREATE_TOPICS_ENABLE", "true") + .withEnv("KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR", "1") + .withEnv("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP", "PLAINTEXT:SASL_PLAINTEXT,BROKER:SASL_PLAINTEXT") + .withEnv("KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL", "PLAIN") + .withEnv("KAFKA_LISTENER_NAME_PLAINTEXT_SASL_ENABLED_MECHANISMS", "PLAIN") + .withEnv("KAFKA_LISTENER_NAME_BROKER_SASL_ENABLED_MECHANISMS", "PLAIN") + // .withEnv("KAFKA_AUTHORIZER_CLASS_NAME", "kafka.security.authorizer.AclAuthorizer") + .withEnv("KAFKA_SUPER_USERS", "User:admin") + .withEnv("KAFKA_OPTS", "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf") + .withCopyFileToContainer(MountableFile.forClasspathResource("kafka_server_jaas.conf"), "/etc/kafka/kafka_server_jaas.conf") + .withCopyFileToContainer(MountableFile.forClasspathResource("kafka_client_jass_cli.properties"), "/etc/kafka/kafka_client_jass_cli.properties") + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(KAFKA_IMAGE_NAME))); + Startables.deepStart(Stream.of(kafkaContainer)).join(); + log.info("Kafka container started successfully"); + Awaitility.given() + .ignoreExceptions() + .atLeast(100, TimeUnit.MILLISECONDS) + .pollInterval(500, TimeUnit.MILLISECONDS) + .atMost(180, TimeUnit.SECONDS) + .untilAsserted(this::initKafkaProducer); + + log.info("Write 100 records to topic test_topic_source"); + generateTestData(DEFAULT_TEST_TOPIC_SOURCE,0, 100); + + + } + + @TestTemplate + public void testKafkaAsSourceConsume(TestContainer container) { + generateTestData("test_topic_json", 0, 10); + CompletableFuture.supplyAsync( + () -> { + try { + return container.executeJob("/kafka_source.yaml"); + } catch (Exception e) { + log.error("Commit task exception :" + e.getMessage()); + throw new RuntimeException(e); + } + }); + + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + String logs = container.getServerLogs(); + Assertions.assertEquals(StringUtils.countMatches(logs, "PrintSinkFunction"), 10); + }); + } + + @TestTemplate + public void testKafkaAsSourceConsumeErrorSchema(TestContainer container) { + generateTestData("test_topic_error_json", 0, 10); + CompletableFuture.supplyAsync( + () -> { + try { + Container.ExecResult execResult = container.executeJob("/kafka_source_error_schema.yaml"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + return execResult; + } catch (Exception e) { + log.error("Commit task exception :" + e.getMessage()); + throw new RuntimeException(e); + } + }); + + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + String logs = container.getServerLogs(); + Assertions.assertTrue(StringUtils.contains(logs, "NumberFormatException")); + }); + } + + @TestTemplate + public void testKafkaAsSink(TestContainer container) throws IOException, InterruptedException { + CompletableFuture.supplyAsync( + () -> { + try { + Container.ExecResult execResult = container.executeJob("/kafka_sink.yaml"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + return execResult; + } catch (Exception e) { + log.error("Commit task exception :" + e.getMessage()); + throw new RuntimeException(e); + } + }); + + + List data = Lists.newArrayList(); + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + data.addAll(getKafkaConsumerListData("test_sink_topic")); + Assertions.assertEquals(10, data.size()); // Check if all 10 records are consumed + }); + + } + + @TestTemplate + public void testKafkaAsSinkProducerQuota(TestContainer container) throws IOException, InterruptedException { + //Create topic with 3 partitions + executeShell("kafka-topics --create --topic SESSION-RECORD-QUOTA-TEST --bootstrap-server kafkaCluster:9092 --partitions 3 --replication-factor 1 --command-config /etc/kafka/kafka_client_jass_cli.properties"); + //Set producer quota to 5KB/s + executeShell("kafka-configs --bootstrap-server kafkaCluster:9092 --alter --add-config 'producer_byte_rate=5120' --entity-type users --entity-name admin --entity-type clients --entity-name SESSION-RECORD-QUOTA-TEST --command-config /etc/kafka/kafka_client_jass_cli.properties "); + + CompletableFuture.supplyAsync( + () -> { + try { + Container.ExecResult execResult = container.executeJob("/kafka_producer_quota.yaml"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + return execResult; + } catch (Exception e) { + log.error("Commit task exception :" + e.getMessage()); + throw new RuntimeException(e); + } + }); + + List data = Lists.newArrayList(); + await().atMost(300000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + data.addAll(getKafkaConsumerListData("SESSION-RECORD-QUOTA-TEST")); + Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "TimeoutException") && data.size()>1000); + }); + + } + + + + @TestTemplate + public void testKafkaAsSinkHandleErrorJsonFormat(TestContainer container) throws IOException, InterruptedException { + CompletableFuture. supplyAsync( + () -> { + try { + Container.ExecResult execResult = container.executeJob("/kafka_sink_handle_error_json_format.yaml"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + return execResult; + } catch (Exception e) { + log.error("Commit task exception :" + e.getMessage()); + throw new RuntimeException(e); + } + }); + + + List data = Lists.newArrayList(); + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + data.addAll(getKafkaConsumerListData("test_handle_error_json_format_topic")); + Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "UnsupportedOperationException")); + Assertions.assertEquals(0, data.size()); + }); + + + + } + + @TestTemplate + public void testKafkaSinkSkipErrorJsonFormat(TestContainer container) throws IOException, InterruptedException { + CompletableFuture.supplyAsync( + () -> { + try { + Container.ExecResult execResult = container.executeJob("/kafka_sink_skip_error_json_format.yaml"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + return execResult; + } catch (Exception e) { + log.error("Commit task exception :" + e.getMessage()); + throw new RuntimeException(e); + } + }); + + + List data = Lists.newArrayList(); + await().atMost(60000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + data.addAll(getKafkaConsumerListData("test_skip_error_json_format_topic")); + Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "NullPointerException")); + Assertions.assertEquals(0, data.size()); + }); + } + + private void generateTestData(String topic, int start, int end) { + StructType dataType = Types.parseStructType("id: int, client_ip: string, server_ip: string, flag: string"); + JsonSerializer serializer = new JsonSerializer(dataType); + for (int i = start; i < end; i++) { + Map row = Map + .of("id", i, + "client_ip", "192.168.40.12", + "server_ip", "8.8.8.8" , + "flag", Boolean.FALSE.booleanValue()); + ProducerRecord record = + new ProducerRecord<>(topic, serializer.serialize(row)); + producer.send(record); + } + + } + + + @AfterAll + @Override + public void tearDown() throws Exception { + if (producer != null) { + producer.close(); + } + if (kafkaContainer != null) { + kafkaContainer.close(); + } + + } + + private void initKafkaProducer() { + Properties properties = new Properties(); + String bootstrapServers = kafkaContainer.getBootstrapServers(); + properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + properties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); + properties.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + properties.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"admin\";"); + producer = new KafkaProducer<>(properties); + } + + private Properties kafkaConsumerConfig(String consumeGroup) { + Properties properties = new Properties(); + properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); + properties.put(ConsumerConfig.GROUP_ID_CONFIG, consumeGroup); + properties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT"); + properties.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + properties.put(SaslConfigs.SASL_JAAS_CONFIG, "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"admin\";"); + properties.put( + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, + OffsetResetStrategy.EARLIEST.toString().toLowerCase()); + properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + return properties; + } + + private Properties kafkaByteConsumerConfig() { + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "test-consume-group"); + props.put( + ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, + OffsetResetStrategy.EARLIEST.toString().toLowerCase()); + props.setProperty( + ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class.getName()); + props.setProperty( + ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, + ByteArrayDeserializer.class.getName()); + return props; + } + + private Map getKafkaConsumerData(String topicName) { + Map data = new HashMap<>(); + try (KafkaConsumer consumer = new KafkaConsumer<>(kafkaConsumerConfig(DEFAULT_TEST_TOPIC_CONSUME_GROUP))) { + consumer.subscribe(Arrays.asList(topicName)); + Map offsets = + consumer.endOffsets(Arrays.asList(new TopicPartition(topicName, 0))); + Long endOffset = offsets.entrySet().iterator().next().getValue(); + Long lastProcessedOffset = -1L; + + do { + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ConsumerRecord record : records) { + if (lastProcessedOffset < record.offset()) { + data.put(record.key(), record.value()); + } + lastProcessedOffset = record.offset(); + } + } while (lastProcessedOffset < endOffset - 1); + } + return data; + } + + private List getKafkaConsumerListData(String topicName) { + List data = new ArrayList<>(); + try (KafkaConsumer consumer = new KafkaConsumer<>(kafkaConsumerConfig(DEFAULT_TEST_TOPIC_CONSUME_GROUP))) { + consumer.subscribe(Arrays.asList(topicName)); + Map offsets = + consumer.endOffsets(Arrays.asList(new TopicPartition(topicName, 0))); + Long endOffset = offsets.entrySet().iterator().next().getValue(); + Long lastProcessedOffset = -1L; + + do { + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ConsumerRecord record : records) { + if (lastProcessedOffset < record.offset()) { + data.add(record.value()); + } + lastProcessedOffset = record.offset(); + } + } while (lastProcessedOffset < endOffset - 1); + } + return data; + } + + private void executeShell(String command) { + try { + Container.ExecResult result = kafkaContainer.execInContainer("/bin/sh", "-c", command); + log.info("Execute shell command result: {},{}", result.getStdout(), result.getStderr()); + + } catch (Exception e) { + log.error("Execute shell command error: {}", e.getMessage()); + } + } + +} diff --git a/pom.xml b/pom.xml index 5e3e128..69ea712 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,30 @@ + + + nexus3 + Team Nexus Repository + http://192.168.40.153:8081/repository/public/ + + + cloudera + https://repository.cloudera.com/artifactory/cloudera-repos/ + + + + + + true + platform-releases + http://192.168.40.153:8081/repository/platform-release/ + + + platform-snapshots + http://192.168.40.153:8081/repository/platform-snapshot/ + + + @@ -967,28 +991,6 @@ - - - nexus3 - Team Nexus Repository - http://192.168.40.153:8081/repository/public/ - - - cloudera - https://repository.cloudera.com/artifactory/cloudera-repos/ - - - - - true - platform-releases - http://192.168.40.153:8081/repository/platform-release/ - - - platform-snapshots - http://192.168.40.153:8081/repository/platform-snapshot/ - - -- cgit v1.2.3 From 906a4a1bfe7d2dee5ed7144d20f05ba55546b02c Mon Sep 17 00:00:00 2001 From: wangkuan Date: Thu, 1 Aug 2024 15:25:42 +0800 Subject: [fix][core]修复mean函数必须配置parameters的问题 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/udf/UnixTimestampConverter.java | 3 --- .../com/geedgenetworks/core/udf/udaf/Mean.java | 2 +- .../core/udf/test/aggregate/MeanTest.java | 22 ++++++++++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java index a8171b3..bdb41e0 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java @@ -32,9 +32,6 @@ public class UnixTimestampConverter implements ScalarFunction { if(udfContext.getLookup_fields().size() != 1){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "The function output fields only support 1 value"); } - if(udfContext.getParameters() == null){ - throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "The function must contain parameters"); - } if(!udfContext.getParameters().containsKey("precision")){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "parameters must containkey precision"); } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java index 380f598..88e4be6 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java @@ -29,7 +29,7 @@ public class Mean implements AggregateFunction { else { outputField = lookupField; } - if(!udfContext.getParameters().isEmpty()) { + if(udfContext.getParameters()!= null && !udfContext.getParameters().isEmpty()) { precision = Integer.parseInt(udfContext.getParameters().getOrDefault("precision", "-1").toString()); if (precision > 0) { StringBuilder pattern = new StringBuilder("#."); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java index 48c4e0f..807b7db 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java @@ -41,6 +41,7 @@ public class MeanTest { Integer[] intArr2 = new Integer[]{1, 6, 3}; excute(intArr1, 0); excute2(intArr2, 2); + excute3(intArr1); } private static void excute(Number[] arr,int precision) throws ParseException { @@ -93,5 +94,26 @@ public class MeanTest { assertEquals(NumberFormat.getInstance().parse((result.getMetricsFields().get("field_mean").toString())),NumberFormat.getInstance().parse("3.33")); } + private static void excute3(Number[] arr) throws ParseException { + UDFContext udfContext = new UDFContext(); + udfContext.setLookup_fields(List.of("field")); + udfContext.setOutput_fields(Collections.singletonList("field_mean")); + Mean mean = new Mean(); + Map metricsFields = new HashMap<>(); + Accumulator accumulator = new Accumulator(); + accumulator.setMetricsFields(metricsFields); + Accumulator agg = mean.open(udfContext,accumulator); + + for (Number o : arr) { + Event event = new Event(); + Map extractedFields = new HashMap<>(); + extractedFields.put("field", o); + event.setExtractedFields(extractedFields); + agg = mean.add(event, agg); + + } + Accumulator result = mean.getResult(agg); + assertEquals(NumberFormat.getInstance().parse((result.getMetricsFields().get("field_mean").toString())),NumberFormat.getInstance().parse("2.5")); + } } \ No newline at end of file -- cgit v1.2.3 From c0a2e18f14fd495d80e047b90e5afb0abcb9602c Mon Sep 17 00:00:00 2001 From: wangkuan Date: Thu, 1 Aug 2024 18:22:54 +0800 Subject: [improve][core]aggregate processor增加window_timestamp_field配置 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/grootstream_job_example.yaml | 1 + .../com/geedgenetworks/common/config/AggregateConfigOptions.java | 6 ++++++ .../src/main/java/com/geedgenetworks/core/pojo/AggregateConfig.java | 3 +-- .../core/processor/aggregate/ProcessWindowFunctionImpl.java | 4 +++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/config/grootstream_job_example.yaml b/config/grootstream_job_example.yaml index 74239c9..4726af0 100644 --- a/config/grootstream_job_example.yaml +++ b/config/grootstream_job_example.yaml @@ -24,6 +24,7 @@ processing_pipelines: output_fields: group_by_fields: [server_ip,server_port] window_type: tumbling_processing_time # tumbling_event_time,sliding_processing_time,sliding_event_time + window_timestamp_field: recv_time window_size: 60 window_slide: 10 #滑动窗口步长 functions: diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/AggregateConfigOptions.java b/groot-common/src/main/java/com/geedgenetworks/common/config/AggregateConfigOptions.java index 3998a3b..0b0379d 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/AggregateConfigOptions.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/AggregateConfigOptions.java @@ -5,6 +5,8 @@ import com.geedgenetworks.common.udf.UDFContext; import java.util.List; +import static com.geedgenetworks.common.Event.WINDOW_START_TIMESTAMP; + public interface AggregateConfigOptions { Option TYPE = Options.key("type") .stringType() @@ -46,4 +48,8 @@ public interface AggregateConfigOptions { .noDefaultValue() .withDescription("The size of sliding window."); + Option WINDOW_TIMESTAMP_FIELD = Options.key("window_timestamp_field") + .stringType() + .noDefaultValue() + .withDescription("which field to be set the start time of window."); } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/pojo/AggregateConfig.java b/groot-core/src/main/java/com/geedgenetworks/core/pojo/AggregateConfig.java index 8cccbbd..d3cbaac 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/pojo/AggregateConfig.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/pojo/AggregateConfig.java @@ -13,10 +13,9 @@ public class AggregateConfig extends ProcessorConfig { private List group_by_fields; - private String timestamp_field; + private String window_timestamp_field; private String window_type; private Integer window_size; - private Integer max_out_of_orderness; private Integer window_slide; private List functions; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunctionImpl.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunctionImpl.java index eaa712a..cd5c485 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunctionImpl.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunctionImpl.java @@ -44,7 +44,9 @@ public class ProcessWindowFunctionImpl extends org.apache.flink.streaming.api.fu internalMetrics.incrementOutEvents(); internalMetrics.incrementErrorEvents(accumulator.getErrorCount()); internalMetrics.incrementInEvents(accumulator.getInEvents()); - + if (aggregateConfig.getWindow_timestamp_field() != null) { + event.getExtractedFields().put(aggregateConfig.getWindow_timestamp_field(), context.window().getStart()); + } if (aggregateConfig.getOutput_fields() != null && !aggregateConfig.getOutput_fields().isEmpty()) { event.setExtractedFields( -- cgit v1.2.3 From 034abcaa214ce4f2211ce5adb02e30aa781cd067 Mon Sep 17 00:00:00 2001 From: doufenghu Date: Thu, 1 Aug 2024 19:27:35 +0800 Subject: [docs][core] add aggregate processor documents. --- docs/connector/connector.md | 16 +- docs/filter/aviator.md | 2 +- docs/processor/aggregate-processor.md | 71 +++++++++ docs/processor/udaf.md | 149 ++++++++++++++++++ docs/processor/udf.md | 74 ++++----- groot-common/src/main/resources/udf.plugins | 4 +- .../geedgenetworks/example/GrootStreamExample.java | 2 +- .../examples/inline_to_print_with_aggregation.yaml | 41 +++++ ...sion_record_mock_to_print_with_aggregation.yaml | 167 +++++++++++++++++++++ 9 files changed, 478 insertions(+), 48 deletions(-) create mode 100644 docs/processor/aggregate-processor.md create mode 100644 docs/processor/udaf.md create mode 100644 groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml create mode 100644 groot-examples/end-to-end-example/src/main/resources/examples/session_record_mock_to_print_with_aggregation.yaml diff --git a/docs/connector/connector.md b/docs/connector/connector.md index 08ec673..1123385 100644 --- a/docs/connector/connector.md +++ b/docs/connector/connector.md @@ -19,14 +19,14 @@ sources: ${prop_key}: ${prop_value} ``` -| Name | Type | Required | Default | Description | -|--------------------------|---------------|----------|---------|--------------------------------------------------------------------------------------------------------------------------| -| type | String | Yes | (none) | The type of the source connector. The `SourceTableFactory` will use this value as identifier to create source connector. | -| schema | Map | No | (none) | The source table schema, config through fields or local_file or url. | -| watermark_timestamp | String | No | (none) | watermark timestamp field name, if need use eventTime. | -| watermark_timestamp_unit | String | No | ms | watermark field timestamp unit, options:ms(milliseconds),s(seconds). is required if watermark_timestamp is not none. | -| watermark_lag | Long | No | (none) | watermark out-of-order milliseconds. is required if watermark_timestamp is not none. | -| properties | Map of String | Yes | (none) | The source connector customize properties, more details see the [Source](source) documentation. | +| Name | Type | Required | Default | Description | +|--------------------------|---------------|----------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| type | String | Yes | (none) | The type of the source connector. The `SourceTableFactory` will use this value as identifier to create source connector. | +| schema | Map | No | (none) | The source table schema, config through fields or local_file or url. | +| watermark_timestamp | String | No | (none) | Specify the field name as the watermark field. It is used to track event time and generate watermarks. | +| watermark_timestamp_unit | String | No | ms | The watermark field timestamp unit. The optional values are `ms`, `s`. | +| watermark_lag | Long | No | (none) | The watermark out-of-order milliseconds (Allowed Latenness). It defines the maximum amount of time (in milliseconds) by which events can be late but still be considered for processing. | +| properties | Map of String | Yes | (none) | The source connector customize properties, more details see the [Source](source) documentation. | ## Schema Field Projection diff --git a/docs/filter/aviator.md b/docs/filter/aviator.md index 54fe24b..e7f6c2b 100644 --- a/docs/filter/aviator.md +++ b/docs/filter/aviator.md @@ -30,7 +30,7 @@ This example read data from inline source and print to console. It will filter t filters: # [object] Define filter operator filter_operator: # [object] AviatorFilter operator name - type: com.geedgenetworks.core.filter.AviatorFilter + type: aviator properties: expression: event.server_ip == '8.8.8.8' || event.decoded_as == 'HTTP' # [string] Aviator expression, it return true or false. diff --git a/docs/processor/aggregate-processor.md b/docs/processor/aggregate-processor.md new file mode 100644 index 0000000..d9bcdb0 --- /dev/null +++ b/docs/processor/aggregate-processor.md @@ -0,0 +1,71 @@ +# Aggregate Processor + +> Processing pipelines for aggregate processor + +## Description + +Aggregate processor is used to aggregate the data from source to sink. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDAFs(User-defined Aggregate functions) into a pipeline. +Within the pipeline, events are processed by each Function in order, top‑>down. The UDAF usage detail can be found in [UDAF](udaf.md). + +## Options +Note：Default will output internal fields `__window_start_timestamp` and `__window_end_timestamp` if not set output_fields. + +| name | type | required | default value | +|--------------------------|--------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| type | String | Yes | The type of the processor, now only support `com.geedgenetworks.core.processor.projection.AggregateProcessor` | +| output_fields | Array | No | Array of String. The list of fields that need to be kept. Fields not in the list will be removed. | +| remove_fields | Array | No | Array of String. The list of fields that need to be removed. | +| group_by_fields | Array | yes | Array of String. The list of fields that need to be grouped. | +| window_type | String | yes | The type of window, now only support `tumbling_processing_time`, `tumbling_event_time`, `sliding_processing_time`, `sliding_event_time`. if window_type is `tumbling/sliding_event_time,` you need to set watermark. | +| window_size | Long | yes | The duration of the window in seconds. | +| window_slide | Long | yes | The duration of the window slide in seconds. | +| window_timestamp_field | String | No | Set the output timestamp field name, with the unit in seconds. It is mapped to the internal field __window_start_timestamp. | +| functions | Array | No | Array of Object. The list of functions that need to be applied to the data. | + +## Usage Example + +This example use aggregate processor to aggregate the fields `received_bytes` by `client_ip` and using NUMBER_SUM function to sum all `received_bytes` in 10 seconds window. + +```yaml +sources: + inline_source: + type: inline + properties: + data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.168.10.198","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931}]' + format: json + json.ignore.parse.errors: false + + +processing_pipelines: + aggregate_processor: + type: aggregate + group_by_fields: [ client_ip ] + window_type: tumbling_processing_time + window_size: 10 + functions: + - function: NUMBER_SUM + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_sum ] + +sinks: + print_sink: + type: print + properties: + format: json + mode: log_warn + +application: + env: + name: example-inline-to-print-with-aggregation + parallelism: 3 + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [aggregate_processor] + - name: aggregate_processor + downstream: [ print_sink ] + - name: print_sink + downstream: [] +``` + diff --git a/docs/processor/udaf.md b/docs/processor/udaf.md new file mode 100644 index 0000000..e22846f --- /dev/null +++ b/docs/processor/udaf.md @@ -0,0 +1,149 @@ +# UDAF + +> The functions for aggregate processors. + +## Function of content + +- [Collect List](#Collect-List) +- [Collect Set](#Collect-Set) +- [First Value](#First-Value) +- [Last Value](#Last-Value) +- [Long Count](#Long-Count) +- [MEAN](#Mean) +- [Number SUM](#Number-SUM) + + +## Description + +UDF(User Defined Aggregate Function) is used to extend the functions of aggregate processor. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDAFs into a pipeline. Within the pipeline, events are processed by each Function in order, top‑>down. +The deference between UDF and UDAF is: +- UDF is used to process each event, and the output is also an event. UDAF is used to process a group of events, and the output is also an event. +- A UDF is designed to perform a transformation or calculation on a single event. A UDAF is designed to perform an aggregation over a group of events, such as summing values, calculating an average, or finding a maximum. It processes multiple events of input data and produces a single aggregated result. + +## UDAF Definition + The UDAF basic properties are the same as UDF, such as `name`, `event`, `context`，more detail can be found in [UDF](udf.md). But Aggregate Processor have some methods to process the data is: +- `void add()`: Add a new event to the aggregation. +- `void getResult()`: Get the result of the aggregation. + +## Functions + +### Collect List + +COLLECT_LIST is used to collect the value of the field in the group of events. + +```COLLECT_LIST(filter, lookup_fields, output_fields)``` + +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: COLLECT_LIST + lookup_fields: [client_ip] + output_fields: [client_ip_list] +``` + +### Collect Set + +COLLECT_SET is used to collect the unique value of the field in the group of events. + +```COLLECT_SET(filter, lookup_fields, output_fields)``` + +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: COLLECT_SET + lookup_fields: [client_ip] + output_fields: [client_ip_set] +``` + +### First Value + +FIRST_VALUE is used to get the first value of the field in the group of events. + +```FIRST_VALUE(filter, lookup_fields, output_fields)``` +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: FIRST_VALUE + lookup_fields: [client_ip] + output_fields: [first_client_ip] +``` +### Last Value + +LAST_VALUE is used to get the last value of the field in the group of events. + +```LAST_VALUE(filter, lookup_fields, output_fields)``` +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: LAST_VALUE + lookup_fields: [client_ip] + output_fields: [last_client_ip] +``` + +### Long Count + +LONG_COUNT is used to count the number of events in the group of events. + +```LONG_COUNT(filter, lookup_fields, output_fields)``` +- filter: optional +- lookup_fields: optional. +- output_fields: required. + +### Example + +```yaml +- function: LONG_COUNT + output_fields: [sessions] +``` + +### Mean + +MEAN is used to calculate the mean value of the field in the group of events. The lookup field value must be a number. + +```MEAN(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. +- parameters: optional. + - precision: `` required. The precision of the mean value. Default is 2. + +### Example + +```yaml +- function: MEAN + lookup_fields: [received_bytes] + output_fields: [received_bytes_mean] +``` + +### Number SUM + +NUMBER_SUM is used to sum the value of the field in the group of events. The lookup field value must be a number. + +```NUMBER_SUM(filter, lookup_fields, output_fields)``` +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: NUMBER_SUM + lookup_fields: [received_bytes] + output_fields: [received_bytes_sum] +``` \ No newline at end of file diff --git a/docs/processor/udf.md b/docs/processor/udf.md index 2a705fd..cf305ef 100644 --- a/docs/processor/udf.md +++ b/docs/processor/udf.md @@ -1,7 +1,7 @@ # UDF > The functions for projection processors. -> + ## Function of content - [Asn Lookup](#asn-lookup) @@ -40,7 +40,7 @@ A UDF includes the following parts: name, event(processing data), context, evalu - open function: Initialize the resources used by the function. - close function: Release the resources used by the function. -### Functions +## Functions Function define common parameters: `filter`, `lookup_fields`, `output_fields`, `parameters`, and will return a Map value of the event. ``` FUNCTION_NAME(filter, lookup_fields, output_fields[, parameters])``` @@ -54,8 +54,8 @@ Asn lookup function is used to lookup the asn information by ip address. You nee - lookup_fields: required - output_fields: required - parameters: required -- kb_name: required. The name of the knowledge base. -- option: required. Now only support `IP_TO_ASN`. + - kb_name: required. The name of the knowledge base. + - option: required. Now only support `IP_TO_ASN`. Example: @@ -77,8 +77,8 @@ Base64 decode function is used to decode the base64 encoded string. - lookup_fields: not required - output_fields: required - parameters: required -- value_field: `` required. -- charset_field:`` optional. Default is `UTF-8`. + - value_field: `` required. + - charset_field:`` optional. Default is `UTF-8`. Example: @@ -99,7 +99,7 @@ Base64 encode function is commonly used to encode the binary data to base64 stri - lookup_fields: not required - output_fields: required - parameters: required -- value_field: `` required. + - value_field: `` required. Example: @@ -119,7 +119,7 @@ Current unix timestamp function is used to get the current unix timestamp. - lookup_fields: not required - output_fields: required - parameters: optional -- precision: `` optional. Default is `seconds`. Enum: `milliseconds`, `seconds`. + - precision: `` optional. Default is `seconds`. Enum: `milliseconds`, `seconds`. Example: @@ -139,7 +139,7 @@ Domain function is used to extract the domain from the url. - lookup_fields: required. Support more than one fields. All fields will be processed from left to right, and the result will be overwritten if the field processed value is not null. - output_fields: required - parameters: required -- option: `` required. Enum: `TOP_LEVEL_DOMAIN`, `FIRST_SIGNIFICANT_SUBDOMAIN`. + - option: `` required. Enum: `TOP_LEVEL_DOMAIN`, `FIRST_SIGNIFICANT_SUBDOMAIN`. #### Option @@ -182,7 +182,7 @@ Eval function is used to adds or removes fields from events by evaluating an val - lookup_fields: not required - output_fields: required - parameters: required -- value_expression: `` required. Enter a value expression to set the field’s value – this can be a constant. + - value_expression: `` required. Enter a value expression to set the field’s value – this can be a constant. Example 1: Add a field `ingestion_time` with value `recv_time`: @@ -213,10 +213,10 @@ Flatten the fields of nested structure to the top level. The new fields name are - lookup_fields: optional - output_fields: not required - parameters: optional -- prefix: `` optional. Prefix string for flattened field names. Default is empty. -- depth: `` optional. Number representing the nested levels to consider for flattening. Minimum 1. Default is `5`. -- delimiter: `` optional. The string used to join nested keys Default is `.`. -- json_string_keys: `` optional. The keys of the json string fields. It indicates keys that contain JSON strings and should be parsed and flattened. Default is empty. + - prefix: `` optional. Prefix string for flattened field names. Default is empty. + - depth: `` optional. Number representing the nested levels to consider for flattening. Minimum 1. Default is `5`. + - delimiter: `` optional. The string used to join nested keys Default is `.`. + - json_string_keys: `` optional. The keys of the json string fields. It indicates keys that contain JSON strings and should be parsed and flattened. Default is empty. Example 1: @@ -259,7 +259,7 @@ From unix timestamp function is used to convert the unix timestamp to date time - lookup_fields: required - output_fields: required - parameters: optional -- precision: `` optional. Default is `seconds`. Enum: `milliseconds`, `seconds`. + - precision: `` optional. Default is `seconds`. Enum: `milliseconds`, `seconds`. #### Precision @@ -303,16 +303,16 @@ GeoIP lookup function is used to lookup the geoip information by ip address. You - lookup_fields: required - output_fields: optional - parameters: required -- kb_name: `` required. The name of the knowledge base. -- option: `` required. Enum: `IP_TO_COUNTRY`, `IP_TO_PROVINCE`, `IP_TO_CITY`, `IP_TO_SUBDIVISION_ADDR`, `IP_TO_DETAIL`, `IP_TO_LATLNG`, `IP_TO_PROVIDER`, `IP_TO_JSON`, `IP_TO_OBJECT`. -- geolocation_field_mapping : `>` optional. The option is required when the option is `IP_TO_OBJECT`. The mapping of the geolocation fields. The key is the field name of the knowledge base , and the value is the field name of the event. -- COUNTRY: `` optional. -- PROVINCE: `` optional. -- CITY: `` optional. -- LONGITUDE: `` optional. -- LATITUDE: `` optional. -- ISP: `` optional. -- ORGANIZATION: `` optional. + - kb_name: `` required. The name of the knowledge base. + - option: `` required. Enum: `IP_TO_COUNTRY`, `IP_TO_PROVINCE`, `IP_TO_CITY`, `IP_TO_SUBDIVISION_ADDR`, `IP_TO_DETAIL`, `IP_TO_LATLNG`, `IP_TO_PROVIDER`, `IP_TO_JSON`, `IP_TO_OBJECT`. + - geolocation_field_mapping : `>` optional. The option is required when the option is `IP_TO_OBJECT`. The mapping of the geolocation fields. The key is the field name of the knowledge base , and the value is the field name of the event. + - COUNTRY: `` optional. + - PROVINCE: `` optional. + - CITY: `` optional. + - LONGITUDE: `` optional. + - LATITUDE: `` optional. + - ISP: `` optional. + - ORGANIZATION: `` optional. #### Option @@ -369,7 +369,7 @@ JSON extract function is used to extract the value from json string. - lookup_fields: required - output_fields: required - parameters: required -- value_expression: `` required. The json path expression. + - value_expression: `` required. The json path expression. Example: @@ -390,7 +390,7 @@ Path combine function is used to combine the file path. The path value can be co - lookup_fields: required - output_fields: required - parameters: required -- path: `` required. + - path: `` required. Example: @@ -411,11 +411,11 @@ Rename function is used to rename or reformat(e.g. by replacing character unders - lookup_fields: not required - output_fields: not required - parameters: required -- parent_fields: `` optional. Specify fields whose children will inherit the Rename fields and Rename expression operations. -- rename_fields: `Map` required. The key is the original field name, and the value is the new field name. -- current_field_name: `` required. The original field name. -- new_field_name: `` required. The new field name. -- rename_expression: `` optional. AviatorScript expression whose returned value will be used to rename fields. + - parent_fields: `` optional. Specify fields whose children will inherit the Rename fields and Rename expression operations. + - rename_fields: `Map` required. The key is the original field name, and the value is the new field name. + - current_field_name: `` required. The original field name. + - new_field_name: `` required. The new field name. + - rename_expression: `` optional. AviatorScript expression whose returned value will be used to rename fields. ``` A single Function can include both rename_fields (to rename specified field names) and rename_expression (to globally rename fields). However, the Rename fields strategy will execute first. @@ -462,7 +462,7 @@ Snowflake ID function is used to generate the snowflake id. The snowflake id is - lookup_fields: not required - output_fields: required - parameters: optional -- data_center_id_num: `` optional. Default is `0`, range is `0-31`. + - data_center_id_num: `` optional. Default is `0`, range is `0-31`. Example: @@ -482,9 +482,9 @@ String joiner function joins multiple string fields using a delimiter, prefix, a - lookup_fields: required. Support more than one fields. - output_fields: required - parameters: optional -- delimiter: `` optional. Default is `,`. -- prefix: `` optional. Default is empty string. -- suffix: `` optional. Default is empty string. + - delimiter: `` optional. Default is `,`. + - prefix: `` optional. Default is empty string. + - suffix: `` optional. Default is empty string. Example: @@ -507,7 +507,7 @@ Unix timestamp converter function is used to convert the unix timestamp precisio - lookup_fields: required - output_fields: required - parameters: required -- precision: `` required. Enum: `milliseconds`, `seconds`, `minutes`. The minutes precision is used to generate Unix timestamp, round it to the minute level, and output it in seconds format. + - precision: `` required. Enum: `milliseconds`, `seconds`, `minutes`. The minutes precision is used to generate Unix timestamp, round it to the minute level, and output it in seconds format. - Example: _`__timestamp` Internal field, from source ingestion time or current unix timestamp. diff --git a/groot-common/src/main/resources/udf.plugins b/groot-common/src/main/resources/udf.plugins index 0eb24cb..1b7fca4 100644 --- a/groot-common/src/main/resources/udf.plugins +++ b/groot-common/src/main/resources/udf.plugins @@ -18,4 +18,6 @@ com.geedgenetworks.core.udf.udaf.NumberSum com.geedgenetworks.core.udf.udaf.CollectList com.geedgenetworks.core.udf.udaf.CollectSet com.geedgenetworks.core.udf.udaf.LongCount -com.geedgenetworks.core.udf.udaf.Mean \ No newline at end of file +com.geedgenetworks.core.udf.udaf.Mean +com.geedgenetworks.core.udf.udaf.LastValue +com.geedgenetworks.core.udf.udaf.FirstValue \ No newline at end of file diff --git a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java index 2e21e49..690f21c 100644 --- a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java +++ b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java @@ -13,7 +13,7 @@ import java.nio.file.Paths; public class GrootStreamExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException { - String configPath = args.length > 0 ? args[0] : "/examples/session_record_mock_to_print.yaml"; + String configPath = args.length > 0 ? args[0] : "/examples/session_record_mock_to_print_with_aggregation.yaml"; String configFile = getTestConfigFile(configPath); ExecuteCommandArgs executeCommandArgs = new ExecuteCommandArgs(); executeCommandArgs.setConfigFile(configFile); diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml new file mode 100644 index 0000000..6f08be2 --- /dev/null +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml @@ -0,0 +1,41 @@ +sources: + inline_source: + type: inline + properties: + data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.168.10.198","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931}]' + format: json + json.ignore.parse.errors: false + + + +processing_pipelines: + aggregate_processor: + type: aggregate + group_by_fields: [ client_ip ] + window_type: tumbling_processing_time + window_size: 10 + functions: + - function: NUMBER_SUM + lookup_fields: [ received_bytes] + output_fields: [ received_bytes_sum ] + +sinks: + print_sink: + type: print + properties: + format: json + mode: log_warn + +application: + env: + name: example-inline-to-print-with-aggregation + parallelism: 3 + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [aggregate_processor] + - name: aggregate_processor + downstream: [ print_sink ] + - name: print_sink + downstream: [] \ No newline at end of file diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/session_record_mock_to_print_with_aggregation.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/session_record_mock_to_print_with_aggregation.yaml new file mode 100644 index 0000000..a3629c1 --- /dev/null +++ b/groot-examples/end-to-end-example/src/main/resources/examples/session_record_mock_to_print_with_aggregation.yaml @@ -0,0 +1,167 @@ +sources: # [object] Define connector source + mock_source: + type: mock + #watermark_timestamp: __timestamp + #watermark_timestamp_unit: s + #watermark_lag: 10000 + properties: + mock.desc.file.path: ./config/template/mock_schema/session_record_mock_desc.json + rows.per.second: 10 + +preprocessing_pipelines: + etl_processor: + type: projection + functions: + - function: SNOWFLAKE_ID + lookup_fields: [''] + output_fields: [log_id] + parameters: + data_center_id_num: 1 + - function: UNIX_TIMESTAMP_CONVERTER + lookup_fields: [ __timestamp ] + output_fields: [ recv_time ] + parameters: + precision: seconds + - function: SNOWFLAKE_ID + lookup_fields: [ '' ] + output_fields: [ session_id ] + parameters: + data_center_id_num: 2 + - function: EVAL + output_fields: [ ingestion_time ] + parameters: + value_expression: recv_time + + - function: DOMAIN + lookup_fields: [ http_host, ssl_sni, dtls_sni, quic_sni ] + output_fields: [ server_domain ] + parameters: + option: FIRST_SIGNIFICANT_SUBDOMAIN + + + - function: ASN_LOOKUP + lookup_fields: [ client_ip ] + output_fields: [ client_asn ] + parameters: + kb_name: tsg_ip_asn + option: IP_TO_ASN + + - function: ASN_LOOKUP + lookup_fields: [ server_ip ] + output_fields: [ server_asn ] + parameters: + kb_name: tsg_ip_asn + option: IP_TO_ASN + + - function: GEOIP_LOOKUP + lookup_fields: [ client_ip ] + output_fields: [] + parameters: + kb_name: tsg_ip_location + option: IP_TO_OBJECT + geolocation_field_mapping: + COUNTRY: client_country + PROVINCE: client_super_administrative_area + CITY: client_administrative_area + + - function: GEOIP_LOOKUP + lookup_fields: [ server_ip ] + output_fields: [] + parameters: + kb_name: tsg_ip_location + option: IP_TO_OBJECT + geolocation_field_mapping: + COUNTRY: server_country + PROVINCE: server_super_administrative_area + CITY: server_administrative_area + + - function: CURRENT_UNIX_TIMESTAMP + output_fields: [ processing_time ] + parameters: + precision: seconds + + - function: UNIX_TIMESTAMP_CONVERTER + lookup_fields: [recv_time] + output_fields: [recv_time] + parameters: + precision: seconds + interval: 60 + + +processing_pipelines: + aggregate_processor: + type: aggregate + group_by_fields: [recv_time, sled_ip] + window_type: tumbling_processing_time + window_size: 60 + functions: + - function: NUMBER_SUM + lookup_fields: [received_bytes, sent_bytes] + output_fields: [received_bytes_sum] + + - function: LONG_COUNT + lookup_fields: [received_bytes] + output_fields: [sessions] + + - function: MEAN + lookup_fields: [received_bytes] + output_fields: [received_bytes_mean] + parameters: + precision: 2 + + - function: FIRST_VALUE + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_first ] + + - function: LAST_VALUE + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_last ] + + - function: COLLECT_LIST + lookup_fields: [received_bytes] + output_fields: [received_bytes_set] + + +sinks: + print_sink: + type: print + properties: + mode: log_info + format: json + + kafka_sink: + type: kafka + properties: + topic: SESSION-RECORD + kafka.bootstrap.servers: 192.168.44.12:9094 + kafka.retries: 0 + kafka.linger.ms: 10 + kafka.request.timeout.ms: 30000 + kafka.batch.size: 262144 + kafka.buffer.memory: 134217728 + kafka.max.request.size: 10485760 + kafka.compression.type: snappy + kafka.security.protocol: SASL_PLAINTEXT + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + format: json + json.ignore.parse.errors: false + log.failures.only: true + + +application: # [object] Define job configuration + env: + name: session_record_mock_to_print_with_aggregation + parallelism: 3 + shade.identifier: aes + pipeline: + object-reuse: true + topology: + - name: mock_source + downstream: [ etl_processor ] + - name: etl_processor + downstream: [ aggregate_processor ] + - name: aggregate_processor + downstream: [ print_sink ] + - name: print_sink + downstream: [] \ No newline at end of file -- cgit v1.2.3 From 778ee36be04ce08c8230d0b3247b4166f90a05ae Mon Sep 17 00:00:00 2001 From: wangkuan Date: Fri, 2 Aug 2024 10:16:58 +0800 Subject: [improve][core]优化部分函数校验逻辑 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java | 3 +++ groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java | 1 - groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java | 4 +++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java index 3153ef7..84c2c2a 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java @@ -24,6 +24,9 @@ public class Flatten implements ScalarFunction { @Override public void open(RuntimeContext runtimeContext, UDFContext udfContext) { + if(udfContext.getParameters()==null){ + throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); + } prefix = udfContext.getParameters().getOrDefault("prefix", "").toString(); delimiter = udfContext.getParameters().getOrDefault("delimiter", ".").toString(); flattenKeys = new HashSet<>(); diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java index e48a503..874735d 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java @@ -31,7 +31,6 @@ public class PathCombine implements ScalarFunction { if (udfContext.getParameters() != null && !udfContext.getParameters().isEmpty()) { String paths = udfContext.getParameters().getOrDefault("path","").toString(); - // 使用逗号分隔项并转换为数组 if (!paths.isEmpty()) { List pathList; try { diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java index ba9b4d2..6a77c3a 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java @@ -26,7 +26,9 @@ public class Rename implements ScalarFunction { @Override public void open(RuntimeContext runtimeContext, UDFContext udfContext) { - + if(udfContext.getParameters()==null ){ + throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); + } String parentFields = udfContext.getParameters().getOrDefault("parent_fields", "").toString(); this.parentFields = new HashSet<>(); if (!parentFields.isEmpty()) { -- cgit v1.2.3 From df7c5dc10d1f6d89abf20ca81a43d0028233b888 Mon Sep 17 00:00:00 2001 From: wangkuan Date: Fri, 2 Aug 2024 11:27:24 +0800 Subject: [improve][core][bootstrap]调整结构,合并Processor接口 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../bootstrap/execution/AbstractExecutor.java | 14 +++++--------- .../bootstrap/execution/AbstractProcessorExecutor.java | 14 +++++++------- .../java/com/geedgenetworks/core/processor/Processor.java | 14 ++++++++++++++ .../core/processor/aggregate/AggregateProcessor.java | 13 ++----------- .../core/processor/aggregate/AggregateProcessorImpl.java | 9 ++------- .../core/processor/projection/ProjectionProcessor.java | 11 ++--------- .../core/processor/projection/ProjectionProcessorImpl.java | 8 +++----- .../services/com.geedgenetworks.core.processor.Processor | 2 ++ ...dgenetworks.core.processor.aggregate.AggregateProcessor | 1 - ...enetworks.core.processor.projection.ProjectionProcessor | 1 - 10 files changed, 37 insertions(+), 50 deletions(-) create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/processor/Processor.java create mode 100644 groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor delete mode 100644 groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.aggregate.AggregateProcessor delete mode 100644 groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.projection.ProjectionProcessor diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java index 04a6a94..64c66b6 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java @@ -2,6 +2,7 @@ package com.geedgenetworks.bootstrap.execution; import com.geedgenetworks.common.utils.ReflectionUtils; import com.geedgenetworks.core.filter.Filter; +import com.geedgenetworks.core.processor.Processor; import com.geedgenetworks.core.processor.aggregate.AggregateProcessor; import com.geedgenetworks.core.processor.projection.ProjectionProcessor; import com.typesafe.config.Config; @@ -20,8 +21,7 @@ public abstract class AbstractExecutor protected final Config operatorConfig; protected final Map operatorMap; protected final Map filterMap = new HashMap<>(); - protected final Map projectionProcessorMap = new HashMap<>(); - protected final Map aggregateProcessorMap = new HashMap<>(); + protected final Map processorMap = new HashMap<>(); protected AbstractExecutor(List jarPaths, Config operatorConfig) { this.operatorConfig = operatorConfig; @@ -30,13 +30,9 @@ public abstract class AbstractExecutor for (Filter filter : filters) { this.filterMap.put(filter.type(), filter); } - ServiceLoader projectionProcessors = ServiceLoader.load(ProjectionProcessor.class); - for (ProjectionProcessor projectionProcessor : projectionProcessors) { - this.projectionProcessorMap.put(projectionProcessor.type(), projectionProcessor); - } - ServiceLoader aggregateProcessors = ServiceLoader.load(AggregateProcessor.class); - for (AggregateProcessor aggregateProcessor : aggregateProcessors) { - this.aggregateProcessorMap.put(aggregateProcessor.type(), aggregateProcessor); + ServiceLoader processors = ServiceLoader.load(Processor.class); + for (Processor processor : processors) { + this.processorMap.put(processor.type(), processor); } } diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java index 42a4828..bd8b75c 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java @@ -47,8 +47,8 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor { + + SingleOutputStreamOperator processorFunction( + SingleOutputStreamOperator singleOutputStreamOperator, + T processorConfig, ExecutionConfig config) + throws Exception; + String type(); +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java index 9acf8fc..0846ffe 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java @@ -1,16 +1,7 @@ package com.geedgenetworks.core.processor.aggregate; import com.geedgenetworks.core.pojo.AggregateConfig; -import com.geedgenetworks.common.Event; +import com.geedgenetworks.core.processor.Processor; +public interface AggregateProcessor extends Processor { -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; - -public interface AggregateProcessor { - - SingleOutputStreamOperator aggregateProcessorFunction( - SingleOutputStreamOperator singleOutputStreamOperator, - AggregateConfig aggregateConfig, ExecutionConfig config) - throws Exception; - String type(); } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorImpl.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorImpl.java index 2f086b0..bc87c32 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorImpl.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorImpl.java @@ -11,17 +11,12 @@ import org.apache.flink.streaming.api.windowing.assigners.SlidingProcessingTimeW import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows; import org.apache.flink.streaming.api.windowing.time.Time; - import static com.geedgenetworks.common.Constants.*; -public class AggregateProcessorImpl implements AggregateProcessor { +public class AggregateProcessorImpl implements AggregateProcessor { @Override - public SingleOutputStreamOperator aggregateProcessorFunction( - SingleOutputStreamOperator grootEventSingleOutputStreamOperator, - AggregateConfig aggregateConfig, ExecutionConfig config) - throws Exception { - + public SingleOutputStreamOperator processorFunction(SingleOutputStreamOperator grootEventSingleOutputStreamOperator, AggregateConfig aggregateConfig, ExecutionConfig config) throws Exception { if (aggregateConfig.getParallelism() != 0) { switch (aggregateConfig.getWindow_type()) { case TUMBLING_PROCESSING_TIME: diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java index 862ba5e..f15d481 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java @@ -1,15 +1,8 @@ package com.geedgenetworks.core.processor.projection; -import com.geedgenetworks.common.Event; import com.geedgenetworks.core.pojo.ProjectionConfig; +import com.geedgenetworks.core.processor.Processor; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +public interface ProjectionProcessor extends Processor{ -public interface ProjectionProcessor { - - SingleOutputStreamOperator projectionProcessorFunction( - SingleOutputStreamOperator grootEventSingleOutputStreamOperator, - ProjectionConfig projectionConfig) - throws Exception; - String type(); } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorImpl.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorImpl.java index 79b0e0d..6b46a7b 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorImpl.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorImpl.java @@ -3,16 +3,14 @@ package com.geedgenetworks.core.processor.projection; import com.geedgenetworks.common.Event; import com.geedgenetworks.core.pojo.ProjectionConfig; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; public class ProjectionProcessorImpl implements ProjectionProcessor { - @Override - public SingleOutputStreamOperator projectionProcessorFunction( - SingleOutputStreamOperator grootEventSingleOutputStreamOperator, - ProjectionConfig projectionConfig) - throws Exception{ + @Override + public SingleOutputStreamOperator processorFunction(SingleOutputStreamOperator grootEventSingleOutputStreamOperator, ProjectionConfig projectionConfig, ExecutionConfig config) throws Exception { if (projectionConfig.getParallelism() != 0) { return grootEventSingleOutputStreamOperator .process(new ProjectionProcessFunction(projectionConfig)) diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor new file mode 100644 index 0000000..727b42b --- /dev/null +++ b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor @@ -0,0 +1,2 @@ +com.geedgenetworks.core.processor.aggregate.AggregateProcessorImpl +com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl \ No newline at end of file diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.aggregate.AggregateProcessor b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.aggregate.AggregateProcessor deleted file mode 100644 index 426a1a9..0000000 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.aggregate.AggregateProcessor +++ /dev/null @@ -1 +0,0 @@ -com.geedgenetworks.core.processor.aggregate.AggregateProcessorImpl \ No newline at end of file diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.projection.ProjectionProcessor b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.projection.ProjectionProcessor deleted file mode 100644 index ede2c8c..0000000 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.projection.ProjectionProcessor +++ /dev/null @@ -1 +0,0 @@ -com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl \ No newline at end of file -- cgit v1.2.3 From ca9b39be199b3d093a8a1366257bf49876ddbee4 Mon Sep 17 00:00:00 2001 From: wangkuan Date: Fri, 2 Aug 2024 14:57:15 +0800 Subject: [improve][core]修改初始化聚合函数逻辑，使函数只构造一次，优化性能 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../aggregate/AggregateProcessorFunction.java | 23 +++++++++++----------- .../core/processor/projection/UdfEntity.java | 2 ++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java index b535faf..c07374e 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java @@ -37,19 +37,11 @@ public class AggregateProcessorFunction implements org.apache.flink.api.common.f public AggregateProcessorFunction(AggregateConfig aggregateConfig, ExecutionConfig config) { udfClassNameLists = JSON.parseObject(config.getGlobalJobParameters().toMap().get(Constants.SYSPROP_UDF_PLUGIN_CONFIG), List.class); udfContexts = aggregateConfig.getFunctions(); - groupByFields = aggregateConfig.getGroup_by_fields(); - } - - @Override - public Accumulator createAccumulator() { - - functions = Lists.newLinkedList(); if (udfContexts == null || udfContexts.isEmpty()) { throw new RuntimeException(); } - Map map = new HashMap<>(); - Accumulator accumulator = new Accumulator(); - accumulator.setMetricsFields(map); + groupByFields = aggregateConfig.getGroup_by_fields(); + functions = Lists.newLinkedList(); Map udfClassReflect = getClassReflect(udfClassNameLists); try { for (UDFContext udfContext : udfContexts) { @@ -59,7 +51,6 @@ public class AggregateProcessorFunction implements org.apache.flink.api.common.f if (udfClassReflect.containsKey(udfContext.getFunction())) { Class cls = Class.forName(udfClassReflect.get(udfContext.getFunction())); AggregateFunction aggregateFunction = (AggregateFunction) cls.getConstructor().newInstance(); - aggregateFunction.open(udfContext, accumulator); // 函数如果包含filter,对表达式进行编译 if (udfContext.getFilter() != null) { AviatorEvaluatorInstance instance = AviatorEvaluator.getInstance(); @@ -72,6 +63,7 @@ public class AggregateProcessorFunction implements org.apache.flink.api.common.f udfEntity.setFilterExpression(filterExpression); udfEntity.setName(udfContext.getFunction()); udfEntity.setClassName(udfClassReflect.get(udfContext.getFunction())); + udfEntity.setUdfContext(udfContext); functions.add(udfEntity); } else { throw new GrootStreamRuntimeException(CommonErrorCode.UNSUPPORTED_OPERATION, @@ -83,7 +75,16 @@ public class AggregateProcessorFunction implements org.apache.flink.api.common.f throw new GrootStreamRuntimeException(CommonErrorCode.UNSUPPORTED_OPERATION, "Initialization UDAF failed!", e); } + } + @Override + public Accumulator createAccumulator() { + Map map = new HashMap<>(); + Accumulator accumulator = new Accumulator(); + accumulator.setMetricsFields(map); + for (UdfEntity udfEntity : functions) { + udfEntity.getAggregateFunction().open(udfEntity.getUdfContext(), accumulator); + } return accumulator; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java index c36a785..34267a6 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java @@ -3,6 +3,7 @@ package com.geedgenetworks.core.processor.projection; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.ScalarFunction; +import com.geedgenetworks.common.udf.UDFContext; import com.googlecode.aviator.Expression; import lombok.Data; @@ -15,4 +16,5 @@ public class UdfEntity implements Serializable { private Expression filterExpression; private String name; private String className; + private UDFContext udfContext; } -- cgit v1.2.3 From e2a63517699d10f86041d1dcdb4b9fb8e81030d3 Mon Sep 17 00:00:00 2001 From: doufenghu Date: Fri, 2 Aug 2024 15:19:49 +0800 Subject: [tests][e2e-clickhouse] Replace the dependency library clickhouse.jdbc.version with version 0.6.3. --- groot-tests/test-e2e-clickhouse/pom.xml | 28 +++++++++++++++++++++- .../test/e2e/clickhouse/ClickHouseIT.java | 3 +-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/groot-tests/test-e2e-clickhouse/pom.xml b/groot-tests/test-e2e-clickhouse/pom.xml index c576100..aef4470 100644 --- a/groot-tests/test-e2e-clickhouse/pom.xml +++ b/groot-tests/test-e2e-clickhouse/pom.xml @@ -16,7 +16,9 @@ 11 11 UTF-8 - 0.3.2-patch9 + 0.6.3 + 4.0.3 + 5.2.1 @@ -36,6 +38,21 @@ test + + com.zaxxer + HikariCP + ${hikaricp.version} + test + + + + org.lz4 + lz4-java + 1.8.0 + test + + + com.clickhouse clickhouse-jdbc @@ -43,6 +60,14 @@ test + + + org.apache.httpcomponents.client5 + httpclient5 + ${apache-httpclient.version} + test + + com.geedgenetworks connector-clickhouse @@ -50,6 +75,7 @@ test + org.xerial.snappy snappy-java diff --git a/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java index f752d57..6cc01b4 100644 --- a/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java +++ b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java @@ -51,8 +51,7 @@ public class ClickHouseIT extends TestSuiteBase implements TestResource { private static final Config CONFIG = getInitClickhouseConfig(); private ClickHouseContainer clickHouseContainer; private Connection connection; - private static final ListTEST_DATASET = - generateTestDataSet(); + private static final ListTEST_DATASET = generateTestDataSet(); private static final String[] default_columns = new String[] { "id", "c_map", -- cgit v1.2.3 From 955c88293f44066674132e18badc1962f45fafba Mon Sep 17 00:00:00 2001 From: doufenghu Date: Sun, 11 Aug 2024 23:31:30 +0800 Subject: [tests][e2e-clickhouse] Support the ingestion of common data types by a Flink job --- .../bootstrap/utils/ConfigShadeTest.java | 1 + .../inline_all_data_type_to_clickhouse.yaml | 78 +++++++ .../test/e2e/clickhouse/ClickHouseIT.java | 227 ++++++++++----------- .../test/resources/clickhouse_data_type_sink.yaml | 78 +++++++ .../src/test/resources/init/clickhouse_init.conf | 143 ------------- .../test/resources/init/clickhouse_test_sql.conf | 98 +++++++++ .../src/test/resources/init/init-clickhouse.sql | 4 + .../src/test/resources/init/users.xml | 29 +++ plugin-mapping.properties | 2 +- pom.xml | 2 +- 10 files changed, 400 insertions(+), 262 deletions(-) create mode 100644 groot-examples/end-to-end-example/src/main/resources/examples/inline_all_data_type_to_clickhouse.yaml create mode 100644 groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml delete mode 100644 groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_init.conf create mode 100644 groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf create mode 100644 groot-tests/test-e2e-clickhouse/src/test/resources/init/init-clickhouse.sql create mode 100644 groot-tests/test-e2e-clickhouse/src/test/resources/init/users.xml diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java index ccdd224..336842f 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java @@ -67,5 +67,6 @@ public class ConfigShadeTest { Assertions.assertEquals(decryptPassword, PASSWORD); System.out.println( ConfigShadeUtils.encryptOption("aes", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"galaxy2019\";")); System.out.println( ConfigShadeUtils.decryptOption("aes", "454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817")); + System.out.println( ConfigShadeUtils.encryptOption("aes", "testuser")); } } diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/inline_all_data_type_to_clickhouse.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_all_data_type_to_clickhouse.yaml new file mode 100644 index 0000000..e449a08 --- /dev/null +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_all_data_type_to_clickhouse.yaml @@ -0,0 +1,78 @@ +sources: + inline_source: + type: inline + schema: + fields: + - name: id + type: bigint + - name: c_array_string + type: array + - name: c_array_short + type: array + - name: c_array_int + type: array + - name: c_array_long + type: array + - name: c_array_float + type: array + - name: c_array_double + type: array + - name: c_string + type: string + - name: c_int8 + type: int + - name: c_int16 + type: int + - name: c_int32 + type: int + - name: c_int64 + type: int + - name: c_float32 + type: float + - name: c_float64 + type: double + - name: c_decimal + type: double + - name: c_date + type: string + - name: c_datetime + type: string + - name: c_nullable + type: int + - name: c_lowcardinality + type: string + properties: + # + # [string] Event Data, it will be parsed to Map by the specified format. + # + data: '[{"id":0,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":0,"c_lowcardinality":"string"},{"id":1,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":1,"c_lowcardinality":"string"},{"id":2,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":2,"c_lowcardinality":"string"},{"id":3,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":3,"c_lowcardinality":"string"},{"id":4,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":4,"c_lowcardinality":"string"},{"id":5,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":5,"c_lowcardinality":"string"},{"id":6,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":6,"c_lowcardinality":"string"},{"id":7,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":7,"c_lowcardinality":"string"},{"id":8,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":8,"c_lowcardinality":"string"},{"id":9,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":9,"c_lowcardinality":"string"},{"id":10,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":10,"c_lowcardinality":"string"},{"id":11,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":11,"c_lowcardinality":"string"},{"id":12,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":12,"c_lowcardinality":"string"},{"id":13,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":13,"c_lowcardinality":"string"},{"id":14,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":14,"c_lowcardinality":"string"},{"id":15,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":15,"c_lowcardinality":"string"},{"id":16,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":16,"c_lowcardinality":"string"},{"id":17,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":17,"c_lowcardinality":"string"},{"id":18,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":18,"c_lowcardinality":"string"},{"id":19,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":19,"c_lowcardinality":"string"},{"id":20,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":20,"c_lowcardinality":"string"},{"id":21,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":21,"c_lowcardinality":"string"},{"id":22,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":22,"c_lowcardinality":"string"},{"id":23,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":23,"c_lowcardinality":"string"},{"id":24,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":24,"c_lowcardinality":"string"},{"id":25,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":25,"c_lowcardinality":"string"},{"id":26,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":26,"c_lowcardinality":"string"},{"id":27,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":27,"c_lowcardinality":"string"},{"id":28,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":28,"c_lowcardinality":"string"},{"id":29,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":29,"c_lowcardinality":"string"},{"id":30,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":30,"c_lowcardinality":"string"},{"id":31,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":31,"c_lowcardinality":"string"},{"id":32,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":32,"c_lowcardinality":"string"},{"id":33,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":33,"c_lowcardinality":"string"},{"id":34,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":34,"c_lowcardinality":"string"},{"id":35,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":35,"c_lowcardinality":"string"},{"id":36,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":36,"c_lowcardinality":"string"},{"id":37,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":37,"c_lowcardinality":"string"},{"id":38,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":38,"c_lowcardinality":"string"},{"id":39,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":39,"c_lowcardinality":"string"},{"id":40,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":40,"c_lowcardinality":"string"},{"id":41,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":41,"c_lowcardinality":"string"},{"id":42,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":42,"c_lowcardinality":"string"},{"id":43,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":43,"c_lowcardinality":"string"},{"id":44,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":44,"c_lowcardinality":"string"},{"id":45,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":45,"c_lowcardinality":"string"},{"id":46,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":46,"c_lowcardinality":"string"},{"id":47,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":47,"c_lowcardinality":"string"},{"id":48,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":48,"c_lowcardinality":"string"},{"id":49,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":49,"c_lowcardinality":"string"},{"id":50,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":50,"c_lowcardinality":"string"},{"id":51,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":51,"c_lowcardinality":"string"},{"id":52,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":52,"c_lowcardinality":"string"},{"id":53,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":53,"c_lowcardinality":"string"},{"id":54,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":54,"c_lowcardinality":"string"},{"id":55,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":55,"c_lowcardinality":"string"},{"id":56,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":56,"c_lowcardinality":"string"},{"id":57,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":57,"c_lowcardinality":"string"},{"id":58,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":58,"c_lowcardinality":"string"},{"id":59,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":59,"c_lowcardinality":"string"},{"id":60,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":60,"c_lowcardinality":"string"},{"id":61,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":61,"c_lowcardinality":"string"},{"id":62,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":62,"c_lowcardinality":"string"},{"id":63,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":63,"c_lowcardinality":"string"},{"id":64,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":64,"c_lowcardinality":"string"},{"id":65,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":65,"c_lowcardinality":"string"},{"id":66,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":66,"c_lowcardinality":"string"},{"id":67,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":67,"c_lowcardinality":"string"},{"id":68,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":68,"c_lowcardinality":"string"},{"id":69,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":69,"c_lowcardinality":"string"},{"id":70,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":70,"c_lowcardinality":"string"},{"id":71,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":71,"c_lowcardinality":"string"},{"id":72,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":72,"c_lowcardinality":"string"},{"id":73,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":73,"c_lowcardinality":"string"},{"id":74,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":74,"c_lowcardinality":"string"},{"id":75,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":75,"c_lowcardinality":"string"},{"id":76,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":76,"c_lowcardinality":"string"},{"id":77,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":77,"c_lowcardinality":"string"},{"id":78,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":78,"c_lowcardinality":"string"},{"id":79,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":79,"c_lowcardinality":"string"},{"id":80,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":80,"c_lowcardinality":"string"},{"id":81,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":81,"c_lowcardinality":"string"},{"id":82,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":82,"c_lowcardinality":"string"},{"id":83,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":83,"c_lowcardinality":"string"},{"id":84,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":84,"c_lowcardinality":"string"},{"id":85,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":85,"c_lowcardinality":"string"},{"id":86,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":86,"c_lowcardinality":"string"},{"id":87,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":87,"c_lowcardinality":"string"},{"id":88,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":88,"c_lowcardinality":"string"},{"id":89,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":89,"c_lowcardinality":"string"},{"id":90,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":90,"c_lowcardinality":"string"},{"id":91,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":91,"c_lowcardinality":"string"},{"id":92,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":92,"c_lowcardinality":"string"},{"id":93,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":93,"c_lowcardinality":"string"},{"id":94,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":94,"c_lowcardinality":"string"},{"id":95,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":95,"c_lowcardinality":"string"},{"id":96,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":96,"c_lowcardinality":"string"},{"id":97,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":97,"c_lowcardinality":"string"},{"id":98,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":98,"c_lowcardinality":"string"},{"id":99,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":99,"c_lowcardinality":"string"}]' + format: json + interval.per.row: 1s + repeat.count: 10 + json.ignore.parse.errors: false + + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: 192.168.44.12:9001 + table: default.sink_table + batch.size: 10 + batch.byte.size: 200MB + batch.interval: 1s + connection.user: e54c9568586180eede1506eecf3574e9 + connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e + +application: # [object] Define job configuration + env: + name: example-inline-to-clickhouse + parallelism: 1 + shade.identifier: aes + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [ clickhouse_sink ] + - name: clickhouse_sink + downstream: [] \ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java index 6cc01b4..8eab377 100644 --- a/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java +++ b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java @@ -1,11 +1,13 @@ package com.geedgenetworks.test.e2e.clickhouse; +import com.alibaba.fastjson2.JSON; import com.geedgenetworks.test.common.TestResource; import com.geedgenetworks.test.common.TestSuiteBase; import com.geedgenetworks.test.common.container.ContainerUtil; import com.geedgenetworks.test.common.container.TestContainer; import com.geedgenetworks.test.common.container.TestContainerId; import com.geedgenetworks.test.common.junit.DisabledOnContainer; +import com.google.common.collect.Maps; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import lombok.extern.slf4j.Slf4j; @@ -15,10 +17,12 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestTemplate; import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.containers.Container; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.lifecycle.Startables; import org.testcontainers.shaded.org.apache.commons.io.IOUtils; import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; import java.io.File; import java.io.IOException; @@ -26,14 +30,14 @@ import java.io.InputStream; import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.sql.*; -import java.sql.Date; -import java.time.LocalDate; -import java.time.LocalDateTime; import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.awaitility.Awaitility.await; + @Slf4j @DisabledOnContainer( value = {TestContainerId.FLINK_1_17}, @@ -41,7 +45,7 @@ import java.util.stream.Stream; public class ClickHouseIT extends TestSuiteBase implements TestResource { private static final String CLICKHOUSE_DOCKER_IMAGE = "clickhouse/clickhouse-server:23.3.19.32"; private static final String DRIVER_CLASS = "com.clickhouse.jdbc.ClickHouseDriver"; - private static final String INIT_CLICKHOUSE_PATH = "/init/clickhouse_init.conf"; + private static final String INIT_CLICKHOUSE_PATH = "/init/clickhouse_test_sql.conf"; private static final String DATABASE = "default"; private static final String SOURCE_TABLE = "source_table"; private static final String SINK_TABLE = "sink_table"; @@ -51,39 +55,27 @@ public class ClickHouseIT extends TestSuiteBase implements TestResource { private static final Config CONFIG = getInitClickhouseConfig(); private ClickHouseContainer clickHouseContainer; private Connection connection; - private static final ListTEST_DATASET = generateTestDataSet(); private static final String[] default_columns = new String[] { - "id", - "c_map", - "c_array_string", - "c_array_short", - "c_array_int", - "c_array_long", - "c_array_float", - "c_array_double", - "c_string", - "c_boolean", - "c_int8", - "c_int16", - "c_int32", - "c_int64", - "c_float32", - "c_float64", - "c_decimal", - "c_date", - "c_datetime", - "c_nullable", - "c_lowcardinality", - "c_nested.int", - "c_nested.double", - "c_nested.string", - "c_int128", - "c_uint128", - "c_int256", - "c_uint256", - "c_point", - "c_ring" + "id", + "c_array_string", + "c_array_short", + "c_array_int", + "c_array_long", + "c_array_float", + "c_array_double", + "c_string", + "c_int8", + "c_int16", + "c_int32", + "c_int64", + "c_float32", + "c_float64", + "c_decimal", + "c_nullable", + "c_lowcardinality" }; + private static final List>TEST_DATASET = generateTestDataSet(); + @BeforeAll @Override @@ -92,10 +84,17 @@ public class ClickHouseIT extends TestSuiteBase implements TestResource { new ClickHouseContainer(CLICKHOUSE_DOCKER_IMAGE) .withNetwork(NETWORK) .withNetworkAliases(HOST) + .withCopyFileToContainer(MountableFile.forClasspathResource("init/users.xml"), "/etc/clickhouse-server/users.xml") + .withCopyFileToContainer(MountableFile.forClasspathResource("init/init-clickhouse.sql"), "/docker-entrypoint-initdb.d/init-clickhouse.sql") .withLogConsumer( new Slf4jLogConsumer( DockerLoggerFactory.getLogger(CLICKHOUSE_DOCKER_IMAGE))); + Startables.deepStart(Stream.of(this.clickHouseContainer)).join(); + System.out.println("Clickhouse JDBC URL: " + this.clickHouseContainer.getJdbcUrl()); + System.out.println("Clickhouse username: " + this.clickHouseContainer.getUsername()); + System.out.println("Clickhouse password: " + this.clickHouseContainer.getPassword()); + log.info("Clickhouse container started"); Awaitility.given() .ignoreExceptions() @@ -104,20 +103,51 @@ public class ClickHouseIT extends TestSuiteBase implements TestResource { .untilAsserted(this::initConnection); this.initializeClickhouseTable(); this.batchInsertData(); + log.info(JSON.toJSONString(TEST_DATASET)); } + private void initConnection() + throws SQLException, ClassNotFoundException, InstantiationException, + IllegalAccessException { + final Properties info = new Properties(); + info.put("user", this.clickHouseContainer.getUsername()); + info.put("password", this.clickHouseContainer.getPassword()); + this.connection = + ((Driver) Class.forName(DRIVER_CLASS).newInstance()) + .connect(this.clickHouseContainer.getJdbcUrl(), info); + + } @TestTemplate - public void testClickhouse(TestContainer container) throws Exception { + public void testClickHouse(TestContainer container) throws Exception { assertHasData(SOURCE_TABLE); + clearTable(SOURCE_TABLE); + } - //assertHasData(SINK_TABLE); - //compareResult(); - //clearSinkTable(); + @TestTemplate + public void testClickHouseDataTypeSinkTable(TestContainer container) throws Exception { + CompletableFuture.supplyAsync( + () -> { + try { + Container.ExecResult execResult = container.executeJob("/clickhouse_data_type_sink.yaml"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + return execResult; + } catch (Exception e) { + log.error("Commit task exception:" + e.getMessage()); + throw new RuntimeException(e); + } + }); + await().atMost(300000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + assertHasData(SINK_TABLE); + clearTable(SINK_TABLE); + }); } + private void assertHasData(String table) { try (Statement statement = connection.createStatement()) { String sql = String.format("select * from %s.%s limit 1", DATABASE, table); @@ -128,9 +158,9 @@ public class ClickHouseIT extends TestSuiteBase implements TestResource { } } - private void clearSinkTable() { + private void clearTable(String table) { try (Statement statement = connection.createStatement()) { - statement.execute(String.format("truncate table %s.%s", DATABASE, SINK_TABLE)); + statement.execute(String.format("truncate table %s.%s", DATABASE, table)); } catch (SQLException e) { throw new RuntimeException("Test clickhouse server image error", e); } @@ -187,38 +217,24 @@ public class ClickHouseIT extends TestSuiteBase implements TestResource { try { this.connection.setAutoCommit(true); preparedStatement = this.connection.prepareStatement(sql); - for (Object[] row : TEST_DATASET) { - preparedStatement.setLong(1, (Long) row[0]); - preparedStatement.setObject(2, row[1]); - preparedStatement.setArray(3, toSqlArray(row[2])); - preparedStatement.setArray(4, toSqlArray(row[3])); - preparedStatement.setArray(5, toSqlArray(row[4])); - preparedStatement.setArray(6, toSqlArray(row[5])); - preparedStatement.setArray(7, toSqlArray(row[6])); - preparedStatement.setArray(8, toSqlArray(row[7])); - preparedStatement.setString(9, (String) row[8]); - preparedStatement.setBoolean(10, (Boolean) row[9]); - preparedStatement.setByte(11, (Byte) row[10]); - preparedStatement.setShort(12, (Short) row[11]); - preparedStatement.setInt(13, (Integer) row[12]); - preparedStatement.setLong(14, (Long) row[13]); - preparedStatement.setFloat(15, (Float) row[14]); - preparedStatement.setDouble(16, (Double) row[15]); - preparedStatement.setBigDecimal(17, (BigDecimal) row[16]); - preparedStatement.setDate(18, Date.valueOf((LocalDate) row[17])); - preparedStatement.setTimestamp( - 19, Timestamp.valueOf((LocalDateTime) row[18])); - preparedStatement.setInt(20, (Integer) row[19]); - preparedStatement.setString(21, (String) row[20]); - preparedStatement.setArray(22, toSqlArray(row[21])); - preparedStatement.setArray(23, toSqlArray(row[22])); - preparedStatement.setArray(24, toSqlArray(row[23])); - preparedStatement.setObject(25, row[24]); - preparedStatement.setObject(26, row[25]); - preparedStatement.setObject(27, row[26]); - preparedStatement.setObject(28, row[27]); - preparedStatement.setObject(29, row[28]); - preparedStatement.setObject(30, row[29]); + for (Map row : TEST_DATASET) { + preparedStatement.setLong(1, (Long) row.get(default_columns[0])); + preparedStatement.setArray(2, toSqlArray(row.get(default_columns[1]))); + preparedStatement.setArray(3, toSqlArray(row.get(default_columns[2]))); + preparedStatement.setArray(4, toSqlArray(row.get(default_columns[3]))); + preparedStatement.setArray(5, toSqlArray(row.get(default_columns[4]))); + preparedStatement.setArray(6, toSqlArray(row.get(default_columns[5]))); + preparedStatement.setArray(7, toSqlArray(row.get(default_columns[6]))); + preparedStatement.setString(8, (String) row.get(default_columns[7])); + preparedStatement.setByte(9, (Byte) row.get(default_columns[8])); + preparedStatement.setShort(10, (Short) row.get(default_columns[9])); + preparedStatement.setInt(11, (Integer) row.get(default_columns[10])); + preparedStatement.setLong(12, (Long) row.get(default_columns[11])); + preparedStatement.setFloat(13, (Float) row.get(default_columns[12])); + preparedStatement.setDouble(14, (Double) row.get(default_columns[13])); + preparedStatement.setBigDecimal(15, (BigDecimal) row.get(default_columns[14])); + preparedStatement.setInt(16, (Integer) row.get(default_columns[15])); + preparedStatement.setString(17, (String) row.get(default_columns[16])); preparedStatement.addBatch(); } @@ -277,56 +293,33 @@ public class ClickHouseIT extends TestSuiteBase implements TestResource { return connection.createArrayOf(sqlType, elements); } - private static List generateTestDataSet() { - List rows = new ArrayList<>(); + private static List> generateTestDataSet() { + List> rows = new ArrayList<>(); for (int i = 0; i < 100; ++i) { - Object[] row = new Object[] { - (long) i, - Collections.singletonMap("key", Integer.parseInt("1")), - new String[] {"string"}, - new Short[] {Short.parseShort("1")}, - new Integer[] {Integer.parseInt("1")}, - new Long[] {Long.parseLong("1")}, - new Float[] {Float.parseFloat("1.1")}, - new Double[] {Double.parseDouble("1.1")}, - "string", - Boolean.FALSE, - Byte.parseByte("1"), - Short.parseShort("1"), - Integer.parseInt("1"), - Long.parseLong("1"), - Float.parseFloat("1.1"), - Double.parseDouble("1.1"), - BigDecimal.valueOf(11L, 1), - LocalDate.now(), - LocalDateTime.now(), - i, - "string", - new Integer[] {Integer.parseInt("1")}, - new Double[] {Double.parseDouble("1.1")}, - new String[] {"1"}, - "170141183460469231731687303715884105727", - "340282366920938463463374607431768211455", - "57896044618658097711785492504343953926634992332820282019728792003956564819967", - "115792089237316195423570985008687907853269984665640564039457584007913129639935", - new double[] {1, 2}, - new double[][] {{2, 3}, {4, 5}} - }; + Map row = Maps.newLinkedHashMap(); + row.put(default_columns[0], (long) i); + row.put(default_columns[1], new String[] {"string"}); + row.put(default_columns[2], new Short[] {Short.parseShort("1")}); + row.put(default_columns[3], new Integer[] {Integer.parseInt("1")}); + row.put(default_columns[4], new Long[] {Long.parseLong("1")}); + row.put(default_columns[5], new Float[] {Float.parseFloat("1.1")}); + row.put(default_columns[6], new Double[] {Double.parseDouble("1.1")}); + row.put(default_columns[7], "string"); + row.put(default_columns[8], Byte.parseByte("1")); + row.put(default_columns[9], Short.parseShort("1")); + row.put(default_columns[10], Integer.parseInt("1")); + row.put(default_columns[11], Long.parseLong("1")); + row.put(default_columns[12], Float.parseFloat("1.1")); + row.put(default_columns[13], Double.parseDouble("1.1")); + row.put(default_columns[14], BigDecimal.valueOf(11L, 1)); + row.put(default_columns[15], i); + row.put(default_columns[16], "string"); rows.add(row); } return rows; } - private void initConnection() - throws SQLException, ClassNotFoundException, InstantiationException, - IllegalAccessException { - final Properties info = new Properties(); - info.put("user", this.clickHouseContainer.getUsername()); - info.put("password", this.clickHouseContainer.getPassword()); - this.connection = - ((Driver) Class.forName(DRIVER_CLASS).newInstance()) - .connect(this.clickHouseContainer.getJdbcUrl(), info); - } + private void initializeClickhouseTable() { try { diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml b/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml new file mode 100644 index 0000000..eb020c1 --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml @@ -0,0 +1,78 @@ +sources: + inline_source: + type: inline + schema: + fields: + - name: id + type: bigint + - name: c_array_string + type: array + - name: c_array_short + type: array + - name: c_array_int + type: array + - name: c_array_long + type: array + - name: c_array_float + type: array + - name: c_array_double + type: array + - name: c_string + type: string + - name: c_int8 + type: int + - name: c_int16 + type: int + - name: c_int32 + type: int + - name: c_int64 + type: int + - name: c_float32 + type: float + - name: c_float64 + type: double + - name: c_decimal + type: double + - name: c_date + type: string + - name: c_datetime + type: string + - name: c_nullable + type: int + - name: c_lowcardinality + type: string + properties: + # + # [string] Event Data, it will be parsed to Map by the specified format. + # + data: '[{"id":0,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":0,"c_lowcardinality":"string"},{"id":1,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":1,"c_lowcardinality":"string"},{"id":2,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":2,"c_lowcardinality":"string"},{"id":3,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":3,"c_lowcardinality":"string"},{"id":4,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":4,"c_lowcardinality":"string"},{"id":5,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":5,"c_lowcardinality":"string"},{"id":6,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":6,"c_lowcardinality":"string"},{"id":7,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":7,"c_lowcardinality":"string"},{"id":8,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":8,"c_lowcardinality":"string"},{"id":9,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":9,"c_lowcardinality":"string"},{"id":10,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":10,"c_lowcardinality":"string"},{"id":11,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":11,"c_lowcardinality":"string"},{"id":12,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":12,"c_lowcardinality":"string"},{"id":13,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":13,"c_lowcardinality":"string"},{"id":14,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":14,"c_lowcardinality":"string"},{"id":15,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":15,"c_lowcardinality":"string"},{"id":16,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":16,"c_lowcardinality":"string"},{"id":17,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":17,"c_lowcardinality":"string"},{"id":18,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":18,"c_lowcardinality":"string"},{"id":19,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":19,"c_lowcardinality":"string"},{"id":20,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":20,"c_lowcardinality":"string"},{"id":21,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":21,"c_lowcardinality":"string"},{"id":22,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":22,"c_lowcardinality":"string"},{"id":23,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":23,"c_lowcardinality":"string"},{"id":24,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":24,"c_lowcardinality":"string"},{"id":25,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":25,"c_lowcardinality":"string"},{"id":26,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":26,"c_lowcardinality":"string"},{"id":27,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":27,"c_lowcardinality":"string"},{"id":28,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":28,"c_lowcardinality":"string"},{"id":29,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":29,"c_lowcardinality":"string"},{"id":30,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":30,"c_lowcardinality":"string"},{"id":31,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":31,"c_lowcardinality":"string"},{"id":32,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":32,"c_lowcardinality":"string"},{"id":33,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":33,"c_lowcardinality":"string"},{"id":34,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":34,"c_lowcardinality":"string"},{"id":35,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":35,"c_lowcardinality":"string"},{"id":36,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":36,"c_lowcardinality":"string"},{"id":37,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":37,"c_lowcardinality":"string"},{"id":38,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":38,"c_lowcardinality":"string"},{"id":39,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":39,"c_lowcardinality":"string"},{"id":40,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":40,"c_lowcardinality":"string"},{"id":41,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":41,"c_lowcardinality":"string"},{"id":42,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":42,"c_lowcardinality":"string"},{"id":43,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":43,"c_lowcardinality":"string"},{"id":44,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":44,"c_lowcardinality":"string"},{"id":45,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":45,"c_lowcardinality":"string"},{"id":46,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":46,"c_lowcardinality":"string"},{"id":47,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":47,"c_lowcardinality":"string"},{"id":48,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":48,"c_lowcardinality":"string"},{"id":49,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":49,"c_lowcardinality":"string"},{"id":50,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":50,"c_lowcardinality":"string"},{"id":51,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":51,"c_lowcardinality":"string"},{"id":52,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":52,"c_lowcardinality":"string"},{"id":53,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":53,"c_lowcardinality":"string"},{"id":54,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":54,"c_lowcardinality":"string"},{"id":55,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":55,"c_lowcardinality":"string"},{"id":56,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":56,"c_lowcardinality":"string"},{"id":57,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":57,"c_lowcardinality":"string"},{"id":58,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":58,"c_lowcardinality":"string"},{"id":59,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":59,"c_lowcardinality":"string"},{"id":60,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":60,"c_lowcardinality":"string"},{"id":61,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":61,"c_lowcardinality":"string"},{"id":62,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":62,"c_lowcardinality":"string"},{"id":63,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":63,"c_lowcardinality":"string"},{"id":64,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":64,"c_lowcardinality":"string"},{"id":65,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":65,"c_lowcardinality":"string"},{"id":66,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":66,"c_lowcardinality":"string"},{"id":67,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":67,"c_lowcardinality":"string"},{"id":68,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":68,"c_lowcardinality":"string"},{"id":69,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":69,"c_lowcardinality":"string"},{"id":70,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":70,"c_lowcardinality":"string"},{"id":71,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":71,"c_lowcardinality":"string"},{"id":72,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":72,"c_lowcardinality":"string"},{"id":73,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":73,"c_lowcardinality":"string"},{"id":74,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":74,"c_lowcardinality":"string"},{"id":75,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":75,"c_lowcardinality":"string"},{"id":76,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":76,"c_lowcardinality":"string"},{"id":77,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":77,"c_lowcardinality":"string"},{"id":78,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":78,"c_lowcardinality":"string"},{"id":79,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":79,"c_lowcardinality":"string"},{"id":80,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":80,"c_lowcardinality":"string"},{"id":81,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":81,"c_lowcardinality":"string"},{"id":82,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":82,"c_lowcardinality":"string"},{"id":83,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":83,"c_lowcardinality":"string"},{"id":84,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":84,"c_lowcardinality":"string"},{"id":85,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":85,"c_lowcardinality":"string"},{"id":86,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":86,"c_lowcardinality":"string"},{"id":87,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":87,"c_lowcardinality":"string"},{"id":88,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":88,"c_lowcardinality":"string"},{"id":89,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":89,"c_lowcardinality":"string"},{"id":90,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":90,"c_lowcardinality":"string"},{"id":91,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":91,"c_lowcardinality":"string"},{"id":92,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":92,"c_lowcardinality":"string"},{"id":93,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":93,"c_lowcardinality":"string"},{"id":94,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":94,"c_lowcardinality":"string"},{"id":95,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":95,"c_lowcardinality":"string"},{"id":96,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":96,"c_lowcardinality":"string"},{"id":97,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":97,"c_lowcardinality":"string"},{"id":98,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":98,"c_lowcardinality":"string"},{"id":99,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":99,"c_lowcardinality":"string"}]' + format: json + interval.per.row: 1s + repeat.count: 100 + json.ignore.parse.errors: false + + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: clickhouse:9000 + table: default.sink_table + batch.size: 10 + batch.byte.size: 200MB + batch.interval: 1s + connection.user: ee9b0016824d59c8c191aa9633e4b61e + connection.password: ee9b0016824d59c8c191aa9633e4b61e + +application: # [object] Define job configuration + env: + name: example-inline-to-clickhouse + parallelism: 1 + shade.identifier: aes + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [ clickhouse_sink ] + - name: clickhouse_sink + downstream: [] \ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_init.conf b/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_init.conf deleted file mode 100644 index 78f2daa..0000000 --- a/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_init.conf +++ /dev/null @@ -1,143 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -source_table = """ -set allow_experimental_geo_types = 1; -create table if not exists `default`.source_table( - `id` Int64, - `c_map` Map(String, Int32), - `c_array_string` Array(String), - `c_array_short` Array(Int16), - `c_array_int` Array(Int32), - `c_array_long` Array(Int64), - `c_array_float` Array(Float32), - `c_array_double` Array(Float64), - `c_string` String, - `c_boolean` Boolean, - `c_int8` Int8, - `c_int16` Int16, - `c_int32` Int32, - `c_int64` Int64, - `c_float32` Float32, - `c_float64` Float64, - `c_decimal` Decimal(9,4), - `c_date` Date, - `c_datetime` DateTime64, - `c_nullable` Nullable(Int32), - `c_lowcardinality` LowCardinality(String), - `c_nested` Nested - ( - `int` UInt32, - `double` Float64, - `string` String - ), - `c_int128` Int128, - `c_uint128` UInt128, - `c_int256` Int256, - `c_uint256` UInt256, - `c_point` Point, - `c_ring` Ring -)engine=Memory; -""" - -sink_table = """ -create table if not exists `default`.sink_table( - `id` Int64, - `c_map` Map(String, Int32), - `c_array_string` Array(String), - `c_array_short` Array(Int16), - `c_array_int` Array(Int32), - `c_array_long` Array(Int64), - `c_array_float` Array(Float32), - `c_array_double` Array(Float64), - `c_string` String, - `c_boolean` Boolean, - `c_int8` Int8, - `c_int16` Int16, - `c_int32` Int32, - `c_int64` Int64, - `c_float32` Float32, - `c_float64` Float64, - `c_decimal` Decimal(9,4), - `c_date` Date, - `c_datetime` DateTime64, - `c_nullable` Nullable(Int32), - `c_lowcardinality` LowCardinality(String), - `c_nested` Nested - ( - `int` UInt32, - `double` Float64, - `string` String - ), - `c_int128` Int128, - `c_uint128` UInt128, - `c_int256` Int256, - `c_uint256` UInt256, - `c_point` Point, - `c_ring` Ring -)engine=Memory; -""" - -insert_sql = """ -insert into `default`.source_table -( - `id`, - `c_map`, - `c_array_string`, - `c_array_short`, - `c_array_int`, - `c_array_long`, - `c_array_float`, - `c_array_double`, - `c_string`, - `c_boolean`, - `c_int8`, - `c_int16`, - `c_int32`, - `c_int64`, - `c_float32`, - `c_float64`, - `c_decimal`, - `c_date`, - `c_datetime`, - `c_nullable`, - `c_lowcardinality`, - `c_nested.int`, - `c_nested.double`, - `c_nested.string`, - `c_int128`, - `c_uint128`, - `c_int256`, - `c_uint256`, - `c_point`, - `c_ring` -) -values -(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) -""" - -compare_sql = """ -select - %s - from ( - select * from default.source_table -union all - select * from default.sink_table - ) -group by %s -having count(*) < 2 -""" \ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf b/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf new file mode 100644 index 0000000..c24e632 --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf @@ -0,0 +1,98 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +source_table = """ +set allow_experimental_geo_types = 1; +create table if not exists `default`.source_table( + `id` Int64, + `c_array_string` Array(String), + `c_array_short` Array(Int16), + `c_array_int` Array(Int32), + `c_array_long` Array(Int64), + `c_array_float` Array(Float32), + `c_array_double` Array(Float64), + `c_string` String, + `c_int8` Int8, + `c_int16` Int16, + `c_int32` Int32, + `c_int64` Int64, + `c_float32` Float32, + `c_float64` Float64, + `c_decimal` Decimal(9,4), + `c_nullable` Nullable(Int32), + `c_lowcardinality` LowCardinality(String) +)engine=Memory; +""" + +sink_table = """ +create table if not exists `default`.sink_table( + `id` Int64, + `c_array_string` Array(String), + `c_array_short` Array(Int16), + `c_array_int` Array(Int32), + `c_array_long` Array(Int64), + `c_array_float` Array(Float32), + `c_array_double` Array(Float64), + `c_string` String, + `c_int8` Int8, + `c_int16` Int16, + `c_int32` Int32, + `c_int64` Int64, + `c_float32` Float32, + `c_float64` Float64, + `c_decimal` Decimal(9,4), + `c_nullable` Nullable(Int32), + `c_lowcardinality` LowCardinality(String) +)engine=Memory; +""" + +insert_sql = """ +insert into `default`.source_table +( + `id`, + `c_array_string`, + `c_array_short`, + `c_array_int`, + `c_array_long`, + `c_array_float`, + `c_array_double`, + `c_string`, + `c_int8`, + `c_int16`, + `c_int32`, + `c_int64`, + `c_float32`, + `c_float64`, + `c_decimal`, + `c_nullable`, + `c_lowcardinality` +) +values +(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) +""" + +compare_sql = """ +select + %s + from ( + select * from default.source_table +union all + select * from default.sink_table + ) +group by %s +having count(*) < 2 +""" \ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/init/init-clickhouse.sql b/groot-tests/test-e2e-clickhouse/src/test/resources/init/init-clickhouse.sql new file mode 100644 index 0000000..fd9daac --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/init/init-clickhouse.sql @@ -0,0 +1,4 @@ +show databases; +-- ALTER USER default IDENTIFIED WITH plaintext_password BY 'testuser'; +CREATE USER testuser IDENTIFIED WITH plaintext_password BY 'testuser'; +GRANT ALL ON *.* TO testuser; \ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/init/users.xml b/groot-tests/test-e2e-clickhouse/src/test/resources/init/users.xml new file mode 100644 index 0000000..86a590d --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/init/users.xml @@ -0,0 +1,29 @@ + + + + + default + 1 + 1 + 1 + 1 + + ::/0 + + + ALTER TABLE ON *.* + CREATE USER ON *.* + GRANT ON *.* + + + + + + + 10000000000 + 1 + random + 8 + + + diff --git a/plugin-mapping.properties b/plugin-mapping.properties index 4e2eb81..7cc144d 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -1,7 +1,7 @@ #Connectors grootstream.source.kafka = connector-kafka grootstream.sink.kafka = connector-kafka -grootstream.source.clickhouse= connector-clickhouse +grootstream.sink.clickhouse = connector-clickhouse grootstream.source.ipfix = connector-ipfix-collector grootstream.source.mock = connector-mock grootstream.source.file = connector-file \ No newline at end of file diff --git a/pom.xml b/pom.xml index 69ea712..a8f919b 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ 3.1.1 1.3.0 4.0.4 - 1.20.0 + 1.20.1 4.2.0 2.40.0 1.7.25 -- cgit v1.2.3 From b6d37a57e16fee20b2725333dd9740fbe03dc142 Mon Sep 17 00:00:00 2001 From: wangkuan Date: Tue, 13 Aug 2024 11:10:40 +0800 Subject: [feature][core][common][bootstrap]支持tableProcessor,增加UnRoll函数 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/udf.plugins | 3 +- .../execution/AbstractProcessorExecutor.java | 57 ++++++++- .../common/config/TableConfigOptions.java | 34 +++++ .../geedgenetworks/common/udf/TableFunction.java | 20 +++ groot-common/src/main/resources/udf.plugins | 3 +- .../com/geedgenetworks/core/pojo/TableConfig.java | 15 +++ .../core/processor/projection/UdfEntity.java | 2 + .../core/processor/table/TableProcessor.java | 7 ++ .../processor/table/TableProcessorFunction.java | 138 +++++++++++++++++++++ .../core/processor/table/TableProcessorImpl.java | 33 +++++ .../com/geedgenetworks/core/udf/udtf/UnRoll.java | 132 ++++++++++++++++++++ .../com.geedgenetworks.core.processor.Processor | 3 +- .../core/udf/test/table/UnRollFunctionTest.java | 132 ++++++++++++++++++++ 13 files changed, 572 insertions(+), 7 deletions(-) create mode 100644 groot-common/src/main/java/com/geedgenetworks/common/config/TableConfigOptions.java create mode 100644 groot-common/src/main/java/com/geedgenetworks/common/udf/TableFunction.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/pojo/TableConfig.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorImpl.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/UnRoll.java create mode 100644 groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnRollFunctionTest.java diff --git a/config/udf.plugins b/config/udf.plugins index 2978bbe..9eb32c4 100644 --- a/config/udf.plugins +++ b/config/udf.plugins @@ -21,4 +21,5 @@ com.geedgenetworks.core.udf.udaf.CollectSet com.geedgenetworks.core.udf.udaf.LongCount com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue -com.geedgenetworks.core.udf.udaf.FirstValue \ No newline at end of file +com.geedgenetworks.core.udf.udaf.FirstValue +com.geedgenetworks.core.udf.udtf.UnRoll \ No newline at end of file diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java index bd8b75c..66c0b0f 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java @@ -2,15 +2,14 @@ package com.geedgenetworks.bootstrap.execution; import com.alibaba.fastjson.JSONObject; import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.config.AggregateConfigOptions; -import com.geedgenetworks.common.config.CheckConfigUtil; -import com.geedgenetworks.common.config.CheckResult; -import com.geedgenetworks.common.config.ProjectionConfigOptions; +import com.geedgenetworks.common.config.*; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; import com.geedgenetworks.core.pojo.AggregateConfig; import com.geedgenetworks.core.pojo.ProcessorConfig; import com.geedgenetworks.core.pojo.ProjectionConfig; +import com.geedgenetworks.core.pojo.TableConfig; +import com.geedgenetworks.core.processor.table.TableProcessor; import com.geedgenetworks.core.processor.aggregate.AggregateProcessor; import com.geedgenetworks.core.processor.projection.ProjectionProcessor; import com.typesafe.config.Config; @@ -35,6 +34,9 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor 0) { + tableConfig.setParallelism(node.getParallelism()); + } + try { + dataStream = + tableProcessor.processorFunction( + dataStream, tableConfig, jobRuntimeEnvironment.getStreamExecutionEnvironment().getConfig()); + } catch (Exception e) { + throw new JobExecuteException("Create orderby pipeline instance failed!", e); + } + return dataStream; + } protected SingleOutputStreamOperator executeAggregateProcessor(SingleOutputStreamOperator dataStream, Node node, AggregateConfig aggregateConfig) throws JobExecuteException { AggregateProcessor aggregateProcessor; @@ -114,6 +141,9 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor value, Config config) { + + CheckResult result = CheckConfigUtil.checkAtLeastOneExists(config.getConfig(key), + TableConfigOptions.OUTPUT_FIELDS.key(), + TableConfigOptions.REMOVE_FIELDS.key(), + TableConfigOptions.FUNCTIONS.key()); + if (!result.isSuccess()) { + throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, String.format( + "Table processor: %s, At least one of [%s] should be specified.", + key, String.join(",", + TableConfigOptions.OUTPUT_FIELDS.key(), + TableConfigOptions.REMOVE_FIELDS.key(), + TableConfigOptions.FUNCTIONS.key()))); + } + + TableConfig tableConfig = new JSONObject(value).toJavaObject(TableConfig.class); + tableConfig.setName(key); + return tableConfig; + } } diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/TableConfigOptions.java b/groot-common/src/main/java/com/geedgenetworks/common/config/TableConfigOptions.java new file mode 100644 index 0000000..480496d --- /dev/null +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/TableConfigOptions.java @@ -0,0 +1,34 @@ +package com.geedgenetworks.common.config; + +import com.alibaba.fastjson2.TypeReference; +import com.geedgenetworks.common.udf.UDFContext; + +import java.util.List; + +public interface TableConfigOptions { + Option TYPE = Options.key("type") + .stringType() + .noDefaultValue() + .withDescription("The type of processor."); + + Option> OUTPUT_FIELDS = Options.key("output_fields") + .listType() + .noDefaultValue() + .withDescription("The fields to be outputted."); + + Option> REMOVE_FIELDS = Options.key("remove_fields") + .listType() + .noDefaultValue() + .withDescription("The fields to be removed."); + + Option> FUNCTIONS = Options.key("functions") + .type(new TypeReference>() {}) + .noDefaultValue() + .withDescription("The functions to be executed."); + + + + + + +} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/udf/TableFunction.java b/groot-common/src/main/java/com/geedgenetworks/common/udf/TableFunction.java new file mode 100644 index 0000000..e602291 --- /dev/null +++ b/groot-common/src/main/java/com/geedgenetworks/common/udf/TableFunction.java @@ -0,0 +1,20 @@ +package com.geedgenetworks.common.udf; + +import com.geedgenetworks.common.Event; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.util.Collector; + +import java.io.Serializable; +import java.util.List; + +public interface TableFunction extends Serializable { + + void open(RuntimeContext runtimeContext, UDFContext udfContext); + + List evaluate(Event event); + + String functionName(); + + void close(); + +} diff --git a/groot-common/src/main/resources/udf.plugins b/groot-common/src/main/resources/udf.plugins index 1b7fca4..f5a4c3f 100644 --- a/groot-common/src/main/resources/udf.plugins +++ b/groot-common/src/main/resources/udf.plugins @@ -20,4 +20,5 @@ com.geedgenetworks.core.udf.udaf.CollectSet com.geedgenetworks.core.udf.udaf.LongCount com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue -com.geedgenetworks.core.udf.udaf.FirstValue \ No newline at end of file +com.geedgenetworks.core.udf.udaf.FirstValue +com.geedgenetworks.core.udf.udtf.UnRoll \ No newline at end of file diff --git a/groot-core/src/main/java/com/geedgenetworks/core/pojo/TableConfig.java b/groot-core/src/main/java/com/geedgenetworks/core/pojo/TableConfig.java new file mode 100644 index 0000000..3efb8e1 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/pojo/TableConfig.java @@ -0,0 +1,15 @@ +package com.geedgenetworks.core.pojo; + +import com.geedgenetworks.common.udf.UDFContext; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +public class TableConfig extends ProcessorConfig { + + private List functions; + private String format; +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java index 34267a6..ab6a6f5 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java @@ -3,6 +3,7 @@ package com.geedgenetworks.core.processor.projection; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.ScalarFunction; +import com.geedgenetworks.common.udf.TableFunction; import com.geedgenetworks.common.udf.UDFContext; import com.googlecode.aviator.Expression; import lombok.Data; @@ -13,6 +14,7 @@ import java.io.Serializable; public class UdfEntity implements Serializable { private ScalarFunction scalarFunction; private AggregateFunction aggregateFunction; + private TableFunction tableFunction; private Expression filterExpression; private String name; private String className; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java new file mode 100644 index 0000000..4078997 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java @@ -0,0 +1,7 @@ +package com.geedgenetworks.core.processor.table; + +import com.geedgenetworks.core.pojo.TableConfig; +import com.geedgenetworks.core.processor.Processor; +public interface TableProcessor extends Processor { + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java new file mode 100644 index 0000000..7b6a5e2 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java @@ -0,0 +1,138 @@ +package com.geedgenetworks.core.processor.table; + +import com.alibaba.fastjson.JSON; +import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.exception.CommonErrorCode; +import com.geedgenetworks.common.exception.GrootStreamRuntimeException; +import com.geedgenetworks.common.udf.TableFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.common.utils.ColumnUtil; +import com.geedgenetworks.core.metrics.InternalMetrics; +import com.geedgenetworks.core.pojo.TableConfig; +import com.geedgenetworks.core.processor.projection.UdfEntity; +import com.google.common.collect.Lists; +import com.googlecode.aviator.AviatorEvaluator; +import com.googlecode.aviator.AviatorEvaluatorInstance; +import com.googlecode.aviator.Expression; +import com.googlecode.aviator.Options; +import com.googlecode.aviator.exception.ExpressionRuntimeException; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.common.functions.RichFlatMapFunction; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.Collector; +import org.checkerframework.checker.units.qual.A; + +import java.util.*; + +import static com.geedgenetworks.core.utils.UDFUtils.filterExecute; +import static com.geedgenetworks.core.utils.UDFUtils.getClassReflect; +@Slf4j +public class TableProcessorFunction extends RichFlatMapFunction { + private LinkedList functions; + private final TableConfig tableConfig; + private transient InternalMetrics internalMetrics; + + public TableProcessorFunction(TableConfig tableConfig) { + this.tableConfig = tableConfig; + } + + @Override + public void open(Configuration parameters) { + functions = Lists.newLinkedList(); + try { + this.internalMetrics = new InternalMetrics(getRuntimeContext()); + List udfContexts = tableConfig.getFunctions(); + if (udfContexts == null || udfContexts.isEmpty()) { + return; + } + Configuration configuration = (Configuration) getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); + List udfClassNameLists = JSON.parseObject(configuration.toMap().get(Constants.SYSPROP_UDF_PLUGIN_CONFIG), List.class); + Map udfClassReflect = getClassReflect(udfClassNameLists); + for (UDFContext udfContext : udfContexts) { + Expression filterExpression = null; + UdfEntity udfEntity = new UdfEntity(); + // 平台注册的函数包含任务中配置的函数则对函数进行实例化 + if (udfClassReflect.containsKey(udfContext.getFunction())) { + Class cls = Class.forName(udfClassReflect.get(udfContext.getFunction())); + TableFunction tableFunction = (TableFunction) cls.getConstructor().newInstance(); + tableFunction.open(getRuntimeContext(), udfContext); + // 函数如果包含filter,对表达式进行编译 + if (udfContext.getFilter() != null) { + AviatorEvaluatorInstance instance = AviatorEvaluator.getInstance(); + instance.setCachedExpressionByDefault(true); + instance.setOption(Options.OPTIMIZE_LEVEL, AviatorEvaluator.EVAL); + instance.setFunctionMissing(null); + filterExpression = instance.compile(udfContext.getFilter(), true); + } + udfEntity.setTableFunction(tableFunction); + udfEntity.setFilterExpression(filterExpression); + udfEntity.setName(udfContext.getFunction()); + udfEntity.setClassName(udfClassReflect.get(udfContext.getFunction())); + functions.add(udfEntity); + } else { + throw new GrootStreamRuntimeException(CommonErrorCode.UNSUPPORTED_OPERATION, + "Unsupported UDTF: " + udfContext.getFunction()); + } + + } + } catch (Exception e) { + throw new GrootStreamRuntimeException(CommonErrorCode.UNSUPPORTED_OPERATION, "Initialization UDTF failed!", e); + } + } + + + @Override + public void flatMap(Event event, Collector out) throws Exception { + internalMetrics.incrementInEvents(); + int errorCount = 0; + List events = new ArrayList<>(); + events.add(event); + for (UdfEntity udfEntity : functions) { + List newEvents = new ArrayList<>(); + for(int i=0;i0){ + internalMetrics.incrementErrorEvents(); + } + for(Event newEvent:events){ + if (tableConfig.getOutput_fields() != null + && !tableConfig.getOutput_fields().isEmpty()) { + newEvent.setExtractedFields( + ColumnUtil.columnSelector( + newEvent.getExtractedFields(), tableConfig.getOutput_fields())); + } + if (tableConfig.getRemove_fields() != null + && !tableConfig.getRemove_fields().isEmpty()) { + newEvent.setExtractedFields( + ColumnUtil.columnRemover( + newEvent.getExtractedFields(), tableConfig.getRemove_fields())); + } + if (!event.isDropped()) { + out.collect(newEvent); + internalMetrics.incrementOutEvents(); + } else { + internalMetrics.incrementDroppedEvents(); + } + + } + + + } +} \ No newline at end of file diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorImpl.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorImpl.java new file mode 100644 index 0000000..f36f8db --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorImpl.java @@ -0,0 +1,33 @@ +package com.geedgenetworks.core.processor.table; + +import com.geedgenetworks.common.Event; +import com.geedgenetworks.core.pojo.TableConfig; +import com.geedgenetworks.core.processor.projection.ProjectionProcessFunction; +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; + +import java.time.Duration; + +public class TableProcessorImpl implements TableProcessor { + + @Override + public SingleOutputStreamOperator processorFunction(SingleOutputStreamOperator grootEventSingleOutputStreamOperator, TableConfig tableConfig, ExecutionConfig config) throws Exception { + + if (tableConfig.getParallelism() != 0) { + return grootEventSingleOutputStreamOperator + .flatMap(new TableProcessorFunction(tableConfig)) + .setParallelism(tableConfig.getParallelism()) + .name(tableConfig.getName()); + } else { + return grootEventSingleOutputStreamOperator + .flatMap(new TableProcessorFunction(tableConfig)) + .name(tableConfig.getName()); + } + } + + @Override + public String type() { + return "table"; + } +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/UnRoll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/UnRoll.java new file mode 100644 index 0000000..82bea7b --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/UnRoll.java @@ -0,0 +1,132 @@ +package com.geedgenetworks.core.udf.udtf; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.exception.CommonErrorCode; +import com.geedgenetworks.common.exception.GrootStreamRuntimeException; +import com.geedgenetworks.common.udf.TableFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.googlecode.aviator.AviatorEvaluator; +import com.googlecode.aviator.AviatorEvaluatorInstance; +import com.googlecode.aviator.Expression; +import com.googlecode.aviator.Options; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.common.functions.RuntimeContext; + +import java.util.*; + + +@Slf4j +public class UnRoll implements TableFunction { + + private String lookupFieldName; + private String outputFieldName; + private Expression compiledExp; + private String expression; + private String outputFieldType; + + + @Override + public void open(RuntimeContext runtimeContext, UDFContext udfContext) { + if(udfContext.getLookup_fields()==null || udfContext.getOutput_fields()==null){ + throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); + } + this.lookupFieldName = udfContext.getLookup_fields().get(0); + this.outputFieldName = udfContext.getOutput_fields().get(0); + this.outputFieldType="object"; + if(udfContext.getParameters()==null ){ + expression=""; + } + else { + this.outputFieldType=udfContext.getParameters().getOrDefault("output_field_type", "object").toString().trim(); + expression=udfContext.getParameters().getOrDefault("path", "").toString().trim(); + } + if(!expression.isEmpty()){ + AviatorEvaluatorInstance instance = AviatorEvaluator.getInstance(); + instance.setCachedExpressionByDefault(true); + instance.setOption(Options.OPTIMIZE_LEVEL, AviatorEvaluator.EVAL); + instance.setFunctionMissing(null); + compiledExp = instance.compile("event."+expression, true); + } + + } + + @Override + public List evaluate(Event event) { + try { + if(event.getExtractedFields().containsKey(lookupFieldName)){ + Object object; + if(event.getExtractedFields().get(lookupFieldName) instanceof String){ + object = JSONObject.parseObject((String) event.getExtractedFields().get(lookupFieldName), Object.class); + } + else { + object = event.getExtractedFields().get(lookupFieldName); + } + try { + if(compiledExp!=null){ + Object obj = compiledExp.execute(compiledExp.newEnv("event", object)); + if(obj instanceof List) { + return parseList(obj,event); + } + else if(obj instanceof String){ + object = JSONObject.parseObject((String)obj, Object.class); + if(object instanceof List){ + return parseList(object,event); + } + else { + log.error("Invalid unroll ! Object is not instance of list. expression=" +expression); + } + }else { + log.error("Invalid unroll ! Object is not instance of String or List. expression=" +expression); + } + } + else { + if(object instanceof List){ + return parseList(object,event); + } + else { + log.error("Invalid unroll ! Object is not instance of list. "); + } + } + }catch (Exception e) { + log.error("Invalid unroll ! expression=" +expression + " Exception :" + e.getMessage()); + } + } + }catch (Exception e) { + log.error("Invalid parseObject ! expression=" +expression + " Exception :" + e.getMessage()); + } + return Collections.singletonList(event); + } + + private List parseList(Object object,Event event) { + List list = (List) object; + List eventList = new ArrayList<>(); + for (Object obj : list) { + Event newEvent = new Event(); + newEvent.setExtractedFields(new HashMap<>()); + newEvent.getExtractedFields().putAll(event.getExtractedFields()); + newEvent.getExtractedFields().remove(lookupFieldName); + if("string".equals(outputFieldType)) { + String jsonString = JSON.toJSONString(obj); + newEvent.getExtractedFields().put(outputFieldName, jsonString); + } + else { + newEvent.getExtractedFields().put(outputFieldName, obj); + } + eventList.add(newEvent); + } + return eventList; + } + + @Override + public String functionName() { + return "UNROLL"; + } + + @Override + public void close() { + + } + +} diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor index 727b42b..1f32ffa 100644 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor +++ b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor @@ -1,2 +1,3 @@ com.geedgenetworks.core.processor.aggregate.AggregateProcessorImpl -com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl \ No newline at end of file +com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl +com.geedgenetworks.core.processor.table.TableProcessorImpl \ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnRollFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnRollFunctionTest.java new file mode 100644 index 0000000..5686a42 --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnRollFunctionTest.java @@ -0,0 +1,132 @@ +package com.geedgenetworks.core.udf.test.table; + +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.udf.udtf.UnRoll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class UnRollFunctionTest { + + private static Map nestedMap; + @BeforeAll + public static void setUp() { + nestedMap = Map.of( + "k1", List.of( + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") + ), + "k2", Map.of("name", "[{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"c2s_destination_mac\":\"10:70:fd:03:c2:6d\",\"s2c_source_mac\":\"10:70:fd:03:c2:6d\",\"s2c_destination_mac\":\"10:70:fd:03:c2:6c\"},{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"c2s_destination_mac\":\"10:70:fd:03:c2:6d\",\"s2c_source_mac\":\"10:70:fd:03:c2:6d\",\"s2c_destination_mac\":\"10:70:fd:03:c2:6c\"}]", "lastName", "{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}"), + "k3","[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}]", + "k4","{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}", + "k5", Map.of("name",List.of( + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") + ) + ), "k6", Map.of("name",List.of( + Map.of("name2", List.of( + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") + ), "source_mac", "52:d4:18:c7:e5:11"), + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") + ) + ) + ); + } + + // 测试方法 + @Test + public void testUnrollFunction1() { + + + UDFContext udfContext = new UDFContext(); + udfContext.setLookup_fields(List.of("k1")); + udfContext.setOutput_fields(List.of("newk1")); + Map params = new HashMap<>(); + params.put("path", ""); + udfContext.setParameters(params); + UnRoll unroll = new UnRoll(); + unroll.open(null, udfContext); + Event event = new Event(); + event.setExtractedFields(nestedMap); + List result = unroll.evaluate(event); + assertEquals(2, result.size()); + Map map1 = (Map) result.get(0).getExtractedFields().get("newk1"); + assertEquals("52:d4:18:c7:e5:11", map1.get("source_mac")); + Map map2 = (Map) result.get(1).getExtractedFields().get("newk1"); + assertEquals("ff:ff:ff:ff:ff:ff", map2.get("source_mac")); + + + + } + + @Test + public void testUnrollFunction2() { + UDFContext udfContext = new UDFContext(); + UnRoll unroll = new UnRoll(); + Event event = new Event(); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k3")); + udfContext.setOutput_fields(List.of("newk3")); + unroll.open(null, udfContext); + List result3 = unroll.evaluate(event); + assertEquals(2, result3.size()); + Map map3 = (Map) result3.get(0).getExtractedFields().get("newk3"); + assertEquals("52:d4:18:c7:e5:11", map3.get("source_mac")); + Map map4 = (Map) result3.get(1).getExtractedFields().get("newk3"); + assertEquals("ff:ff:ff:ff:ff:ff", map4.get("source_mac")); + + + Map params = new HashMap<>(); + params.put("path", "name"); + udfContext.setParameters(params); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k2")); + udfContext.setOutput_fields(List.of("newk2")); + unroll.open(null, udfContext); + List result2 = unroll.evaluate(event); + assertEquals(2, result2.size()); + Map map5 = (Map) result2.get(0).getExtractedFields().get("newk2"); + assertEquals("52:d4:18:c7:e5:11", map5.get("source_mac")); + Map map6 = (Map) result2.get(1).getExtractedFields().get("newk2"); + assertEquals("ff:ff:ff:ff:ff:ff", map6.get("source_mac")); + + + Map params1 = new HashMap<>(); + params.put("path", "name.0.name2"); + udfContext.setParameters(params1); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k6")); + udfContext.setOutput_fields(List.of("newk6")); + unroll.open(null, udfContext); + List result6 = unroll.evaluate(event); + assertEquals(2, result6.size()); + Map map9 = (Map) result6.get(0).getExtractedFields().get("newk6"); + assertEquals("52:d4:18:c7:e5:11", map9.get("source_mac")); + Map map10 = (Map) result6.get(1).getExtractedFields().get("newk6"); + assertEquals("ff:ff:ff:ff:ff:ff", map10.get("source_mac")); + } + @Test + public void testUnrollFunction4() { + + + UDFContext udfContext = new UDFContext(); + UnRoll unroll = new UnRoll(); + Event event = new Event(); + Map params = new HashMap<>(); + udfContext.setParameters(params); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k4")); + udfContext.setOutput_fields(List.of("newk4")); + unroll.open(null, udfContext); + List result2 = unroll.evaluate(event); + assertEquals(1, result2.size()); + + } +} -- cgit v1.2.3 From e9d132716800f4b8ef46273b254c173c3d450864 Mon Sep 17 00:00:00 2001 From: wangkuan Date: Tue, 13 Aug 2024 17:14:57 +0800 Subject: [improve][core]拆分聚合函数open方法,较少调用次数,优化性能 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/geedgenetworks/common/udf/AggregateFunction.java | 7 ++++--- .../processor/aggregate/AggregateProcessorFunction.java | 7 ++++--- .../java/com/geedgenetworks/core/udf/udaf/CollectList.java | 14 +++++++------- .../java/com/geedgenetworks/core/udf/udaf/CollectSet.java | 10 +++++----- .../java/com/geedgenetworks/core/udf/udaf/FirstValue.java | 13 +++++++------ .../java/com/geedgenetworks/core/udf/udaf/LastValue.java | 10 ++++------ .../java/com/geedgenetworks/core/udf/udaf/LongCount.java | 11 ++++------- .../main/java/com/geedgenetworks/core/udf/udaf/Mean.java | 12 +++++------- .../java/com/geedgenetworks/core/udf/udaf/NumberSum.java | 9 ++++++--- .../core/udf/test/aggregate/CollectListTest.java | 5 +++-- .../core/udf/test/aggregate/CollectSetTest.java | 4 ++-- .../core/udf/test/aggregate/FirstValueTest.java | 6 +++--- .../core/udf/test/aggregate/LastValueTest.java | 6 +++--- .../core/udf/test/aggregate/LongCountTest.java | 6 +++--- .../geedgenetworks/core/udf/test/aggregate/MeanTest.java | 14 +++++++------- .../core/udf/test/aggregate/NumberSumTest.java | 6 +++--- 16 files changed, 70 insertions(+), 70 deletions(-) diff --git a/groot-common/src/main/java/com/geedgenetworks/common/udf/AggregateFunction.java b/groot-common/src/main/java/com/geedgenetworks/common/udf/AggregateFunction.java index 98450fd..455073f 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/udf/AggregateFunction.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/udf/AggregateFunction.java @@ -7,7 +7,9 @@ import java.io.Serializable; public interface AggregateFunction extends Serializable { - Accumulator open(UDFContext udfContext,Accumulator acc); + void open(UDFContext udfContext); + + Accumulator initAccumulator(Accumulator acc); Accumulator add(Event val, Accumulator acc); @@ -15,6 +17,5 @@ public interface AggregateFunction extends Serializable { Accumulator getResult(Accumulator acc); - void close(); - + default void close(){}; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java index c07374e..803fefc 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java @@ -71,23 +71,24 @@ public class AggregateProcessorFunction implements org.apache.flink.api.common.f } } + for (UdfEntity udfEntity : functions) { + udfEntity.getAggregateFunction().open(udfEntity.getUdfContext()); + } } catch (Exception e) { throw new GrootStreamRuntimeException(CommonErrorCode.UNSUPPORTED_OPERATION, "Initialization UDAF failed!", e); } } - @Override public Accumulator createAccumulator() { Map map = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(map); for (UdfEntity udfEntity : functions) { - udfEntity.getAggregateFunction().open(udfEntity.getUdfContext(), accumulator); + udfEntity.getAggregateFunction().initAccumulator(accumulator); } return accumulator; } - @Override public Accumulator add(Event event, Accumulator accumulator) { diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java index 4a43163..423eff9 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java @@ -22,6 +22,8 @@ import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.processor.projection.UdfEntity; + import java.util.*; /** @@ -32,9 +34,8 @@ public class CollectList implements AggregateFunction { private String lookupField; private String outputField; - @Override - public Accumulator open(UDFContext udfContext,Accumulator acc) { + public void open(UDFContext udfContext) { if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -45,11 +46,14 @@ public class CollectList implements AggregateFunction { else { outputField = lookupField; } + + } + @Override + public Accumulator initAccumulator(Accumulator acc) { acc.getMetricsFields().put(outputField, new ArrayList<>()); return acc; } - @Override public Accumulator add(Event event, Accumulator acc) { if(event.getExtractedFields().containsKey(lookupField)){ @@ -71,8 +75,4 @@ public class CollectList implements AggregateFunction { return acc; } - @Override - public void close() { - - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java index a425118..b4dfb14 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java @@ -8,6 +8,7 @@ import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.UDFContext; +import java.util.ArrayList; import java.util.HashSet; import java.util.Set; @@ -21,7 +22,7 @@ public class CollectSet implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc) { + public void open(UDFContext udfContext) { if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -32,11 +33,13 @@ public class CollectSet implements AggregateFunction { else { outputField = lookupField; } + } + @Override + public Accumulator initAccumulator(Accumulator acc) { acc.getMetricsFields().put(outputField, new HashSet<>()); return acc; } - @Override public Accumulator add(Event event, Accumulator acc) { if(event.getExtractedFields().containsKey(lookupField)){ @@ -58,8 +61,5 @@ public class CollectSet implements AggregateFunction { return acc; } - @Override - public void close() { - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java index 27490ef..6301a01 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java @@ -23,6 +23,8 @@ import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.UDFContext; +import java.util.ArrayList; + /** * Collects elements within a group and returns the list of aggregated objects */ @@ -33,7 +35,7 @@ public class FirstValue implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc) { + public void open(UDFContext udfContext) { if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -44,9 +46,12 @@ public class FirstValue implements AggregateFunction { else { outputField = lookupField; } - return acc; } + @Override + public Accumulator initAccumulator(Accumulator acc) { + return acc; + } @Override public Accumulator add(Event event, Accumulator acc) { @@ -66,8 +71,4 @@ public class FirstValue implements AggregateFunction { return acc; } - @Override - public void close() { - - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java index 4adafd4..f27a2e6 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java @@ -36,7 +36,7 @@ public class LastValue implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc) { + public void open(UDFContext udfContext) { if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -47,10 +47,12 @@ public class LastValue implements AggregateFunction { else { outputField = lookupField; } + } + @Override + public Accumulator initAccumulator(Accumulator acc) { return acc; } - @Override public Accumulator add(Event event, Accumulator acc) { if(event.getExtractedFields().containsKey(lookupField)){ @@ -69,8 +71,4 @@ public class LastValue implements AggregateFunction { return acc; } - @Override - public void close() { - - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java index 5662935..ea33271 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java @@ -13,15 +13,16 @@ public class LongCount implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc){ + public void open(UDFContext udfContext){ if(udfContext.getOutput_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } outputField = udfContext.getOutput_fields().get(0); + } + @Override + public Accumulator initAccumulator(Accumulator acc) { return acc; } - - @Override public Accumulator add(Event event, Accumulator acc) { @@ -39,9 +40,5 @@ public class LongCount implements AggregateFunction { return acc; } - @Override - public void close() { - - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java index 88e4be6..2a615ef 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java @@ -17,7 +17,7 @@ public class Mean implements AggregateFunction { private Integer precision; private DecimalFormat df; @Override - public Accumulator open(UDFContext udfContext,Accumulator acc){ + public void open(UDFContext udfContext){ if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); @@ -41,11 +41,14 @@ public class Mean implements AggregateFunction { }else { precision = -1; } + + } + @Override + public Accumulator initAccumulator(Accumulator acc) { acc.getMetricsFields().put(outputField,new OnlineStatistics()); return acc; } - @Override public Accumulator add(Event event, Accumulator acc) { @@ -76,9 +79,4 @@ public class Mean implements AggregateFunction { return acc; } - @Override - public void close() { - - } - } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java index 4ed3143..01e9a5b 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java @@ -6,6 +6,7 @@ import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.pojo.OnlineStatistics; public class NumberSum implements AggregateFunction { @@ -14,7 +15,7 @@ public class NumberSum implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc){ + public void open(UDFContext udfContext){ if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -25,10 +26,12 @@ public class NumberSum implements AggregateFunction { else { outputField = lookupField; } - return acc; } - + @Override + public Accumulator initAccumulator(Accumulator acc) { + return acc; + } @Override public Accumulator add(Event event, Accumulator acc) { diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java index a01edb3..b0d846b 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java @@ -33,7 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class CollectListTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { List arr = List.of("192.168.1.1", "192.168.1.2", "192.168.1.3", "192.168.1.4"); excute(arr); @@ -49,7 +49,8 @@ public class CollectListTest { Map metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = collectList.open(udfContext,accumulator); + collectList.open(udfContext); + Accumulator agg = collectList.initAccumulator(accumulator); for (String o : arr) { Event event = new Event(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java index ae69d7c..ea4fe8d 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java @@ -47,8 +47,8 @@ public class CollectSetTest { Map metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = collectSet.open(udfContext,accumulator); - + collectSet.open(udfContext); + Accumulator agg = collectSet.initAccumulator(accumulator); for (String o : arr) { Event event = new Event(); Map extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java index 2c4d460..506f6de 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java @@ -31,7 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class FirstValueTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { List arr = List.of("192.168.1.1", "192.168.1.2", "192.168.1.3", "192.168.1.4","192.168.1.4"); excute(arr); @@ -47,8 +47,8 @@ public class FirstValueTest { Map metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = firstValue.open(udfContext,accumulator); - + firstValue.open(udfContext); + Accumulator agg = firstValue.initAccumulator(accumulator); for (String o : arr) { Event event = new Event(); Map extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java index e9609f7..f8306cd 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class LastValueTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { List arr = List.of("192.168.1.1", "192.168.1.2", "192.168.1.3", "192.168.1.4","192.168.1.4"); excute(arr); @@ -50,8 +50,8 @@ public class LastValueTest { Map metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = lastValue.open(udfContext,accumulator); - + lastValue.open(udfContext); + Accumulator agg = lastValue.initAccumulator(accumulator); for (String o : arr) { Event event = new Event(); Map extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java index 3bde558..3c02499 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java @@ -35,7 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class LongCountTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { Long[] longArr = new Long[]{1L, 2L, 3L, 4L}; excute(longArr); @@ -49,8 +49,8 @@ public class LongCountTest { Map metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = longCount.open(udfContext,accumulator); - + longCount.open(udfContext); + Accumulator agg = longCount.initAccumulator(accumulator); for (Number o : arr) { Event event = new Event(); Map extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java index 807b7db..6deed0f 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java @@ -35,7 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class MeanTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { Integer[] intArr1 = new Integer[]{1, 2, 3, 4}; Integer[] intArr2 = new Integer[]{1, 6, 3}; @@ -55,8 +55,8 @@ public class MeanTest { Map metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = mean.open(udfContext,accumulator); - + mean.open(udfContext); + Accumulator agg = mean.initAccumulator(accumulator); for (Number o : arr) { Event event = new Event(); Map extractedFields = new HashMap<>(); @@ -80,8 +80,8 @@ public class MeanTest { Map metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = mean.open(udfContext,accumulator); - + mean.open(udfContext); + Accumulator agg = mean.initAccumulator(accumulator); for (Number o : arr) { Event event = new Event(); Map extractedFields = new HashMap<>(); @@ -103,8 +103,8 @@ public class MeanTest { Map metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = mean.open(udfContext,accumulator); - + mean.open(udfContext); + Accumulator agg = mean.initAccumulator(accumulator); for (Number o : arr) { Event event = new Event(); Map extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java index a1cd54e..d0d3d2c 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java @@ -31,7 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class NumberSumTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { Integer[] intArr = new Integer[]{1, 2, 3, 4}; Long[] longArr = new Long[]{1L, 2L, 3L, 4L}; @@ -52,8 +52,8 @@ public class NumberSumTest { Map metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = numberSum.open(udfContext,accumulator); - + numberSum.open(udfContext); + Accumulator agg = numberSum.initAccumulator(accumulator); for (Number o : arr) { Event event = new Event(); Map extractedFields = new HashMap<>(); -- cgit v1.2.3 From 184940fea67a14fb4cf5313a78ee5e705f13680b Mon Sep 17 00:00:00 2001 From: wangkuan Date: Wed, 14 Aug 2024 15:13:52 +0800 Subject: [improve][core]修改UnRoll函数名称为Unroll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/udf.plugins | 2 +- groot-common/src/main/resources/udf.plugins | 2 +- .../com/geedgenetworks/core/udf/udtf/UnRoll.java | 132 --------------------- .../com/geedgenetworks/core/udf/udtf/Unroll.java | 132 +++++++++++++++++++++ .../core/udf/test/table/UnRollFunctionTest.java | 132 --------------------- .../core/udf/test/table/UnrollFunctionTest.java | 132 +++++++++++++++++++++ 6 files changed, 266 insertions(+), 266 deletions(-) delete mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/UnRoll.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java delete mode 100644 groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnRollFunctionTest.java create mode 100644 groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java diff --git a/config/udf.plugins b/config/udf.plugins index 9eb32c4..0d6c0fe 100644 --- a/config/udf.plugins +++ b/config/udf.plugins @@ -22,4 +22,4 @@ com.geedgenetworks.core.udf.udaf.LongCount com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue com.geedgenetworks.core.udf.udaf.FirstValue -com.geedgenetworks.core.udf.udtf.UnRoll \ No newline at end of file +com.geedgenetworks.core.udf.udtf.Unroll \ No newline at end of file diff --git a/groot-common/src/main/resources/udf.plugins b/groot-common/src/main/resources/udf.plugins index f5a4c3f..ca0a7ac 100644 --- a/groot-common/src/main/resources/udf.plugins +++ b/groot-common/src/main/resources/udf.plugins @@ -21,4 +21,4 @@ com.geedgenetworks.core.udf.udaf.LongCount com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue com.geedgenetworks.core.udf.udaf.FirstValue -com.geedgenetworks.core.udf.udtf.UnRoll \ No newline at end of file +com.geedgenetworks.core.udf.udtf.Unroll \ No newline at end of file diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/UnRoll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/UnRoll.java deleted file mode 100644 index 82bea7b..0000000 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/UnRoll.java +++ /dev/null @@ -1,132 +0,0 @@ -package com.geedgenetworks.core.udf.udtf; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONObject; -import com.geedgenetworks.common.Event; -import com.geedgenetworks.common.exception.CommonErrorCode; -import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.common.udf.TableFunction; -import com.geedgenetworks.common.udf.UDFContext; -import com.googlecode.aviator.AviatorEvaluator; -import com.googlecode.aviator.AviatorEvaluatorInstance; -import com.googlecode.aviator.Expression; -import com.googlecode.aviator.Options; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.api.common.functions.RuntimeContext; - -import java.util.*; - - -@Slf4j -public class UnRoll implements TableFunction { - - private String lookupFieldName; - private String outputFieldName; - private Expression compiledExp; - private String expression; - private String outputFieldType; - - - @Override - public void open(RuntimeContext runtimeContext, UDFContext udfContext) { - if(udfContext.getLookup_fields()==null || udfContext.getOutput_fields()==null){ - throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); - } - this.lookupFieldName = udfContext.getLookup_fields().get(0); - this.outputFieldName = udfContext.getOutput_fields().get(0); - this.outputFieldType="object"; - if(udfContext.getParameters()==null ){ - expression=""; - } - else { - this.outputFieldType=udfContext.getParameters().getOrDefault("output_field_type", "object").toString().trim(); - expression=udfContext.getParameters().getOrDefault("path", "").toString().trim(); - } - if(!expression.isEmpty()){ - AviatorEvaluatorInstance instance = AviatorEvaluator.getInstance(); - instance.setCachedExpressionByDefault(true); - instance.setOption(Options.OPTIMIZE_LEVEL, AviatorEvaluator.EVAL); - instance.setFunctionMissing(null); - compiledExp = instance.compile("event."+expression, true); - } - - } - - @Override - public List evaluate(Event event) { - try { - if(event.getExtractedFields().containsKey(lookupFieldName)){ - Object object; - if(event.getExtractedFields().get(lookupFieldName) instanceof String){ - object = JSONObject.parseObject((String) event.getExtractedFields().get(lookupFieldName), Object.class); - } - else { - object = event.getExtractedFields().get(lookupFieldName); - } - try { - if(compiledExp!=null){ - Object obj = compiledExp.execute(compiledExp.newEnv("event", object)); - if(obj instanceof List) { - return parseList(obj,event); - } - else if(obj instanceof String){ - object = JSONObject.parseObject((String)obj, Object.class); - if(object instanceof List){ - return parseList(object,event); - } - else { - log.error("Invalid unroll ! Object is not instance of list. expression=" +expression); - } - }else { - log.error("Invalid unroll ! Object is not instance of String or List. expression=" +expression); - } - } - else { - if(object instanceof List){ - return parseList(object,event); - } - else { - log.error("Invalid unroll ! Object is not instance of list. "); - } - } - }catch (Exception e) { - log.error("Invalid unroll ! expression=" +expression + " Exception :" + e.getMessage()); - } - } - }catch (Exception e) { - log.error("Invalid parseObject ! expression=" +expression + " Exception :" + e.getMessage()); - } - return Collections.singletonList(event); - } - - private List parseList(Object object,Event event) { - List list = (List) object; - List eventList = new ArrayList<>(); - for (Object obj : list) { - Event newEvent = new Event(); - newEvent.setExtractedFields(new HashMap<>()); - newEvent.getExtractedFields().putAll(event.getExtractedFields()); - newEvent.getExtractedFields().remove(lookupFieldName); - if("string".equals(outputFieldType)) { - String jsonString = JSON.toJSONString(obj); - newEvent.getExtractedFields().put(outputFieldName, jsonString); - } - else { - newEvent.getExtractedFields().put(outputFieldName, obj); - } - eventList.add(newEvent); - } - return eventList; - } - - @Override - public String functionName() { - return "UNROLL"; - } - - @Override - public void close() { - - } - -} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java new file mode 100644 index 0000000..3931743 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java @@ -0,0 +1,132 @@ +package com.geedgenetworks.core.udf.udtf; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.exception.CommonErrorCode; +import com.geedgenetworks.common.exception.GrootStreamRuntimeException; +import com.geedgenetworks.common.udf.TableFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.googlecode.aviator.AviatorEvaluator; +import com.googlecode.aviator.AviatorEvaluatorInstance; +import com.googlecode.aviator.Expression; +import com.googlecode.aviator.Options; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.common.functions.RuntimeContext; + +import java.util.*; + + +@Slf4j +public class Unroll implements TableFunction { + + private String lookupFieldName; + private String outputFieldName; + private Expression compiledExp; + private String expression; + private String outputFieldType; + + + @Override + public void open(RuntimeContext runtimeContext, UDFContext udfContext) { + if(udfContext.getLookup_fields()==null || udfContext.getOutput_fields()==null){ + throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); + } + this.lookupFieldName = udfContext.getLookup_fields().get(0); + this.outputFieldName = udfContext.getOutput_fields().get(0); + this.outputFieldType="object"; + if(udfContext.getParameters()==null ){ + expression=""; + } + else { + this.outputFieldType=udfContext.getParameters().getOrDefault("output_field_type", "object").toString().trim(); + expression=udfContext.getParameters().getOrDefault("path", "").toString().trim(); + } + if(!expression.isEmpty()){ + AviatorEvaluatorInstance instance = AviatorEvaluator.getInstance(); + instance.setCachedExpressionByDefault(true); + instance.setOption(Options.OPTIMIZE_LEVEL, AviatorEvaluator.EVAL); + instance.setFunctionMissing(null); + compiledExp = instance.compile("event."+expression, true); + } + + } + + @Override + public List evaluate(Event event) { + try { + if(event.getExtractedFields().containsKey(lookupFieldName)){ + Object object; + if(event.getExtractedFields().get(lookupFieldName) instanceof String){ + object = JSONObject.parseObject((String) event.getExtractedFields().get(lookupFieldName), Object.class); + } + else { + object = event.getExtractedFields().get(lookupFieldName); + } + try { + if(compiledExp!=null){ + Object obj = compiledExp.execute(compiledExp.newEnv("event", object)); + if(obj instanceof List) { + return parseList(obj,event); + } + else if(obj instanceof String){ + object = JSONObject.parseObject((String)obj, Object.class); + if(object instanceof List){ + return parseList(object,event); + } + else { + log.error("Invalid unroll ! Object is not instance of list. expression=" +expression); + } + }else { + log.error("Invalid unroll ! Object is not instance of String or List. expression=" +expression); + } + } + else { + if(object instanceof List){ + return parseList(object,event); + } + else { + log.error("Invalid unroll ! Object is not instance of list. "); + } + } + }catch (Exception e) { + log.error("Invalid unroll ! expression=" +expression + " Exception :" + e.getMessage()); + } + } + }catch (Exception e) { + log.error("Invalid parseObject ! expression=" +expression + " Exception :" + e.getMessage()); + } + return Collections.singletonList(event); + } + + private List parseList(Object object,Event event) { + List list = (List) object; + List eventList = new ArrayList<>(); + for (Object obj : list) { + Event newEvent = new Event(); + newEvent.setExtractedFields(new HashMap<>()); + newEvent.getExtractedFields().putAll(event.getExtractedFields()); + newEvent.getExtractedFields().remove(lookupFieldName); + if("string".equals(outputFieldType)) { + String jsonString = JSON.toJSONString(obj); + newEvent.getExtractedFields().put(outputFieldName, jsonString); + } + else { + newEvent.getExtractedFields().put(outputFieldName, obj); + } + eventList.add(newEvent); + } + return eventList; + } + + @Override + public String functionName() { + return "UNROLL"; + } + + @Override + public void close() { + + } + +} diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnRollFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnRollFunctionTest.java deleted file mode 100644 index 5686a42..0000000 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnRollFunctionTest.java +++ /dev/null @@ -1,132 +0,0 @@ -package com.geedgenetworks.core.udf.test.table; - -import com.geedgenetworks.common.Event; -import com.geedgenetworks.common.udf.UDFContext; -import com.geedgenetworks.core.udf.udtf.UnRoll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class UnRollFunctionTest { - - private static Map nestedMap; - @BeforeAll - public static void setUp() { - nestedMap = Map.of( - "k1", List.of( - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") - ), - "k2", Map.of("name", "[{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"c2s_destination_mac\":\"10:70:fd:03:c2:6d\",\"s2c_source_mac\":\"10:70:fd:03:c2:6d\",\"s2c_destination_mac\":\"10:70:fd:03:c2:6c\"},{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"c2s_destination_mac\":\"10:70:fd:03:c2:6d\",\"s2c_source_mac\":\"10:70:fd:03:c2:6d\",\"s2c_destination_mac\":\"10:70:fd:03:c2:6c\"}]", "lastName", "{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}"), - "k3","[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}]", - "k4","{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}", - "k5", Map.of("name",List.of( - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") - ) - ), "k6", Map.of("name",List.of( - Map.of("name2", List.of( - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") - ), "source_mac", "52:d4:18:c7:e5:11"), - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") - ) - ) - ); - } - - // 测试方法 - @Test - public void testUnrollFunction1() { - - - UDFContext udfContext = new UDFContext(); - udfContext.setLookup_fields(List.of("k1")); - udfContext.setOutput_fields(List.of("newk1")); - Map params = new HashMap<>(); - params.put("path", ""); - udfContext.setParameters(params); - UnRoll unroll = new UnRoll(); - unroll.open(null, udfContext); - Event event = new Event(); - event.setExtractedFields(nestedMap); - List result = unroll.evaluate(event); - assertEquals(2, result.size()); - Map map1 = (Map) result.get(0).getExtractedFields().get("newk1"); - assertEquals("52:d4:18:c7:e5:11", map1.get("source_mac")); - Map map2 = (Map) result.get(1).getExtractedFields().get("newk1"); - assertEquals("ff:ff:ff:ff:ff:ff", map2.get("source_mac")); - - - - } - - @Test - public void testUnrollFunction2() { - UDFContext udfContext = new UDFContext(); - UnRoll unroll = new UnRoll(); - Event event = new Event(); - event.setExtractedFields(nestedMap); - udfContext.setLookup_fields(List.of("k3")); - udfContext.setOutput_fields(List.of("newk3")); - unroll.open(null, udfContext); - List result3 = unroll.evaluate(event); - assertEquals(2, result3.size()); - Map map3 = (Map) result3.get(0).getExtractedFields().get("newk3"); - assertEquals("52:d4:18:c7:e5:11", map3.get("source_mac")); - Map map4 = (Map) result3.get(1).getExtractedFields().get("newk3"); - assertEquals("ff:ff:ff:ff:ff:ff", map4.get("source_mac")); - - - Map params = new HashMap<>(); - params.put("path", "name"); - udfContext.setParameters(params); - event.setExtractedFields(nestedMap); - udfContext.setLookup_fields(List.of("k2")); - udfContext.setOutput_fields(List.of("newk2")); - unroll.open(null, udfContext); - List result2 = unroll.evaluate(event); - assertEquals(2, result2.size()); - Map map5 = (Map) result2.get(0).getExtractedFields().get("newk2"); - assertEquals("52:d4:18:c7:e5:11", map5.get("source_mac")); - Map map6 = (Map) result2.get(1).getExtractedFields().get("newk2"); - assertEquals("ff:ff:ff:ff:ff:ff", map6.get("source_mac")); - - - Map params1 = new HashMap<>(); - params.put("path", "name.0.name2"); - udfContext.setParameters(params1); - event.setExtractedFields(nestedMap); - udfContext.setLookup_fields(List.of("k6")); - udfContext.setOutput_fields(List.of("newk6")); - unroll.open(null, udfContext); - List result6 = unroll.evaluate(event); - assertEquals(2, result6.size()); - Map map9 = (Map) result6.get(0).getExtractedFields().get("newk6"); - assertEquals("52:d4:18:c7:e5:11", map9.get("source_mac")); - Map map10 = (Map) result6.get(1).getExtractedFields().get("newk6"); - assertEquals("ff:ff:ff:ff:ff:ff", map10.get("source_mac")); - } - @Test - public void testUnrollFunction4() { - - - UDFContext udfContext = new UDFContext(); - UnRoll unroll = new UnRoll(); - Event event = new Event(); - Map params = new HashMap<>(); - udfContext.setParameters(params); - event.setExtractedFields(nestedMap); - udfContext.setLookup_fields(List.of("k4")); - udfContext.setOutput_fields(List.of("newk4")); - unroll.open(null, udfContext); - List result2 = unroll.evaluate(event); - assertEquals(1, result2.size()); - - } -} diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java new file mode 100644 index 0000000..68b3683 --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java @@ -0,0 +1,132 @@ +package com.geedgenetworks.core.udf.test.table; + +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.udf.udtf.Unroll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class UnrollFunctionTest { + + private static Map nestedMap; + @BeforeAll + public static void setUp() { + nestedMap = Map.of( + "k1", List.of( + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") + ), + "k2", Map.of("name", "[{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"c2s_destination_mac\":\"10:70:fd:03:c2:6d\",\"s2c_source_mac\":\"10:70:fd:03:c2:6d\",\"s2c_destination_mac\":\"10:70:fd:03:c2:6c\"},{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"c2s_destination_mac\":\"10:70:fd:03:c2:6d\",\"s2c_source_mac\":\"10:70:fd:03:c2:6d\",\"s2c_destination_mac\":\"10:70:fd:03:c2:6c\"}]", "lastName", "{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}"), + "k3","[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}]", + "k4","{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}", + "k5", Map.of("name",List.of( + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") + ) + ), "k6", Map.of("name",List.of( + Map.of("name2", List.of( + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") + ), "source_mac", "52:d4:18:c7:e5:11"), + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") + ) + ) + ); + } + + // 测试方法 + @Test + public void testUnrollFunction1() { + + + UDFContext udfContext = new UDFContext(); + udfContext.setLookup_fields(List.of("k1")); + udfContext.setOutput_fields(List.of("newk1")); + Map params = new HashMap<>(); + params.put("path", ""); + udfContext.setParameters(params); + Unroll unroll = new Unroll(); + unroll.open(null, udfContext); + Event event = new Event(); + event.setExtractedFields(nestedMap); + List result = unroll.evaluate(event); + assertEquals(2, result.size()); + Map map1 = (Map) result.get(0).getExtractedFields().get("newk1"); + assertEquals("52:d4:18:c7:e5:11", map1.get("source_mac")); + Map map2 = (Map) result.get(1).getExtractedFields().get("newk1"); + assertEquals("ff:ff:ff:ff:ff:ff", map2.get("source_mac")); + + + + } + + @Test + public void testUnrollFunction2() { + UDFContext udfContext = new UDFContext(); + Unroll unroll = new Unroll(); + Event event = new Event(); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k3")); + udfContext.setOutput_fields(List.of("newk3")); + unroll.open(null, udfContext); + List result3 = unroll.evaluate(event); + assertEquals(2, result3.size()); + Map map3 = (Map) result3.get(0).getExtractedFields().get("newk3"); + assertEquals("52:d4:18:c7:e5:11", map3.get("source_mac")); + Map map4 = (Map) result3.get(1).getExtractedFields().get("newk3"); + assertEquals("ff:ff:ff:ff:ff:ff", map4.get("source_mac")); + + + Map params = new HashMap<>(); + params.put("path", "name"); + udfContext.setParameters(params); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k2")); + udfContext.setOutput_fields(List.of("newk2")); + unroll.open(null, udfContext); + List result2 = unroll.evaluate(event); + assertEquals(2, result2.size()); + Map map5 = (Map) result2.get(0).getExtractedFields().get("newk2"); + assertEquals("52:d4:18:c7:e5:11", map5.get("source_mac")); + Map map6 = (Map) result2.get(1).getExtractedFields().get("newk2"); + assertEquals("ff:ff:ff:ff:ff:ff", map6.get("source_mac")); + + + Map params1 = new HashMap<>(); + params.put("path", "name.0.name2"); + udfContext.setParameters(params1); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k6")); + udfContext.setOutput_fields(List.of("newk6")); + unroll.open(null, udfContext); + List result6 = unroll.evaluate(event); + assertEquals(2, result6.size()); + Map map9 = (Map) result6.get(0).getExtractedFields().get("newk6"); + assertEquals("52:d4:18:c7:e5:11", map9.get("source_mac")); + Map map10 = (Map) result6.get(1).getExtractedFields().get("newk6"); + assertEquals("ff:ff:ff:ff:ff:ff", map10.get("source_mac")); + } + @Test + public void testUnrollFunction4() { + + + UDFContext udfContext = new UDFContext(); + Unroll unroll = new Unroll(); + Event event = new Event(); + Map params = new HashMap<>(); + udfContext.setParameters(params); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k4")); + udfContext.setOutput_fields(List.of("newk4")); + unroll.open(null, udfContext); + List result2 = unroll.evaluate(event); + assertEquals(1, result2.size()); + + } +} -- cgit v1.2.3 From f7927f9ce7e651f925db8fed2cb4e89f5b5e10c8 Mon Sep 17 00:00:00 2001 From: wangkuan Date: Thu, 15 Aug 2024 11:50:18 +0800 Subject: [feature][core]新增JsonUnroll函数，优化Unroll函数 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/udf.plugins | 1 + .../geedgenetworks/common/utils/JsonPathUtil.java | 33 ++++++ groot-common/src/main/resources/udf.plugins | 1 + .../geedgenetworks/core/udf/udtf/JsonUnroll.java | 125 +++++++++++++++++++++ .../com/geedgenetworks/core/udf/udtf/Unroll.java | 107 +++++++----------- .../udf/test/table/JsonUnrollFunctionTest.java | 105 +++++++++++++++++ .../core/udf/test/table/UnrollFunctionTest.java | 72 +++--------- 7 files changed, 319 insertions(+), 125 deletions(-) create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java create mode 100644 groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java diff --git a/config/udf.plugins b/config/udf.plugins index 0d6c0fe..31d1b21 100644 --- a/config/udf.plugins +++ b/config/udf.plugins @@ -22,4 +22,5 @@ com.geedgenetworks.core.udf.udaf.LongCount com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue com.geedgenetworks.core.udf.udaf.FirstValue +com.geedgenetworks.core.udf.udtf.JsonUnroll com.geedgenetworks.core.udf.udtf.Unroll \ No newline at end of file diff --git a/groot-common/src/main/java/com/geedgenetworks/common/utils/JsonPathUtil.java b/groot-common/src/main/java/com/geedgenetworks/common/utils/JsonPathUtil.java index dcba58c..0823ddc 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/utils/JsonPathUtil.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/utils/JsonPathUtil.java @@ -2,6 +2,8 @@ package com.geedgenetworks.common.utils; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; import com.alibaba.fastjson2.JSONPath; import com.alibaba.fastjson2.JSONReader; @@ -42,4 +44,35 @@ public class JsonPathUtil { } return flattenResult; } + public static Object get(JSONObject jsonObject, String expr) { + Object Result = ""; + try { + Result = jsonObject.getByPath(expr); + } catch (Exception e) { + logger.error( + "The label resolution exception or [expr] analytic expression error" + + e.getMessage()); + } + return Result; + } + public static JSONObject set(JSONObject jsonObject, String expr,Object value) { + try { + JSONPath.set(jsonObject, expr, value); + } catch (Exception e) { + logger.error( + "JSONObject set value exception or [expr] expression error" + + e.getMessage()); + } + return jsonObject; + } + public static JSONObject remove(JSONObject jsonObject, String expr) { + try { + JSONPath.remove(jsonObject, expr); + } catch (Exception e) { + logger.error( + "JSONObject remove value exception or [expr] expression error" + + e.getMessage()); + } + return jsonObject; + } } diff --git a/groot-common/src/main/resources/udf.plugins b/groot-common/src/main/resources/udf.plugins index ca0a7ac..18446c9 100644 --- a/groot-common/src/main/resources/udf.plugins +++ b/groot-common/src/main/resources/udf.plugins @@ -21,4 +21,5 @@ com.geedgenetworks.core.udf.udaf.LongCount com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue com.geedgenetworks.core.udf.udaf.FirstValue +com.geedgenetworks.core.udf.udtf.JsonUnroll com.geedgenetworks.core.udf.udtf.Unroll \ No newline at end of file diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java new file mode 100644 index 0000000..2e8eb7e --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java @@ -0,0 +1,125 @@ +package com.geedgenetworks.core.udf.udtf; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.exception.CommonErrorCode; +import com.geedgenetworks.common.exception.GrootStreamRuntimeException; +import com.geedgenetworks.common.udf.TableFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.common.utils.JsonPathUtil; +import com.googlecode.aviator.AviatorEvaluator; +import com.googlecode.aviator.AviatorEvaluatorInstance; +import com.googlecode.aviator.Expression; +import com.googlecode.aviator.Options; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.common.functions.RuntimeContext; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + + +@Slf4j +public class JsonUnroll implements TableFunction { + + private String lookupFieldName; + private String outputFieldName; + private String path; + private String new_Path; + + + @Override + public void open(RuntimeContext runtimeContext, UDFContext udfContext) { + if(udfContext.getLookup_fields()==null ){ + throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); + } + this.lookupFieldName = udfContext.getLookup_fields().get(0); + if(udfContext.getOutput_fields()!=null && !udfContext.getOutput_fields().isEmpty()) { + this.outputFieldName = udfContext.getOutput_fields().get(0); + } + else { + outputFieldName = lookupFieldName; + } + if(udfContext.getParameters()==null ){ + path=""; + new_Path=""; + } + else { + path=udfContext.getParameters().getOrDefault("path", "").toString().trim(); + new_Path=udfContext.getParameters().getOrDefault("new_path", path).toString().trim(); + } + } + + @Override + public List evaluate(Event event) { + try { + if(event.getExtractedFields().containsKey(lookupFieldName) ){ + try { + if(path.isEmpty()){ + JSONArray jsonArray = JSONArray.parseArray(event.getExtractedFields().get(lookupFieldName).toString()); + return parseList(jsonArray,event); + }else { + JSONObject jsonObject = JSONObject.parseObject(event.getExtractedFields().get(lookupFieldName).toString()); + Object obj = JsonPathUtil.get(jsonObject,path); + if(obj instanceof List || obj instanceof Array) { + List list = (List) obj; + List eventList = new ArrayList<>(); + for (Object o : list) { + JSONObject newJsonObject = new JSONObject(); + newJsonObject.putAll(jsonObject); + JsonPathUtil.remove(newJsonObject,path); + JsonPathUtil.set(newJsonObject,new_Path,o); + String jsonString = JSON.toJSONString(newJsonObject); + Event newEvent = new Event(); + newEvent.setExtractedFields(new HashMap<>()); + newEvent.getExtractedFields().putAll(event.getExtractedFields()); + newEvent.getExtractedFields().remove(lookupFieldName); + newEvent.getExtractedFields().put(outputFieldName, jsonString); + eventList.add(newEvent); + } + return eventList; + } + else { + log.error("Invalid unroll ! expression=" +path + " Exception :" + " expression should return a list or array"); + } + } + + }catch (Exception e) { + log.error("Invalid unroll ! expression=" +path + " Exception :" + e.getMessage()); + } + } + }catch (Exception e) { + log.error("Invalid parseObject ! expression=" +path + " Exception :" + e.getMessage()); + } + return Collections.singletonList(event); + } + + private List parseList(Object object,Event event) { + List list = (List) object; + List eventList = new ArrayList<>(); + for (Object obj : list) { + Event newEvent = new Event(); + newEvent.setExtractedFields(new HashMap<>()); + newEvent.getExtractedFields().putAll(event.getExtractedFields()); + newEvent.getExtractedFields().remove(lookupFieldName); + newEvent.getExtractedFields().put(outputFieldName, JSON.toJSONString(obj)); + eventList.add(newEvent); + } + return eventList; + } + + @Override + public String functionName() { + return "JSON_UNROLL"; + } + + @Override + public void close() { + + } + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java index 3931743..5becb8e 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java @@ -1,16 +1,10 @@ package com.geedgenetworks.core.udf.udtf; -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONObject; import com.geedgenetworks.common.Event; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.udf.TableFunction; import com.geedgenetworks.common.udf.UDFContext; -import com.googlecode.aviator.AviatorEvaluator; -import com.googlecode.aviator.AviatorEvaluatorInstance; -import com.googlecode.aviator.Expression; -import com.googlecode.aviator.Options; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; @@ -22,79 +16,54 @@ public class Unroll implements TableFunction { private String lookupFieldName; private String outputFieldName; - private Expression compiledExp; - private String expression; - private String outputFieldType; + private String regex; @Override public void open(RuntimeContext runtimeContext, UDFContext udfContext) { - if(udfContext.getLookup_fields()==null || udfContext.getOutput_fields()==null){ + if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } - this.lookupFieldName = udfContext.getLookup_fields().get(0); - this.outputFieldName = udfContext.getOutput_fields().get(0); - this.outputFieldType="object"; - if(udfContext.getParameters()==null ){ - expression=""; + this.lookupFieldName = udfContext.getLookup_fields().get(0); + if(udfContext.getOutput_fields()!=null && !udfContext.getOutput_fields().isEmpty()) { + this.outputFieldName = udfContext.getOutput_fields().get(0); } else { - this.outputFieldType=udfContext.getParameters().getOrDefault("output_field_type", "object").toString().trim(); - expression=udfContext.getParameters().getOrDefault("path", "").toString().trim(); + outputFieldName = lookupFieldName; } - if(!expression.isEmpty()){ - AviatorEvaluatorInstance instance = AviatorEvaluator.getInstance(); - instance.setCachedExpressionByDefault(true); - instance.setOption(Options.OPTIMIZE_LEVEL, AviatorEvaluator.EVAL); - instance.setFunctionMissing(null); - compiledExp = instance.compile("event."+expression, true); + if(udfContext.getParameters()==null ){ + regex=""; + } + else { + this.regex=udfContext.getParameters().getOrDefault("regex", "").toString().trim(); } - } @Override public List evaluate(Event event) { try { - if(event.getExtractedFields().containsKey(lookupFieldName)){ - Object object; - if(event.getExtractedFields().get(lookupFieldName) instanceof String){ - object = JSONObject.parseObject((String) event.getExtractedFields().get(lookupFieldName), Object.class); + if(event.getExtractedFields().containsKey(lookupFieldName)) { + + if(regex.isEmpty()){ + if (event.getExtractedFields().get(lookupFieldName) instanceof List ) { + return parseList(event.getExtractedFields().get(lookupFieldName), event); + } else if(event.getExtractedFields().get(lookupFieldName) instanceof Object[]){ + return parseArray(event.getExtractedFields().get(lookupFieldName), event); + }else { + log.error("Invalid unroll ! Object is not instance of list or array. expression=" + regex); + } } else { - object = event.getExtractedFields().get(lookupFieldName); - } - try { - if(compiledExp!=null){ - Object obj = compiledExp.execute(compiledExp.newEnv("event", object)); - if(obj instanceof List) { - return parseList(obj,event); - } - else if(obj instanceof String){ - object = JSONObject.parseObject((String)obj, Object.class); - if(object instanceof List){ - return parseList(object,event); - } - else { - log.error("Invalid unroll ! Object is not instance of list. expression=" +expression); - } - }else { - log.error("Invalid unroll ! Object is not instance of String or List. expression=" +expression); - } - } - else { - if(object instanceof List){ - return parseList(object,event); - } - else { - log.error("Invalid unroll ! Object is not instance of list. "); - } + if (event.getExtractedFields().get(lookupFieldName) instanceof String) { + String[] array =((String) event.getExtractedFields().get(lookupFieldName)).split(regex); + return parseArray(array, event); + }else { + log.error("Invalid unroll ! Object is not instance of String. expression=" + regex); } - }catch (Exception e) { - log.error("Invalid unroll ! expression=" +expression + " Exception :" + e.getMessage()); } } }catch (Exception e) { - log.error("Invalid parseObject ! expression=" +expression + " Exception :" + e.getMessage()); + log.error("Invalid parseObject ! expression=" +regex + " Exception :" + e.getMessage()); } return Collections.singletonList(event); } @@ -107,18 +76,24 @@ public class Unroll implements TableFunction { newEvent.setExtractedFields(new HashMap<>()); newEvent.getExtractedFields().putAll(event.getExtractedFields()); newEvent.getExtractedFields().remove(lookupFieldName); - if("string".equals(outputFieldType)) { - String jsonString = JSON.toJSONString(obj); - newEvent.getExtractedFields().put(outputFieldName, jsonString); - } - else { - newEvent.getExtractedFields().put(outputFieldName, obj); - } + newEvent.getExtractedFields().put(outputFieldName, obj); + eventList.add(newEvent); + } + return eventList; + } + private List parseArray(Object object, Event event) { + List eventList = new ArrayList<>(); + Object[] objects = (Object[]) object; + for (Object obj : objects) { + Event newEvent = new Event(); + newEvent.setExtractedFields(new HashMap<>()); + newEvent.getExtractedFields().putAll(event.getExtractedFields()); + newEvent.getExtractedFields().remove(lookupFieldName); + newEvent.getExtractedFields().put(outputFieldName, obj); eventList.add(newEvent); } return eventList; } - @Override public String functionName() { return "UNROLL"; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java new file mode 100644 index 0000000..02f0b66 --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java @@ -0,0 +1,105 @@ +package com.geedgenetworks.core.udf.test.table; + +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.udf.udtf.JsonUnroll; +import com.geedgenetworks.core.udf.udtf.Unroll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class JsonUnrollFunctionTest { + + private static Map nestedMap; + @BeforeAll + public static void setUp() { + nestedMap = Map.of( + "k1","[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}]", + "k2","{\"k2_1\":\"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}]\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}", + "k3","{\n" + + " \"k3_1\": {\n" + + " \"k3_1_1\": [\n" + + " {\n" + + " \"tunnels_schema_type\": \"ETHERNET\",\n" + + " \"source_mac\": \"52:d4:18:c7:e5:11\"\n" + + " },\n" + + " {\n" + + " \"tunnels_schema_type\": \"ETHERNET\",\n" + + " \"source_mac\": \"ff:ff:ff:ff:ff:ff\"\n" + + " }\n" + + " ],\n" + + " \"k3_1_2\": {\n" + + " \"tunnels_schema_type\": \"ETHERNET\",\n" + + " \"source_mac\": 19.95\n" + + " }\n" + + " }\n" + + "}", + "k4","" + + ); + } + + // 测试方法 + + @Test + public void testJsonUnrollFunction1() { + UDFContext udfContext = new UDFContext(); + JsonUnroll unroll = new JsonUnroll(); + Event event = new Event(); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k1")); + udfContext.setOutput_fields(List.of("newk1")); + unroll.open(null, udfContext); + List result3 = unroll.evaluate(event); + assertEquals(2, result3.size()); + assertEquals("{\"destination_mac\":\"ff:ff:ff:ff:ff:ff\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"tunnels_schema_type\":\"ETHERNET\"}", result3.get(0).getExtractedFields().get("newk1").toString()); + assertEquals("{\"destination_mac\":\"ff:ff:ff:ff:ff:ff\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"tunnels_schema_type\":\"ETHERNET\"}", result3.get(1).getExtractedFields().get("newk1").toString()); + + + + + } + @Test + public void testJsonUnrollFunction2() { + + + UDFContext udfContext = new UDFContext(); + JsonUnroll unroll = new JsonUnroll(); + Event event = new Event(); + Map params = new HashMap<>(); + udfContext.setParameters(params); + params.put("path", "$.k3_1.k3_1_1"); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k3")); + udfContext.setOutput_fields(List.of("newk3")); + unroll.open(null, udfContext); + List result2 = unroll.evaluate(event); + assertEquals(2, result2.size()); + assertEquals("{\"k3_1\":{\"k3_1_2\":{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":19.95},\"k3_1_1\":{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\"}}}",result2.get(0).getExtractedFields().get("newk3").toString()); + assertEquals("{\"k3_1\":{\"k3_1_2\":{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":19.95},\"k3_1_1\":{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\"}}}",result2.get(1).getExtractedFields().get("newk3").toString()); + + } + @Test + public void testJsonUnrollFunction3() { + + + UDFContext udfContext = new UDFContext(); + JsonUnroll unroll = new JsonUnroll(); + Event event = new Event(); + Map params = new HashMap<>(); + udfContext.setParameters(params); + params.put("path", "$.k4_1.k4_1_1"); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k4")); + udfContext.setOutput_fields(List.of("newk4")); + unroll.open(null, udfContext); + List result2 = unroll.evaluate(event); + assertEquals(1, result2.size()); + + } +} diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java index 68b3683..2f4da76 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java @@ -22,21 +22,9 @@ public class UnrollFunctionTest { Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") ), - "k2", Map.of("name", "[{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"c2s_destination_mac\":\"10:70:fd:03:c2:6d\",\"s2c_source_mac\":\"10:70:fd:03:c2:6d\",\"s2c_destination_mac\":\"10:70:fd:03:c2:6c\"},{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"c2s_destination_mac\":\"10:70:fd:03:c2:6d\",\"s2c_source_mac\":\"10:70:fd:03:c2:6d\",\"s2c_destination_mac\":\"10:70:fd:03:c2:6c\"}]", "lastName", "{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}"), - "k3","[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}]", - "k4","{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}", - "k5", Map.of("name",List.of( - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") - ) - ), "k6", Map.of("name",List.of( - Map.of("name2", List.of( - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") - ), "source_mac", "52:d4:18:c7:e5:11"), - Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") - ) - ) + "k2","{\"source_mac\":\"52:d4:18:c7:e5:10\"},{\"source_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"source_mac\":\"52:d4:18:c7:e5:11\"}", + "k3","" + ); } @@ -48,9 +36,6 @@ public class UnrollFunctionTest { UDFContext udfContext = new UDFContext(); udfContext.setLookup_fields(List.of("k1")); udfContext.setOutput_fields(List.of("newk1")); - Map params = new HashMap<>(); - params.put("path", ""); - udfContext.setParameters(params); Unroll unroll = new Unroll(); unroll.open(null, udfContext); Event event = new Event(); @@ -62,68 +47,37 @@ public class UnrollFunctionTest { Map map2 = (Map) result.get(1).getExtractedFields().get("newk1"); assertEquals("ff:ff:ff:ff:ff:ff", map2.get("source_mac")); - - } - @Test public void testUnrollFunction2() { + + UDFContext udfContext = new UDFContext(); Unroll unroll = new Unroll(); Event event = new Event(); - event.setExtractedFields(nestedMap); - udfContext.setLookup_fields(List.of("k3")); - udfContext.setOutput_fields(List.of("newk3")); - unroll.open(null, udfContext); - List result3 = unroll.evaluate(event); - assertEquals(2, result3.size()); - Map map3 = (Map) result3.get(0).getExtractedFields().get("newk3"); - assertEquals("52:d4:18:c7:e5:11", map3.get("source_mac")); - Map map4 = (Map) result3.get(1).getExtractedFields().get("newk3"); - assertEquals("ff:ff:ff:ff:ff:ff", map4.get("source_mac")); - - Map params = new HashMap<>(); - params.put("path", "name"); + params.put("regex", ","); + udfContext.setParameters(params); udfContext.setParameters(params); event.setExtractedFields(nestedMap); udfContext.setLookup_fields(List.of("k2")); - udfContext.setOutput_fields(List.of("newk2")); + udfContext.setOutput_fields(List.of("k2")); unroll.open(null, udfContext); List result2 = unroll.evaluate(event); - assertEquals(2, result2.size()); - Map map5 = (Map) result2.get(0).getExtractedFields().get("newk2"); - assertEquals("52:d4:18:c7:e5:11", map5.get("source_mac")); - Map map6 = (Map) result2.get(1).getExtractedFields().get("newk2"); - assertEquals("ff:ff:ff:ff:ff:ff", map6.get("source_mac")); - + assertEquals(3, result2.size()); + assertEquals("{\"source_mac\":\"52:d4:18:c7:e5:10\"}", result2.get(0).getExtractedFields().get("k2")); - Map params1 = new HashMap<>(); - params.put("path", "name.0.name2"); - udfContext.setParameters(params1); - event.setExtractedFields(nestedMap); - udfContext.setLookup_fields(List.of("k6")); - udfContext.setOutput_fields(List.of("newk6")); - unroll.open(null, udfContext); - List result6 = unroll.evaluate(event); - assertEquals(2, result6.size()); - Map map9 = (Map) result6.get(0).getExtractedFields().get("newk6"); - assertEquals("52:d4:18:c7:e5:11", map9.get("source_mac")); - Map map10 = (Map) result6.get(1).getExtractedFields().get("newk6"); - assertEquals("ff:ff:ff:ff:ff:ff", map10.get("source_mac")); } @Test - public void testUnrollFunction4() { + public void testUnrollFunction3() { UDFContext udfContext = new UDFContext(); Unroll unroll = new Unroll(); Event event = new Event(); - Map params = new HashMap<>(); - udfContext.setParameters(params); event.setExtractedFields(nestedMap); - udfContext.setLookup_fields(List.of("k4")); - udfContext.setOutput_fields(List.of("newk4")); + udfContext.setLookup_fields(List.of("k3")); + udfContext.setOutput_fields(List.of("newk3")); unroll.open(null, udfContext); List result2 = unroll.evaluate(event); assertEquals(1, result2.size()); -- cgit v1.2.3 From cbdd7f96b4e342c9e8b296e231ea655fca139f5d Mon Sep 17 00:00:00 2001 From: lifengchao Date: Thu, 15 Aug 2024 15:28:24 +0800 Subject: [feature][core]添加sketche聚合函数和sketche mock类型 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/udf.plugins | 7 +- groot-common/src/main/resources/udf.plugins | 7 +- groot-connectors/connector-kafka/pom.xml | 11 +++ .../connectors/mock/faker/FakerUtils.java | 18 ++++ .../connectors/mock/faker/HdrHistogramFaker.java | 35 +++++++ .../connectors/mock/faker/HlldFaker.java | 39 ++++++++ groot-core/pom.xml | 5 + .../core/udf/udaf/HdrHistogram/HdrHistogram.java | 42 +++++++++ .../HdrHistogram/HdrHistogramBaseAggregate.java | 101 ++++++++++++++++++++ .../udaf/HdrHistogram/HdrHistogramQuantile.java | 36 ++++++++ .../udaf/HdrHistogram/HdrHistogramQuantiles.java | 51 +++++++++++ .../geedgenetworks/core/udf/udaf/hlld/Hlld.java | 40 ++++++++ .../udf/udaf/hlld/HlldApproxCountDistinct.java | 25 +++++ .../core/udf/udaf/hlld/HlldBaseAggregate.java | 98 ++++++++++++++++++++ .../HdrHistogram/HdrHistogramQuantileTest.java | 89 ++++++++++++++++++ .../HdrHistogram/HdrHistogramQuantilesTest.java | 98 ++++++++++++++++++++ .../udf/udaf/HdrHistogram/HdrHistogramTest.java | 102 +++++++++++++++++++++ .../udf/udaf/hlld/HlldApproxCountDistinctTest.java | 87 ++++++++++++++++++ .../core/udf/udaf/hlld/HlldTest.java | 86 +++++++++++++++++ pom.xml | 5 + 20 files changed, 980 insertions(+), 2 deletions(-) create mode 100644 groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HdrHistogramFaker.java create mode 100644 groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HlldFaker.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinct.java create mode 100644 groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java create mode 100644 groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java create mode 100644 groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java create mode 100644 groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java create mode 100644 groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java create mode 100644 groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java diff --git a/config/udf.plugins b/config/udf.plugins index 31d1b21..772d2bc 100644 --- a/config/udf.plugins +++ b/config/udf.plugins @@ -23,4 +23,9 @@ com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue com.geedgenetworks.core.udf.udaf.FirstValue com.geedgenetworks.core.udf.udtf.JsonUnroll -com.geedgenetworks.core.udf.udtf.Unroll \ No newline at end of file +com.geedgenetworks.core.udf.udtf.Unroll +com.geedgenetworks.core.udf.udaf.hlld.Hlld +com.geedgenetworks.core.udf.udaf.hlld.HlldApproxCountDistinct +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogram +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantile +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantiles \ No newline at end of file diff --git a/groot-common/src/main/resources/udf.plugins b/groot-common/src/main/resources/udf.plugins index 18446c9..0062c1a 100644 --- a/groot-common/src/main/resources/udf.plugins +++ b/groot-common/src/main/resources/udf.plugins @@ -22,4 +22,9 @@ com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue com.geedgenetworks.core.udf.udaf.FirstValue com.geedgenetworks.core.udf.udtf.JsonUnroll -com.geedgenetworks.core.udf.udtf.Unroll \ No newline at end of file +com.geedgenetworks.core.udf.udtf.Unroll +com.geedgenetworks.core.udf.udaf.hlld.Hlld +com.geedgenetworks.core.udf.udaf.hlld.HlldApproxCountDistinct +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogram +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantile +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantiles \ No newline at end of file diff --git a/groot-connectors/connector-kafka/pom.xml b/groot-connectors/connector-kafka/pom.xml index 7ec7d86..448383b 100644 --- a/groot-connectors/connector-kafka/pom.xml +++ b/groot-connectors/connector-kafka/pom.xml @@ -15,6 +15,17 @@ org.apache.flink flink-connector-kafka_${scala.version} ${flink.version} + + + org.xerial.snappy + snappy-java + + + + + org.xerial.snappy + snappy-java + 1.1.8.3 diff --git a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java index 09cc8f8..5101fa1 100644 --- a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java +++ b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java @@ -46,6 +46,10 @@ public class FakerUtils { return wrapFaker(parseIPv4Faker(obj), obj); } else if ("Expression".equalsIgnoreCase(type)) { return wrapFaker(parseExpressionFaker(obj), obj); + } else if ("Hlld".equalsIgnoreCase(type)) { + return wrapFaker(parseHlldFaker(obj), obj); + } else if ("HdrHistogram".equalsIgnoreCase(type)) { + return wrapFaker(parseHdrHistogramFaker(obj), obj); } else if ("Object".equalsIgnoreCase(type)) { return wrapFaker(parseObjectFaker(obj.getJSONArray("fields")), obj); } else if ("Union".equalsIgnoreCase(type)) { @@ -109,6 +113,20 @@ public class FakerUtils { return new ExpressionFaker(expression); } + private static Faker parseHlldFaker(JSONObject obj) { + long itemCount = obj.getLongValue("itemCount", 1000000L); + int batchCount = obj.getIntValue("batchCount", 10000); + int precision = obj.getIntValue("precision", 12); + return new HlldFaker(itemCount, batchCount, precision); + } + + private static Faker parseHdrHistogramFaker(JSONObject obj) { + int max = obj.getIntValue("max", 100000); + int batchCount = obj.getIntValue("batchCount", 1000); + int numberOfSignificantValueDigits = obj.getIntValue("numberOfSignificantValueDigits", 1); + return new HdrHistogramFaker(max, batchCount, numberOfSignificantValueDigits); + } + private static Faker parseIPv4Faker(JSONObject obj) { String start = obj.getString("start"); String end = obj.getString("end"); diff --git a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HdrHistogramFaker.java b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HdrHistogramFaker.java new file mode 100644 index 0000000..393bf8e --- /dev/null +++ b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HdrHistogramFaker.java @@ -0,0 +1,35 @@ +package com.geedgenetworks.connectors.mock.faker; + +import com.geedgenetworks.sketch.util.StringUtils; +import org.HdrHistogram.HistogramSketch; + +import java.util.concurrent.ThreadLocalRandom; + +public class HdrHistogramFaker extends Faker { + private final int max; + private final int batchCount; + private final int numberOfSignificantValueDigits; + private HistogramSketch his; + + public HdrHistogramFaker(int max, int batchCount, int numberOfSignificantValueDigits) { + this.max = max; + this.batchCount = batchCount; + this.numberOfSignificantValueDigits = numberOfSignificantValueDigits; + } + + @Override + public void init(int instanceCount, int instanceIndex) throws Exception { + his = new HistogramSketch(1L, max, numberOfSignificantValueDigits, false); + } + + @Override + public Object geneValue() throws Exception { + his.reset(); + ThreadLocalRandom random = ThreadLocalRandom.current(); + for (int i = 0; i < batchCount; i++) { + his.recordValue(random.nextInt(max)); + } + return StringUtils.encodeBase64String(his.toBytes()); + } + +} diff --git a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HlldFaker.java b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HlldFaker.java new file mode 100644 index 0000000..5af2b35 --- /dev/null +++ b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HlldFaker.java @@ -0,0 +1,39 @@ +package com.geedgenetworks.connectors.mock.faker; + +import com.geedgenetworks.sketch.hlld.Hll; +import com.geedgenetworks.sketch.util.StringUtils; + +import java.util.concurrent.ThreadLocalRandom; + +public class HlldFaker extends Faker { + private final long itemCount; + private final int batchCount; + private final int precision; + private Hll hll; + + public HlldFaker(long itemCount, int batchCount, int precision) { + this.itemCount = itemCount; + this.batchCount = batchCount; + this.precision = precision; + } + + public HlldFaker(long itemCount, int batchCount) { + this(itemCount, batchCount, 12); + } + + @Override + public void init(int instanceCount, int instanceIndex) throws Exception { + hll = new Hll(precision); + } + + @Override + public Object geneValue() throws Exception { + hll.reset(); + ThreadLocalRandom random = ThreadLocalRandom.current(); + for (int i = 0; i < batchCount; i++) { + hll.add(random.nextLong(itemCount)); + } + return StringUtils.encodeBase64String(hll.toBytes()); + } + +} diff --git a/groot-core/pom.xml b/groot-core/pom.xml index 18ae33b..e723fa5 100644 --- a/groot-core/pom.xml +++ b/groot-core/pom.xml @@ -65,6 +65,11 @@ provided + + com.geedgenetworks + sketches + + com.alibaba.nacos nacos-client diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java new file mode 100644 index 0000000..368e8c1 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java @@ -0,0 +1,42 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.sketch.util.StringUtils; +import org.HdrHistogram.Histogramer; + +import java.util.Map; + +public class HdrHistogram extends HdrHistogramBaseAggregate { + boolean outputBase64; + + @Override + public void open(UDFContext c) { + super.open(c); + Map params = c.getParameters(); + outputBase64 = "base64".equalsIgnoreCase(params.getOrDefault("output_format", "base64").toString()); + } + + @Override + public Accumulator getResult(Accumulator acc) { + Object agg = acc.getMetricsFields().get(outputField); + if (agg == null) { + return acc; + } + + byte[] bytes = ((Histogramer) agg).toBytes(); + if (outputBase64) { + acc.getMetricsFields().put(outputField, StringUtils.encodeBase64String(bytes)); + } else { + acc.getMetricsFields().put(outputField, bytes); + } + + return acc; + } + + @Override + public String functionName() { + return "HDR_HISTOGRAM"; + } + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java new file mode 100644 index 0000000..1648fa5 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java @@ -0,0 +1,101 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.AggregateFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.sketch.util.StringUtils; +import org.HdrHistogram.ArrayHistogram; +import org.HdrHistogram.DirectMapHistogram; +import org.HdrHistogram.Histogramer; +import org.apache.commons.collections.CollectionUtils; + +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public abstract class HdrHistogramBaseAggregate implements AggregateFunction { + protected String inputField; + protected String outputField; + protected boolean inputSketch; + protected long lowestDiscernibleValue; + protected long highestTrackableValue; + protected int numberOfSignificantValueDigits; + protected boolean autoResize; + + @Override + public void open(UDFContext c) { + inputField = c.getLookup_fields().get(0); + if (CollectionUtils.isNotEmpty(c.getOutput_fields())) { + outputField = c.getOutput_fields().get(0); + } else { + outputField = inputField; + } + Map params = c.getParameters(); + lowestDiscernibleValue = Long.parseLong(params.getOrDefault("lowestDiscernibleValue", "1").toString()); + highestTrackableValue = Long.parseLong(params.getOrDefault("highestTrackableValue", "2").toString()); + numberOfSignificantValueDigits = Integer.parseInt(params.getOrDefault("numberOfSignificantValueDigits", "1").toString()); + autoResize = Boolean.valueOf(params.getOrDefault("autoResize", "true").toString()); + inputSketch = "sketch".equalsIgnoreCase(params.getOrDefault("input_type", "sketch").toString()); + } + + @Override + public Accumulator initAccumulator(Accumulator acc) { + return acc; + } + + @Override + public Accumulator add(Event event, Accumulator acc) { + Object value = event.getExtractedFields().get(inputField); + if (value == null) { + return acc; + } + + if (inputSketch) { + updateHdrMerge(acc, value); + } else { + updateHdr(acc, value); + } + + return acc; + } + + protected void updateHdr(Accumulator acc, Object value) { + Map aggs = acc.getMetricsFields(); + ArrayHistogram his = (ArrayHistogram) aggs.get(outputField); + if (his == null) { + his = new ArrayHistogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); + his.setAutoResize(autoResize); + aggs.put(outputField, his); + } + + his.recordValue(((Number) value).longValue()); + } + + + protected void updateHdrMerge(Accumulator acc, Object value) { + Map aggs = acc.getMetricsFields(); + ArrayHistogram his = (ArrayHistogram) aggs.get(outputField); + if (his == null) { + his = new ArrayHistogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); + his.setAutoResize(autoResize); + aggs.put(outputField, his); + } + + Histogramer h; + if (value instanceof String) { + byte[] bytes = StringUtils.decodeBase64(((String) value).getBytes(StandardCharsets.UTF_8)); + h = DirectMapHistogram.wrapBytes(bytes); + } else if (value instanceof byte[]) { + h = DirectMapHistogram.wrapBytes((byte[]) value); + } else if (value instanceof Histogramer) { + h = (Histogramer) value; + } else { + throw new IllegalArgumentException("Unsupported type " + value.getClass()); + } + + his.merge(h); + } + + @Override + public void close() {} +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java new file mode 100644 index 0000000..b9f7d5b --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java @@ -0,0 +1,36 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.udf.UDFContext; +import org.HdrHistogram.Histogramer; + +import java.util.Map; + +public class HdrHistogramQuantile extends HdrHistogramBaseAggregate { + Double probability; + + @Override + public void open(UDFContext c) { + super.open(c); + Map params = c.getParameters(); + probability = Double.parseDouble(params.getOrDefault("probability", "0.5").toString()); + } + + @Override + public Accumulator getResult(Accumulator acc) { + Object agg = acc.getMetricsFields().get(outputField); + if (agg == null) { + return acc; + } + + long percentile = ((Histogramer) agg).getValueAtPercentile(probability * 100); + acc.getMetricsFields().put(outputField, percentile); + return acc; + } + + @Override + public String functionName() { + return "APPROX_QUANTILE_HDR"; + } + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java new file mode 100644 index 0000000..ccfffd3 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java @@ -0,0 +1,51 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram; + +import com.alibaba.fastjson2.JSON; +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.udf.UDFContext; +import org.HdrHistogram.Histogramer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class HdrHistogramQuantiles extends HdrHistogramBaseAggregate { + double[] probabilities; + + @Override + public void open(UDFContext c) { + super.open(c); + Map params = c.getParameters(); + Object ps = params.get("probabilities"); + if(ps == null){ + throw new IllegalArgumentException("probabilities param is requested"); + } + List floats = JSON.parseArray(ps instanceof String ? ps.toString(): JSON.toJSONString(ps), Double.class); + probabilities = new double[floats.size()]; + for (int i = 0; i < floats.size(); i++) { + probabilities[i] = floats.get(i); + } + } + + @Override + public Accumulator getResult(Accumulator acc) { + Object agg = acc.getMetricsFields().get(outputField); + if (agg == null) { + return acc; + } + + Histogramer his = ((Histogramer) agg); + final List counts = new ArrayList<>(probabilities.length); + for (int i = 0; i < probabilities.length; i++) { + counts.add(his.getValueAtPercentile(probabilities[i] * 100)); + } + acc.getMetricsFields().put(outputField, counts); + return acc; + } + + @Override + public String functionName() { + return "APPROX_QUANTILES_HDR"; + } + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java new file mode 100644 index 0000000..e373a7a --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java @@ -0,0 +1,40 @@ +package com.geedgenetworks.core.udf.udaf.hlld; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.sketch.hlld.Hll; +import com.geedgenetworks.sketch.util.StringUtils; + +import java.util.Map; + +public class Hlld extends HlldBaseAggregate { + boolean outputBase64; + + @Override + public void open(UDFContext c) { + super.open(c); + Map params = c.getParameters(); + outputBase64 = "base64".equalsIgnoreCase(params.getOrDefault("output_format", "base64").toString()); + } + + @Override + public Accumulator getResult(Accumulator acc) { + Hll hll = getResultHll(acc); + if (hll == null) { + return acc; + } + + if (outputBase64) { + acc.getMetricsFields().put(outputField, StringUtils.encodeBase64String(hll.toBytes())); + } else { + acc.getMetricsFields().put(outputField, hll.toBytes()); + } + + return acc; + } + + @Override + public String functionName() { + return "HLLD"; + } +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinct.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinct.java new file mode 100644 index 0000000..ec003f8 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinct.java @@ -0,0 +1,25 @@ +package com.geedgenetworks.core.udf.udaf.hlld; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.sketch.hlld.Hll; + +public class HlldApproxCountDistinct extends HlldBaseAggregate { + + @Override + public Accumulator getResult(Accumulator acc) { + Hll hll = getResultHll(acc); + if (hll == null) { + return acc; + } + + acc.getMetricsFields().put(outputField, (long)hll.size()); + + return acc; + } + + @Override + public String functionName() { + return "APPROX_COUNT_DISTINCT_HLLD"; + } + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java new file mode 100644 index 0000000..71d61dc --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java @@ -0,0 +1,98 @@ +package com.geedgenetworks.core.udf.udaf.hlld; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.AggregateFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.sketch.hlld.Hll; +import com.geedgenetworks.sketch.hlld.HllUnion; +import com.geedgenetworks.sketch.hlld.HllUtils; +import org.apache.commons.collections.CollectionUtils; + +import java.util.Map; + +public abstract class HlldBaseAggregate implements AggregateFunction { + protected String inputField; + protected String outputField; + protected boolean inputSketch; + protected int precision = 12; + + @Override + public void open(UDFContext c) { + inputField = c.getLookup_fields().get(0); + if (CollectionUtils.isNotEmpty(c.getOutput_fields())) { + outputField = c.getOutput_fields().get(0); + } else { + outputField = inputField; + } + Map params = c.getParameters(); + precision = Integer.parseInt(params.getOrDefault("precision", "12").toString()); + inputSketch = "sketch".equalsIgnoreCase(params.getOrDefault("input_type", "sketch").toString()); + } + + @Override + public Accumulator initAccumulator(Accumulator acc) { + return acc; + } + + @Override + public Accumulator add(Event event, Accumulator acc) { + Object value = event.getExtractedFields().get(inputField); + if (value == null) { + return acc; + } + + if (inputSketch) { + updateHllUnion(acc, value); + } else { + updateHll(acc, value); + } + + return acc; + } + + protected Hll getResultHll(Accumulator acc){ + Object agg = acc.getMetricsFields().get(outputField); + if (agg == null) { + return null; + } + + return inputSketch ? ((HllUnion) agg).getResult() : (Hll) agg; + } + + protected void updateHll(Accumulator acc, Object value) { + Map aggs = acc.getMetricsFields(); + Hll hll = (Hll) aggs.get(outputField); + if (hll == null) { + hll = new Hll(precision); + aggs.put(outputField, hll); + } + + if (value instanceof Integer || value instanceof Long) { + hll.add(((Number) value).longValue()); + } else if (value instanceof Float || value instanceof Double) { + hll.add(((Number) value).doubleValue()); + } else if (value instanceof String) { + hll.add((String) value); + } else if (value instanceof byte[]) { + hll.add((byte[]) value); + } else { + throw new IllegalArgumentException("Unsupported type " + value.getClass()); + } + } + + protected void updateHllUnion(Accumulator acc, Object value) { + Map aggs = acc.getMetricsFields(); + HllUnion hllUnion = (HllUnion) aggs.get(outputField); + if (hllUnion == null) { + hllUnion = new HllUnion(precision); + aggs.put(outputField, hllUnion); + } + + Hll hll = HllUtils.deserializeHll(value); + hllUnion.update(hll); + } + + @Override + public void close() {} +} diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java new file mode 100644 index 0000000..33f7bad --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java @@ -0,0 +1,89 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.AggregateFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.sketch.util.StringUtils; +import org.HdrHistogram.ArrayHistogram; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +public class HdrHistogramQuantileTest { + AggregateFunction agg; + Accumulator acc; + Event event; + + @Test + public void inputRegular() { + double probability = 0.5; + initData( "regular", 2, probability); + long count = 100000; + Map fields = event.getExtractedFields(); + for (int i = 1; i <= count; i++) { + fields.put("ms", i); + agg.add(event, acc); + } + + long expect = (long) (count * probability); + long rst = (long)agg.getResult(acc).getMetricsFields().get("ms_his"); + double error = Math.abs(rst - expect) / (double) expect; + System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error)); + assertTrue(error <= 0.05); + } + + @Test + public void inputSketch() { + double probability = 0.5; + initData( "sketch", 2, probability); + long count = 100000; + Map fields = event.getExtractedFields(); + + ArrayHistogram his = new ArrayHistogram(2); + for (int i = 1; i <= count; i++) { + his.recordValue(i); + } + fields.put("ms", StringUtils.encodeBase64String(his.toBytes())); + agg.add(event, acc); + + his = new ArrayHistogram(2); + for (int i = 1; i <= count; i++) { + his.recordValue(i); + } + fields.put("ms", his.toBytes()); + agg.add(event, acc); + + long expect = (long) (count * probability); + long rst = (long)agg.getResult(acc).getMetricsFields().get("ms_his"); + double error = Math.abs(rst - expect) / (double) expect; + System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error)); + assertTrue(error <= 0.05); + } + + private void initData(String input_type, int numberOfSignificantValueDigits, double probability){ + agg = new HdrHistogramQuantile(); + UDFContext c = new UDFContext(); + Map parameters = new HashMap<>(); + parameters.put("input_type", input_type); + parameters.put("numberOfSignificantValueDigits", numberOfSignificantValueDigits); + parameters.put("probability", probability); + c.setParameters(parameters); + c.setLookup_fields(Collections.singletonList("ms")); + c.setOutput_fields(Collections.singletonList("ms_his")); + + agg.open(c); + Map map = new HashMap<>(); + acc = new Accumulator(); + acc.setMetricsFields(map); + agg.initAccumulator(acc); + + event = new Event(); + Map fields = new HashMap<>(); + event.setExtractedFields(fields); + } +} \ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java new file mode 100644 index 0000000..4eefd9a --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java @@ -0,0 +1,98 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.AggregateFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.sketch.util.StringUtils; +import org.HdrHistogram.ArrayHistogram; +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class HdrHistogramQuantilesTest { + AggregateFunction agg; + Accumulator acc; + Event event; + + @Test + public void inputRegular() { + double[] probabilities = new double[]{0, 0.1, 0.25, 0.5, 0.75, 1}; + initData( "regular", 2, probabilities); + long count = 100000; + Map fields = event.getExtractedFields(); + for (int i = 1; i <= count; i++) { + fields.put("ms", i); + agg.add(event, acc); + } + + long[] expects = Arrays.stream(probabilities).mapToLong(x -> (long) (count * x)).toArray(); + + List rsts = (List)agg.getResult(acc).getMetricsFields().get("ms_his"); + for (int i = 0; i < expects.length; i++) { + long rst = rsts.get(i); + long expect = expects[i]; + double probability = probabilities[i]; + double error = probability <= 0.01? 0 : Math.abs(rst - expect) / (double) expect; + System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error)); + assertTrue(error <= 0.05); + } + } + + @Test + public void inputSketch() { + double[] probabilities = new double[]{0, 0.1, 0.25, 0.5, 0.75, 1}; + initData( "sketch", 2, probabilities); + long count = 100000; + Map fields = event.getExtractedFields(); + long[] expects = Arrays.stream(probabilities).mapToLong(x -> (long) (count * x)).toArray(); + + ArrayHistogram his = new ArrayHistogram(2); + for (int i = 1; i <= count; i++) { + his.recordValue(i); + } + fields.put("ms", StringUtils.encodeBase64String(his.toBytes())); + agg.add(event, acc); + + his = new ArrayHistogram(2); + for (int i = 1; i <= count; i++) { + his.recordValue(i); + } + fields.put("ms", his.toBytes()); + agg.add(event, acc); + + List rsts = (List)agg.getResult(acc).getMetricsFields().get("ms_his"); + for (int i = 0; i < expects.length; i++) { + long rst = rsts.get(i); + long expect = expects[i]; + double probability = probabilities[i]; + double error = probability <= 0.01? 0 : Math.abs(rst - expect) / (double) expect; + System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error)); + assertTrue(error <= 0.05); + } + } + + private void initData(String input_type, int numberOfSignificantValueDigits, double[] probabilities){ + agg = new HdrHistogramQuantiles(); + UDFContext c = new UDFContext(); + Map parameters = new HashMap<>(); + parameters.put("input_type", input_type); + parameters.put("numberOfSignificantValueDigits", numberOfSignificantValueDigits); + parameters.put("probabilities", probabilities); + c.setParameters(parameters); + c.setLookup_fields(Collections.singletonList("ms")); + c.setOutput_fields(Collections.singletonList("ms_his")); + + agg.open(c); + Map map = new HashMap<>(); + acc = new Accumulator(); + acc.setMetricsFields(map); + agg.initAccumulator(acc); + + event = new Event(); + Map fields = new HashMap<>(); + event.setExtractedFields(fields); + } +} \ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java new file mode 100644 index 0000000..f177ca5 --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java @@ -0,0 +1,102 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.AggregateFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.sketch.util.StringUtils; +import org.HdrHistogram.ArrayHistogram; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class HdrHistogramTest { + AggregateFunction agg; + Accumulator acc; + Event event; + + @Test + public void inputRegular() { + double[] probabilities = new double[]{0, 0.1, 0.25, 0.5, 0.75, 1}; + initData( "regular", 2, "base64"); + long count = 100000; + Map fields = event.getExtractedFields(); + for (int i = 1; i <= count; i++) { + fields.put("ms", i); + agg.add(event, acc); + } + + long[] expects = Arrays.stream(probabilities).mapToLong(x -> (long) (count * x)).toArray(); + String str = (String) agg.getResult(acc).getMetricsFields().get("ms_his"); + ArrayHistogram his = ArrayHistogram.fromBytes(StringUtils.decodeBase64(str.getBytes(StandardCharsets.UTF_8))); + + for (int i = 0; i < expects.length; i++) { + long rst = his.getValueAtPercentile(probabilities[i] * 100); + long expect = expects[i]; + double probability = probabilities[i]; + double error = probability <= 0.01? 0 : Math.abs(rst - expect) / (double) expect; + System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error)); + assertTrue(error <= 0.05); + } + } + + @Test + public void inputSketch() { + double[] probabilities = new double[]{0, 0.1, 0.25, 0.5, 0.75, 1}; + initData( "sketch", 2, "binary"); + long count = 100000; + Map fields = event.getExtractedFields(); + long[] expects = Arrays.stream(probabilities).mapToLong(x -> (long) (count * x)).toArray(); + + ArrayHistogram his = new ArrayHistogram(2); + for (int i = 1; i <= count; i++) { + his.recordValue(i); + } + fields.put("ms", StringUtils.encodeBase64String(his.toBytes())); + agg.add(event, acc); + + his = new ArrayHistogram(2); + for (int i = 1; i <= count; i++) { + his.recordValue(i); + } + fields.put("ms", his.toBytes()); + agg.add(event, acc); + + byte[] bytes = (byte[]) agg.getResult(acc).getMetricsFields().get("ms_his"); + ArrayHistogram h = ArrayHistogram.fromBytes(bytes); + + for (int i = 0; i < expects.length; i++) { + long rst = h.getValueAtPercentile(probabilities[i] * 100); + long expect = expects[i]; + double probability = probabilities[i]; + double error = probability <= 0.01? 0 : Math.abs(rst - expect) / (double) expect; + System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error)); + assertTrue(error <= 0.05); + } + } + + private void initData(String input_type, int numberOfSignificantValueDigits, String output_format){ + agg = new HdrHistogram(); + UDFContext c = new UDFContext(); + Map parameters = new HashMap<>(); + parameters.put("input_type", input_type); + parameters.put("numberOfSignificantValueDigits", numberOfSignificantValueDigits); + parameters.put("output_format", output_format); + c.setParameters(parameters); + c.setLookup_fields(Collections.singletonList("ms")); + c.setOutput_fields(Collections.singletonList("ms_his")); + + agg.open(c); + Map map = new HashMap<>(); + acc = new Accumulator(); + acc.setMetricsFields(map); + agg.initAccumulator(acc); + + event = new Event(); + Map fields = new HashMap<>(); + event.setExtractedFields(fields); + } +} \ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java new file mode 100644 index 0000000..eae356d --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java @@ -0,0 +1,87 @@ +package com.geedgenetworks.core.udf.udaf.hlld; + + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.AggregateFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.sketch.hlld.Hll; +import com.geedgenetworks.sketch.util.StringUtils; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertTrue; + + +public class HlldApproxCountDistinctTest { + AggregateFunction agg; + Accumulator acc; + Event event; + + + @Test + public void inputRegular() { + initData(14, "regular"); + long count = 100000; + Map fields = event.getExtractedFields(); + for (int i = 0; i < count; i++) { + fields.put("ip", i); + agg.add(event, acc); + } + + long rst = (long)agg.getResult(acc).getMetricsFields().get("ip_cnt"); + double error = Math.abs(rst - count) / (double) count; + System.out.println(String.format("%d,%d,%.4f", count , rst , error)); + assertTrue(error <= 0.05); + } + + @Test + public void inputSketch() { + initData(14, "sketch"); + long count = 150000; + Map fields = event.getExtractedFields(); + + Hll hll = new Hll(12); + for (int i = 0; i < 100000; i++) { + hll.add(i); + } + fields.put("ip", StringUtils.encodeBase64String(hll.toBytes())); + agg.add(event, acc); + + hll = new Hll(13); + for (int i = 50000; i < 150000; i++) { + hll.add(i); + } + fields.put("ip", hll.toBytes()); + agg.add(event, acc); + + long rst = (long)agg.getResult(acc).getMetricsFields().get("ip_cnt"); + double error = Math.abs(rst - count) / (double) count; + System.out.println(String.format("%d,%d,%.4f", count , rst , error)); + assertTrue(error <= 0.05); + } + + private void initData(int precision, String input_type){ + agg = new HlldApproxCountDistinct(); + UDFContext c = new UDFContext(); + Map parameters = new HashMap<>(); + parameters.put("precision", precision); + parameters.put("input_type", input_type); + c.setParameters(parameters); + c.setLookup_fields(Collections.singletonList("ip")); + c.setOutput_fields(Collections.singletonList("ip_cnt")); + + agg.open(c); + Map map = new HashMap<>(); + acc = new Accumulator(); + acc.setMetricsFields(map); + agg.initAccumulator(acc); + + event = new Event(); + Map fields = new HashMap<>(); + event.setExtractedFields(fields); + } +} \ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java new file mode 100644 index 0000000..f489ee4 --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java @@ -0,0 +1,86 @@ +package com.geedgenetworks.core.udf.udaf.hlld; + +import com.geedgenetworks.common.Accumulator; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.AggregateFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.sketch.hlld.Hll; +import com.geedgenetworks.sketch.util.StringUtils; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class HlldTest { + AggregateFunction agg; + Accumulator acc; + Event event; + + @Test + public void inputRegular() { + initData(14, "regular", "base64"); + long count = 100000; + Map fields = event.getExtractedFields(); + for (int i = 0; i < count; i++) { + fields.put("ip", i); + agg.add(event, acc); + } + + String hllStr = (String)agg.getResult(acc).getMetricsFields().get("ip_cnt"); + long rst = (long) Hll.fromBytes(StringUtils.decodeBase64(hllStr.getBytes(StandardCharsets.UTF_8))).size(); + double error = Math.abs(rst - count) / (double) count; + System.out.println(String.format("%d,%d,%.4f", count , rst , error)); + assertTrue(error <= 0.05); + } + + @Test + public void inputSketch() { + initData(14, "sketch", "binary"); + long count = 150000; + Map fields = event.getExtractedFields(); + for (int i = 0; i < 100000; i++) { + Hll hll = new Hll(12); + hll.add(i); + fields.put("ip", StringUtils.encodeBase64String(hll.toBytes())); + agg.add(event, acc); + } + for (int i = 50000; i < 150000; i++) { + Hll hll = new Hll(13); + hll.add(i); + fields.put("ip", hll.toBytes()); + agg.add(event, acc); + } + + byte[] hllBytes = (byte[])agg.getResult(acc).getMetricsFields().get("ip_cnt"); + long rst = (long) Hll.fromBytes(hllBytes).size(); + double error = Math.abs(rst - count) / (double) count; + System.out.println(String.format("%d,%d,%.4f", count , rst , error)); + assertTrue(error <= 0.05); + } + + private void initData(int precision, String input_type, String output_format){ + agg = new Hlld(); + UDFContext c = new UDFContext(); + Map parameters = new HashMap<>(); + parameters.put("precision", precision); + parameters.put("input_type", input_type); + parameters.put("output_format", output_format); + c.setParameters(parameters); + c.setLookup_fields(Collections.singletonList("ip")); + c.setOutput_fields(Collections.singletonList("ip_cnt")); + + agg.open(c); + Map map = new HashMap<>(); + acc = new Accumulator(); + acc.setMetricsFields(map); + agg.initAccumulator(acc); + + event = new Event(); + Map fields = new HashMap<>(); + event.setExtractedFields(fields); + } +} diff --git a/pom.xml b/pom.xml index a8f919b..53bcf1f 100644 --- a/pom.xml +++ b/pom.xml @@ -268,6 +268,11 @@ fastjson ${fastjson2.version} + + com.geedgenetworks + sketches + 1.0.0 + com.alibaba.nacos nacos-client -- cgit v1.2.3 From 07332297c1306aa0dac649c7d15bf131e8edbc7e Mon Sep 17 00:00:00 2001 From: doufenghu Date: Thu, 15 Aug 2024 16:18:41 +0800 Subject: [docs][udf] Update scalar UDFs, user-defined aggregate functions (UDAFs) description. --- README.md | 2 +- docs/processor/aggregate-processor.md | 4 ++-- docs/processor/projection-processor.md | 4 ++-- docs/processor/udf.md | 4 ++-- docs/user-guide.md | 2 +- .../java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java | 1 + .../main/java/com/geedgenetworks/example/GrootStreamExample.java | 4 +++- .../src/main/resources/examples/inline_to_kafka.yaml | 6 +++--- .../src/test/resources/clickhouse_data_type_sink.yaml | 3 ++- 9 files changed, 17 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 633f803..d6485d9 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Groot Stream Platform helps you process netflow data - logs, metrics etc. - in r Configure a job, you'll set up Sources, Filters, Processing Pipeline, and Sinks, and will assemble several built-in functions into a Processing Pipeline. The job will then be deployed to a Flink cluster for execution. - **Source**: The data source of the job, which can be a Kafka topic, a IPFIX Collector, or a file. - **Filter**: Filters data based on specified conditions. -- **Types of Pipelines**: The fundamental unit of data stream processing is the processor, categorized by functionality into stateless and stateful processors. Each processor can be assemble `UDFs`(User-defined functions) or `UDAFs`(User-defined aggregation functions) into a pipeline. There are 3 types of pipelines at different stages of the data processing process: +- **Types of Pipelines**: The fundamental unit of data stream processing is the processor, categorized by functionality into stateless and stateful processors. Each processor can be assemble `UDFs`(User-defined functions) into a pipeline. There are 3 types of pipelines at different stages of the data processing process: - **Pre-processing Pipeline**: Optional. These pipelines that are attached to a source to normalize the events before they enter the processing pipeline. - **Processing Pipeline**: Event processing pipeline. - **Post-processing Pipeline**: Optional. These pipelines that are attached to a sink to normalize the events before they're written to the sink. diff --git a/docs/processor/aggregate-processor.md b/docs/processor/aggregate-processor.md index d9bcdb0..af82d4e 100644 --- a/docs/processor/aggregate-processor.md +++ b/docs/processor/aggregate-processor.md @@ -1,11 +1,11 @@ # Aggregate Processor -> Processing pipelines for aggregate processor +> Processing pipelines for aggregate processors using UDAFs ## Description Aggregate processor is used to aggregate the data from source to sink. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDAFs(User-defined Aggregate functions) into a pipeline. -Within the pipeline, events are processed by each Function in order, top‑>down. The UDAF usage detail can be found in [UDAF](udaf.md). +Within the pipeline, events are processed by each Function in order, top‑>down. More details can be found in user-defined aggregate functions [(UDAFs)](udaf.md). ## Options Note：Default will output internal fields `__window_start_timestamp` and `__window_end_timestamp` if not set output_fields. diff --git a/docs/processor/projection-processor.md b/docs/processor/projection-processor.md index bc4b249..4319f36 100644 --- a/docs/processor/projection-processor.md +++ b/docs/processor/projection-processor.md @@ -1,12 +1,12 @@ # Projection Processor -> Processing pipelines for projection processor +> Processing pipelines for projection processors using scalar UDFs ## Description Projection processor is used to project the data from source to sink. It can be used to filter, remove, and transform fields. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDFs(User-defined functions) into a pipeline. -Within the pipeline, events are processed by each Function in order, top‑>down. The UDF usage detail can be found in [UDF](udf.md). +Within the pipeline, events are processed by each Function in order, top‑>down. More details can be found in User Defined Functions [(UDFs)](udf.md). ## Options diff --git a/docs/processor/udf.md b/docs/processor/udf.md index cf305ef..170d86f 100644 --- a/docs/processor/udf.md +++ b/docs/processor/udf.md @@ -1,6 +1,6 @@ # UDF -> The functions for projection processors. +> The functions for projection processors. ## Function of content @@ -24,7 +24,7 @@ ## Description -UDF(User Defined Function) is used to extend the functions of projection processor. The UDF is a part of the processing pipeline. It can be used in the pre-processing pipeline, processing pipeline, and post-processing pipeline. +Scalar UDF(User Defined Function) is used to extend the functions of projection processor. The UDF is a part of the processing pipeline. It can be used in the pre-processing pipeline, processing pipeline, and post-processing pipeline. ## UDF Definition diff --git a/docs/user-guide.md b/docs/user-guide.md index 9d5b1c7..e35616f 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -137,7 +137,7 @@ Based on the filter expression, the event will be passed to downstream if the ex ## Processing Pipelines Processing pipelines are used to define the event processing logic of the job. It can be categorized by functionality into stateless and stateful processors. Based processing order, it can be categorized into pre-processing pipeline, processing pipeline and post-processing pipeline. Each processor can assemble `UDFs`(User-defined functions) into a pipeline. The detail of processor is listed in [Processor](processor). - +UDF supports [scalar UDFs](processor/udf.md) , user-defined aggregate functions [(UDAFs)](processor/udaf.md), and user-defined table functions (UDTFs). ## Sinks Sink is used to define where GrootStream needs to output data. Multiple sinks can be defined in a job. The supported sinks are listed in [Sink Connectors](connector/sink). Each sink has its own specific parameters to define how to output data, and GrootStream also extracts the properties that each sink will use, such as the `topic` and `kafka.bootstrap.servers` of the `Kafka` sink. diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java index 336842f..c3746a4 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java @@ -68,5 +68,6 @@ public class ConfigShadeTest { System.out.println( ConfigShadeUtils.encryptOption("aes", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"galaxy2019\";")); System.out.println( ConfigShadeUtils.decryptOption("aes", "454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817")); System.out.println( ConfigShadeUtils.encryptOption("aes", "testuser")); + System.out.println( ConfigShadeUtils.encryptOption("aes", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"olap\" password=\"galaxy2019\";")); } } diff --git a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java index 690f21c..f666ee8 100644 --- a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java +++ b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java @@ -13,11 +13,13 @@ import java.nio.file.Paths; public class GrootStreamExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException { - String configPath = args.length > 0 ? args[0] : "/examples/session_record_mock_to_print_with_aggregation.yaml"; + String configPath = args.length > 0 ? args[0] : "/examples/inline_to_kafka.yaml"; String configFile = getTestConfigFile(configPath); ExecuteCommandArgs executeCommandArgs = new ExecuteCommandArgs(); executeCommandArgs.setConfigFile(configFile); executeCommandArgs.setCheckConfig(false); + executeCommandArgs.setEncrypt(true); + executeCommandArgs.setDecrypt(false); executeCommandArgs.setVersion(false); executeCommandArgs.setDeployMode(DeployMode.RUN); executeCommandArgs.setTargetType(TargetType.LOCAL); diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml index a5c5ece..517d29b 100644 --- a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml @@ -46,7 +46,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json log.failures.only: true @@ -64,7 +64,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 format: json log.failures.only: true @@ -72,7 +72,7 @@ application: # [object] Define job configuration env: name: example-inline-to-kafka parallelism: 3 - shade.identifier: default + shade.identifier: aes pipeline: object-reuse: true topology: diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml b/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml index eb020c1..fcff8a3 100644 --- a/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml @@ -57,7 +57,8 @@ sinks: type: clickhouse properties: host: clickhouse:9000 - table: default.sink_table + table: sink_table + connection.database: default batch.size: 10 batch.byte.size: 200MB batch.interval: 1s -- cgit v1.2.3 From 06c7bda268c1aa038bb98e75ec99370ca5fa4ecf Mon Sep 17 00:00:00 2001 From: doufenghu Date: Mon, 19 Aug 2024 16:31:10 +0800 Subject: [improve][e2e-clickhouse] Close ResultSet after used. --- .../test/e2e/clickhouse/ClickHouseIT.java | 66 +++++++++++----------- .../test/resources/clickhouse_data_type_sink.yaml | 6 +- .../test/resources/init/clickhouse_test_sql.conf | 17 ------ .../com/geedgenetworks/test/e2e/kafka/KafkaIT.java | 6 +- 4 files changed, 39 insertions(+), 56 deletions(-) diff --git a/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java index 8eab377..20caace 100644 --- a/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java +++ b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java @@ -143,15 +143,16 @@ public class ClickHouseIT extends TestSuiteBase implements TestResource { .untilAsserted( () -> { assertHasData(SINK_TABLE); + compareResult(); clearTable(SINK_TABLE); }); } private void assertHasData(String table) { - try (Statement statement = connection.createStatement()) { - String sql = String.format("select * from %s.%s limit 1", DATABASE, table); - ResultSet source = statement.executeQuery(sql); + String sql = String.format("select * from %s.%s limit 1", DATABASE, table); + try (Statement statement = connection.createStatement(); + ResultSet source = statement.executeQuery(sql);) { Assertions.assertTrue(source.next()); } catch (SQLException e) { throw new RuntimeException("test clickhouse server image error", e); @@ -167,43 +168,42 @@ public class ClickHouseIT extends TestSuiteBase implements TestResource { } private void compareResult() throws SQLException, IOException { - String sourceSql = "select * from " + SOURCE_TABLE + " order by id"; + String sourceSql = "select * from " + SOURCE_TABLE + " order by id "; String sinkSql = "select * from " + SINK_TABLE + " order by id"; - List columnList = - Arrays.stream(default_columns).collect(Collectors.toList()); - Statement sourceStatement = connection.createStatement(); - Statement sinkStatement = connection.createStatement(); - ResultSet sourceResultSet = sourceStatement.executeQuery(sourceSql); - ResultSet sinkResultSet = sinkStatement.executeQuery(sinkSql); - Assertions.assertEquals( - sourceResultSet.getMetaData().getColumnCount(), - sinkResultSet.getMetaData().getColumnCount()); - while (sourceResultSet.next()) { - if (sinkResultSet.next()) { - for (String column : columnList) { - Object source = sourceResultSet.getObject(column); - Object sink = sinkResultSet.getObject(column); - if (!Objects.deepEquals(source, sink)) { - InputStream sourceAsciiStream = sourceResultSet.getBinaryStream(column); - InputStream sinkAsciiStream = sinkResultSet.getBinaryStream(column); - String sourceValue = - IOUtils.toString(sourceAsciiStream, StandardCharsets.UTF_8); - String sinkValue = - IOUtils.toString(sinkAsciiStream, StandardCharsets.UTF_8); - Assertions.assertEquals(sourceValue, sinkValue); - } - Assertions.assertTrue(true); + try (Statement sourceStatement = connection.createStatement(); + Statement sinkStatement = connection.createStatement(); + ResultSet sourceResultSet = sourceStatement.executeQuery(sourceSql); + ResultSet sinkResultSet = sinkStatement.executeQuery(sinkSql)) { + Assertions.assertEquals( + sourceResultSet.getMetaData().getColumnCount(), + sinkResultSet.getMetaData().getColumnCount()); + String columns = String.join(",", default_columns); + Assertions.assertTrue( + compare(String.format(CONFIG.getString(COMPARE_SQL), columns, columns))); + } + + sourceSql = "select count(distinct id) as count from " + SOURCE_TABLE; + sinkSql = "select count(distinct id) as count from " + SINK_TABLE; + + try (Statement sourceStatement = connection.createStatement(); + Statement sinkStatement = connection.createStatement(); + ResultSet sourceResultSet = sourceStatement.executeQuery(sourceSql); + ResultSet sinkResultSet = sinkStatement.executeQuery(sinkSql)) { + while (sourceResultSet.next()) { + if (sinkResultSet.next()) { + int sinkUniqueIds = sinkResultSet.getInt("count"); + int sourceUniqueIds = sourceResultSet.getInt("count"); + Assertions.assertEquals(sinkUniqueIds, sourceUniqueIds); } } } - String columns = String.join(",", default_columns); - Assertions.assertTrue( - compare(String.format(CONFIG.getString(COMPARE_SQL), columns, columns))); + + } private Boolean compare(String sql) { - try (Statement statement = connection.createStatement()) { - ResultSet resultSet = statement.executeQuery(sql); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql);) { return !resultSet.next(); } catch (SQLException e) { throw new RuntimeException("result compare error", e); diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml b/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml index fcff8a3..3406a67 100644 --- a/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml @@ -47,8 +47,8 @@ sources: # data: '[{"id":0,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":0,"c_lowcardinality":"string"},{"id":1,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":1,"c_lowcardinality":"string"},{"id":2,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":2,"c_lowcardinality":"string"},{"id":3,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":3,"c_lowcardinality":"string"},{"id":4,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":4,"c_lowcardinality":"string"},{"id":5,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":5,"c_lowcardinality":"string"},{"id":6,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":6,"c_lowcardinality":"string"},{"id":7,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":7,"c_lowcardinality":"string"},{"id":8,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":8,"c_lowcardinality":"string"},{"id":9,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":9,"c_lowcardinality":"string"},{"id":10,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":10,"c_lowcardinality":"string"},{"id":11,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":11,"c_lowcardinality":"string"},{"id":12,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":12,"c_lowcardinality":"string"},{"id":13,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":13,"c_lowcardinality":"string"},{"id":14,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":14,"c_lowcardinality":"string"},{"id":15,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":15,"c_lowcardinality":"string"},{"id":16,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":16,"c_lowcardinality":"string"},{"id":17,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":17,"c_lowcardinality":"string"},{"id":18,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":18,"c_lowcardinality":"string"},{"id":19,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":19,"c_lowcardinality":"string"},{"id":20,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":20,"c_lowcardinality":"string"},{"id":21,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":21,"c_lowcardinality":"string"},{"id":22,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":22,"c_lowcardinality":"string"},{"id":23,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":23,"c_lowcardinality":"string"},{"id":24,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":24,"c_lowcardinality":"string"},{"id":25,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":25,"c_lowcardinality":"string"},{"id":26,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":26,"c_lowcardinality":"string"},{"id":27,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":27,"c_lowcardinality":"string"},{"id":28,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":28,"c_lowcardinality":"string"},{"id":29,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":29,"c_lowcardinality":"string"},{"id":30,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":30,"c_lowcardinality":"string"},{"id":31,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":31,"c_lowcardinality":"string"},{"id":32,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":32,"c_lowcardinality":"string"},{"id":33,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":33,"c_lowcardinality":"string"},{"id":34,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":34,"c_lowcardinality":"string"},{"id":35,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":35,"c_lowcardinality":"string"},{"id":36,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":36,"c_lowcardinality":"string"},{"id":37,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":37,"c_lowcardinality":"string"},{"id":38,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":38,"c_lowcardinality":"string"},{"id":39,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":39,"c_lowcardinality":"string"},{"id":40,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":40,"c_lowcardinality":"string"},{"id":41,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":41,"c_lowcardinality":"string"},{"id":42,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":42,"c_lowcardinality":"string"},{"id":43,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":43,"c_lowcardinality":"string"},{"id":44,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":44,"c_lowcardinality":"string"},{"id":45,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":45,"c_lowcardinality":"string"},{"id":46,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":46,"c_lowcardinality":"string"},{"id":47,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":47,"c_lowcardinality":"string"},{"id":48,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":48,"c_lowcardinality":"string"},{"id":49,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":49,"c_lowcardinality":"string"},{"id":50,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":50,"c_lowcardinality":"string"},{"id":51,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":51,"c_lowcardinality":"string"},{"id":52,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":52,"c_lowcardinality":"string"},{"id":53,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":53,"c_lowcardinality":"string"},{"id":54,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":54,"c_lowcardinality":"string"},{"id":55,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":55,"c_lowcardinality":"string"},{"id":56,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":56,"c_lowcardinality":"string"},{"id":57,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":57,"c_lowcardinality":"string"},{"id":58,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":58,"c_lowcardinality":"string"},{"id":59,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":59,"c_lowcardinality":"string"},{"id":60,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":60,"c_lowcardinality":"string"},{"id":61,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":61,"c_lowcardinality":"string"},{"id":62,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":62,"c_lowcardinality":"string"},{"id":63,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":63,"c_lowcardinality":"string"},{"id":64,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":64,"c_lowcardinality":"string"},{"id":65,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":65,"c_lowcardinality":"string"},{"id":66,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":66,"c_lowcardinality":"string"},{"id":67,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":67,"c_lowcardinality":"string"},{"id":68,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":68,"c_lowcardinality":"string"},{"id":69,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":69,"c_lowcardinality":"string"},{"id":70,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":70,"c_lowcardinality":"string"},{"id":71,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":71,"c_lowcardinality":"string"},{"id":72,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":72,"c_lowcardinality":"string"},{"id":73,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":73,"c_lowcardinality":"string"},{"id":74,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":74,"c_lowcardinality":"string"},{"id":75,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":75,"c_lowcardinality":"string"},{"id":76,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":76,"c_lowcardinality":"string"},{"id":77,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":77,"c_lowcardinality":"string"},{"id":78,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":78,"c_lowcardinality":"string"},{"id":79,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":79,"c_lowcardinality":"string"},{"id":80,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":80,"c_lowcardinality":"string"},{"id":81,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":81,"c_lowcardinality":"string"},{"id":82,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":82,"c_lowcardinality":"string"},{"id":83,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":83,"c_lowcardinality":"string"},{"id":84,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":84,"c_lowcardinality":"string"},{"id":85,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":85,"c_lowcardinality":"string"},{"id":86,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":86,"c_lowcardinality":"string"},{"id":87,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":87,"c_lowcardinality":"string"},{"id":88,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":88,"c_lowcardinality":"string"},{"id":89,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":89,"c_lowcardinality":"string"},{"id":90,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":90,"c_lowcardinality":"string"},{"id":91,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":91,"c_lowcardinality":"string"},{"id":92,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":92,"c_lowcardinality":"string"},{"id":93,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":93,"c_lowcardinality":"string"},{"id":94,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":94,"c_lowcardinality":"string"},{"id":95,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":95,"c_lowcardinality":"string"},{"id":96,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":96,"c_lowcardinality":"string"},{"id":97,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":97,"c_lowcardinality":"string"},{"id":98,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":98,"c_lowcardinality":"string"},{"id":99,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":99,"c_lowcardinality":"string"}]' format: json - interval.per.row: 1s - repeat.count: 100 + interval.per.row: 10ms + repeat.count: -1 json.ignore.parse.errors: false @@ -59,7 +59,7 @@ sinks: host: clickhouse:9000 table: sink_table connection.database: default - batch.size: 10 + batch.size: 100 batch.byte.size: 200MB batch.interval: 1s connection.user: ee9b0016824d59c8c191aa9633e4b61e diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf b/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf index c24e632..f132795 100644 --- a/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf @@ -1,20 +1,3 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - source_table = """ set allow_experimental_geo_types = 1; create table if not exists `default`.source_table( diff --git a/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java b/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java index 1c40e01..50ff9d8 100644 --- a/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java +++ b/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java @@ -163,8 +163,8 @@ public class KafkaIT extends TestSuiteBase implements TestResource { public void testKafkaAsSinkProducerQuota(TestContainer container) throws IOException, InterruptedException { //Create topic with 3 partitions executeShell("kafka-topics --create --topic SESSION-RECORD-QUOTA-TEST --bootstrap-server kafkaCluster:9092 --partitions 3 --replication-factor 1 --command-config /etc/kafka/kafka_client_jass_cli.properties"); - //Set producer quota to 5KB/s - executeShell("kafka-configs --bootstrap-server kafkaCluster:9092 --alter --add-config 'producer_byte_rate=5120' --entity-type users --entity-name admin --entity-type clients --entity-name SESSION-RECORD-QUOTA-TEST --command-config /etc/kafka/kafka_client_jass_cli.properties "); + //Set producer quota to 2KB/s + executeShell("kafka-configs --bootstrap-server kafkaCluster:9092 --alter --add-config 'producer_byte_rate=2048' --entity-type users --entity-name admin --entity-type clients --entity-name SESSION-RECORD-QUOTA-TEST --command-config /etc/kafka/kafka_client_jass_cli.properties "); CompletableFuture.supplyAsync( () -> { @@ -183,7 +183,7 @@ public class KafkaIT extends TestSuiteBase implements TestResource { .untilAsserted( () -> { data.addAll(getKafkaConsumerListData("SESSION-RECORD-QUOTA-TEST")); - Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "TimeoutException") && data.size()>1000); + Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "TimeoutException") && data.size()>100); }); } -- cgit v1.2.3 From 43bc690b73d2df56cb29cea7235c650d60af82d6 Mon Sep 17 00:00:00 2001 From: doufenghu Date: Mon, 19 Aug 2024 16:33:30 +0800 Subject: [docs][mock-connector] add HLLD and HDRHistogram mock data type description. --- config/template/grootstream_job_template.yaml | 36 ++- .../mock_schema/object_statistics_mock_desc.json | 186 +++++++----- .../mock_schema/statistics_rule_mock_desc.json | 320 +++++++++++++-------- docs/connector/connector.md | 94 +++--- docs/images/groot_stream_architecture.jpg | Bin 5054004 -> 5263679 bytes .../geedgenetworks/example/GrootStreamExample.java | 4 +- 6 files changed, 414 insertions(+), 226 deletions(-) diff --git a/config/template/grootstream_job_template.yaml b/config/template/grootstream_job_template.yaml index 9f64abe..7cf50c8 100644 --- a/config/template/grootstream_job_template.yaml +++ b/config/template/grootstream_job_template.yaml @@ -151,7 +151,7 @@ preprocessing_pipelines: # [object] Define Processors for preprocessing pipeline # It will be accomplished the common processing for the event by the user-defined functions. # processing_pipelines: # [object] Define Processors for processing pipelines. - processor: # [object] Define projection processor name, must be unique. + projection_processor: # [object] Define projection processor name, must be unique. type: projection # [string] Processor Type remove_fields: output_fields: @@ -276,7 +276,7 @@ processing_pipelines: # [object] Define Processors for processing pipelines. lookup_fields: [ client_asn,server_asn ] output_fields: [ asn_list ] - metrics_processor: # [object] metrics processing Pipeline + projection_metrics_processor: # [object] metrics processing Pipeline type: projection output_fields: properties: @@ -323,6 +323,38 @@ processing_pipelines: # [object] Define Processors for processing pipelines. parameters: data_center_id_num: 1 + aggregate_processor: # [object] Define aggregate processor name, must be unique. + type: aggregate + group_by_fields: [ recv_time, sled_ip ] # [array of string] Group By Fields + window_type: tumbling_processing_time # [string] Window Type, tumbling_processing_time, tumbling_event_time, sliding_processing_time, sliding_event_time + window_size: 60 + functions: + - function: NUMBER_SUM + lookup_fields: [ received_bytes, sent_bytes ] + output_fields: [ received_bytes_sum ] + + - function: LONG_COUNT + lookup_fields: [ received_bytes ] + output_fields: [ sessions ] + + - function: MEAN + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_mean ] + parameters: + precision: 2 + + - function: FIRST_VALUE + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_first ] + + - function: LAST_VALUE + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_last ] + + - function: COLLECT_LIST + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_set ] + postprocessing_pipelines: # [object] Define Processors for postprocessing pipelines. postprocessor: # [object] Define projection processor name, must be unique. type: projection diff --git a/config/template/mock_schema/object_statistics_mock_desc.json b/config/template/mock_schema/object_statistics_mock_desc.json index 8542767..fbe5eb5 100644 --- a/config/template/mock_schema/object_statistics_mock_desc.json +++ b/config/template/mock_schema/object_statistics_mock_desc.json @@ -1,77 +1,113 @@ [ - {"name": "name", "type": "String", "options":["object_statistics"]}, - {"name": "timestamp_ms", "type": "Timestamp", "unit":"millis"}, - {"name": "tags", "type": "Object", "fields": [ - {"name": "object_id", "type":"Number","min":1,"max":100}, - {"name": "item_id", "type":"Number","min":1,"max":100000}, - {"name": "object_type", "type":"String","options":["ip","asn","port","url","fqdn","account","subscriberid","keywords","application","fqdn_category"]}, - { - "name": "device_id", - "type": "String", - "options": [ - "9800165603191151", - "9800165603247024", - "9800165802621377", - "9800165603191148" - ] - }, - { - "name": "device_group", - "type": "String", - "options": [ - "OLAP-MOCK-DG-1", - "OLAP-MOCK-DG-2", - "OLAP-MOCK-DG-3", - "OLAP-MOCK-DG-4" - ] - },{ - "name": "data_center", - "type": "String", - "options": [ - "OLAP-MOCK-DC-1", - "OLAP-MOCK-DC-2", - "OLAP-MOCK-DC-3", - "OLAP-MOCK-DC-4" - ] - },{ - "name": "vsys_id", - "type": "Number", - "options": [ - 2048 - ] - } - ] }, - {"name": "fields", "type": "Object", "fields": [ - { - "name": "out_bytes", - "type": "Number", - "min": 1, - "max": 15000 - }, - { - "name": "in_bytes", - "type": "Number", - "min": 1, - "max": 60000 - },{ - "name": "bytes", - "type": "Eval", - "expression": "in_bytes+out_bytes" - },{ - "name": "new_in_sessions", - "type": "Number", - "min": 1, - "max": 10 - },{ - "name": "new_out_sessions", - "type": "Number", - "min": 1, - "max": 100 - },{ - "name": "sessions", - "type": "Eval", - "expression": "new_in_sessions+new_out_sessions" - } - ] } - + { + "name": "name", + "type": "String", + "options": [ + "object_statistics" + ] + }, + { + "name": "timestamp_ms", + "type": "Timestamp", + "unit": "millis" + }, + { + "name": "object_id", + "type": "Number", + "min": 1, + "max": 100 + }, + { + "name": "item_id", + "type": "Number", + "min": 1, + "max": 100000 + }, + { + "name": "object_type", + "type": "String", + "options": [ + "ip", + "asn", + "port", + "url", + "fqdn", + "account", + "subscriberid", + "keywords", + "application", + "fqdn_category" + ] + }, + { + "name": "device_id", + "type": "String", + "options": [ + "9800165603191151", + "9800165603247024", + "9800165802621377", + "9800165603191148" + ] + }, + { + "name": "device_group", + "type": "String", + "options": [ + "OLAP-MOCK-DG-1", + "OLAP-MOCK-DG-2", + "OLAP-MOCK-DG-3", + "OLAP-MOCK-DG-4" + ] + }, + { + "name": "data_center", + "type": "String", + "options": [ + "OLAP-MOCK-DC-1", + "OLAP-MOCK-DC-2", + "OLAP-MOCK-DC-3", + "OLAP-MOCK-DC-4" + ] + }, + { + "name": "vsys_id", + "type": "Number", + "options": [ + 2048 + ] + }, + { + "name": "out_bytes", + "type": "Number", + "min": 1, + "max": 15000 + }, + { + "name": "in_bytes", + "type": "Number", + "min": 1, + "max": 60000 + }, + { + "name": "bytes", + "type": "Eval", + "expression": "in_bytes+out_bytes" + }, + { + "name": "new_in_sessions", + "type": "Number", + "min": 1, + "max": 10 + }, + { + "name": "new_out_sessions", + "type": "Number", + "min": 1, + "max": 100 + }, + { + "name": "sessions", + "type": "Eval", + "expression": "new_in_sessions+new_out_sessions" + } ] \ No newline at end of file diff --git a/config/template/mock_schema/statistics_rule_mock_desc.json b/config/template/mock_schema/statistics_rule_mock_desc.json index c3207ec..91f54e9 100644 --- a/config/template/mock_schema/statistics_rule_mock_desc.json +++ b/config/template/mock_schema/statistics_rule_mock_desc.json @@ -1,122 +1,218 @@ [ - {"name": "name", "type": "String", "options":["statistics_rule"]}, - {"name": "timestamp_ms", "type": "Timestamp", "unit":"millis"}, - {"name": "tags", "type": "Object", "fields": [ - { "name": "unionFields", "type": "Union", "random": false, "unionFields": [ - { "weight": 2, "fields": [ - {"name": "rule_id", "type":"Number", "random": false, "options": [ - 1 - ]}, - {"name": "chart_id", "type":"Number", "random": false,"options": [ - 1 - ]}, - {"name": "template_id", "type":"Number", "random": false, "options": [ - 1 - ]} - ] + { + "name": "name", + "type": "String", + "options": [ + "statistics_rule" + ] + }, + { + "name": "timestamp_ms", + "type": "Timestamp", + "unit": "millis" + }, + { + "name": "unionFields", + "type": "Union", + "random": false, + "unionFields": [ + { + "weight": 2, + "fields": [ + { + "name": "rule_id", + "type": "Number", + "random": false, + "options": [ + 1 + ] + }, + { + "name": "chart_id", + "type": "Number", + "random": false, + "options": [ + 1 + ] + }, + { + "name": "template_id", + "type": "Number", + "random": false, + "options": [ + 1 + ] + } + ] }, - { "weight": 2, "fields": [ - {"name": "rule_id", "type":"Number", "random": false,"options": [ - 4 - ]}, - {"name": "chart_id", "type":"Number", "random": false,"options": [ - 4 - ]}, - {"name": "template_id", "type":"Number", "random": false,"options": [ - 4 - ]}, - { "name": "server_ip", - "type": "IPv4", - "start": "1.0.0.0", - "end": "162.105.10.255"} - ] + { + "weight": 2, + "fields": [ + { + "name": "rule_id", + "type": "Number", + "random": false, + "options": [ + 4 + ] + }, + { + "name": "chart_id", + "type": "Number", + "random": false, + "options": [ + 4 + ] + }, + { + "name": "template_id", + "type": "Number", + "random": false, + "options": [ + 4 + ] + }, + { + "name": "server_ip", + "type": "IPv4", + "start": "1.0.0.0", + "end": "162.105.10.255" + } + ] }, - { "weight": 2, "fields": [ - {"name": "rule_id", "type":"Number", "random": false,"options": [ - 7 - ]}, - {"name": "chart_id", "type":"Number", "random": false,"options": [ - 7 - ]}, - {"name": "template_id", "type":"Number", "random": false,"options": [ - 7 - ]}, - { - "name": "application", - "type": "String", - "options": [ - "ntp", - "stun", - "unknown", - "teredo", - "qq_web_qq", - "kugou", - "quic" - ] - } - ] + { + "weight": 2, + "fields": [ + { + "name": "rule_id", + "type": "Number", + "random": false, + "options": [ + 7 + ] + }, + { + "name": "chart_id", + "type": "Number", + "random": false, + "options": [ + 7 + ] + }, + { + "name": "template_id", + "type": "Number", + "random": false, + "options": [ + 7 + ] + }, + { + "name": "application", + "type": "String", + "options": [ + "ntp", + "stun", + "unknown", + "teredo", + "qq_web_qq", + "kugou", + "quic" + ] + } + ] } ] - }, - { "name": "version", "type": "Number", "options": [1] }, - { - "name": "device_group", - "type": "String", - "options": [ - "OLAP-MOCK-DG-1", - "OLAP-MOCK-DG-2", - "OLAP-MOCK-DG-3", - "OLAP-MOCK-DG-4" - ] - },{ - "name": "data_center", - "type": "String", - "options": [ - "OLAP-MOCK-DC-1", - "OLAP-MOCK-DC-2", - "OLAP-MOCK-DC-3", - "OLAP-MOCK-DC-4" - ] - },{ - "name": "vsys_id", - "type": "Number", - "options": [ - 2048 - ] - } - ] }, - {"name": "fields", "type": "Object", "fields": [ - { "name": "unionFields", "type": "Union", "random": false, "unionFields": [ - { "weight": 2, "fields": [ - { "name": "client_ip_sketch", "type": "String", "options": [ - "AQwAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAACAAAAAAAAAAAAAAAACAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAwAAAIAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAA==" - ,"AQwAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ,"AQwAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ,"AQwAABAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ,"AQwAABAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] } - ] + }, + { + "name": "version", + "type": "Number", + "options": [ + 1 + ] + }, + { + "name": "device_group", + "type": "String", + "options": [ + "OLAP-MOCK-DG-1", + "OLAP-MOCK-DG-2", + "OLAP-MOCK-DG-3", + "OLAP-MOCK-DG-4" + ] + }, + { + "name": "data_center", + "type": "String", + "options": [ + "OLAP-MOCK-DC-1", + "OLAP-MOCK-DC-2", + "OLAP-MOCK-DC-3", + "OLAP-MOCK-DC-4" + ] + }, + { + "name": "vsys_id", + "type": "Number", + "options": [ + 2048 + ] + }, + { + "name": "unionFields", + "type": "Union", + "random": false, + "unionFields": [ + { + "weight": 2, + "fields": [ + { + "name": "client_ip_sketch", + "type": "String", + "options": [ + "AQwAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAACAAAAAAAAAAAAAAAACAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAwAAAIAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAA==", + "AQwAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "AQwAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "AQwAABAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "AQwAABAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + }, + { + "name": "unique_count_1", + "type": "Hlld", + "itemCount": 100000, + "batchCount": 1000 + }, + { + "name": "distribution_1", + "type": "HdrHistogram", + "max": 100000, + "batchCount": 1000 + } + ] }, - { "weight": 2, "fields": [ - { - "name": "bytes", - "type": "Number", - "min": 1, - "max": 15000 - } - ] + { + "weight": 2, + "fields": [ + { + "name": "bytes", + "type": "Number", + "min": 1, + "max": 15000 + } + ] }, - { "weight": 2, "fields": [ - { - "name": "sessions", - "type": "Number", - "min": 1, - "max": 200 - } - ] + { + "weight": 2, + "fields": [ + { + "name": "sessions", + "type": "Number", + "min": 1, + "max": 200 + } + ] } ] - } - ] } - + } ] \ No newline at end of file diff --git a/docs/connector/connector.md b/docs/connector/connector.md index 1123385..766b73e 100644 --- a/docs/connector/connector.md +++ b/docs/connector/connector.md @@ -85,41 +85,49 @@ schema: The mock data type is used to define the template of the mock data. -| Mock Type | Parameter | Result Type | Default | Description | -|-----------------------------------------|-------------|-----------------------|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| **[Number](#Number)** | - | **int/bigint/double** | - | **Randomly generate a number.** | -| - | min | number | 0 | The minimum value (include). | -| - | max | number | int32.max | The maximum value (exclusive). | -| - | options | array of number | (none) | The optional values. If set, the random value will be selected from the options and `start` and `end` will be ignored. | -| - | random | boolean | true | Default is random mode. If set to false, the value will be generated in order. | -| **[Sequence](#Sequence)** | - | **bigint** | - | **Generate a sequence number based on a specific step value .** | -| - | start | bigint | 0 | The first number in the sequence (include). | -| - | step | bigint | 1 | The number to add to each subsequent value. | -| **[UniqueSequence](#UniqueSequence)** | - | **bigint** | - | **Generate a global unique sequence number.** | -| - | start | bigint | 0 | The first number in the sequence (include). | -| **[String](#String)** | - | string | - | **Randomly generate a string.** | -| - | regex | string | [a-zA-Z]{0,5} | The regular expression. | -| - | options | array of string | (none) | The optional values. If set, the random value will be selected from the options and `regex` will be ignored. | -| - | random | boolean | true | Default is random mode. If set to false, the options value will be generated in order. | -| **[Timestamp](#Timestamp)** | - | **bigint** | - | **Generate a unix timestamp in milliseconds or seconds.** | -| - | unit | string | second | The unit of the timestamp. The optional values are `second`, `millis`. | -| **[FormatTimestamp](#FormatTimestamp)** | - | **string** | - | **Generate a formatted timestamp.** | -| - | format | string | yyyy-MM-dd HH:mm:ss | The format to output. | -| - | utc | boolean | false | Default is local time. If set to true, the time will be converted to UTC time. | -| **[IPv4](#IPv4)** | - | **string** | - | **Randomly generate a IPv4 address.** | -| - | start | string | 0.0.0.0 | The minimum value of the IPv4 address(include). | -| - | end | string | 255.255.255.255 | The maximum value of the IPv4 address(include). | -| **[Expression](#Expression)** | - | string | - | **Use library [Datafaker](https://www.datafaker.net/documentation/expressions/) expressions to generate fake data.** | -| - | expression | string | (none) | The datafaker expression used #{expression}. | -| **[Eval](#Eval)** | - | **string** | - | **Use AviatorScript value expression to generate data.** | -| - | expression | string | (none) | Support basic arithmetic operations and function calls. More details sess [AviatorScript](https://www.yuque.com/boyan-avfmj/aviatorscript). | -| **[Object](#Object)** | - | **struct/object** | - | **Generate a object data structure. It used to define the nested structure of the mock data.** | -| - | fields | array of object | (none) | The fields of the object. | -| **[Union](#Union)** | - | - | - | **Generate a union data structure with multiple mock data type fields.** | -| - | unionFields | array of object | (none) | The fields of the object. | -| - | - fields | - array of object | (none) | | -| - | - weight | - int | 0 | The weight of the generated object. | -| | random | boolean | true | Default is random mode. If set to false, the options value will be generated in order. | +| Mock Type | Parameter | Result Type | Default | Description | +|-----------------------------------------|---------------------------------|-----------------------|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **[Number](#Number)** | - | **int/bigint/double** | - | **Randomly generate a number.** | +| | min | number | 0 | The minimum value (inclusive). | +| | max | number | int32.max | The maximum value (exclusive). | +| | options | array of number | (none) | The optional values. If set, the random value will be selected from the options and `min` and `max` will be ignored. | +| | random | boolean | true | Default is random mode. If set to false, the value will be generated in order. | +| **[Sequence](#Sequence)** | - | **bigint** | - | **Generate a sequence number based on a specific step value.** | +| | start | bigint | 0 | The first number in the sequence (inclusive). | +| | step | bigint | 1 | The number to add to each subsequent value. | +| **[UniqueSequence](#UniqueSequence)** | - | **bigint** | - | **Generate a globally unique sequence number.** | +| | start | bigint | 0 | The first number in the sequence (inclusive). | +| **[String](#String)** | - | string | - | **Randomly generate a string.** | +| | regex | string | [a-zA-Z]{0,5} | The regular expression used to generate the string. | +| | options | array of string | (none) | The optional values. If set, the random value will be selected from the options and `regex` will be ignored. | +| | random | boolean | true | Default is random mode. If set to false, the options value will be generated in order. | +| **[Timestamp](#Timestamp)** | - | **bigint** | - | **Generate a Unix timestamp in milliseconds or seconds.** | +| | unit | string | second | The unit of the timestamp. Options are `second` or `millis`. | +| **[FormatTimestamp](#FormatTimestamp)** | - | **string** | - | **Generate a formatted timestamp.** | +| | format | string | yyyy-MM-dd HH:mm:ss | The format to output the timestamp in. | +| | utc | boolean | false | Default is local time. If set to true, the time will be converted to UTC time. | +| **[IPv4](#IPv4)** | - | **string** | - | **Randomly generate an IPv4 address.** | +| | start | string | 0.0.0.0 | The minimum value of the IPv4 address (inclusive). | +| | end | string | 255.255.255.255 | The maximum value of the IPv4 address (inclusive). | +| **[Expression](#Expression)** | - | string | - | **Use library [Datafaker](https://www.datafaker.net/documentation/expressions/) expressions to generate fake data.** | +| | expression | string | (none) | The Datafaker expression to use, in the format `#{expression}`. | +| **[Hlld](#HLLD)** | - | **string** | - | **Generate a IP Address HyperLogLog data structure and store it as a base64 string. Use library [HLLD](https://github.com/armon/hlld).** | +| | itemCount | bigint | 1000000 | The total number of items. | +| | batchCount | int | 10000 | The number of items in each batch. | +| | precision | int | 12 | The precision of the HyperLogLog data structure. Allowed range is [4, 18]. | +| **[HdrHistogram](#HdrHistogram)** | - | **string** | - | **Generate a Latency HdrHistogram data structure and store it as a base64 string. Use library [HdrHistogram](https://github.com/HdrHistogram/HdrHistogram).** | +| | max | bigint | 100000 | The maximum value of the histogram. | +| | batchCount | int | 1000 | The random number of items in each batch. | +| | numberOfSignificantValueDigits | int | 1 | The precision of the histogram data structure. Allowed range is [1, 5]. | +| **[Eval](#Eval)** | - | **string** | - | **Use AviatorScript value expression to generate data.** | +| | expression | string | (none) | Support basic arithmetic operations and function calls. More details in [AviatorScript](https://www.yuque.com/boyan-avfmj/aviatorscript). | +| **[Object](#Object)** | - | **struct/object** | - | **Generate an object data structure. Used to define the nested structure of the mock data.** | +| | fields | array of object | (none) | The fields of the object. | +| **[Union](#Union)** | - | - | - | **Generate a union data structure with multiple mock data type fields.** | +| | unionFields | array of object | (none) | The fields of the union. | +| | weight | int | 0 | The weight of the generated object. | +| | random | boolean | true | Default is random mode. If set to false, the options value will be generated in order. | + ### Common Parameters @@ -250,6 +258,22 @@ Mock data type supports some common parameters. {"name":"phoneNumber","type":"Expression","expression":"#{phoneNumber.phoneNumber}"} ``` +### HLLD + +- Generate a IP Address HyperLogLog data structure, stored as a base64 string. At most 1000 IP addresses are generated in each batch. + +```json +{"name":"hll","type":"Hlld","itemCount":1000000,"batchCount":1000,"precision":12} +``` + +### HdrHistogram + +- Generate a Latency HdrHistogram data structure, stored as a base64 string. The maximum value of the histogram is 100000, and at most 1000 items are generated in each batch. + +```json +{"name":"distribution","type":"HdrHistogram","max":100000,"batchCount":1000,"numberOfSignificantValueDigits":1} +``` + ### Eval - Generate a value by using AviatorScript expression. Commonly used for arithmetic operations. diff --git a/docs/images/groot_stream_architecture.jpg b/docs/images/groot_stream_architecture.jpg index 1fff0e5..d8f1d4b 100644 Binary files a/docs/images/groot_stream_architecture.jpg and b/docs/images/groot_stream_architecture.jpg differ diff --git a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java index f666ee8..c637e36 100644 --- a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java +++ b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java @@ -13,12 +13,12 @@ import java.nio.file.Paths; public class GrootStreamExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException { - String configPath = args.length > 0 ? args[0] : "/examples/inline_to_kafka.yaml"; + String configPath = args.length > 0 ? args[0] : "/examples/object_statistics_mock_to_print.yaml"; String configFile = getTestConfigFile(configPath); ExecuteCommandArgs executeCommandArgs = new ExecuteCommandArgs(); executeCommandArgs.setConfigFile(configFile); executeCommandArgs.setCheckConfig(false); - executeCommandArgs.setEncrypt(true); + executeCommandArgs.setEncrypt(false); executeCommandArgs.setDecrypt(false); executeCommandArgs.setVersion(false); executeCommandArgs.setDeployMode(DeployMode.RUN); -- cgit v1.2.3 From a1f2fd8385f418af0a55867e12747967e7b836f9 Mon Sep 17 00:00:00 2001 From: doufenghu Date: Mon, 19 Aug 2024 18:54:33 +0800 Subject: [docs][table processor] add table-processor and udtf description. --- config/udf.plugins | 6 +- docs/processor/aggregate-processor.md | 2 +- docs/processor/table-processor.md | 61 +++++++ docs/processor/udaf.md | 180 ++++++++++++++++++++- docs/processor/udtf.md | 66 ++++++++ groot-common/src/main/resources/udf.plugins | 6 +- .../geedgenetworks/example/GrootStreamExample.java | 2 +- .../examples/inline_to_print_use_udtf.yaml | 36 +++++ 8 files changed, 349 insertions(+), 10 deletions(-) create mode 100644 docs/processor/table-processor.md create mode 100644 docs/processor/udtf.md create mode 100644 groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_use_udtf.yaml diff --git a/config/udf.plugins b/config/udf.plugins index 772d2bc..e4f940f 100644 --- a/config/udf.plugins +++ b/config/udf.plugins @@ -22,10 +22,10 @@ com.geedgenetworks.core.udf.udaf.LongCount com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue com.geedgenetworks.core.udf.udaf.FirstValue -com.geedgenetworks.core.udf.udtf.JsonUnroll -com.geedgenetworks.core.udf.udtf.Unroll com.geedgenetworks.core.udf.udaf.hlld.Hlld com.geedgenetworks.core.udf.udaf.hlld.HlldApproxCountDistinct com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogram com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantile -com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantiles \ No newline at end of file +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantiles +com.geedgenetworks.core.udf.udtf.JsonUnroll +com.geedgenetworks.core.udf.udtf.Unroll \ No newline at end of file diff --git a/docs/processor/aggregate-processor.md b/docs/processor/aggregate-processor.md index af82d4e..5ab0ae0 100644 --- a/docs/processor/aggregate-processor.md +++ b/docs/processor/aggregate-processor.md @@ -12,7 +12,7 @@ Note：Default will output internal fields `__window_start_timestamp` and `__win | name | type | required | default value | |--------------------------|--------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| type | String | Yes | The type of the processor, now only support `com.geedgenetworks.core.processor.projection.AggregateProcessor` | +| type | String | Yes | The type of the processor, now only support `com.geedgenetworks.core.processor.aggregate.AggregateProcessor` | | output_fields | Array | No | Array of String. The list of fields that need to be kept. Fields not in the list will be removed. | | remove_fields | Array | No | Array of String. The list of fields that need to be removed. | | group_by_fields | Array | yes | Array of String. The list of fields that need to be grouped. | diff --git a/docs/processor/table-processor.md b/docs/processor/table-processor.md new file mode 100644 index 0000000..7b3066c --- /dev/null +++ b/docs/processor/table-processor.md @@ -0,0 +1,61 @@ +# Table Processor + +> Processing pipelines for table processors using UDTFs + +## Description + +Table processor is used to process the data from source to sink. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDTFs(User-defined Table functions) into a pipeline. Within the pipeline, events are processed by each Function in order, top‑>down. More details can be found in user-defined table functions [(UDTFs)](udtf.md). + +## Options + +| name | type | required | default value | +|-----------------|--------|----------|------------------------------------------------------------------------------------------------------| +| type | String | Yes | The type of the processor, now only support `com.geedgenetworks.core.processor.table.TableProcessor` | +| output_fields | Array | No | Array of String. The list of fields that ne ed to be kept. Fields not in the list will be removed. | +| remove_fields | Array | No | Array of String. The list of fields that need to be removed. | +| functions | Array | No | Array of Object. The list of functions that need to be applied to the data. | + +## Usage Example +This example uses a table processor to unroll the encapsulation field, converting one row into multiple rows. + +```yaml +sources: + inline_source: + type: inline + properties: + data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931,"encapsulation":"[{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"c2s_source_mac\":\"48:73:97:96:38:27\",\"c2s_destination_mac\":\"58:b3:8f:fa:3b:11\",\"s2c_source_mac\":\"58:b3:8f:fa:3b:11\",\"s2c_destination_mac\":\"48:73:97:96:38:27\"}]"},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.168.10.198","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931,"device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"center-xxg-tsgx\"},{\"tag\":\"device_group\",\"value\":\"group-xxg-tsgx\"}]}"}]' + format: json + json.ignore.parse.errors: false + +processing_pipelines: + table_processor: + type: table + functions: + - function: JSON_UNROLL + lookup_fields: [ encapsulation] + output_fields: [ encapsulation ] + +sinks: + print_sink: + type: print + properties: + format: json + mode: log_warn + +application: + env: + name: example-inline-to-print-use-udtf + parallelism: 3 + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [table_processor] + - name: table_processor + downstream: [ print_sink ] + - name: print_sink + downstream: [] + +``` + + diff --git a/docs/processor/udaf.md b/docs/processor/udaf.md index e22846f..dd1dd70 100644 --- a/docs/processor/udaf.md +++ b/docs/processor/udaf.md @@ -11,7 +11,11 @@ - [Long Count](#Long-Count) - [MEAN](#Mean) - [Number SUM](#Number-SUM) - +- [HLLD](#HLLD) +- [Approx Count Distinct HLLD](#Approx-Count-Distinct-HLLD) +- [HDR Histogram](#HDR-Histogram) +- [Approx Quantile HDR](#APPROX_QUANTILE_HDR) +- [Approx Quantiles HDR](#APPROX_QUANTILES_HDR) ## Description @@ -146,4 +150,176 @@ NUMBER_SUM is used to sum the value of the field in the group of events. The loo - function: NUMBER_SUM lookup_fields: [received_bytes] output_fields: [received_bytes_sum] -``` \ No newline at end of file +``` + +### HLLD +hlld is a high-performance C server which is used to expose HyperLogLog sets and operations over them to networked clients. More details can be found in [hlld](https://github.com/armon/hlld). + +```HLLD(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `` optional. input field type can be `regular` or `sketch`. Default is `sketch`. regular field data type includes `string`, `int`, `long`, `float`, `double` etc. + - precision: `` optional. The precision of the hlld value. Default is 12. + - output_format: `` optional. The output format can be either `base64(encoded string)` or `binary(byte[])`. The default is `base64`. + +### Example + Merge multiple string field into a HyperLogLog data structure. +```yaml + - function: HLLD + lookup_fields: [client_ip] + output_fields: [client_ip_hlld] + parameters: + input_type: regular + +``` + Merge multiple `unique_count ` metric type fields into a HyperLogLog data structure +```yaml + - function: HLLD + lookup_fields: [client_ip_hlld] + output_fields: [client_ip_hlld] + parameters: + input_type: sketch +``` + +### Approx Count Distinct HLLD +Approx Count Distinct HLLD is used to count the approximate number of distinct values in the group of events. + +```APPROX_COUNT_DISTINCT_HLLD(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `` optional. Refer to `HLLD` function. + - precision: `` optional. Refer to `HLLD` function. + +### Example + +```yaml +- function: APPROX_COUNT_DISTINCT_HLLD + lookup_fields: [client_ip] + output_fields: [unique_client_ip] + parameters: + input_type: regular +``` + +```yaml +- function: APPROX_COUNT_DISTINCT_HLLD + lookup_fields: [client_ip_hlld] + output_fields: [unique_client_ip] + parameters: + input_type: sketch +``` + +### HDR Histogram + +A High Dynamic Range (HDR) Histogram. More details can be found in [HDR Histogram](https://github.com/HdrHistogram/HdrHistogram). + +```HDR_HISTOGRAM(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `` optional. input field type can be `regular` or `sketch`. Default is `sketch`. regular field is a number. + - lowestDiscernibleValue: `` optional. The lowest trackable value. Default is 1. + - highestTrackableValue: `` optional. The highest trackable value. Default is 2. + - numberOfSignificantValueDigits: `` optional. The number of significant value digits. Default is 1. The range is 1 to 5. + - autoResize: `` optional. If true, the highestTrackableValue will auto-resize. Default is true. + - output_format: `` optional. The output format can be either `base64(encoded string)` or `binary(byte[])`. The default is `base64`. + +### Example + + ```yaml + - function: HDR_HISTOGRAM + lookup_fields: [latency_ms] + output_fields: [latency_ms_histogram] + parameters: + input_type: regular + lowestDiscernibleValue: 1 + highestTrackableValue: 3600000 + numberOfSignificantValueDigits: 3 + ``` + ```yaml + - function: HDR_HISTOGRAM + lookup_fields: [latency_ms_histogram] + output_fields: [latency_ms_histogram] + parameters: + input_type: sketch + ``` + +### Approx Quantile HDR + +Approx Quantile HDR is used to calculate the approximate quantile value of the field in the group of events. + +```APPROX_QUANTILE_HDR(filter, lookup_fields, output_fields, quantile[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `` optional. Refer to `HDR_HISTOGRAM` function. + - lowestDiscernibleValue: `` optional. Refer to `HDR_HISTOGRAM` function. + - highestTrackableValue: `` required. Refer to `HDR_HISTOGRAM` function. + - numberOfSignificantValueDigits: `` optional. Refer to `HDR_HISTOGRAM` function. + - autoResize: `` optional. Refer to `HDR_HISTOGRAM` function. + - probability: `` optional. The probability of the quantile. Default is 0.5. + +### Example + + ```yaml + - function: APPROX_QUANTILE_HDR + lookup_fields: [latency_ms] + output_fields: [latency_ms_p95] + parameters: + input_type: regular + probability: 0.95 + ``` + + ```yaml + - function: APPROX_QUANTILE_HDR + lookup_fields: [latency_ms_HDR] + output_fields: [latency_ms_p95] + parameters: + input_type: sketch + probability: 0.95 + + ``` + +### Approx Quantiles HDR + +Approx Quantiles HDR is used to calculate the approximate quantile values of the field in the group of events. + +```APPROX_QUANTILES_HDR(filter, lookup_fields, output_fields, quantiles[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `` optional. Refer to `HDR_HISTOGRAM` function. + - lowestDiscernibleValue: `` optional. Refer to `HDR_HISTOGRAM` function. + - highestTrackableValue: `` required. Refer to `HDR_HISTOGRAM` function. + - numberOfSignificantValueDigits: `` optional. Refer to `HDR_HISTOGRAM` function. + - autoResize: `` optional. Refer to `HDR_HISTOGRAM` function. + - probabilities: `>` required. The list of probabilities of the quantiles. Range is 0 to 1. + +### Example + +```yaml +- function: APPROX_QUANTILES_HDR + lookup_fields: [latency_ms] + output_fields: [latency_ms_quantiles] + parameters: + input_type: regular + probabilities: [0.5, 0.95, 0.99] +``` + +```yaml +- function: APPROX_QUANTILES_HDR + lookup_fields: [latency_ms_HDR] + output_fields: [latency_ms_quantiles] + parameters: + input_type: sketch + probabilities: [0.5, 0.95, 0.99] +``` + + + diff --git a/docs/processor/udtf.md b/docs/processor/udtf.md new file mode 100644 index 0000000..a6e8444 --- /dev/null +++ b/docs/processor/udtf.md @@ -0,0 +1,66 @@ +# UDTF + +> The functions for table processors. + +## Function of content + +- [UNROLL](#unroll) +- [JSON_UNROLL](#json_unroll) + +## Description + +The UDTFs(user-defined table functions) are used to process the data from source to sink. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDTFs into a pipeline. Within the pipeline, events are processed by each Function in order, top‑>down. +Unlike scalar functions, which return a single value, UDTFs are particularly useful when you need to explode or unroll data, transforming a single input row into multiple output rows. + +## UDTF Definition + + The UDTFs and UDFs share similar input and context structures, please refer to [UDF](udf.md). + +## Functions + +### UNROLL + +The Unroll Function handles an array field—or an expression evaluating to an array—and unrolls it into individual events. + +```UNROLL(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required +- output_fields: required +- parameters: optional + - regex: `` optional. If lookup_fields is a string, the regex parameter is used to split the string into an array. The default value is a comma. + +#### Example + +```yaml +functions: + - function: UNROLL + lookup_fields: [ monitor_rule_list ] + output_fields: [ monitor_rule ] +``` + +### JSON_UNROLL + +The JSON Unroll Function handles a JSON object, unrolls/explodes an array of objects therein into individual events, while also inheriting top level fields. + +```JSON_UNROLL(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required +- output_fields: required +- parameters: optional + - path: `` optional. Path to array to unroll, default is the root of the JSON object. + - new_path: `` optional. Rename path to new_path, default is the same as path. + +#### Example + +```yaml +functions: + - function: JSON_UNROLL + lookup_fields: [ device_tag ] + output_fields: [ device_tag ] + parameters: + - path: tags + - new_path: tag +``` + + + diff --git a/groot-common/src/main/resources/udf.plugins b/groot-common/src/main/resources/udf.plugins index 0062c1a..7544cc7 100644 --- a/groot-common/src/main/resources/udf.plugins +++ b/groot-common/src/main/resources/udf.plugins @@ -21,10 +21,10 @@ com.geedgenetworks.core.udf.udaf.LongCount com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue com.geedgenetworks.core.udf.udaf.FirstValue -com.geedgenetworks.core.udf.udtf.JsonUnroll -com.geedgenetworks.core.udf.udtf.Unroll com.geedgenetworks.core.udf.udaf.hlld.Hlld com.geedgenetworks.core.udf.udaf.hlld.HlldApproxCountDistinct com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogram com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantile -com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantiles \ No newline at end of file +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantiles +com.geedgenetworks.core.udf.udtf.JsonUnroll +com.geedgenetworks.core.udf.udtf.Unroll \ No newline at end of file diff --git a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java index c637e36..d6f736f 100644 --- a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java +++ b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java @@ -13,7 +13,7 @@ import java.nio.file.Paths; public class GrootStreamExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException { - String configPath = args.length > 0 ? args[0] : "/examples/object_statistics_mock_to_print.yaml"; + String configPath = args.length > 0 ? args[0] : "/examples/inline_to_print_use_udtf.yaml"; String configFile = getTestConfigFile(configPath); ExecuteCommandArgs executeCommandArgs = new ExecuteCommandArgs(); executeCommandArgs.setConfigFile(configFile); diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_use_udtf.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_use_udtf.yaml new file mode 100644 index 0000000..edde893 --- /dev/null +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_use_udtf.yaml @@ -0,0 +1,36 @@ +sources: + inline_source: + type: inline + properties: + data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931,"encapsulation":"[{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"c2s_source_mac\":\"48:73:97:96:38:27\",\"c2s_destination_mac\":\"58:b3:8f:fa:3b:11\",\"s2c_source_mac\":\"58:b3:8f:fa:3b:11\",\"s2c_destination_mac\":\"48:73:97:96:38:27\"}]"},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.168.10.198","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931,"device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"center-xxg-tsgx\"},{\"tag\":\"device_group\",\"value\":\"group-xxg-tsgx\"}]}"}]' + format: json + json.ignore.parse.errors: false + +processing_pipelines: + table_processor: + type: table + functions: + - function: JSON_UNROLL + lookup_fields: [ encapsulation] + output_fields: [ encapsulation ] + +sinks: + print_sink: + type: print + properties: + format: json + mode: log_warn + +application: + env: + name: example-inline-to-print-with-aggregation + parallelism: 3 + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [table_processor] + - name: table_processor + downstream: [ print_sink ] + - name: print_sink + downstream: [] \ No newline at end of file -- cgit v1.2.3 From 6564a5e9a43ecd88f5497e2b75a219a8a54101bb Mon Sep 17 00:00:00 2001 From: doufenghu Date: Mon, 19 Aug 2024 19:03:59 +0800 Subject: add APPROX_COUNT_DISTINCT_HLLD test case --- .../main/java/com/geedgenetworks/example/GrootStreamExample.java | 2 +- .../main/resources/examples/inline_to_print_with_aggregation.yaml | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java index d6f736f..0eba408 100644 --- a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java +++ b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java @@ -13,7 +13,7 @@ import java.nio.file.Paths; public class GrootStreamExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException { - String configPath = args.length > 0 ? args[0] : "/examples/inline_to_print_use_udtf.yaml"; + String configPath = args.length > 0 ? args[0] : "/examples/inline_to_print_with_aggregation.yaml"; String configFile = getTestConfigFile(configPath); ExecuteCommandArgs executeCommandArgs = new ExecuteCommandArgs(); executeCommandArgs.setConfigFile(configFile); diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml index 6f08be2..04dddf8 100644 --- a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml @@ -2,7 +2,7 @@ sources: inline_source: type: inline properties: - data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.168.10.198","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931}]' + data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931}]' format: json json.ignore.parse.errors: false @@ -18,6 +18,12 @@ processing_pipelines: - function: NUMBER_SUM lookup_fields: [ received_bytes] output_fields: [ received_bytes_sum ] + - function: APPROX_COUNT_DISTINCT_HLLD + lookup_fields: [ server_ip ] + output_fields: [ server_ip_count ] + parameters: + input_type: regular + sinks: print_sink: -- cgit v1.2.3 From 7bc23562099556f8fa28a45b2481d13cc6afdba8 Mon Sep 17 00:00:00 2001 From: doufenghu Date: Tue, 27 Aug 2024 20:13:59 +0800 Subject: release 1.5.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 53bcf1f..8378a72 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ - 1.5.0-SNAPSHOT + 1.5.0 11 UTF-8 ${java.version} -- cgit v1.2.3