diff options
326 files changed, 1393 insertions, 2054 deletions
diff --git a/config/template/grootstream_job_template.yaml b/config/template/grootstream_job_template.yaml index b26fbb2..c4aa726 100644 --- a/config/template/grootstream_job_template.yaml +++ b/config/template/grootstream_job_template.yaml @@ -151,7 +151,7 @@ preprocessing_pipelines: # [object] Define Processors for preprocessing pipeline # It will be accomplished the common processing for the event by the user-defined functions. # processing_pipelines: # [object] Define Processors for processing pipelines. - z: # [object] Define projection processor name, must be unique. + projection_processor: # [object] Define projection processor name, must be unique. type: projection # [string] Processor Type remove_fields: output_fields: diff --git a/docs/connector/connector.md b/docs/connector/connector.md index ff495d4..c254d53 100644 --- a/docs/connector/connector.md +++ b/docs/connector/connector.md @@ -70,7 +70,7 @@ schema: To retrieve the schema from a local file using its absolute path. -> Ensures that the file path is accessible to all operatorNodes in your Flink cluster. +> Ensures that the file path is accessible to all jobTopologyNodes in your Flink cluster. ```yaml schema: diff --git a/docs/connector/formats/protobuf.md b/docs/connector/formats/protobuf.md index e991fca..467177a 100644 --- a/docs/connector/formats/protobuf.md +++ b/docs/connector/formats/protobuf.md @@ -13,7 +13,7 @@ ## Format Options -> Ensures that the file path is accessible to all operatorNodes in your Flink cluster. +> Ensures that the file path is accessible to all jobTopologyNodes in your Flink cluster. | Name | Type | Required | Default | Description | |-------------------------------|---------|----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| diff --git a/docs/connector/source/file.md b/docs/connector/source/file.md index 75de66b..98c6aee 100644 --- a/docs/connector/source/file.md +++ b/docs/connector/source/file.md @@ -24,7 +24,7 @@ File source custom properties. This example read data of file test source and print to console. -> Ensures that the file path is accessible to all operatorNodes in your Flink cluster. +> Ensures that the file path is accessible to all jobTopologyNodes in your Flink cluster. ```yaml sources: diff --git a/docs/filter/aviator.md b/docs/filter/aviator.md index e7f6c2b..acf98a5 100644 --- a/docs/filter/aviator.md +++ b/docs/filter/aviator.md @@ -11,7 +11,7 @@ | Name | Type | Required | Default | Description | |-------------|--------|----------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| type | String | Yes | (none) | The type of the filter operator. Now only support `com.geedgenetworks.core.filter.AviatorFilter`. | +| type | String | Yes | (none) | The type of the filter operator. Now only support `com.geedgenetworks.core.filter.AviatorFilterProcessor`. | | properties | Map | Yes | (none) | Filter operator properties. | | -expression | String | Yes | (none) | Based on the filter expression, the event will be passed to downstream if the expression is true, otherwise it will be dropped. Build a filter expression need add prefix `event.`, if you want get event field. | diff --git a/docs/grootstream-config.md b/docs/grootstream-config.md index c5e3046..ad54603 100644 --- a/docs/grootstream-config.md +++ b/docs/grootstream-config.md @@ -24,7 +24,7 @@ grootstream: The knowledge base is a collection of libraries that can be used in the groot-stream job's UDFs. File system type can be specified `local`, `http` or `hdfs`. If the value is `http`, must be ` QGW Knowledge Base Repository` URL. The library will be dynamically updated according to the `scheduler.knowledge_base.update.interval.minutes` configuration. -If the value is `local`, the library will be loaded from the local file system. Need to manually upgrade all operatorNodes in the Flink cluster when the library is updated. +If the value is `local`, the library will be loaded from the local file system. Need to manually upgrade all jobTopologyNodes in the Flink cluster when the library is updated. If the value is `hdfs`, the library will be loaded from the HDFS file system. More details about hdfs operation can be found in the [HDFS](./faq.md#hadoop-hdfs-commands-for-beginners). | Name | Type | Required | Default | Description | @@ -36,7 +36,7 @@ If the value is `hdfs`, the library will be loaded from the HDFS file system. Mo ### Define the knowledge base file from a local file -> Ensures that the file path is accessible to all operatorNodes in your Flink cluster. +> Ensures that the file path is accessible to all jobTopologyNodes in your Flink cluster. ```yaml grootstream: @@ -65,7 +65,7 @@ grootstream: ### Define the knowledge base file from a HDFS file system -> Ensure that the HDFS file system is accessible to all operatorNodes in your Flink cluster. +> Ensure that the HDFS file system is accessible to all jobTopologyNodes in your Flink cluster. ```yaml grootstream: diff --git a/docs/processor/projection-processor.md b/docs/processor/projection-processor.md index d11fcc9..62258de 100644 --- a/docs/processor/projection-processor.md +++ b/docs/processor/projection-processor.md @@ -32,7 +32,7 @@ sources: filters: filter_operator: - type: com.geedgenetworks.core.filter.AviatorFilter + type: com.geedgenetworks.core.filter.AviatorFilterProcessor properties: expression: event.server_ip != '12.12.12.12' diff --git a/docs/processor/split-processor.md b/docs/processor/split-processor.md index e1a1163..05d3d92 100644 --- a/docs/processor/split-processor.md +++ b/docs/processor/split-processor.md @@ -10,7 +10,7 @@ Using the flink side Outputs send data from a stream to multiple downstream cons | name | type | required | default value | |-------------------|--------|----------|--------------------------------------------------------------------------------------------| -| type | String | Yes | The type of the processor, now only support ` com.geedgenetworks.core.split.SplitOperator` | +| type | String | Yes | The type of the processor, now only support ` com.geedgenetworks.core.split.SplitProcessor` | | rules | Array | Yes | Array of Object. Defining rules for labeling Side Output Tag | | [rule.]tag | String | Yes | The tag name of the side output | | [rule.]expression | String | Yes | The expression to evaluate the event. | diff --git a/docs/user-guide.md b/docs/user-guide.md index 1db9f91..764628a 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -32,7 +32,7 @@ sources: filters: filter: - type: com.geedgenetworks.core.filter.AviatorFilter + type: com.geedgenetworks.core.filter.AviatorFilterProcessor properties: expression: event.decoded_as == 'BASE' diff --git a/groot-spi/pom.xml b/groot-api/pom.xml index e27a9db..1588f11 100644 --- a/groot-spi/pom.xml +++ b/groot-api/pom.xml @@ -8,8 +8,8 @@ <version>${revision}</version> </parent> - <artifactId>groot-spi</artifactId> - <name>Groot : SPI </name> + <artifactId>groot-api</artifactId> + <name>Groot : API </name> <dependencies> diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/AggregateFunction.java b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/AggregateFunction.java index 9d166f7..e846be1 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/AggregateFunction.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/AggregateFunction.java @@ -1,7 +1,7 @@ -package com.geedgenetworks.spi.common.udf; +package com.geedgenetworks.api.common.udf; import com.geedgenetworks.common.config.Accumulator; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import java.io.Serializable; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/ScalarFunction.java b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/ScalarFunction.java index 35ed3d3..17e299d 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/ScalarFunction.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/ScalarFunction.java @@ -1,10 +1,10 @@ -package com.geedgenetworks.spi.common.udf; +package com.geedgenetworks.api.common.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.configuration.CheckUDFContextUtil; -import com.geedgenetworks.spi.configuration.UDFContextConfigOptions; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.configuration.CheckUDFContextUtil; +import com.geedgenetworks.api.configuration.UDFContextConfigOptions; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import java.io.Serializable; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/TableFunction.java b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/TableFunction.java index a3cd034..8b8a008 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/TableFunction.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/TableFunction.java @@ -1,6 +1,6 @@ -package com.geedgenetworks.spi.common.udf; +package com.geedgenetworks.api.common.udf; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import java.io.Serializable; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/UDFContext.java b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/UDFContext.java index 8c91745..c595212 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/UDFContext.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/UDFContext.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.common.udf; +package com.geedgenetworks.api.common.udf; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/UdfEntity.java b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/UdfEntity.java index eee3051..d8434d6 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/UdfEntity.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/common/udf/UdfEntity.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.common.udf; +package com.geedgenetworks.api.common.udf; import com.googlecode.aviator.Expression; import lombok.Data; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/CheckUDFContextUtil.java b/groot-api/src/main/java/com/geedgenetworks/api/configuration/CheckUDFContextUtil.java index 6fc6607..3d6e53b 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/CheckUDFContextUtil.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/configuration/CheckUDFContextUtil.java @@ -1,7 +1,7 @@ -package com.geedgenetworks.spi.configuration; +package com.geedgenetworks.api.configuration; import com.geedgenetworks.common.config.CheckResult; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; import java.util.Arrays; import java.util.List; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/UDFContextConfigOptions.java b/groot-api/src/main/java/com/geedgenetworks/api/configuration/UDFContextConfigOptions.java index ee120d3..021d198 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/UDFContextConfigOptions.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/configuration/UDFContextConfigOptions.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.configuration; +package com.geedgenetworks.api.configuration; import com.alibaba.fastjson2.TypeReference; import com.geedgenetworks.common.config.Option; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/util/LoadIntervalDataOptions.java b/groot-api/src/main/java/com/geedgenetworks/api/configuration/util/LoadIntervalDataOptions.java index b73a5d2..688fcc0 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/util/LoadIntervalDataOptions.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/configuration/util/LoadIntervalDataOptions.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.configuration.util; +package com.geedgenetworks.api.configuration.util; import java.io.Serializable; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/util/LoadIntervalDataUtil.java b/groot-api/src/main/java/com/geedgenetworks/api/configuration/util/LoadIntervalDataUtil.java index c9ea475..0c92a39 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/util/LoadIntervalDataUtil.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/configuration/util/LoadIntervalDataUtil.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.configuration.util; +package com.geedgenetworks.api.configuration.util; import org.apache.flink.shaded.guava18.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.flink.util.function.SupplierWithException; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/event/Event.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/event/Event.java index 3a1e851..2b04140 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/event/Event.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/event/Event.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.event; +package com.geedgenetworks.api.connector.event; import lombok.Data; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/DynamicSchema.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/DynamicSchema.java index 1bc799e..182218b 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/DynamicSchema.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/DynamicSchema.java @@ -1,7 +1,7 @@ -package com.geedgenetworks.spi.table.schema; +package com.geedgenetworks.api.connector.schema; -import com.geedgenetworks.spi.table.schema.utils.DynamicSchemaManager; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.schema.utils.DynamicSchemaManager; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/HttpDynamicSchema.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/HttpDynamicSchema.java index 4bfd914..bb8069b 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/HttpDynamicSchema.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/HttpDynamicSchema.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.schema; +package com.geedgenetworks.api.connector.schema; import com.geedgenetworks.common.utils.HttpClientPoolUtil; import com.geedgenetworks.shaded.org.apache.http.Header; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/Schema.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/Schema.java index c05e2b1..7be573f 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/Schema.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/Schema.java @@ -1,7 +1,7 @@ -package com.geedgenetworks.spi.table.schema; +package com.geedgenetworks.api.connector.schema; -import com.geedgenetworks.spi.table.schema.utils.DynamicSchemaManager; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.schema.utils.DynamicSchemaManager; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.util.Preconditions; import java.io.Serializable; diff --git a/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/SchemaChangeAware.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/SchemaChangeAware.java new file mode 100644 index 0000000..e9485c0 --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/SchemaChangeAware.java @@ -0,0 +1,7 @@ +package com.geedgenetworks.api.connector.schema; + +import com.geedgenetworks.api.connector.type.StructType; + +public interface SchemaChangeAware { + void schemaChange(StructType dataType); +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/SchemaParser.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/SchemaParser.java index 1a53556..ff43cd3 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/SchemaParser.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/SchemaParser.java @@ -1,13 +1,13 @@ -package com.geedgenetworks.spi.table.schema; +package com.geedgenetworks.api.connector.schema; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; -import com.geedgenetworks.spi.table.type.ArrayType; -import com.geedgenetworks.spi.table.type.DataType; -import com.geedgenetworks.spi.table.type.Types; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.StructType.StructField; +import com.geedgenetworks.api.connector.type.ArrayType; +import com.geedgenetworks.api.connector.type.DataType; +import com.geedgenetworks.api.connector.type.Types; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.StructType.StructField; import java.io.Serializable; import java.util.*; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/StaticSchema.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/StaticSchema.java index 1c485e8..b6c741d 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/StaticSchema.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/StaticSchema.java @@ -1,7 +1,7 @@ -package com.geedgenetworks.spi.table.schema; +package com.geedgenetworks.api.connector.schema; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.type.StructType; public class StaticSchema implements Schema{ private final StructType dataType; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/utils/DynamicSchemaManager.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/utils/DynamicSchemaManager.java index 37a1769..41dc445 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/utils/DynamicSchemaManager.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/schema/utils/DynamicSchemaManager.java @@ -1,9 +1,9 @@ -package com.geedgenetworks.spi.table.schema.utils; +package com.geedgenetworks.api.connector.schema.utils; -import com.geedgenetworks.spi.table.schema.DynamicSchema; -import com.geedgenetworks.spi.table.schema.SchemaChangeAware; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.schema.DynamicSchema; +import com.geedgenetworks.api.connector.schema.SchemaChangeAware; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.runtime.util.ExecutorThreadFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/connector/DecodingFormat.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/serialization/DecodingFormat.java index 6bed4f9..95514ef 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/connector/DecodingFormat.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/serialization/DecodingFormat.java @@ -1,7 +1,7 @@ -package com.geedgenetworks.spi.table.connector; +package com.geedgenetworks.api.connector.serialization; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.DeserializationSchema; public interface DecodingFormat { diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/connector/EncodingFormat.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/serialization/EncodingFormat.java index 6047711..c8f9ce5 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/connector/EncodingFormat.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/serialization/EncodingFormat.java @@ -1,7 +1,7 @@ -package com.geedgenetworks.spi.table.connector; +package com.geedgenetworks.api.connector.serialization; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.SerializationSchema; public interface EncodingFormat { diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/connector/MapDeserialization.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/serialization/MapDeserialization.java index 1c8cc42..b3e3a13 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/connector/MapDeserialization.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/serialization/MapDeserialization.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.connector; +package com.geedgenetworks.api.connector.serialization; import java.io.IOException; import java.util.Map; diff --git a/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkConfig.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkConfig.java new file mode 100644 index 0000000..1c9c27d --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkConfig.java @@ -0,0 +1,14 @@ +package com.geedgenetworks.api.connector.sink; + +import lombok.Data; + +import java.io.Serializable; +import java.util.Map; +@Data +public class SinkConfig implements Serializable { + private String name; + private String type; + private Map<String, Object> schema; + private Map<String, String> properties; + +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkConfigOptions.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkConfigOptions.java index 6e407e6..12011aa 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkConfigOptions.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkConfigOptions.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.sink; +package com.geedgenetworks.api.connector.sink; import com.geedgenetworks.common.config.Option; import com.geedgenetworks.common.config.Options; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkProvider.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkProvider.java index 0c2f425..19c8fe4 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkProvider.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkProvider.java @@ -1,6 +1,6 @@ -package com.geedgenetworks.spi.sink; +package com.geedgenetworks.api.connector.sink; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; diff --git a/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkTableFactory.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkTableFactory.java new file mode 100644 index 0000000..ae5b390 --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/sink/SinkTableFactory.java @@ -0,0 +1,8 @@ +package com.geedgenetworks.api.connector.sink; + + +import com.geedgenetworks.api.factory.ConnectorFactory; + +public interface SinkTableFactory extends ConnectorFactory { + SinkProvider getSinkProvider(Context context); +} diff --git a/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceConfig.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceConfig.java new file mode 100644 index 0000000..d7f7393 --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceConfig.java @@ -0,0 +1,16 @@ +package com.geedgenetworks.api.connector.source; + +import lombok.Data; + +import java.io.Serializable; +import java.util.Map; +@Data +public class SourceConfig implements Serializable { + private String name; + private String type; + private Map<String, Object> schema; + private String watermark_timestamp; + private String watermark_timestamp_unit = "ms"; + private Long watermark_lag; + private Map<String, String> properties; +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/SourceConfigOptions.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceConfigOptions.java index 53cb309..cec53ce 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/SourceConfigOptions.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceConfigOptions.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.configuration; +package com.geedgenetworks.api.connector.source; import com.alibaba.fastjson2.TypeReference; import com.geedgenetworks.common.config.Option; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/source/SourceProvider.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceProvider.java index f7cf68f..37a2d49 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/source/SourceProvider.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceProvider.java @@ -1,7 +1,7 @@ -package com.geedgenetworks.spi.source; +package com.geedgenetworks.api.connector.source; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; diff --git a/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceTableFactory.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceTableFactory.java new file mode 100644 index 0000000..404fdd5 --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/source/SourceTableFactory.java @@ -0,0 +1,7 @@ +package com.geedgenetworks.api.connector.source; + +import com.geedgenetworks.api.factory.ConnectorFactory; + +public interface SourceTableFactory extends ConnectorFactory { + SourceProvider getSourceProvider(Context context); +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/ArrayType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/ArrayType.java index 4f3d663..e8fae36 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/ArrayType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/ArrayType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; public class ArrayType extends DataType { public DataType elementType; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/BinaryType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/BinaryType.java index 8b8b996..3d3b5f0 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/BinaryType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/BinaryType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; public class BinaryType extends DataType { diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/BooleanType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/BooleanType.java index 591391a..99e29e3 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/BooleanType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/BooleanType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; public class BooleanType extends DataType { diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/DataType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/DataType.java index 17bc290..f1f222f 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/DataType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/DataType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; import java.io.Serializable; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/DoubleType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/DoubleType.java index cc3b073..96af23b 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/DoubleType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/DoubleType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; public class DoubleType extends DataType { public DoubleType() { diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/FloatType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/FloatType.java index be79215..5decc23 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/FloatType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/FloatType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; public class FloatType extends DataType { public FloatType() { diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/IntegerType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/IntegerType.java index 31d7080..6dd9864 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/IntegerType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/IntegerType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; public class IntegerType extends DataType { public IntegerType() { diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/LongType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/LongType.java index 8219e2f..fa4bf79 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/LongType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/LongType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; public class LongType extends DataType { public LongType() { diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/StringType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/StringType.java index ef8eaf6..d411aa1 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/StringType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/StringType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; public class StringType extends DataType { diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/StructType.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/StructType.java index 0d58ca8..eeb5aef 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/StructType.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/StructType.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; import org.apache.commons.lang3.StringUtils; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/Types.java b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/Types.java index 8733744..9a1fd45 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/type/Types.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/connector/type/Types.java @@ -1,9 +1,9 @@ -package com.geedgenetworks.spi.table.type; +package com.geedgenetworks.api.connector.type; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; -import com.geedgenetworks.spi.table.type.StructType.StructField; +import com.geedgenetworks.api.connector.type.StructType.StructField; import java.util.ArrayList; import java.util.List; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/TableFactory.java b/groot-api/src/main/java/com/geedgenetworks/api/factory/ConnectorFactory.java index a2ee1d5..1697a24 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/TableFactory.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/factory/ConnectorFactory.java @@ -1,12 +1,12 @@ -package com.geedgenetworks.spi.table.factory; +package com.geedgenetworks.api.factory; -import com.geedgenetworks.spi.table.schema.Schema; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.schema.Schema; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.configuration.Configuration; import java.util.Map; -public interface TableFactory extends Factory { +public interface ConnectorFactory extends Factory { public static class Context { private final Schema schema; diff --git a/groot-api/src/main/java/com/geedgenetworks/api/factory/DecodingFormatFactory.java b/groot-api/src/main/java/com/geedgenetworks/api/factory/DecodingFormatFactory.java new file mode 100644 index 0000000..9d06dc3 --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/factory/DecodingFormatFactory.java @@ -0,0 +1,10 @@ +package com.geedgenetworks.api.factory; + +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import org.apache.flink.configuration.ReadableConfig; + +public interface DecodingFormatFactory extends FormatFactory { + DecodingFormat createDecodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions); +} + + diff --git a/groot-api/src/main/java/com/geedgenetworks/api/factory/EncodingFormatFactory.java b/groot-api/src/main/java/com/geedgenetworks/api/factory/EncodingFormatFactory.java new file mode 100644 index 0000000..fca9273 --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/factory/EncodingFormatFactory.java @@ -0,0 +1,8 @@ +package com.geedgenetworks.api.factory; + +import com.geedgenetworks.api.connector.serialization.EncodingFormat; +import org.apache.flink.configuration.ReadableConfig; + +public interface EncodingFormatFactory extends FormatFactory { + EncodingFormat createEncodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions); +} diff --git a/groot-api/src/main/java/com/geedgenetworks/api/factory/Factory.java b/groot-api/src/main/java/com/geedgenetworks/api/factory/Factory.java new file mode 100644 index 0000000..e8b1da2 --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/factory/Factory.java @@ -0,0 +1,17 @@ +package com.geedgenetworks.api.factory; + +import org.apache.flink.configuration.ConfigOption; + +import java.util.Set; + +public interface Factory { + /** + * Returns the factory identifier. + * If multiple factories exist for different versions, a version should be appended using "-". + * (e.g. {@code kafka-1}). + */ + String type(); + + Set<ConfigOption<?>> requiredOptions(); + Set<ConfigOption<?>> optionalOptions(); +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/FactoryUtil.java b/groot-api/src/main/java/com/geedgenetworks/api/factory/FactoryUtil.java index 0f37fd4..8c5a7eb 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/FactoryUtil.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/factory/FactoryUtil.java @@ -1,7 +1,7 @@ -package com.geedgenetworks.spi.table.factory; +package com.geedgenetworks.api.factory; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.table.connector.EncodingFormat; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; import org.apache.flink.configuration.*; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; @@ -74,12 +74,12 @@ public final class FactoryUtil { } public static TableFactoryHelper createTableFactoryHelper( - TableFactory factory, TableFactory.Context context) { + ConnectorFactory factory, ConnectorFactory.Context context) { return new TableFactoryHelper(factory, context); } public static <T extends Factory> T discoverFactory( - ClassLoader classLoader, Class<T> factoryClass, String factoryIdentifier) { + ClassLoader classLoader, Class<T> factoryClass, String type) { final List<Factory> factories = discoverFactories(classLoader); final List<Factory> foundFactories = @@ -96,7 +96,7 @@ public final class FactoryUtil { final List<Factory> matchingFactories = foundFactories.stream() - .filter(f -> f.factoryIdentifier().equals(factoryIdentifier)) + .filter(f -> f.type().equals(type)) .collect(Collectors.toList()); if (matchingFactories.isEmpty()) { @@ -105,10 +105,10 @@ public final class FactoryUtil { "Could not find any factory for identifier '%s' that implements '%s' in the classpath.\n\n" + "Available factory identifiers are:\n\n" + "%s", - factoryIdentifier, + type, factoryClass.getName(), foundFactories.stream() - .map(Factory::factoryIdentifier) + .map(Factory::type) .distinct() .sorted() .collect(Collectors.joining("\n")))); @@ -119,7 +119,7 @@ public final class FactoryUtil { "Multiple factories for identifier '%s' that implement '%s' found in the classpath.\n\n" + "Ambiguous factory classes are:\n\n" + "%s", - factoryIdentifier, + type, factoryClass.getName(), matchingFactories.stream() .map(f -> f.getClass().getName()) @@ -154,7 +154,12 @@ public final class FactoryUtil { return result; } - public static <T extends TableFactory> T discoverTableFactory( + public static <T extends ProcessorFactory> T discoverProcessorFactory( + Class<T> factoryClass, String type) { + return discoverFactory(Thread.currentThread().getContextClassLoader(), factoryClass, type); + } + + public static <T extends ConnectorFactory> T discoverConnectorFactory( Class<T> factoryClass, String connector) { return discoverFactory(Thread.currentThread().getContextClassLoader(), factoryClass, connector); } @@ -202,12 +207,12 @@ public final class FactoryUtil { } public static class TableFactoryHelper { - private final TableFactory factory; - private final TableFactory.Context context; + private final ConnectorFactory factory; + private final ConnectorFactory.Context context; private final Configuration allOptions; private final Set<String> consumedOptionKeys; - public TableFactoryHelper(TableFactory factory, TableFactory.Context context) { + public TableFactoryHelper(ConnectorFactory factory, ConnectorFactory.Context context) { this.factory = factory; this.context = context; this.allOptions = context.getConfiguration(); @@ -225,7 +230,7 @@ public final class FactoryUtil { public void validate() { validateFactoryOptions(factory, allOptions); validateUnconsumedKeys( - factory.factoryIdentifier(), + factory.type(), allOptions.keySet(), consumedOptionKeys); } @@ -270,7 +275,7 @@ public final class FactoryUtil { throw new ValidationException( String.format( "Error creating sink format '%s' in option space '%s'.", - formatFactory.factoryIdentifier(), + formatFactory.type(), formatPrefix), t); } @@ -308,7 +313,7 @@ public final class FactoryUtil { throw new ValidationException( String.format( "Error creating scan format '%s' in option space '%s'.", - formatFactory.factoryIdentifier(), + formatFactory.type(), formatPrefix), t); } @@ -338,7 +343,7 @@ public final class FactoryUtil { return Optional.of(factory); } private String formatPrefix(Factory formatFactory, ConfigOption<String> formatOption) { - String identifier = formatFactory.factoryIdentifier(); + String identifier = formatFactory.type(); return getFormatPrefix(formatOption, identifier); } diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/FormatFactory.java b/groot-api/src/main/java/com/geedgenetworks/api/factory/FormatFactory.java index 5688d28..9ca8572 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/FormatFactory.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/factory/FormatFactory.java @@ -1,4 +1,5 @@ -package com.geedgenetworks.spi.table.factory; +package com.geedgenetworks.api.factory; public interface FormatFactory extends Factory { + } diff --git a/groot-api/src/main/java/com/geedgenetworks/api/factory/ProcessorFactory.java b/groot-api/src/main/java/com/geedgenetworks/api/factory/ProcessorFactory.java new file mode 100644 index 0000000..3928f02 --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/factory/ProcessorFactory.java @@ -0,0 +1,7 @@ +package com.geedgenetworks.api.factory; + +import com.geedgenetworks.api.processor.Processor; + +public interface ProcessorFactory extends Factory { + Processor<?> createProcessor(); +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/ServiceLoaderUtil.java b/groot-api/src/main/java/com/geedgenetworks/api/factory/ServiceLoaderUtil.java index ab0293a..222146e 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/ServiceLoaderUtil.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/factory/ServiceLoaderUtil.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.table.factory; +package com.geedgenetworks.api.factory; import java.util.*; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/metrics/InternalMetrics.java b/groot-api/src/main/java/com/geedgenetworks/api/metrics/InternalMetrics.java index 172072a..3192655 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/metrics/InternalMetrics.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/metrics/InternalMetrics.java @@ -1,4 +1,4 @@ -package com.geedgenetworks.spi.metrics; +package com.geedgenetworks.api.metrics; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.metrics.Counter; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/Processor.java b/groot-api/src/main/java/com/geedgenetworks/api/processor/Processor.java index ad42566..fede994 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/Processor.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/processor/Processor.java @@ -1,18 +1,15 @@ -package com.geedgenetworks.spi.processor; +package com.geedgenetworks.api.processor; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import com.typesafe.config.Config; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - import java.io.Serializable; -import java.util.Map; + public interface Processor<T extends ProcessorConfig> extends Serializable { DataStream<Event> process(StreamExecutionEnvironment env, DataStream<Event> input, T processorConfig) ; - String type(); - - T checkConfig(String name, Map<String, Object> configProperties, Config typeSafeConfig); + T parseConfig(String name, Config config); } diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProcessorConfig.java b/groot-api/src/main/java/com/geedgenetworks/api/processor/ProcessorConfig.java index e0fb40c..325deac 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProcessorConfig.java +++ b/groot-api/src/main/java/com/geedgenetworks/api/processor/ProcessorConfig.java @@ -1,17 +1,14 @@ -package com.geedgenetworks.spi.processor; +package com.geedgenetworks.api.processor; import lombok.Data; import java.io.Serializable; -import java.util.List; import java.util.Map; @Data public class ProcessorConfig implements Serializable { + private String name; private String type; private int parallelism; - private String name; - private List<String> output_fields; - private List<String> remove_fields; private Map<String, Object> properties; } diff --git a/groot-api/src/main/java/com/geedgenetworks/api/processor/ProcessorConfigOptions.java b/groot-api/src/main/java/com/geedgenetworks/api/processor/ProcessorConfigOptions.java new file mode 100644 index 0000000..f5511c7 --- /dev/null +++ b/groot-api/src/main/java/com/geedgenetworks/api/processor/ProcessorConfigOptions.java @@ -0,0 +1,34 @@ +package com.geedgenetworks.api.processor; + +import com.geedgenetworks.common.config.Option; +import com.geedgenetworks.common.config.Options; +import java.util.Map; + +public interface ProcessorConfigOptions { + Option<String> NAME = Options.key("name") + .stringType() + .noDefaultValue() + .withDescription("The name of the operator."); + + Option<String> TYPE = Options.key("type") + .stringType() + .noDefaultValue() + .withDescription("The type of operator."); + + Option<Integer> PARALLELISM = Options.key("parallelism") + .intType() + .defaultValue(1) + .withDescription("The parallelism of the operator."); + + Option<Map<String, String>> PROPERTIES = Options.key("properties") + .mapType() + .noDefaultValue() + .withDescription("Custom properties for sink."); + + + + + + + +} diff --git a/groot-bootstrap/pom.xml b/groot-bootstrap/pom.xml index 150c941..60e602a 100644 --- a/groot-bootstrap/pom.xml +++ b/groot-bootstrap/pom.xml @@ -30,7 +30,7 @@ <dependency> <groupId>com.geedgenetworks</groupId> - <artifactId>groot-spi</artifactId> + <artifactId>groot-api</artifactId> <version>${revision}</version> </dependency> diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/enums/StageType.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/enums/OperatorType.java index 8b4e154..a32c844 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/enums/StageType.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/enums/OperatorType.java @@ -1,6 +1,6 @@ package com.geedgenetworks.bootstrap.enums; -public enum StageType { +public enum OperatorType { SOURCE("source"), FILTER("filter"), SPLIT("split"), @@ -13,10 +13,10 @@ public enum StageType { public String getType() { return type; } - StageType(String type) {this.type = type;} + OperatorType(String type) {this.type = type;} - public static StageType fromType(String type) { - for (StageType stage : values()) { + public static OperatorType fromType(String type) { + for (OperatorType stage : values()) { if (stage.type.equalsIgnoreCase(type)) { return stage; } diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java index fe440f7..8ad33a2 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java @@ -1,35 +1,25 @@ package com.geedgenetworks.bootstrap.execution; import com.geedgenetworks.common.utils.ReflectionUtils; -import com.geedgenetworks.spi.table.event.Event; -import com.typesafe.config.Config; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.streaming.api.datastream.DataStream; import java.net.URL; import java.net.URLClassLoader; -import java.util.*; import java.util.function.BiConsumer; -public abstract class AbstractExecutor<K, V> - implements Executor<DataStream<Event>, JobRuntimeEnvironment> { - protected JobRuntimeEnvironment jobRuntimeEnvironment; - protected final Config operatorConfig; - protected final Map<K,V> operatorMap; +public abstract class AbstractExecutor<E, C> implements Executor<DataStream<Event>> { + public E environment; + protected final C jobConfig; - protected AbstractExecutor(Config operatorConfig) { - this.operatorConfig = operatorConfig; - this.operatorMap = initialize(operatorConfig); + protected AbstractExecutor(E environment, C jobConfig) { + this.environment = environment; + this.jobConfig = jobConfig; + initialize(jobConfig); } + protected abstract void initialize(C jobConfig); - @Override - public void setRuntimeEnvironment(JobRuntimeEnvironment jobRuntimeEnvironment) { - this.jobRuntimeEnvironment = jobRuntimeEnvironment; - - } - - protected abstract Map<K, V> initialize(Config operatorConfig); - - protected static final BiConsumer<ClassLoader, URL> ADD_URL_TO_CLASSLOADER = + protected static final BiConsumer<ClassLoader, URL> ADD_URL_TO_CLASSLOADER = (classLoader, url) -> { if (classLoader.getClass().getName().endsWith("SafetyNetWrapperClassLoader")) { URLClassLoader c = diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java deleted file mode 100644 index a45380e..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java +++ /dev/null @@ -1,87 +0,0 @@ -package com.geedgenetworks.bootstrap.execution; - -import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.config.*; -import com.geedgenetworks.common.exception.CommonErrorCode; -import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.spi.configuration.ProjectionConfigOptions; -import com.geedgenetworks.spi.processor.Processor; -import com.geedgenetworks.spi.processor.ProcessorConfig; -import com.geedgenetworks.spi.table.event.Event; -import com.typesafe.config.Config; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.net.URL; -import java.util.List; -import java.util.Map; -import java.util.ServiceLoader; - -public abstract class AbstractProcessorExecutor extends AbstractExecutor<String, ProcessorConfig> { - - - protected AbstractProcessorExecutor(Config operatorConfig) { - super(operatorConfig); - } - - @Override - public DataStream<Event> execute(DataStream<Event> input, OperatorNode operatorNode) throws JobExecuteException { - - ProcessorConfig processorConfig = operatorMap.get(operatorNode.getName()); - boolean found = false; // 标志变量 - ServiceLoader<Processor> processors = ServiceLoader.load(Processor.class); - for (Processor processor : processors) { - if(processor.type().equals(processorConfig.getType())){ - found = true; - if (operatorNode.getParallelism() > 0) { - processorConfig.setParallelism(operatorNode.getParallelism()); - } - try { - input = processor.process(jobRuntimeEnvironment.getStreamExecutionEnvironment(), input, processorConfig); - } catch (Exception e) { - throw new JobExecuteException("Create orderby pipeline instance failed!", e); - } - break; - } - } - if (!found) { - throw new JobExecuteException("No matching processor found for type: " + processorConfig.getType()); - } - return input; - } - - protected ProcessorConfig checkConfig(String key, Map<String, Object> value, Config processorsConfig) { - ProcessorConfig ProcessorConfig = new ProcessorConfig(); - boolean found = false; // 标志变量 - CheckResult result = CheckConfigUtil.checkAllExists(processorsConfig.getConfig(key), - ProjectionConfigOptions.TYPE.key()); - if (!result.isSuccess()) { - throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, String.format( - "Postprocessor: %s, Message: %s", - key, result.getMsg())); - } - ServiceLoader<Processor> processors = ServiceLoader.load(Processor.class); - for (Processor processor : processors) { - if(processor.type().equals(value.getOrDefault("type", "").toString())){ - found = true; - try { - ProcessorConfig = processor.checkConfig(key, value, processorsConfig); - - } catch (Exception e) { - throw new JobExecuteException("Create orderby pipeline instance failed!", e); - } - break; - } - } - if (!found) { - throw new JobExecuteException("No matching processor found for type: " + value.getOrDefault("type", "").toString()); - } - return ProcessorConfig; - } - - - - - - - -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/Executor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/Executor.java index e43c949..e36971d 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/Executor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/Executor.java @@ -2,10 +2,7 @@ package com.geedgenetworks.bootstrap.execution; import com.geedgenetworks.bootstrap.exception.JobExecuteException; -public interface Executor<T, ENV extends RuntimeEnvironment> { - - T execute(T dataStream, OperatorNode edge) throws JobExecuteException; - - void setRuntimeEnvironment(ENV runtimeEnvironment); +public interface Executor<T> { + T execute(T dataStream, JobTopologyNode jobTopologyNode) throws JobExecuteException; } diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/FilterExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/FilterExecutor.java deleted file mode 100644 index d70420e..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/FilterExecutor.java +++ /dev/null @@ -1,102 +0,0 @@ -package com.geedgenetworks.bootstrap.execution; - -import com.geedgenetworks.bootstrap.enums.StageType; -import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.config.Constants; -import com.geedgenetworks.common.config.CheckConfigUtil; -import com.geedgenetworks.common.config.CheckResult; -import com.geedgenetworks.common.exception.CommonErrorCode; -import com.geedgenetworks.common.exception.ConfigValidationException; - -import com.geedgenetworks.spi.configuration.FilterConfigOptions; -import com.geedgenetworks.spi.filter.Filter; -import com.geedgenetworks.spi.filter.FilterConfig; -import com.geedgenetworks.spi.table.event.Event; -import com.google.common.collect.Maps; -import com.typesafe.config.Config; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.net.URL; -import java.util.List; -import java.util.Map; -import java.util.ServiceLoader; - -/** - * Initialize config and execute filter operator - */ -@Slf4j -public class FilterExecutor extends AbstractExecutor<String, FilterConfig> { - private static final String PROCESSOR_TYPE = StageType.FILTER.getType(); - - public FilterExecutor(Config config) { - super(config); - } - - @Override - protected Map<String, FilterConfig> initialize(Config operatorConfig) { - Map<String, FilterConfig> filterConfigMap = Maps.newHashMap(); - if (operatorConfig.hasPath(Constants.FILTERS)) { - Config filterConfig = operatorConfig.getConfig(Constants.FILTERS); - filterConfig.root().unwrapped().forEach((key, value) -> { - CheckResult result = CheckConfigUtil.checkAllExists(filterConfig.getConfig(key), - FilterConfigOptions.TYPE.key(), FilterConfigOptions.PROPERTIES.key()); - if (!result.isSuccess()) { - throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, String.format( - "Filter: %s, Message: %s", - key, result.getMsg())); - } - filterConfigMap.put(key, checkConfig(key, (Map<String, Object>) value, filterConfig)); - }); - } - - return filterConfigMap; - } - - @Override - public DataStream<Event> execute(DataStream<Event> input, OperatorNode operatorNode) throws JobExecuteException { - FilterConfig filterConfig = operatorMap.get(operatorNode.getName()); - boolean found = false; // 标志变量 - ServiceLoader<Filter> filters = ServiceLoader.load(Filter.class); - for (Filter filter : filters) { - if(filter.type().equals(filterConfig.getType())){ - found = true; - if (operatorNode.getParallelism() > 0) { - filterConfig.setParallelism(operatorNode.getParallelism()); - } - try { - input = - filter.filterFunction( - input, filterConfig); - } catch (Exception e) { - throw new JobExecuteException("Create filter instance failed!", e); - } - break; - } - } - if (!found) { - throw new JobExecuteException("No matching filter found for type: " + filterConfig.getType()); - } - return input; - } - - protected FilterConfig checkConfig(String key, Map<String, Object> value, Config config) { - FilterConfig filterConfig = new FilterConfig(); - boolean found = false; // 标志变量 - ServiceLoader<Filter> filters = ServiceLoader.load(Filter.class); - for (Filter filter : filters) { - if(filter.type().equals(value.getOrDefault("type", "").toString())){ - found = true; - try { - filterConfig = filter.checkConfig(key, value, config); - } catch (Exception e) { - throw new JobExecuteException("Create split pipeline instance failed!", e); - } - } - } - if (!found) { - throw new JobExecuteException("No matching filter found for type: " + value.getOrDefault("type", "").toString()); - } - return filterConfig; - } -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobExecution.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobExecution.java index cd70f44..ad31d88 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobExecution.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobExecution.java @@ -1,12 +1,12 @@ package com.geedgenetworks.bootstrap.execution; import com.alibaba.fastjson2.JSONObject; -import com.geedgenetworks.bootstrap.enums.StageType; +import com.geedgenetworks.bootstrap.enums.OperatorType; import com.geedgenetworks.bootstrap.exception.JobExecuteException; import com.geedgenetworks.bootstrap.main.GrootStreamRunner; import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.GrootStreamConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.typesafe.config.Config; @@ -29,14 +29,10 @@ import java.util.stream.Stream; public class JobExecution { private final JobRuntimeEnvironment jobRuntimeEnvironment; - private final Executor<DataStream<Event>, JobRuntimeEnvironment> sourceExecutor; - private final Executor<DataStream<Event>, JobRuntimeEnvironment> sinkExecutor; - private final Executor<DataStream<Event>, JobRuntimeEnvironment> filterExecutor; - private final Executor<DataStream<Event>, JobRuntimeEnvironment> splitExecutor; - private final Executor<DataStream<Event>, JobRuntimeEnvironment> preprocessingExecutor; - private final Executor<DataStream<Event>, JobRuntimeEnvironment> processingExecutor; - private final Executor<DataStream<Event>, JobRuntimeEnvironment> postprocessingExecutor; - private final List<OperatorNode> operatorNodes; + private final Executor<DataStream<Event>> sourceExecutor; + private final Executor<DataStream<Event>> sinkExecutor; + private final Executor<DataStream<Event>> processorExecutor; + private final List<JobTopologyNode> jobTopologyNodes; private final List<URL> jarPaths; private final Map<String,String> nodeNameWithSplitTags = new HashMap<>(); @@ -50,25 +46,13 @@ public class JobExecution { } registerPlugin(jobConfig.getConfig(Constants.APPLICATION)); - - this.sourceExecutor = new SourceExecutor(jobConfig); - this.sinkExecutor = new SinkExecutor(jobConfig); - this.filterExecutor = new FilterExecutor(jobConfig); - this.splitExecutor = new SplitExecutor(jobConfig); - this.preprocessingExecutor = new PreprocessingExecutor(jobConfig); - this.processingExecutor = new ProcessingExecutor(jobConfig); - this.postprocessingExecutor = new PostprocessingExecutor(jobConfig); this.jobRuntimeEnvironment = JobRuntimeEnvironment.getInstance(this.registerPlugin(jobConfig, jarPaths), grootStreamConfig); - this.sourceExecutor.setRuntimeEnvironment(jobRuntimeEnvironment); - this.sinkExecutor.setRuntimeEnvironment(jobRuntimeEnvironment); - this.filterExecutor.setRuntimeEnvironment(jobRuntimeEnvironment); - this.splitExecutor.setRuntimeEnvironment(jobRuntimeEnvironment); - this.preprocessingExecutor.setRuntimeEnvironment(jobRuntimeEnvironment); - this.processingExecutor.setRuntimeEnvironment(jobRuntimeEnvironment); - this.postprocessingExecutor.setRuntimeEnvironment(jobRuntimeEnvironment); - this.operatorNodes = buildJobNode(jobConfig); + this.sourceExecutor = new SourceExecutor(jobRuntimeEnvironment, jobConfig); + this.sinkExecutor = new SinkExecutor(jobRuntimeEnvironment, jobConfig); + this.processorExecutor = new ProcessorExecutor(jobRuntimeEnvironment, jobConfig); + this.jobTopologyNodes = buildJobNode(jobConfig); } @@ -88,7 +72,7 @@ public class JobExecution { try { return uri.toURL(); } catch (MalformedURLException e) { - throw new RuntimeException("the uri of jar illegal: " + uri, e); + throw new RuntimeException("The uri of jar illegal:" + uri, e); } }).collect(Collectors.toList()); jarDependencies.forEach(url -> { @@ -153,7 +137,7 @@ public class JobExecution { return config; } - private List<OperatorNode> buildJobNode(Config config) { + private List<JobTopologyNode> buildJobNode(Config config) { Map<String, Object> sources = Maps.newHashMap(); Map<String, Object> sinks = Maps.newHashMap(); @@ -187,34 +171,34 @@ public class JobExecution { List<? extends Config> topology = config.getConfig(Constants.APPLICATION).getConfigList(Constants.APPLICATION_TOPOLOGY); - List<OperatorNode> operatorNodes = Lists.newArrayList(); + List<JobTopologyNode> jobTopologyNodes = Lists.newArrayList(); topology.forEach(item -> { - OperatorNode operatorNode = JSONObject.from(item.root().unwrapped()).toJavaObject(OperatorNode.class); - operatorNodes.add(operatorNode); + JobTopologyNode jobTopologyNode = JSONObject.from(item.root().unwrapped()).toJavaObject(JobTopologyNode.class); + jobTopologyNodes.add(jobTopologyNode); }); - for (OperatorNode operatorNode : operatorNodes) { - if (sources.containsKey(operatorNode.getName())) { - operatorNode.setType(StageType.SOURCE); - } else if (sinks.containsKey(operatorNode.getName())) { - operatorNode.setType(StageType.SINK); - } else if (filters.containsKey(operatorNode.getName())) { - operatorNode.setType(StageType.FILTER); - } else if (splits.containsKey(operatorNode.getName())) { - operatorNode.setType(StageType.SPLIT); - } else if (preprocessingPipelines.containsKey(operatorNode.getName())) { - operatorNode.setType(StageType.PREPROCESSING); - } else if (processingPipelines.containsKey(operatorNode.getName())) { - operatorNode.setType(StageType.PROCESSING); - } else if (postprocessingPipelines.containsKey(operatorNode.getName())) { - operatorNode.setType(StageType.POSTPROCESSING); + for (JobTopologyNode jobTopologyNode : jobTopologyNodes) { + if (sources.containsKey(jobTopologyNode.getName())) { + jobTopologyNode.setType(OperatorType.SOURCE); + } else if (sinks.containsKey(jobTopologyNode.getName())) { + jobTopologyNode.setType(OperatorType.SINK); + } else if (filters.containsKey(jobTopologyNode.getName())) { + jobTopologyNode.setType(OperatorType.FILTER); + } else if (splits.containsKey(jobTopologyNode.getName())) { + jobTopologyNode.setType(OperatorType.SPLIT); + } else if (preprocessingPipelines.containsKey(jobTopologyNode.getName())) { + jobTopologyNode.setType(OperatorType.PREPROCESSING); + } else if (processingPipelines.containsKey(jobTopologyNode.getName())) { + jobTopologyNode.setType(OperatorType.PROCESSING); + } else if (postprocessingPipelines.containsKey(jobTopologyNode.getName())) { + jobTopologyNode.setType(OperatorType.POSTPROCESSING); } else { - throw new JobExecuteException("unsupported process type " + operatorNode.getName()); + throw new JobExecuteException("unsupported process type " + jobTopologyNode.getName()); } } - return operatorNodes; + return jobTopologyNodes; } @@ -223,14 +207,14 @@ public class JobExecution { if (!jobRuntimeEnvironment.isLocalMode() && !jobRuntimeEnvironment.isTestMode()) { jobRuntimeEnvironment.registerPlugin(jarPaths); } - List<OperatorNode> sourceOperatorNodes = operatorNodes - .stream().filter(v -> v.getType().name().equals(StageType.SOURCE.name())).collect(Collectors.toList()); + List<JobTopologyNode> sourceJobTopologyNodes = jobTopologyNodes + .stream().filter(v -> v.getType().name().equals(OperatorType.SOURCE.name())).collect(Collectors.toList()); DataStream<Event> dataStream = null; - for (OperatorNode sourceOperatorNode : sourceOperatorNodes) { - dataStream = sourceExecutor.execute(dataStream, sourceOperatorNode); - for (String nodeName : sourceOperatorNode.getDownstream()) { + for (JobTopologyNode sourceJobTopologyNode : sourceJobTopologyNodes) { + dataStream = sourceExecutor.execute(dataStream, sourceJobTopologyNode); + for (String nodeName : sourceJobTopologyNode.getDownstream()) { buildJobGraph(dataStream, nodeName); } } @@ -251,68 +235,68 @@ public class JobExecution { } private void buildJobGraph(DataStream<Event> dataStream, String downstreamNodeName) { - OperatorNode operatorNode = getNode(downstreamNodeName).orElseGet(() -> { + JobTopologyNode jobTopologyNode = getNode(downstreamNodeName).orElseGet(() -> { throw new JobExecuteException("Can't find downstream node " + downstreamNodeName); }); - if (operatorNode.getType().name().equals(StageType.FILTER.name())) { - if (nodeNameWithSplitTags.containsKey(operatorNode.getName())) { - dataStream = filterExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream) - .getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(operatorNode.getName())) {}), operatorNode); + if (jobTopologyNode.getType().name().equals(OperatorType.FILTER.name())) { + if (nodeNameWithSplitTags.containsKey(jobTopologyNode.getName())) { + dataStream = processorExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream) + .getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(jobTopologyNode.getName())) {}), jobTopologyNode); } else { - dataStream = filterExecutor.execute(dataStream, operatorNode); + dataStream = processorExecutor.execute(dataStream, jobTopologyNode); } - } else if (operatorNode.getType().name().equals(StageType.SPLIT.name())) { - if (operatorNode.getTags().size() == operatorNode.getDownstream().size()) { - for (int i = 0; i < operatorNode.getDownstream().size(); i++) { - nodeNameWithSplitTags.put(operatorNode.getDownstream().get(i), operatorNode.getTags().get(i)); + } else if (jobTopologyNode.getType().name().equals(OperatorType.SPLIT.name())) { + if (jobTopologyNode.getTags().size() == jobTopologyNode.getDownstream().size()) { + for (int i = 0; i < jobTopologyNode.getDownstream().size(); i++) { + nodeNameWithSplitTags.put(jobTopologyNode.getDownstream().get(i), jobTopologyNode.getTags().get(i)); } } else { throw new JobExecuteException("split node downstream size not equal tags size"); } - dataStream = splitExecutor.execute(dataStream, operatorNode); - } else if (operatorNode.getType().name().equals(StageType.PREPROCESSING.name())) { - if (nodeNameWithSplitTags.containsKey(operatorNode.getName())) { - dataStream = preprocessingExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream).getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(operatorNode.getName())){ - }), operatorNode); + dataStream = processorExecutor.execute(dataStream, jobTopologyNode); + } else if (jobTopologyNode.getType().name().equals(OperatorType.PREPROCESSING.name())) { + if (nodeNameWithSplitTags.containsKey(jobTopologyNode.getName())) { + dataStream = processorExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream).getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(jobTopologyNode.getName())){ + }), jobTopologyNode); } else { - dataStream = preprocessingExecutor.execute(dataStream, operatorNode); + dataStream = processorExecutor.execute(dataStream, jobTopologyNode); } - } else if (operatorNode.getType().name().equals(StageType.PROCESSING.name())) { - if (nodeNameWithSplitTags.containsKey(operatorNode.getName())) { - dataStream = processingExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream).getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(operatorNode.getName())) { - }), operatorNode); + } else if (jobTopologyNode.getType().name().equals(OperatorType.PROCESSING.name())) { + if (nodeNameWithSplitTags.containsKey(jobTopologyNode.getName())) { + dataStream = processorExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream).getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(jobTopologyNode.getName())) { + }), jobTopologyNode); } else { - dataStream = processingExecutor.execute(dataStream, operatorNode); + dataStream = processorExecutor.execute(dataStream, jobTopologyNode); } - } else if (operatorNode.getType().name().equals(StageType.POSTPROCESSING.name())) { - if (nodeNameWithSplitTags.containsKey(operatorNode.getName())) { - dataStream = postprocessingExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream).getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(operatorNode.getName())) { - }), operatorNode); + } else if (jobTopologyNode.getType().name().equals(OperatorType.POSTPROCESSING.name())) { + if (nodeNameWithSplitTags.containsKey(jobTopologyNode.getName())) { + dataStream = processorExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream).getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(jobTopologyNode.getName())) { + }), jobTopologyNode); } else { - dataStream = postprocessingExecutor.execute(dataStream, operatorNode); + dataStream = processorExecutor.execute(dataStream, jobTopologyNode); } - } else if (operatorNode.getType().name().equals(StageType.SINK.name())) { - if (nodeNameWithSplitTags.containsKey(operatorNode.getName())) { - dataStream = sinkExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream).getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(operatorNode.getName())) { - }), operatorNode); + } else if (jobTopologyNode.getType().name().equals(OperatorType.SINK.name())) { + if (nodeNameWithSplitTags.containsKey(jobTopologyNode.getName())) { + dataStream = sinkExecutor.execute(((SingleOutputStreamOperator<Event>) dataStream).getSideOutput(new OutputTag<Event>(nodeNameWithSplitTags.get(jobTopologyNode.getName())) { + }), jobTopologyNode); } else { - dataStream = sinkExecutor.execute(dataStream, operatorNode); + dataStream = sinkExecutor.execute(dataStream, jobTopologyNode); } } else { - throw new JobExecuteException("unsupported process type " + operatorNode.getType().name()); + throw new JobExecuteException("unsupported process type " + jobTopologyNode.getType().name()); } - for (String nodeName : operatorNode.getDownstream()) { + for (String nodeName : jobTopologyNode.getDownstream()) { buildJobGraph(dataStream, nodeName); } } - private Optional<OperatorNode> getNode(String name) { - return operatorNodes.stream().filter(v -> v.getName().equals(name)).findFirst(); + private Optional<JobTopologyNode> getNode(String name) { + return jobTopologyNodes.stream().filter(v -> v.getName().equals(name)).findFirst(); } diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/OperatorNode.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobTopologyNode.java index 8c4b392..dcc15e9 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/OperatorNode.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobTopologyNode.java @@ -1,6 +1,6 @@ package com.geedgenetworks.bootstrap.execution; -import com.geedgenetworks.bootstrap.enums.StageType; +import com.geedgenetworks.bootstrap.enums.OperatorType; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; @@ -10,13 +10,16 @@ import java.io.Serializable; import java.util.Collections; import java.util.List; +/** + * Represents an operator node in the execution graph. + */ @Data @NoArgsConstructor @AllArgsConstructor @EqualsAndHashCode -public class OperatorNode implements Serializable { +public class JobTopologyNode implements Serializable { private String name; - private StageType type; + private OperatorType type; private int parallelism; private List<String> downstream = Collections.emptyList(); private List<String> tags = Collections.emptyList(); diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PostprocessingExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PostprocessingExecutor.java deleted file mode 100644 index 10d9188..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PostprocessingExecutor.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.geedgenetworks.bootstrap.execution; - -import com.geedgenetworks.bootstrap.enums.StageType; -import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.config.Constants; -import com.geedgenetworks.spi.processor.ProcessorConfig; -import com.geedgenetworks.spi.table.event.Event; -import com.google.common.collect.Maps; -import com.typesafe.config.Config; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.net.URL; -import java.util.List; -import java.util.Map; - -/** - * Initialize config and execute postprocessor - */ -public class PostprocessingExecutor extends AbstractProcessorExecutor { - private static final String PROCESSOR_TYPE = StageType.POSTPROCESSING.getType(); - - public PostprocessingExecutor(Config config) { - super(config); - } - - @Override - protected Map<String, ProcessorConfig> initialize(Config operatorConfig) { - Map<String, ProcessorConfig> postprocessingConfigMap = Maps.newHashMap(); - if (operatorConfig.hasPath(Constants.POSTPROCESSING_PIPELINES)) { - Config postprocessors = operatorConfig.getConfig(Constants.POSTPROCESSING_PIPELINES); - postprocessors.root().unwrapped().forEach((key, value) -> { - postprocessingConfigMap.put(key, checkConfig(key, (Map<String, Object>) value, postprocessors)); - }); - } - return postprocessingConfigMap; - } - - - @Override - public DataStream<Event> execute(DataStream<Event> input, OperatorNode operatorNode) throws JobExecuteException { - return super.execute(input, operatorNode); - } -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PreprocessingExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PreprocessingExecutor.java deleted file mode 100644 index 9acda99..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PreprocessingExecutor.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.geedgenetworks.bootstrap.execution; - -import com.geedgenetworks.bootstrap.enums.StageType; -import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.config.Constants; -import com.geedgenetworks.spi.processor.ProcessorConfig; -import com.geedgenetworks.spi.table.event.Event; -import com.google.common.collect.Maps; -import com.typesafe.config.Config; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.net.URL; -import java.util.List; -import java.util.Map; - -/** - * Initialize config and execute preprocessor - */ -@Slf4j -public class PreprocessingExecutor extends AbstractProcessorExecutor { - private static final String PROCESSOR_TYPE = StageType.PREPROCESSING.getType(); - - public PreprocessingExecutor(Config config) { - super(config); - } - - @Override - protected Map<String, ProcessorConfig> initialize(Config operatorConfig) { - Map<String, ProcessorConfig> preprocessingConfigMap = Maps.newHashMap(); - if (operatorConfig.hasPath(Constants.PREPROCESSING_PIPELINES)) { - Config preprocessors = operatorConfig.getConfig(Constants.PREPROCESSING_PIPELINES); - preprocessors.root().unwrapped().forEach((key, value) -> { - preprocessingConfigMap.put(key, checkConfig(key, (Map<String, Object>) value, preprocessors)); - }); - } - return preprocessingConfigMap; - } - - @Override - public DataStream<Event> execute(DataStream<Event> input, OperatorNode operatorNode) throws JobExecuteException { - - return super.execute(input, operatorNode); - - } -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessingExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessingExecutor.java deleted file mode 100644 index c49df88..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessingExecutor.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.geedgenetworks.bootstrap.execution; - -import com.geedgenetworks.bootstrap.enums.StageType; -import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.config.Constants; -import com.geedgenetworks.spi.processor.Processor; -import com.geedgenetworks.spi.processor.ProcessorConfig; -import com.geedgenetworks.spi.processor.ProcessorProvider; -import com.geedgenetworks.spi.table.event.Event; -import com.google.common.collect.Maps; -import com.typesafe.config.Config; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.net.URL; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Initialize config and execute processor - */ -public class ProcessingExecutor extends AbstractProcessorExecutor { - private static final String PROCESSOR_TYPE = StageType.PROCESSING.getType(); - //private Map<String, Processor<?>> processors; - - public ProcessingExecutor(Config config) { - super(config); - } - - @Override - protected Map<String, ProcessorConfig> initialize(Config operatorConfig) { - Map<String, ProcessorConfig> processingConfigMap = Maps.newHashMap(); - //processors = new HashMap<>(); - if (operatorConfig.hasPath(Constants.PROCESSING_PIPELINES)) { - Config processingConfig = operatorConfig.getConfig(Constants.PROCESSING_PIPELINES); - processingConfig.root().unwrapped().forEach((key, value) -> { - processingConfigMap.put(key, checkConfig(key, (Map<String, Object>) value, processingConfig)); - //processors.put(key, ProcessorProvider.load(((Map<?, ?>) value).get("type").toString())); - - }); - } - return processingConfigMap; - } - - @Override - public DataStream<Event> execute(DataStream<Event> input, OperatorNode operatorNode) throws JobExecuteException { - return super.execute(input, operatorNode); - } -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessorExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessorExecutor.java new file mode 100644 index 0000000..204866f --- /dev/null +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessorExecutor.java @@ -0,0 +1,102 @@ +package com.geedgenetworks.bootstrap.execution; + +import com.geedgenetworks.api.processor.ProcessorConfigOptions; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.factory.ProcessorFactory; +import com.geedgenetworks.bootstrap.exception.JobExecuteException; +import com.geedgenetworks.common.config.CheckConfigUtil; +import com.geedgenetworks.common.config.CheckResult; +import com.geedgenetworks.common.config.Constants; +import com.geedgenetworks.common.exception.CommonErrorCode; +import com.geedgenetworks.common.exception.ConfigValidationException; +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.processor.ProcessorConfig; +import com.geedgenetworks.api.connector.event.Event; +import com.google.common.collect.Maps; +import com.typesafe.config.Config; +import org.apache.flink.streaming.api.datastream.DataStream; +import java.util.Map; +/** + * Initialize config and execute processor + */ +public class ProcessorExecutor extends AbstractExecutor<JobRuntimeEnvironment, Config> { + private Map<String, ProcessorConfig> operators; + private Map<String, Processor<?>> processors; + + public ProcessorExecutor(JobRuntimeEnvironment environment, Config jobConfig) { + super(environment, jobConfig); + } + + @Override + protected void initialize(Config jobConfig) { + operators = Maps.newHashMap(); + processors = Maps.newHashMap(); + + if (jobConfig.hasPath(Constants.FILTERS)) { + discoveryProcessors(jobConfig.getConfig(Constants.FILTERS)); + } + + if (jobConfig.hasPath(Constants.SPLITS)) { + discoveryProcessors(jobConfig.getConfig(Constants.SPLITS)); + } + + if (jobConfig.hasPath(Constants.PREPROCESSING_PIPELINES)) { + discoveryProcessors(jobConfig.getConfig(Constants.PREPROCESSING_PIPELINES)); + } + + if (jobConfig.hasPath(Constants.PROCESSING_PIPELINES)) { + discoveryProcessors(jobConfig.getConfig(Constants.PROCESSING_PIPELINES)); + } + + if (jobConfig.hasPath(Constants.POSTPROCESSING_PIPELINES)) { + discoveryProcessors(jobConfig.getConfig(Constants.POSTPROCESSING_PIPELINES)); + } + } + + private void discoveryProcessors(Config config) { + + config.root().unwrapped().forEach((key, value) -> { + + CheckResult result = CheckConfigUtil.checkAllExists(config.getConfig(key), + ProcessorConfigOptions.TYPE.key()); + if (!result.isSuccess()) { + throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, String.format( + "Processor: %s, Message: %s", key, result.getMsg())); + } + + Processor processor = FactoryUtil + .discoverProcessorFactory(ProcessorFactory.class, ((Map<?, ?>) value).get("type").toString()).createProcessor(); + processors.put(key, processor); + operators.put(key, processor.parseConfig(key,config.getConfig(key))); + }); + + } + + @Override + public DataStream<Event> execute(DataStream<Event> input, JobTopologyNode jobTopologyNode) throws JobExecuteException { + String name = jobTopologyNode.getName(); + ProcessorConfig operatorConfig = operators.get(name); + if (operatorConfig == null) { + throw new JobExecuteException("No matching operator configuration found for: " + name); + } + + Processor processor = processors.get(operatorConfig.getName()); + + if (processor == null) { + throw new JobExecuteException("No matching processor found for type: " + operatorConfig.getType()); + } + + // Set parallelism if needed + int parallelism = jobTopologyNode.getParallelism(); + if (parallelism > 0) { + operatorConfig.setParallelism(parallelism); + } + + try { + return processor.process(environment.getStreamExecutionEnvironment(), input, operatorConfig); + } catch (Exception e) { + throw new JobExecuteException("Failed to execute processor due to unexpected error.", e); + } + + } +} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SinkExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SinkExecutor.java index 130705a..501fa81 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SinkExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SinkExecutor.java @@ -1,7 +1,7 @@ package com.geedgenetworks.bootstrap.execution; import com.alibaba.fastjson.JSONObject; -import com.geedgenetworks.bootstrap.enums.StageType; +import com.geedgenetworks.bootstrap.enums.OperatorType; import com.geedgenetworks.bootstrap.exception.JobExecuteException; import com.geedgenetworks.bootstrap.utils.SchemaConfigParse; import com.geedgenetworks.common.config.Constants; @@ -9,14 +9,14 @@ import com.geedgenetworks.common.config.CheckConfigUtil; import com.geedgenetworks.common.config.CheckResult; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.spi.sink.SinkConfig; -import com.geedgenetworks.spi.sink.SinkConfigOptions; -import com.geedgenetworks.spi.sink.SinkProvider; -import com.geedgenetworks.spi.sink.SinkTableFactory; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.factory.TableFactory; -import com.geedgenetworks.spi.table.schema.Schema; +import com.geedgenetworks.api.connector.sink.SinkConfig; +import com.geedgenetworks.api.connector.sink.SinkConfigOptions; +import com.geedgenetworks.api.connector.sink.SinkProvider; +import com.geedgenetworks.api.connector.sink.SinkTableFactory; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.factory.ConnectorFactory; +import com.geedgenetworks.api.connector.schema.Schema; import com.google.common.collect.Maps; import com.typesafe.config.Config; import lombok.extern.slf4j.Slf4j; @@ -30,18 +30,17 @@ import java.util.Map; * Initialize config and execute sink connector */ @Slf4j -public class SinkExecutor extends AbstractExecutor<String, SinkConfig> { - private static final String PROCESSOR_TYPE = StageType.SINK.getType(); - - public SinkExecutor(Config config) { - super(config); +public class SinkExecutor extends AbstractExecutor<JobRuntimeEnvironment, Config> { + private static final String PROCESSOR_TYPE = OperatorType.SINK.getType(); + private Map<String, SinkConfig> operators; + public SinkExecutor(JobRuntimeEnvironment environment, Config jobConfig) { + super(environment, jobConfig); } @Override - protected Map<String, SinkConfig> initialize(Config operatorConfig) { - Map<String, SinkConfig> sinkConfigMap = Maps.newHashMap(); - - if (operatorConfig.hasPath(Constants.SINKS)) { - Config sinks = operatorConfig.getConfig(Constants.SINKS); + protected void initialize(Config jobConfig) { + operators = Maps.newHashMap(); + if (jobConfig.hasPath(Constants.SINKS)) { + Config sinks = jobConfig.getConfig(Constants.SINKS); sinks.root().unwrapped().forEach((key,value) -> { CheckResult result = CheckConfigUtil.checkAllExists(sinks.getConfig(key), SinkConfigOptions.TYPE.key(), SinkConfigOptions.PROPERTIES.key()); @@ -53,26 +52,25 @@ public class SinkExecutor extends AbstractExecutor<String, SinkConfig> { SinkConfig sinkConfig = new JSONObject((Map<String, Object>) value).toJavaObject(SinkConfig.class); sinkConfig.setName(key); - sinkConfigMap.put(key, sinkConfig); + operators.put(key, sinkConfig); }); } - return sinkConfigMap; } @Override - public DataStream<Event> execute(DataStream<Event> input, OperatorNode operatorNode) throws JobExecuteException { - SinkConfig sinkConfig = operatorMap.get(operatorNode.getName()); + public DataStream<Event> execute(DataStream<Event> input, JobTopologyNode jobTopologyNode) throws JobExecuteException { + SinkConfig sinkConfig = operators.get(jobTopologyNode.getName()); try { - SinkTableFactory sinkTableFactory = FactoryUtil.discoverTableFactory(SinkTableFactory.class, sinkConfig.getType()); + SinkTableFactory sinkTableFactory = FactoryUtil.discoverConnectorFactory(SinkTableFactory.class, sinkConfig.getType()); Map<String, String> options = sinkConfig.getProperties(); Configuration configuration = Configuration.fromMap(options); Schema schema = null; if(sinkConfig.getSchema() != null && !sinkConfig.getSchema().isEmpty()){ schema = SchemaConfigParse.parseSchemaConfig(sinkConfig.getSchema()); } - TableFactory.Context context = new TableFactory.Context(schema, options, configuration); + ConnectorFactory.Context context = new ConnectorFactory.Context(schema, options, configuration); SinkProvider sinkProvider = sinkTableFactory.getSinkProvider(context); if(!sinkProvider.supportDynamicSchema() && schema != null && schema.isDynamic()){ throw new UnsupportedOperationException(String.format("sink(%s) not support DynamicSchema", sinkConfig.getName())); @@ -83,8 +81,8 @@ public class SinkExecutor extends AbstractExecutor<String, SinkConfig> { } DataStreamSink<?> dataStreamSink = sinkProvider.consumeDataStream(input); - if (operatorNode.getParallelism() > 0) { - dataStreamSink.setParallelism(operatorNode.getParallelism()); + if (jobTopologyNode.getParallelism() > 0) { + dataStreamSink.setParallelism(jobTopologyNode.getParallelism()); } dataStreamSink.name(sinkConfig.getName()); } catch (Exception e) { diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SourceExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SourceExecutor.java index 5109540..ca4fc1d 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SourceExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SourceExecutor.java @@ -1,7 +1,7 @@ package com.geedgenetworks.bootstrap.execution; import com.alibaba.fastjson2.JSONObject; -import com.geedgenetworks.bootstrap.enums.StageType; +import com.geedgenetworks.bootstrap.enums.OperatorType; import com.geedgenetworks.bootstrap.exception.ConfigCheckException; import com.geedgenetworks.bootstrap.exception.JobExecuteException; import com.geedgenetworks.bootstrap.utils.SchemaConfigParse; @@ -10,14 +10,14 @@ import com.geedgenetworks.common.config.CheckConfigUtil; import com.geedgenetworks.common.config.CheckResult; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.spi.configuration.SourceConfigOptions; -import com.geedgenetworks.spi.source.SourceConfig; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.source.SourceTableFactory; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.factory.TableFactory; -import com.geedgenetworks.spi.table.schema.Schema; +import com.geedgenetworks.api.connector.source.SourceConfigOptions; +import com.geedgenetworks.api.connector.source.SourceConfig; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.source.SourceTableFactory; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.factory.ConnectorFactory; +import com.geedgenetworks.api.connector.schema.Schema; import com.google.common.collect.Maps; import com.typesafe.config.*; import lombok.extern.slf4j.Slf4j; @@ -35,17 +35,17 @@ import java.util.Map; * Initialize config and execute source connector */ @Slf4j -public class SourceExecutor extends AbstractExecutor<String, SourceConfig> { - private static final String PROCESSOR_TYPE = StageType.SOURCE.getType(); - - public SourceExecutor(Config config) { - super(config); +public class SourceExecutor extends AbstractExecutor<JobRuntimeEnvironment, Config> { + private static final String PROCESSOR_TYPE = OperatorType.SOURCE.getType(); + private Map<String, SourceConfig> operators; + public SourceExecutor(JobRuntimeEnvironment environment, Config jobConfig) { + super(environment, jobConfig); } @Override - protected Map<String, SourceConfig> initialize(Config operatorConfig) { - Map<String, SourceConfig> sourceConfigMap = Maps.newHashMap(); - if (operatorConfig.hasPath(Constants.SOURCES)) { - Config sources = operatorConfig.getConfig(Constants.SOURCES); + protected void initialize(Config jobConfig) { + operators = Maps.newHashMap(); + if (jobConfig.hasPath(Constants.SOURCES)) { + Config sources = jobConfig.getConfig(Constants.SOURCES); sources.root().unwrapped().forEach((key,value) -> { CheckResult result = CheckConfigUtil.checkAllExists(sources.getConfig(key), SourceConfigOptions.TYPE.key(), SourceConfigOptions.PROPERTIES.key()); @@ -57,27 +57,25 @@ public class SourceExecutor extends AbstractExecutor<String, SourceConfig> { SourceConfig sourceConfig = new JSONObject((Map<String, Object>) value).toJavaObject(SourceConfig.class); sourceConfig.setName(key); - sourceConfigMap.put(key, sourceConfig); + operators.put(key, sourceConfig); }); } - - return sourceConfigMap; } @Override - public DataStream<Event> execute(DataStream<Event> input, OperatorNode operatorNode) throws JobExecuteException { - SourceConfig sourceConfig = operatorMap.get(operatorNode.getName()); + public DataStream<Event> execute(DataStream<Event> input, JobTopologyNode jobTopologyNode) throws JobExecuteException { + SourceConfig sourceConfig = operators.get(jobTopologyNode.getName()); SingleOutputStreamOperator sourceSingleOutputStreamOperator; try { - SourceTableFactory tableFactory = FactoryUtil.discoverTableFactory(SourceTableFactory.class, sourceConfig.getType()); + SourceTableFactory tableFactory = FactoryUtil.discoverConnectorFactory(SourceTableFactory.class, sourceConfig.getType()); Map<String, String> options = sourceConfig.getProperties(); Configuration configuration = Configuration.fromMap(options); Schema schema = null; if(sourceConfig.getSchema() != null && !sourceConfig.getSchema().isEmpty()){ schema = SchemaConfigParse.parseSchemaConfig(sourceConfig.getSchema()); } - TableFactory.Context context = new TableFactory.Context(schema, options, configuration); + ConnectorFactory.Context context = new ConnectorFactory.Context(schema, options, configuration); SourceProvider sourceProvider = tableFactory.getSourceProvider(context); if(!sourceProvider.supportDynamicSchema() && schema != null && schema.isDynamic()){ throw new UnsupportedOperationException(String.format("source(%s) not support DynamicSchema", sourceConfig.getName())); @@ -87,18 +85,18 @@ public class SourceExecutor extends AbstractExecutor<String, SourceConfig> { System.out.println(String.format("source(%s) schema:\n%s", sourceConfig.getName(), schema.getDataType().treeString())); } - sourceSingleOutputStreamOperator = sourceProvider.produceDataStream(jobRuntimeEnvironment.getStreamExecutionEnvironment()).name(sourceConfig.getName()); - if (operatorNode.getParallelism() > 0) { - sourceSingleOutputStreamOperator.setParallelism(operatorNode.getParallelism()); + sourceSingleOutputStreamOperator = sourceProvider.produceDataStream(environment.getStreamExecutionEnvironment()).name(sourceConfig.getName()); + if (jobTopologyNode.getParallelism() > 0) { + sourceSingleOutputStreamOperator.setParallelism(jobTopologyNode.getParallelism()); } - sourceSingleOutputStreamOperator = setWatermarkIfNecessary(sourceSingleOutputStreamOperator, sourceConfig, operatorNode); + sourceSingleOutputStreamOperator = setWatermarkIfNecessary(sourceSingleOutputStreamOperator, sourceConfig, jobTopologyNode); return sourceSingleOutputStreamOperator; } catch (Exception e) { throw new JobExecuteException("Create source instance failed!", e); } } - private SingleOutputStreamOperator<Event> setWatermarkIfNecessary(SingleOutputStreamOperator<Event> dataStream, SourceConfig sourceConfig, OperatorNode operatorNode){ + private SingleOutputStreamOperator<Event> setWatermarkIfNecessary(SingleOutputStreamOperator<Event> dataStream, SourceConfig sourceConfig, JobTopologyNode jobTopologyNode){ final String watermarkTimestamp = sourceConfig.getWatermark_timestamp(); if(StringUtils.isNotBlank(watermarkTimestamp)){ String timestampUnit = sourceConfig.getWatermark_timestamp_unit(); @@ -137,8 +135,8 @@ public class SourceExecutor extends AbstractExecutor<String, SourceConfig> { WatermarkStrategy.<Event>forBoundedOutOfOrderness(Duration.ofMillis(watermarkLag)) .withTimestampAssigner(timestampAssigner) ); - if (operatorNode.getParallelism() > 0) { - dataStream.setParallelism(operatorNode.getParallelism()); + if (jobTopologyNode.getParallelism() > 0) { + dataStream.setParallelism(jobTopologyNode.getParallelism()); } } return dataStream; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SplitExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SplitExecutor.java deleted file mode 100644 index c142614..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SplitExecutor.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.geedgenetworks.bootstrap.execution; - -import com.alibaba.fastjson.JSONObject; -import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.config.Constants; -import com.geedgenetworks.common.config.CheckConfigUtil; -import com.geedgenetworks.common.config.CheckResult; -import com.geedgenetworks.common.exception.CommonErrorCode; -import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.spi.split.Split; -import com.geedgenetworks.spi.split.SplitConfig; -import com.geedgenetworks.spi.split.SplitConfigOptions; -import com.geedgenetworks.spi.table.event.Event; -import com.google.common.collect.Maps; -import com.typesafe.config.Config; -import lombok.extern.slf4j.Slf4j; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.net.URL; -import java.util.List; -import java.util.Map; -import java.util.ServiceLoader; - - -/** - * Initialize config and execute filter operator - */ -@Slf4j -public class SplitExecutor extends AbstractExecutor<String, SplitConfig> { - - - public SplitExecutor(Config config) { - super(config); - } - - @Override - protected Map<String, SplitConfig> initialize(Config operatorConfig) { - Map<String, SplitConfig> splitConfigMap = Maps.newHashMap(); - if (operatorConfig.hasPath(Constants.SPLITS)) { - Config splitsConfig = operatorConfig.getConfig(Constants.SPLITS); - splitsConfig.root().unwrapped().forEach((key, value) -> { - CheckResult result = CheckConfigUtil.checkAllExists(splitsConfig.getConfig(key), - SplitConfigOptions.TYPE.key()); - if (!result.isSuccess()) { - throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, String.format( - "split: %s, Message: %s", - key, result.getMsg())); - } - SplitConfig splitConfig = new JSONObject((Map<String, Object>) value).toJavaObject(SplitConfig.class); - splitConfig.setName(key); - splitConfigMap.put(key, splitConfig); - }); - } - - return splitConfigMap; - } - - @Override - public DataStream<Event> execute(DataStream<Event> input, OperatorNode operatorNode) throws JobExecuteException { - SplitConfig splitConfig = operatorMap.get(operatorNode.getName()); - boolean found = false; // 标志变量 - ServiceLoader<Split> splits = ServiceLoader.load(Split.class); - for (Split split : splits) { - found = true; // 标志变量 - if(split.type().equals(splitConfig.getType())){ - if (operatorNode.getParallelism() > 0) { - splitConfig.setParallelism(operatorNode.getParallelism()); - } - try { - input = - split.splitFunction( - input, splitConfig); - } catch (Exception e) { - throw new JobExecuteException("Create split instance failed!", e); - } - break; - } - } - if (!found) { - throw new JobExecuteException("No matching split found for type: " + splitConfig.getType()); - } - return input; - } - - protected SplitConfig checkConfig(String key, Map<String, Object> value, Config config) { - SplitConfig splitConfig = new SplitConfig(); - boolean found = false; // 标志变量 - ServiceLoader<Split> splits = ServiceLoader.load(Split.class); - for (Split split : splits) { - if(split.type().equals(value.getOrDefault("type", "").toString())){ - found = true; - try { - splitConfig = split.checkConfig(key, value, config); - } catch (Exception e) { - throw new JobExecuteException("Create split pipeline instance failed!", e); - } - break; - } - } - if (!found) { - throw new JobExecuteException("No matching split found for type: " + value.getOrDefault("type", "").toString()); - } - return splitConfig; - } -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/SchemaConfigParse.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/SchemaConfigParse.java index f6b4292..00fcd61 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/SchemaConfigParse.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/SchemaConfigParse.java @@ -3,10 +3,10 @@ package com.geedgenetworks.bootstrap.utils; import com.alibaba.fastjson2.JSON; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.spi.table.schema.Schema; -import com.geedgenetworks.spi.table.schema.SchemaParser; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.schema.Schema; +import com.geedgenetworks.api.connector.schema.SchemaParser; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.Types; import org.apache.commons.io.FileUtils; import java.io.File; diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectSink.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectSink.java index 74b40a4..ccb01a4 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectSink.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectSink.java @@ -1,6 +1,6 @@ package com.geedgenetworks.bootstrap.main.simple.collect; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.streaming.api.functions.sink.SinkFunction; import java.util.*; diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectTableFactory.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectTableFactory.java index 130478e..15d6328 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectTableFactory.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectTableFactory.java @@ -1,8 +1,8 @@ package com.geedgenetworks.bootstrap.main.simple.collect; -import com.geedgenetworks.spi.sink.SinkProvider; -import com.geedgenetworks.spi.sink.SinkTableFactory; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.sink.SinkProvider; +import com.geedgenetworks.api.connector.sink.SinkTableFactory; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; @@ -16,7 +16,7 @@ import java.util.Set; public class CollectTableFactory implements SinkTableFactory { public static final String IDENTIFIER = "collect"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } diff --git a/groot-connectors/connector-clickhouse/src/main/java/com/geedgenetworks/connectors/clickhouse/ClickHouseTableFactory.java b/groot-connectors/connector-clickhouse/src/main/java/com/geedgenetworks/connectors/clickhouse/ClickHouseTableFactory.java index 53cf99a..274061d 100644 --- a/groot-connectors/connector-clickhouse/src/main/java/com/geedgenetworks/connectors/clickhouse/ClickHouseTableFactory.java +++ b/groot-connectors/connector-clickhouse/src/main/java/com/geedgenetworks/connectors/clickhouse/ClickHouseTableFactory.java @@ -1,12 +1,12 @@ package com.geedgenetworks.connectors.clickhouse; import com.geedgenetworks.connectors.clickhouse.sink.EventBatchIntervalClickHouseSink; -import com.geedgenetworks.spi.sink.SinkProvider; -import com.geedgenetworks.spi.sink.SinkTableFactory; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.factory.FactoryUtil.TableFactoryHelper; -import com.geedgenetworks.spi.table.schema.Schema; +import com.geedgenetworks.api.connector.sink.SinkProvider; +import com.geedgenetworks.api.connector.sink.SinkTableFactory; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.factory.FactoryUtil.TableFactoryHelper; +import com.geedgenetworks.api.connector.schema.Schema; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.MemorySize; import org.apache.flink.configuration.ReadableConfig; @@ -25,7 +25,7 @@ import static com.geedgenetworks.connectors.clickhouse.ClickHouseConnectorOption public class ClickHouseTableFactory implements SinkTableFactory { public static final String IDENTIFIER = "clickhouse"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } diff --git a/groot-connectors/connector-clickhouse/src/main/java/com/geedgenetworks/connectors/clickhouse/sink/EventBatchIntervalClickHouseSink.java b/groot-connectors/connector-clickhouse/src/main/java/com/geedgenetworks/connectors/clickhouse/sink/EventBatchIntervalClickHouseSink.java index 8a52fb9..4b64a84 100644 --- a/groot-connectors/connector-clickhouse/src/main/java/com/geedgenetworks/connectors/clickhouse/sink/EventBatchIntervalClickHouseSink.java +++ b/groot-connectors/connector-clickhouse/src/main/java/com/geedgenetworks/connectors/clickhouse/sink/EventBatchIntervalClickHouseSink.java @@ -2,11 +2,11 @@ package com.geedgenetworks.connectors.clickhouse.sink; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.schema.Schema; -import com.geedgenetworks.spi.table.schema.SchemaChangeAware; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.schema.Schema; +import com.geedgenetworks.api.connector.schema.SchemaChangeAware; +import com.geedgenetworks.api.connector.type.StructType; import com.github.housepower.data.Block; import org.apache.flink.configuration.Configuration; diff --git a/groot-connectors/connector-clickhouse/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-connectors/connector-clickhouse/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index 9f8187a..9f8187a 100644 --- a/groot-connectors/connector-clickhouse/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-connectors/connector-clickhouse/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/FileSourceProvider.java b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/FileSourceProvider.java index a946f84..4a3fd77 100644 --- a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/FileSourceProvider.java +++ b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/FileSourceProvider.java @@ -1,8 +1,8 @@ package com.geedgenetworks.connectors.file; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; import org.apache.flink.api.common.serialization.DeserializationSchema; diff --git a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/FileTableFactory.java b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/FileTableFactory.java index 36a7610..02baa51 100644 --- a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/FileTableFactory.java +++ b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/FileTableFactory.java @@ -1,11 +1,11 @@ package com.geedgenetworks.connectors.file; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.source.SourceTableFactory; -import com.geedgenetworks.spi.table.factory.DecodingFormatFactory; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.source.SourceTableFactory; +import com.geedgenetworks.api.factory.DecodingFormatFactory; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ReadableConfig; @@ -18,7 +18,7 @@ public class FileTableFactory implements SourceTableFactory { public static final String IDENTIFIER = "file"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } diff --git a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/HdfsTextFileSource.java b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/HdfsTextFileSource.java index 4faad97..09994f8 100644 --- a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/HdfsTextFileSource.java +++ b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/HdfsTextFileSource.java @@ -1,6 +1,6 @@ package com.geedgenetworks.connectors.file; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; import org.apache.flink.api.common.serialization.DeserializationSchema; diff --git a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/LocalTextFileSource.java b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/LocalTextFileSource.java index c19a5ea..aec3f55 100644 --- a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/LocalTextFileSource.java +++ b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/LocalTextFileSource.java @@ -1,6 +1,6 @@ package com.geedgenetworks.connectors.file; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; import org.apache.flink.api.common.serialization.DeserializationSchema; diff --git a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/MemoryTextFileSource.java b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/MemoryTextFileSource.java index 24ca96c..56444bb 100644 --- a/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/MemoryTextFileSource.java +++ b/groot-connectors/connector-file/src/main/java/com/geedgenetworks/connectors/file/MemoryTextFileSource.java @@ -1,6 +1,6 @@ package com.geedgenetworks.connectors.file; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; import org.slf4j.Logger; diff --git a/groot-connectors/connector-file/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-connectors/connector-file/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index d1c44cc..d1c44cc 100644 --- a/groot-connectors/connector-file/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-connectors/connector-file/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-connectors/connector-ipfix-collector/src/main/java/com/geedgenetworks/connectors/ipfix/collector/IPFixSourceProvider.java b/groot-connectors/connector-ipfix-collector/src/main/java/com/geedgenetworks/connectors/ipfix/collector/IPFixSourceProvider.java index 7e86a2c..7426637 100644 --- a/groot-connectors/connector-ipfix-collector/src/main/java/com/geedgenetworks/connectors/ipfix/collector/IPFixSourceProvider.java +++ b/groot-connectors/connector-ipfix-collector/src/main/java/com/geedgenetworks/connectors/ipfix/collector/IPFixSourceProvider.java @@ -3,12 +3,12 @@ package com.geedgenetworks.connectors.ipfix.collector; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; import com.geedgenetworks.connectors.ipfix.collector.utils.IPFixUtil; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.table.type.*; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.DataType; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.type.*; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.DataType; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.functions.RichFlatMapFunction; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.ReadableConfig; diff --git a/groot-connectors/connector-ipfix-collector/src/main/java/com/geedgenetworks/connectors/ipfix/collector/IPFixTableFactory.java b/groot-connectors/connector-ipfix-collector/src/main/java/com/geedgenetworks/connectors/ipfix/collector/IPFixTableFactory.java index 2865019..b4ca1e7 100644 --- a/groot-connectors/connector-ipfix-collector/src/main/java/com/geedgenetworks/connectors/ipfix/collector/IPFixTableFactory.java +++ b/groot-connectors/connector-ipfix-collector/src/main/java/com/geedgenetworks/connectors/ipfix/collector/IPFixTableFactory.java @@ -1,9 +1,9 @@ package com.geedgenetworks.connectors.ipfix.collector; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.source.SourceTableFactory; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.source.SourceTableFactory; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ReadableConfig; @@ -27,7 +27,7 @@ public class IPFixTableFactory implements SourceTableFactory { } @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } diff --git a/groot-connectors/connector-ipfix-collector/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-connectors/connector-ipfix-collector/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index bcf4133..bcf4133 100644 --- a/groot-connectors/connector-ipfix-collector/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-connectors/connector-ipfix-collector/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/EventKafkaDeserializationSchema.java b/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/EventKafkaDeserializationSchema.java index 65b01d4..35fcde7 100644 --- a/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/EventKafkaDeserializationSchema.java +++ b/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/EventKafkaDeserializationSchema.java @@ -1,7 +1,7 @@ package com.geedgenetworks.connectors.kafka; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; diff --git a/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaSinkProvider.java b/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaSinkProvider.java index b61b376..57a7d70 100644 --- a/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaSinkProvider.java +++ b/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaSinkProvider.java @@ -1,10 +1,10 @@ package com.geedgenetworks.connectors.kafka; import com.geedgenetworks.connectors.kafka.rate.RateLimitingStrategy; -import com.geedgenetworks.spi.table.connector.EncodingFormat; -import com.geedgenetworks.spi.sink.SinkProvider; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; +import com.geedgenetworks.api.connector.sink.SinkProvider; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; diff --git a/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaSourceProvider.java b/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaSourceProvider.java index 81d766e..6b6de05 100644 --- a/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaSourceProvider.java +++ b/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaSourceProvider.java @@ -1,9 +1,9 @@ package com.geedgenetworks.connectors.kafka; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; diff --git a/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaTableFactory.java b/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaTableFactory.java index 0478e00..dca76ed 100644 --- a/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaTableFactory.java +++ b/groot-connectors/connector-kafka/src/main/java/com/geedgenetworks/connectors/kafka/KafkaTableFactory.java @@ -1,18 +1,19 @@ package com.geedgenetworks.connectors.kafka; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; import com.geedgenetworks.connectors.kafka.rate.BlockDropRateLimitingStrategy; import com.geedgenetworks.connectors.kafka.rate.NoRateLimitingStrategy; import com.geedgenetworks.connectors.kafka.rate.RateLimitingStrategy; import com.geedgenetworks.connectors.kafka.rate.RateLimitingStrategyType; -import com.geedgenetworks.spi.sink.SinkProvider; -import com.geedgenetworks.spi.sink.SinkTableFactory; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.source.SourceTableFactory; -import com.geedgenetworks.spi.table.connector.*; -import com.geedgenetworks.spi.table.factory.DecodingFormatFactory; -import com.geedgenetworks.spi.table.factory.EncodingFormatFactory; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.sink.SinkProvider; +import com.geedgenetworks.api.connector.sink.SinkTableFactory; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.source.SourceTableFactory; +import com.geedgenetworks.api.factory.DecodingFormatFactory; +import com.geedgenetworks.api.factory.EncodingFormatFactory; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ReadableConfig; import org.apache.flink.util.Preconditions; @@ -25,7 +26,7 @@ import static com.geedgenetworks.connectors.kafka.KafkaConnectorOptionsUtil.*; public class KafkaTableFactory implements SourceTableFactory, SinkTableFactory { public static final String IDENTIFIER = "kafka"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } diff --git a/groot-connectors/connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/GrootFlinkKafkaProducer.java b/groot-connectors/connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/GrootFlinkKafkaProducer.java index 239d125..22af04f 100644 --- a/groot-connectors/connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/GrootFlinkKafkaProducer.java +++ b/groot-connectors/connector-kafka/src/main/java/org/apache/flink/streaming/connectors/kafka/GrootFlinkKafkaProducer.java @@ -2,7 +2,7 @@ package org.apache.flink.streaming.connectors.kafka; import com.geedgenetworks.connectors.kafka.rate.RateLimitingStatus; import com.geedgenetworks.connectors.kafka.rate.RateLimitingStrategy; -import com.geedgenetworks.spi.metrics.InternalMetrics; +import com.geedgenetworks.api.metrics.InternalMetrics; import org.apache.commons.lang3.StringUtils; import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.PublicEvolving; diff --git a/groot-connectors/connector-kafka/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-connectors/connector-kafka/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index 531df31..531df31 100644 --- a/groot-connectors/connector-kafka/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-connectors/connector-kafka/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/MockSource.java b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/MockSource.java index 49cbc5a..61792a1 100644 --- a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/MockSource.java +++ b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/MockSource.java @@ -1,7 +1,7 @@ package com.geedgenetworks.connectors.mock; import com.geedgenetworks.connectors.mock.faker.ObjectFaker; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; diff --git a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/MockTableFactory.java b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/MockTableFactory.java index a978938..43e3364 100644 --- a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/MockTableFactory.java +++ b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/MockTableFactory.java @@ -2,11 +2,11 @@ package com.geedgenetworks.connectors.mock; import com.geedgenetworks.connectors.mock.faker.FakerUtils; import com.geedgenetworks.connectors.mock.faker.ObjectFaker; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.source.SourceTableFactory; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.source.SourceTableFactory; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.commons.io.FileUtils; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ReadableConfig; @@ -24,7 +24,7 @@ public class MockTableFactory implements SourceTableFactory { public static final String IDENTIFIER = "mock"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } diff --git a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java index 4ef63bf..1d3b517 100644 --- a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java +++ b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java @@ -9,8 +9,8 @@ import com.geedgenetworks.connectors.mock.faker.TimestampFaker.*; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; -import com.geedgenetworks.spi.table.type.DataType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.type.DataType; +import com.geedgenetworks.api.connector.type.Types; import org.apache.flink.util.Preconditions; import java.math.BigDecimal; diff --git a/groot-connectors/connector-mock/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-connectors/connector-mock/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index eea834f..eea834f 100644 --- a/groot-connectors/connector-mock/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-connectors/connector-mock/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-connectors/connector-starrocks/src/main/java/com/geedgenetworks/connectors/starrocks/StarRocksTableFactory.java b/groot-connectors/connector-starrocks/src/main/java/com/geedgenetworks/connectors/starrocks/StarRocksTableFactory.java index c7ada13..3bca2fa 100644 --- a/groot-connectors/connector-starrocks/src/main/java/com/geedgenetworks/connectors/starrocks/StarRocksTableFactory.java +++ b/groot-connectors/connector-starrocks/src/main/java/com/geedgenetworks/connectors/starrocks/StarRocksTableFactory.java @@ -1,10 +1,10 @@ package com.geedgenetworks.connectors.starrocks; -import com.geedgenetworks.spi.sink.SinkProvider; -import com.geedgenetworks.spi.sink.SinkTableFactory; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.FactoryUtil; +import com.geedgenetworks.api.connector.sink.SinkProvider; +import com.geedgenetworks.api.connector.sink.SinkTableFactory; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.FactoryUtil; import com.starrocks.connector.flink.table.sink.EventStarRocksDynamicSinkFunctionV2; import com.starrocks.connector.flink.table.sink.SinkFunctionFactory; import com.starrocks.connector.flink.table.sink.StarRocksSinkOptions; @@ -21,7 +21,7 @@ import java.util.Set; public class StarRocksTableFactory implements SinkTableFactory { public static final String IDENTIFIER = "starrocks"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } diff --git a/groot-connectors/connector-starrocks/src/main/java/com/starrocks/connector/flink/table/sink/EventStarRocksDynamicSinkFunctionV2.java b/groot-connectors/connector-starrocks/src/main/java/com/starrocks/connector/flink/table/sink/EventStarRocksDynamicSinkFunctionV2.java index 63920ab..7bf57ab 100644 --- a/groot-connectors/connector-starrocks/src/main/java/com/starrocks/connector/flink/table/sink/EventStarRocksDynamicSinkFunctionV2.java +++ b/groot-connectors/connector-starrocks/src/main/java/com/starrocks/connector/flink/table/sink/EventStarRocksDynamicSinkFunctionV2.java @@ -1,8 +1,8 @@ package com.starrocks.connector.flink.table.sink; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.event.Event; import com.starrocks.connector.flink.manager.StarRocksSinkBufferEntity; import com.starrocks.connector.flink.manager.StarRocksStreamLoadListener; import com.starrocks.connector.flink.tools.EnvUtils; diff --git a/groot-connectors/connector-starrocks/src/main/java/com/starrocks/connector/flink/table/sink/EventStreamLoadListener.java b/groot-connectors/connector-starrocks/src/main/java/com/starrocks/connector/flink/table/sink/EventStreamLoadListener.java index d1aed43..d7b7ef2 100644 --- a/groot-connectors/connector-starrocks/src/main/java/com/starrocks/connector/flink/table/sink/EventStreamLoadListener.java +++ b/groot-connectors/connector-starrocks/src/main/java/com/starrocks/connector/flink/table/sink/EventStreamLoadListener.java @@ -1,6 +1,6 @@ package com.starrocks.connector.flink.table.sink; -import com.geedgenetworks.spi.metrics.InternalMetrics; +import com.geedgenetworks.api.metrics.InternalMetrics; import com.starrocks.connector.flink.manager.StarRocksStreamLoadListener; import com.starrocks.data.load.stream.StreamLoadResponse; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-connectors/connector-starrocks/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-connectors/connector-starrocks/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index d5d12b5..d5d12b5 100644 --- a/groot-connectors/connector-starrocks/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-connectors/connector-starrocks/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-connectors/pom.xml b/groot-connectors/pom.xml index 939f3bb..7fe10f7 100644 --- a/groot-connectors/pom.xml +++ b/groot-connectors/pom.xml @@ -23,7 +23,7 @@ <dependency> <groupId>com.geedgenetworks</groupId> - <artifactId>groot-spi</artifactId> + <artifactId>groot-api</artifactId> <version>${revision}</version> <scope>provided</scope> </dependency> diff --git a/groot-core/pom.xml b/groot-core/pom.xml index fb92c8d..3a8e712 100644 --- a/groot-core/pom.xml +++ b/groot-core/pom.xml @@ -15,7 +15,7 @@ <dependency> <groupId>com.geedgenetworks</groupId> - <artifactId>groot-spi</artifactId> + <artifactId>groot-api</artifactId> <version>${revision}</version> <scope>provided</scope> </dependency> diff --git a/groot-core/src/main/java/com/geedgenetworks/core/connector/inline/InlineSourceProvider.java b/groot-core/src/main/java/com/geedgenetworks/core/connector/inline/InlineSourceProvider.java index d6e2ba9..4ed69b5 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/connector/inline/InlineSourceProvider.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/connector/inline/InlineSourceProvider.java @@ -1,9 +1,9 @@ package com.geedgenetworks.core.connector.inline; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/connector/inline/InlineTableFactory.java b/groot-core/src/main/java/com/geedgenetworks/core/connector/inline/InlineTableFactory.java index 3117ef9..7eab006 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/connector/inline/InlineTableFactory.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/connector/inline/InlineTableFactory.java @@ -1,12 +1,12 @@ package com.geedgenetworks.core.connector.inline; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.source.SourceTableFactory; -import com.geedgenetworks.spi.table.factory.DecodingFormatFactory; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.source.SourceTableFactory; +import com.geedgenetworks.api.factory.DecodingFormatFactory; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.ReadableConfig; @@ -24,7 +24,7 @@ import static org.apache.flink.configuration.ConfigOptions.key; public class InlineTableFactory implements SourceTableFactory { public static final String IDENTIFIER = "inline"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/connector/print/PrintSinkFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/connector/print/PrintSinkFunction.java index f0a20de..22187bd 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/connector/print/PrintSinkFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/connector/print/PrintSinkFunction.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.connector.print; -import com.geedgenetworks.spi.table.connector.EncodingFormat; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/connector/print/PrintTableFactory.java b/groot-core/src/main/java/com/geedgenetworks/core/connector/print/PrintTableFactory.java index cd8a4f6..e558aeb 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/connector/print/PrintTableFactory.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/connector/print/PrintTableFactory.java @@ -1,17 +1,17 @@ package com.geedgenetworks.core.connector.print; -import com.geedgenetworks.spi.table.connector.EncodingFormat; -import com.geedgenetworks.spi.sink.SinkTableFactory; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; +import com.geedgenetworks.api.connector.sink.SinkTableFactory; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.ReadableConfig; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; -import com.geedgenetworks.spi.sink.SinkProvider; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.factory.EncodingFormatFactory; +import com.geedgenetworks.api.connector.sink.SinkProvider; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.factory.EncodingFormatFactory; import java.util.HashSet; import java.util.Optional; import java.util.Set; @@ -24,7 +24,7 @@ import static com.geedgenetworks.core.connector.print.PrintMode.STDOUT; public class PrintTableFactory implements SinkTableFactory { public static final String IDENTIFIER = "print"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/filter/AviatorFilter.java b/groot-core/src/main/java/com/geedgenetworks/core/filter/AviatorFilter.java deleted file mode 100644 index 6f2eb2f..0000000 --- a/groot-core/src/main/java/com/geedgenetworks/core/filter/AviatorFilter.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.geedgenetworks.core.filter; - -import com.alibaba.fastjson.JSONObject; -import com.geedgenetworks.spi.filter.Filter; -import com.geedgenetworks.spi.filter.FilterConfig; -import com.geedgenetworks.spi.table.event.Event; -import com.typesafe.config.Config; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.util.Map; - -public class AviatorFilter implements Filter<FilterConfig> { - - @Override - public DataStream<Event> filterFunction( - DataStream<Event> singleOutputStreamOperator, FilterConfig FilterConfig) - throws Exception { - - if (FilterConfig.getParallelism() != 0) { - return singleOutputStreamOperator - .filter(new FilterFunction(FilterConfig)) - .setParallelism(FilterConfig.getParallelism()) - .name(FilterConfig.getName()); - } else { - return singleOutputStreamOperator - .filter(new FilterFunction(FilterConfig)) - .name(FilterConfig.getName()); - } - } - - @Override - public String type() { - return "aviator"; - } - - @Override - public FilterConfig checkConfig(String name, Map<String, Object> configProperties, Config typeSafeConfig) { - - FilterConfig filterConfig = new JSONObject(configProperties).toJavaObject(FilterConfig.class); - filterConfig.setName(name); - return filterConfig; - } - -} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/pojo/SinkConfigOld.java b/groot-core/src/main/java/com/geedgenetworks/core/pojo/SinkConfigOld.java deleted file mode 100644 index b2d2647..0000000 --- a/groot-core/src/main/java/com/geedgenetworks/core/pojo/SinkConfigOld.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.geedgenetworks.core.pojo; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -public class SinkConfigOld implements Serializable { - - private int parallelism; - private Map<String, Object> properties; - private String format; - private String type; - private List<String> output_fields; - private String name; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public int getParallelism() { - return parallelism; - } - - public void setParallelism(int parallelism) { - this.parallelism = parallelism; - } - - public Map<String, Object> getProperties() { - return properties; - } - - public void setProperties(Map<String, Object> properties) { - this.properties = properties; - } - - public String getFormat() { - return format; - } - - public void setFormat(String format) { - this.format = format; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public List<String> getOutput_fields() { - return output_fields; - } - - public void setOutput_fields(List<String> output_fields) { - this.output_fields = output_fields; - } -} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/pojo/SourceConfigOld.java b/groot-core/src/main/java/com/geedgenetworks/core/pojo/SourceConfigOld.java deleted file mode 100644 index 9186a22..0000000 --- a/groot-core/src/main/java/com/geedgenetworks/core/pojo/SourceConfigOld.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.geedgenetworks.core.pojo; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -public class SourceConfigOld implements Serializable { - - private String name; - private int parallelism; - private Map<String, Object> properties; - private String watermark_timestamp; - private Long watermark_lag; - private String type; - private String format; - private List<String> output_fields; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public int getParallelism() { - return parallelism; - } - - public void setParallelism(int parallelism) { - this.parallelism = parallelism; - } - - public Map<String, Object> getProperties() { - return properties; - } - - public void setProperties(Map<String, Object> properties) { - this.properties = properties; - } - - public String getWatermark_timestamp() { - return watermark_timestamp; - } - - public void setWatermark_timestamp(String watermark_timestamp) { - this.watermark_timestamp = watermark_timestamp; - } - - public Long getWatermark_lag() { - return watermark_lag; - } - - public void setWatermark_lag(Long watermark_lag) { - this.watermark_lag = watermark_lag; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getFormat() { - return format; - } - - public void setFormat(String format) { - this.format = format; - } - - public List<String> getOutput_fields() { - return output_fields; - } - - public void setOutput_fields(List<String> output_fields) { - this.output_fields = output_fields; - } -} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/pojo/Topology.java b/groot-core/src/main/java/com/geedgenetworks/core/pojo/Topology.java index e40442b..fffbca6 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/pojo/Topology.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/pojo/Topology.java @@ -1,26 +1,13 @@ package com.geedgenetworks.core.pojo; +import lombok.Data; + import java.io.Serializable; import java.util.List; +@Data public class Topology implements Serializable { private int parallelism; private List<String> next; - - public int getParallelism() { - return parallelism; - } - - public void setParallelism(int parallelism) { - this.parallelism = parallelism; - } - - public List<String> getNext() { - return next; - } - - public void setNext(List<String> next) { - this.next = next; - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/pojo/YamlEntity.java b/groot-core/src/main/java/com/geedgenetworks/core/pojo/YamlEntity.java index 94e5ae5..1d69479 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/pojo/YamlEntity.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/pojo/YamlEntity.java @@ -1,8 +1,11 @@ package com.geedgenetworks.core.pojo; +import lombok.Data; + import java.io.Serializable; import java.util.Map; +@Data public class YamlEntity implements Serializable { private Map<String, Object> sources; @@ -12,60 +15,4 @@ public class YamlEntity implements Serializable { private Map<String, Object> pre_processing_pipelines; private Map<String, Object> processing_pipelines; private Map<String, Object> post_processing_pipelines; - - public Map<String, Object> getSources() { - return sources; - } - - public void setSources(Map<String, Object> sources) { - this.sources = sources; - } - - public Map<String, Object> getApplication() { - return application; - } - - public void setApplication(Map<String, Object> application) { - this.application = application; - } - - public Map<String, Object> getSinks() { - return sinks; - } - - public void setSinks(Map<String, Object> sinks) { - this.sinks = sinks; - } - - public Map<String, Object> getFilters() { - return filters; - } - - public void setFilters(Map<String, Object> filters) { - this.filters = filters; - } - - public Map<String, Object> getPre_processing_pipelines() { - return pre_processing_pipelines; - } - - public void setPre_processing_pipelines(Map<String, Object> pre_processing_pipelines) { - this.pre_processing_pipelines = pre_processing_pipelines; - } - - public Map<String, Object> getProcessing_pipelines() { - return processing_pipelines; - } - - public void setProcessing_pipelines(Map<String, Object> processing_pipelines) { - this.processing_pipelines = processing_pipelines; - } - - public Map<String, Object> getPost_processing_pipelines() { - return post_processing_pipelines; - } - - public void setPost_processing_pipelines(Map<String, Object> post_processing_pipelines) { - this.post_processing_pipelines = post_processing_pipelines; - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AbstractFirstAggregation.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AbstractFirstAggregation.java index 125ee7e..4dcea9d 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AbstractFirstAggregation.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AbstractFirstAggregation.java @@ -7,12 +7,11 @@ import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.KeybyEntity; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.common.udf.UdfEntity; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.processor.AggregateConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UdfEntity; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.event.Event; import com.google.common.collect.Lists; import com.googlecode.aviator.AviatorEvaluator; import com.googlecode.aviator.AviatorEvaluatorInstance; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/AggregateConfig.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateConfig.java index 1d64d0d..e8d5c26 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/AggregateConfig.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateConfig.java @@ -1,7 +1,8 @@ -package com.geedgenetworks.spi.processor; +package com.geedgenetworks.core.processor.aggregate; import com.alibaba.fastjson2.annotation.JSONField; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.processor.ProcessorConfig; import lombok.Data; import lombok.EqualsAndHashCode; @@ -10,7 +11,8 @@ import java.util.List; @EqualsAndHashCode(callSuper = true) @Data public class AggregateConfig extends ProcessorConfig { - + private List<String> output_fields; + private List<String> remove_fields; private List<String> group_by_fields; private String window_timestamp_field; private String window_type; diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/AggregateConfigOptions.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateConfigOptions.java index 116ffa7..4f88d55 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/AggregateConfigOptions.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateConfigOptions.java @@ -1,9 +1,9 @@ -package com.geedgenetworks.spi.configuration; +package com.geedgenetworks.core.processor.aggregate; import com.alibaba.fastjson2.TypeReference; +import com.geedgenetworks.api.common.udf.UDFContext; import com.geedgenetworks.common.config.Option; import com.geedgenetworks.common.config.Options; -import com.geedgenetworks.spi.common.udf.UDFContext; import java.util.List; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java index cbc80a2..498d833 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java @@ -6,11 +6,10 @@ import com.geedgenetworks.common.config.CheckResult; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.configuration.AggregateConfigOptions; -import com.geedgenetworks.spi.processor.AggregateConfig; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.processor.Processor; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.processor.Processor; import com.typesafe.config.Config; +import com.typesafe.config.ConfigUtil; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; @@ -103,13 +102,9 @@ public class AggregateProcessor implements Processor<AggregateConfig> { @Override - public String type() { - return "aggregate"; - } + public AggregateConfig parseConfig(String name, Config config) { - @Override - public AggregateConfig checkConfig(String name, Map<String, Object> configProperties, Config typeSafeConfig) { - CheckResult result = CheckConfigUtil.checkAllExists(typeSafeConfig.getConfig(name), + CheckResult result = CheckConfigUtil.checkAllExists(config, AggregateConfigOptions.GROUP_BY_FIELDS.key(), AggregateConfigOptions.WINDOW_TYPE.key(), AggregateConfigOptions.FUNCTIONS.key(), @@ -123,7 +118,7 @@ public class AggregateProcessor implements Processor<AggregateConfig> { AggregateConfigOptions.FUNCTIONS.key()))); } - AggregateConfig aggregateConfig = new JSONObject(configProperties).toJavaObject(AggregateConfig.class); + AggregateConfig aggregateConfig = new JSONObject(config.root().unwrapped()).toJavaObject(AggregateConfig.class); aggregateConfig.setName(name); return aggregateConfig; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFactory.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFactory.java new file mode 100644 index 0000000..03cc1e1 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFactory.java @@ -0,0 +1,30 @@ +package com.geedgenetworks.core.processor.aggregate; + +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.factory.ProcessorFactory; +import org.apache.flink.configuration.ConfigOption; + +import java.util.Set; + +public class AggregateProcessorFactory implements ProcessorFactory { + + @Override + public String type() { + return "aggregate"; + } + + @Override + public Set<ConfigOption<?>> requiredOptions() { + return Set.of(); + } + + @Override + public Set<ConfigOption<?>> optionalOptions() { + return Set.of(); + } + + @Override + public Processor<?> createProcessor() { + return new AggregateProcessor(); + } +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java index 0b22faa..cf54c3f 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java @@ -5,11 +5,10 @@ import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.common.udf.UdfEntity; -import com.geedgenetworks.spi.processor.AggregateConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UdfEntity; +import com.geedgenetworks.api.connector.event.Event; import com.google.common.collect.Lists; import com.googlecode.aviator.AviatorEvaluator; import com.googlecode.aviator.AviatorEvaluatorInstance; @@ -17,7 +16,6 @@ import com.googlecode.aviator.Expression; import com.googlecode.aviator.Options; import com.googlecode.aviator.exception.ExpressionRuntimeException; import lombok.extern.slf4j.Slf4j; -import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import java.util.HashMap; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/FirstAggregationEventTime.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/FirstAggregationEventTime.java index 269428a..6e53bd1 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/FirstAggregationEventTime.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/FirstAggregationEventTime.java @@ -3,8 +3,7 @@ package com.geedgenetworks.core.processor.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.config.KeybyEntity; -import com.geedgenetworks.spi.processor.AggregateConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.ProcessFunction; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/FirstAggregationProcessingTime.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/FirstAggregationProcessingTime.java index 77a95ee..2cd7c61 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/FirstAggregationProcessingTime.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/FirstAggregationProcessingTime.java @@ -4,8 +4,7 @@ package com.geedgenetworks.core.processor.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.config.KeybyEntity; -import com.geedgenetworks.spi.processor.AggregateConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.functions.ProcessFunction; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/KeySelector.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/KeySelector.java index ba9c0ca..2b5f1e3 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/KeySelector.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/KeySelector.java @@ -2,7 +2,7 @@ package com.geedgenetworks.core.processor.aggregate; import cn.hutool.crypto.SecureUtil; import com.geedgenetworks.common.config.KeybyEntity; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import java.util.HashMap; import java.util.List; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunction.java index 2aab0e6..7e1bc8c 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunction.java @@ -3,15 +3,14 @@ package com.geedgenetworks.core.processor.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.config.KeybyEntity; import com.geedgenetworks.common.utils.ColumnUtil; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.processor.AggregateConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.windowing.windows.TimeWindow; import org.apache.flink.util.Collector; -import static com.geedgenetworks.spi.table.event.Event.WINDOW_END_TIMESTAMP; -import static com.geedgenetworks.spi.table.event.Event.WINDOW_START_TIMESTAMP; +import static com.geedgenetworks.api.connector.event.Event.WINDOW_END_TIMESTAMP; +import static com.geedgenetworks.api.connector.event.Event.WINDOW_START_TIMESTAMP; public class ProcessWindowFunction extends org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction< Accumulator, // 输入类型 diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/SecondAggregateProcessorFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/SecondAggregateProcessorFunction.java index b9b53b7..86cf3f6 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/SecondAggregateProcessorFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/SecondAggregateProcessorFunction.java @@ -5,10 +5,9 @@ import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.common.udf.UdfEntity; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.processor.AggregateConfig; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UdfEntity; +import com.geedgenetworks.api.common.udf.AggregateFunction; import com.google.common.collect.Lists; import com.googlecode.aviator.AviatorEvaluator; import com.googlecode.aviator.AviatorEvaluatorInstance; @@ -16,7 +15,6 @@ import com.googlecode.aviator.Expression; import com.googlecode.aviator.Options; import com.googlecode.aviator.exception.ExpressionRuntimeException; import lombok.extern.slf4j.Slf4j; -import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import java.util.HashMap; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/AviatorFilterProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/AviatorFilterProcessor.java new file mode 100644 index 0000000..8953c94 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/AviatorFilterProcessor.java @@ -0,0 +1,32 @@ +package com.geedgenetworks.core.processor.filter; + +import com.alibaba.fastjson.JSONObject; +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.connector.event.Event; +import com.typesafe.config.Config; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +public class AviatorFilterProcessor implements Processor<FilterConfig> { + + @Override + public DataStream<Event> process(StreamExecutionEnvironment env, + DataStream<Event> input, FilterConfig FilterConfig) { + SingleOutputStreamOperator<Event> resultStream = + input.filter(new FilterFunction(FilterConfig)).name(FilterConfig.getName()); + + if (FilterConfig.getParallelism() != 0) { + resultStream.setParallelism(FilterConfig.getParallelism()); + } + return resultStream; + } + + @Override + public FilterConfig parseConfig(String name, Config config) { + FilterConfig filterConfig = new JSONObject(config.root().unwrapped()).toJavaObject(FilterConfig.class); + filterConfig.setName(name); + return filterConfig; + } + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/AviatorFilterProcessorFactory.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/AviatorFilterProcessorFactory.java new file mode 100644 index 0000000..ea0c60b --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/AviatorFilterProcessorFactory.java @@ -0,0 +1,30 @@ +package com.geedgenetworks.core.processor.filter; + +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.factory.ProcessorFactory; +import org.apache.flink.configuration.ConfigOption; + +import java.util.Set; + +public class AviatorFilterProcessorFactory implements ProcessorFactory { + + @Override + public Processor<?> createProcessor() { + return new AviatorFilterProcessor(); + } + + @Override + public String type() { + return "aviator"; + } + + @Override + public Set<ConfigOption<?>> requiredOptions() { + return Set.of(); + } + + @Override + public Set<ConfigOption<?>> optionalOptions() { + return Set.of(); + } +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/FilterConfig.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/FilterConfig.java new file mode 100644 index 0000000..6291860 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/FilterConfig.java @@ -0,0 +1,13 @@ +package com.geedgenetworks.core.processor.filter; + +import com.geedgenetworks.api.processor.ProcessorConfig; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +public class FilterConfig extends ProcessorConfig { + private List<String> output_fields; +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/FilterConfigOptions.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/FilterConfigOptions.java new file mode 100644 index 0000000..3f7e01a --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/FilterConfigOptions.java @@ -0,0 +1,15 @@ +package com.geedgenetworks.core.processor.filter; + +import com.geedgenetworks.common.config.Option; +import com.geedgenetworks.common.config.Options; + +import java.util.List; +import java.util.Map; + +public interface FilterConfigOptions { + Option<List<String>> OUTPUT_FIELDS = Options.key("output_fields") + .listType() + .noDefaultValue() + .withDescription("The fields to be outputted."); + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/filter/FilterFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/FilterFunction.java index ec6a13c..9d5d6f3 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/filter/FilterFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/filter/FilterFunction.java @@ -1,9 +1,8 @@ -package com.geedgenetworks.core.filter; +package com.geedgenetworks.core.processor.filter; import com.geedgenetworks.common.utils.ColumnUtil; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.filter.FilterConfig; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.event.Event; import com.googlecode.aviator.AviatorEvaluator; import com.googlecode.aviator.AviatorEvaluatorInstance; import com.googlecode.aviator.Expression; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionConfig.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionConfig.java new file mode 100644 index 0000000..fd7ff41 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionConfig.java @@ -0,0 +1,17 @@ +package com.geedgenetworks.core.processor.projection; + +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.processor.ProcessorConfig; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +public class ProjectionConfig extends ProcessorConfig { + private List<String> output_fields; + private List<String> remove_fields; + private List<UDFContext> functions; + +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/ProjectionConfigOptions.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionConfigOptions.java index d0ade99..0607e16 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/ProjectionConfigOptions.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionConfigOptions.java @@ -1,17 +1,13 @@ -package com.geedgenetworks.spi.configuration; +package com.geedgenetworks.core.processor.projection; import com.alibaba.fastjson2.TypeReference; +import com.geedgenetworks.api.common.udf.UDFContext; import com.geedgenetworks.common.config.Option; import com.geedgenetworks.common.config.Options; -import com.geedgenetworks.spi.common.udf.UDFContext; import java.util.List; public interface ProjectionConfigOptions { - Option<String> TYPE = Options.key("type") - .stringType() - .noDefaultValue() - .withDescription("The type of processor."); Option<List<String>> OUTPUT_FIELDS = Options.key("output_fields") .listType() diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessFactory.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessFactory.java deleted file mode 100644 index b8b4201..0000000 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessFactory.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.geedgenetworks.core.processor.projection; - -import com.geedgenetworks.spi.processor.Processor; -import com.geedgenetworks.spi.processor.ProcessorFactory; - -public class ProjectionProcessFactory implements ProcessorFactory { - - @Override - public String type() { - return "projection"; - } - - @Override - public Processor<?> createProcessor() { - return new ProjectionProcessor(); - } -} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessFunction.java index 004bd78..db0070f 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessFunction.java @@ -8,12 +8,11 @@ import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.utils.ColumnUtil; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseScheduler; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.common.udf.UdfEntity; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.processor.ProjectionConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UdfEntity; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.event.Event; import com.google.common.collect.Lists; import com.googlecode.aviator.AviatorEvaluator; import com.googlecode.aviator.AviatorEvaluatorInstance; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java index d182bd8..eb32786 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java @@ -5,41 +5,31 @@ import com.geedgenetworks.common.config.CheckConfigUtil; import com.geedgenetworks.common.config.CheckResult; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; - -import com.geedgenetworks.spi.configuration.ProjectionConfigOptions; -import com.geedgenetworks.spi.processor.Processor; -import com.geedgenetworks.spi.processor.ProjectionConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.connector.event.Event; import com.typesafe.config.Config; import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import java.util.Map; - public class ProjectionProcessor implements Processor<ProjectionConfig> { - @Override - public DataStream<Event> process(StreamExecutionEnvironment env, DataStream<Event> input, ProjectionConfig projectionConfig) { + public DataStream<Event> process(StreamExecutionEnvironment env, + DataStream<Event> input, ProjectionConfig projectionConfig) { + SingleOutputStreamOperator<Event> resultStream = + input.process(new ProjectionProcessFunction(projectionConfig)).name(projectionConfig.getName()); + if (projectionConfig.getParallelism() != 0) { - return input - .process(new ProjectionProcessFunction(projectionConfig)) - .setParallelism(projectionConfig.getParallelism()) - .name(projectionConfig.getName()); - } else { - return input - .process(new ProjectionProcessFunction(projectionConfig)) - .name(projectionConfig.getName()); + resultStream.setParallelism(projectionConfig.getParallelism()); } - } - @Override - public String type() { - return "projection"; + return resultStream; } @Override - public ProjectionConfig checkConfig(String name, Map<String, Object> configProperties, Config typeSafeConfig) { - CheckResult result = CheckConfigUtil.checkAtLeastOneExists(typeSafeConfig.getConfig(name), + public ProjectionConfig parseConfig(String name, Config config) { + + CheckResult result = CheckConfigUtil.checkAtLeastOneExists(config, ProjectionConfigOptions.OUTPUT_FIELDS.key(), ProjectionConfigOptions.REMOVE_FIELDS.key(), ProjectionConfigOptions.FUNCTIONS.key()); @@ -52,7 +42,7 @@ public class ProjectionProcessor implements Processor<ProjectionConfig> { ProjectionConfigOptions.FUNCTIONS.key()))); } - ProjectionConfig projectionConfig = new JSONObject(configProperties).toJavaObject(ProjectionConfig.class); + ProjectionConfig projectionConfig = new JSONObject(config.root().unwrapped()).toJavaObject(ProjectionConfig.class); projectionConfig.setName(name); return projectionConfig; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorFactory.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorFactory.java new file mode 100644 index 0000000..706eeea --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorFactory.java @@ -0,0 +1,30 @@ +package com.geedgenetworks.core.processor.projection; + +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.factory.ProcessorFactory; +import org.apache.flink.configuration.ConfigOption; + +import java.util.Set; + +public class ProjectionProcessorFactory implements ProcessorFactory { + + @Override + public Processor<?> createProcessor() { + return new ProjectionProcessor(); + } + + @Override + public String type() { + return "projection"; + } + + @Override + public Set<ConfigOption<?>> requiredOptions() { + return Set.of(); + } + + @Override + public Set<ConfigOption<?>> optionalOptions() { + return Set.of(); + } +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/RuleContext.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/RuleContext.java index a6b9b98..99076f9 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/common/udf/RuleContext.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/RuleContext.java @@ -1,6 +1,6 @@ -package com.geedgenetworks.spi.common.udf; +package com.geedgenetworks.core.processor.split; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import com.googlecode.aviator.Expression; import lombok.Data; import org.apache.flink.util.OutputTag; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitConfig.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitConfig.java new file mode 100644 index 0000000..908cca4 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitConfig.java @@ -0,0 +1,13 @@ +package com.geedgenetworks.core.processor.split; + +import com.geedgenetworks.api.processor.ProcessorConfig; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +public class SplitConfig extends ProcessorConfig { + private List<RuleContext> rules; +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/split/SplitConfigOptions.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitConfigOptions.java index 8c62328..51bb90c 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/split/SplitConfigOptions.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitConfigOptions.java @@ -1,18 +1,12 @@ -package com.geedgenetworks.spi.split; +package com.geedgenetworks.core.processor.split; import com.alibaba.fastjson2.TypeReference; import com.geedgenetworks.common.config.Option; import com.geedgenetworks.common.config.Options; -import com.geedgenetworks.spi.common.udf.RuleContext; import java.util.List; public interface SplitConfigOptions { - Option<String> TYPE = Options.key("type") - .stringType() - .noDefaultValue() - .withDescription("The type of route ."); - Option<List<RuleContext>> RULES = Options.key("rules") .type(new TypeReference<List<RuleContext>>() {}) .noDefaultValue() diff --git a/groot-core/src/main/java/com/geedgenetworks/core/split/SplitFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitFunction.java index 64465a2..2e0fda6 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/split/SplitFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitFunction.java @@ -1,9 +1,7 @@ -package com.geedgenetworks.core.split; +package com.geedgenetworks.core.processor.split; -import com.geedgenetworks.spi.common.udf.RuleContext; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.split.SplitConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.event.Event; import com.googlecode.aviator.AviatorEvaluator; import com.googlecode.aviator.AviatorEvaluatorInstance; import com.googlecode.aviator.Expression; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitProcessor.java new file mode 100644 index 0000000..e4ecb18 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitProcessor.java @@ -0,0 +1,35 @@ +package com.geedgenetworks.core.processor.split; + +import com.alibaba.fastjson.JSONObject; +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.connector.event.Event; +import com.typesafe.config.Config; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +import java.util.Map; + +public class SplitProcessor implements Processor<SplitConfig> { + + @Override + public DataStream<Event> process(StreamExecutionEnvironment env, + DataStream<Event> input, SplitConfig splitConfig) { + + SingleOutputStreamOperator<Event> resultStream = + input.process(new SplitFunction(splitConfig)).name(splitConfig.getName()); + + if (splitConfig.getParallelism() != 0) { + resultStream.setParallelism(splitConfig.getParallelism()); + } + return resultStream; + } + + @Override + public SplitConfig parseConfig(String name, Config config) { + SplitConfig splitConfig = new JSONObject(config.root().unwrapped()).toJavaObject(SplitConfig.class); + splitConfig.setName(name); + return splitConfig; + } + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitProcessorFactory.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitProcessorFactory.java new file mode 100644 index 0000000..ff85a45 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/split/SplitProcessorFactory.java @@ -0,0 +1,30 @@ +package com.geedgenetworks.core.processor.split; + +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.factory.ProcessorFactory; +import org.apache.flink.configuration.ConfigOption; + +import java.util.Set; + +public class SplitProcessorFactory implements ProcessorFactory { + + @Override + public Processor<?> createProcessor() { + return new SplitProcessor(); + } + + @Override + public String type() { + return "split"; + } + + @Override + public Set<ConfigOption<?>> requiredOptions() { + return Set.of(); + } + + @Override + public Set<ConfigOption<?>> optionalOptions() { + return Set.of(); + } +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableConfig.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableConfig.java new file mode 100644 index 0000000..e3c483a --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableConfig.java @@ -0,0 +1,16 @@ +package com.geedgenetworks.core.processor.table; + +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.processor.ProcessorConfig; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +public class TableConfig extends ProcessorConfig { + private List<String> output_fields; + private List<String> remove_fields; + private List<UDFContext> functions; +} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/TableConfigOptions.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableConfigOptions.java index 758d8ec..0c46d1f 100644 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/TableConfigOptions.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableConfigOptions.java @@ -1,17 +1,13 @@ -package com.geedgenetworks.spi.configuration; +package com.geedgenetworks.core.processor.table; import com.alibaba.fastjson2.TypeReference; +import com.geedgenetworks.api.common.udf.UDFContext; import com.geedgenetworks.common.config.Option; import com.geedgenetworks.common.config.Options; -import com.geedgenetworks.spi.common.udf.UDFContext; import java.util.List; public interface TableConfigOptions { - Option<String> TYPE = Options.key("type") - .stringType() - .noDefaultValue() - .withDescription("The type of processor."); Option<List<String>> OUTPUT_FIELDS = Options.key("output_fields") .listType() diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java index 1db565b..273f6de 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java @@ -5,12 +5,11 @@ import com.geedgenetworks.common.config.CheckConfigUtil; import com.geedgenetworks.common.config.CheckResult; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.spi.configuration.TableConfigOptions; -import com.geedgenetworks.spi.processor.Processor; -import com.geedgenetworks.spi.processor.TableConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.connector.event.Event; import com.typesafe.config.Config; import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import java.util.Map; @@ -20,26 +19,19 @@ public class TableProcessor implements Processor<TableConfig> { @Override public DataStream<Event> process(StreamExecutionEnvironment env, DataStream<Event> input, TableConfig tableConfig) { + + SingleOutputStreamOperator<Event> resultStream = + input.flatMap(new TableProcessorFunction(tableConfig)).name(tableConfig.getName()); + if (tableConfig.getParallelism() != 0) { - return input - .flatMap(new TableProcessorFunction(tableConfig)) - .setParallelism(tableConfig.getParallelism()) - .name(tableConfig.getName()); - } else { - return input - .flatMap(new TableProcessorFunction(tableConfig)) - .name(tableConfig.getName()); + resultStream.setParallelism(tableConfig.getParallelism()); } + return resultStream; } @Override - public String type() { - return "table"; - } - - @Override - public TableConfig checkConfig(String name, Map<String, Object> configProperties, Config typeSafeConfig) { - CheckResult result = CheckConfigUtil.checkAtLeastOneExists(typeSafeConfig.getConfig(name), + public TableConfig parseConfig(String name, Config config) { + CheckResult result = CheckConfigUtil.checkAtLeastOneExists(config, TableConfigOptions.OUTPUT_FIELDS.key(), TableConfigOptions.REMOVE_FIELDS.key(), TableConfigOptions.FUNCTIONS.key()); @@ -52,7 +44,7 @@ public class TableProcessor implements Processor<TableConfig> { TableConfigOptions.FUNCTIONS.key()))); } - TableConfig tableConfig = new JSONObject(configProperties).toJavaObject(TableConfig.class); + TableConfig tableConfig = new JSONObject(config.root().unwrapped()).toJavaObject(TableConfig.class); tableConfig.setName(name); return tableConfig; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFactory.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFactory.java new file mode 100644 index 0000000..c9e1e81 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFactory.java @@ -0,0 +1,29 @@ +package com.geedgenetworks.core.processor.table; +import com.geedgenetworks.api.processor.Processor; +import com.geedgenetworks.api.factory.ProcessorFactory; +import org.apache.flink.configuration.ConfigOption; + +import java.util.Set; + +public class TableProcessorFactory implements ProcessorFactory { + + @Override + public Processor<?> createProcessor() { + return new TableProcessor(); + } + + @Override + public String type() { + return "table"; + } + + @Override + public Set<ConfigOption<?>> requiredOptions() { + return Set.of(); + } + + @Override + public Set<ConfigOption<?>> optionalOptions() { + return Set.of(); + } +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java index ecb76f4..b840739 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java @@ -5,12 +5,11 @@ import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.utils.ColumnUtil; -import com.geedgenetworks.spi.common.udf.TableFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.common.udf.UdfEntity; -import com.geedgenetworks.spi.metrics.InternalMetrics; -import com.geedgenetworks.spi.processor.TableConfig; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.TableFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UdfEntity; +import com.geedgenetworks.api.metrics.InternalMetrics; +import com.geedgenetworks.api.connector.event.Event; import com.google.common.collect.Lists; import com.googlecode.aviator.AviatorEvaluator; import com.googlecode.aviator.AviatorEvaluatorInstance; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/split/SplitOperator.java b/groot-core/src/main/java/com/geedgenetworks/core/split/SplitOperator.java deleted file mode 100644 index bed2ebe..0000000 --- a/groot-core/src/main/java/com/geedgenetworks/core/split/SplitOperator.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.geedgenetworks.core.split; - -import com.alibaba.fastjson.JSONObject; -import com.geedgenetworks.spi.split.Split; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.split.SplitConfig; -import com.typesafe.config.Config; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.util.Map; - -public class SplitOperator implements Split<SplitConfig> { - - @Override - public DataStream<Event> splitFunction( - DataStream<Event> dataStream, SplitConfig splitConfig) - throws Exception { - if (splitConfig.getParallelism() != 0) { - return dataStream - .process(new SplitFunction(splitConfig)) - .setParallelism(splitConfig.getParallelism()) - .name(splitConfig.getName()); - } else { - return dataStream - .process(new SplitFunction(splitConfig)) - .name(splitConfig.getName()); - } - } - @Override - public String type() { - return "split"; - } - - @Override - public SplitConfig checkConfig(String name, Map<String, Object> configProperties, Config config) { - SplitConfig splitConfig = new JSONObject(configProperties).toJavaObject(SplitConfig.class); - splitConfig.setName(name); - return splitConfig; - } - -} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java index d0b5ac2..808c6b2 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/AsnLookup.java @@ -8,11 +8,11 @@ import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.udf.knowlegdebase.handler.AsnKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.configuration.CheckUDFContextUtil; -import com.geedgenetworks.spi.configuration.UDFContextConfigOptions; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.configuration.CheckUDFContextUtil; +import com.geedgenetworks.api.configuration.UDFContextConfigOptions; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/CurrentUnixTimestamp.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/CurrentUnixTimestamp.java index 65727ab..e59a7c9 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/CurrentUnixTimestamp.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/CurrentUnixTimestamp.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/DecodeBase64.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/DecodeBase64.java index 41c93e7..f1ff72f 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/DecodeBase64.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/DecodeBase64.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import com.geedgenetworks.utils.StringUtil; import lombok.extern.slf4j.Slf4j; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Domain.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Domain.java index 52f05a8..5e41135 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Domain.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Domain.java @@ -3,11 +3,11 @@ import com.geedgenetworks.common.config.CheckResult; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.shaded.com.google.common.net.InternetDomainName; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.configuration.CheckUDFContextUtil; -import com.geedgenetworks.spi.configuration.UDFContextConfigOptions; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.configuration.CheckUDFContextUtil; +import com.geedgenetworks.api.configuration.UDFContextConfigOptions; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; import java.util.List; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Drop.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Drop.java index 059dc16..8b1c7a2 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Drop.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Drop.java @@ -3,11 +3,11 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.config.CheckResult; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.configuration.CheckUDFContextUtil; -import com.geedgenetworks.spi.configuration.UDFContextConfigOptions; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.configuration.CheckUDFContextUtil; +import com.geedgenetworks.api.configuration.UDFContextConfigOptions; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/EncodeBase64.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/EncodeBase64.java index 8ea19d8..7f1fe94 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/EncodeBase64.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/EncodeBase64.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Encrypt.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Encrypt.java index 6f69ab1..80a2460 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Encrypt.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Encrypt.java @@ -15,13 +15,13 @@ import com.geedgenetworks.core.utils.*; import com.geedgenetworks.shaded.org.apache.http.HttpHeaders; import com.geedgenetworks.shaded.org.apache.http.HttpStatus; import com.geedgenetworks.shaded.org.apache.http.message.BasicHeader; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.configuration.CheckUDFContextUtil; -import com.geedgenetworks.spi.configuration.UDFContextConfigOptions; -import com.geedgenetworks.spi.configuration.util.LoadIntervalDataOptions; -import com.geedgenetworks.spi.configuration.util.LoadIntervalDataUtil; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.configuration.CheckUDFContextUtil; +import com.geedgenetworks.api.configuration.UDFContextConfigOptions; +import com.geedgenetworks.api.configuration.util.LoadIntervalDataOptions; +import com.geedgenetworks.api.configuration.util.LoadIntervalDataUtil; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Eval.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Eval.java index a8fdc52..b0d2f73 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Eval.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Eval.java @@ -4,9 +4,9 @@ import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.expressions.Calc; import com.geedgenetworks.core.expressions.EvalExecutor; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import java.util.List; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java index 821124e..38e0e98 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java @@ -4,9 +4,9 @@ import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/FromUnixTimestamp.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/FromUnixTimestamp.java index fab4fed..e31e44f 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/FromUnixTimestamp.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/FromUnixTimestamp.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/GenerateStringArray.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/GenerateStringArray.java index e15b989..7db582b 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/GenerateStringArray.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/GenerateStringArray.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import java.util.*; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java index 67b1d01..9c26527 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/GeoIpLookup.java @@ -9,11 +9,11 @@ import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.alibaba.fastjson2.JSON; import com.geedgenetworks.model.LocationResponse; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.configuration.CheckUDFContextUtil; -import com.geedgenetworks.spi.configuration.UDFContextConfigOptions; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.configuration.CheckUDFContextUtil; +import com.geedgenetworks.api.configuration.UDFContextConfigOptions; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Hmac.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Hmac.java index 43a7622..970d5b4 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Hmac.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Hmac.java @@ -5,11 +5,11 @@ import cn.hutool.crypto.digest.HmacAlgorithm; import com.geedgenetworks.common.config.CheckResult; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.configuration.CheckUDFContextUtil; -import com.geedgenetworks.spi.configuration.UDFContextConfigOptions; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.configuration.CheckUDFContextUtil; +import com.geedgenetworks.api.configuration.UDFContextConfigOptions; +import com.geedgenetworks.api.connector.event.Event; import com.geedgenetworks.utils.StringUtil; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/JsonExtract.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/JsonExtract.java index 69e9d7e..a64b3d2 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/JsonExtract.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/JsonExtract.java @@ -3,9 +3,9 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.utils.JsonPathUtil; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; public class JsonExtract implements ScalarFunction { diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java index 1c75b0c..18aa591 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java @@ -6,9 +6,9 @@ import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CommonConfig; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java index 8271bfa..662129c 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java @@ -3,9 +3,9 @@ package com.geedgenetworks.core.udf; import com.alibaba.fastjson2.JSONArray; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import com.googlecode.aviator.AviatorEvaluator; import com.googlecode.aviator.AviatorEvaluatorInstance; import com.googlecode.aviator.Expression; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/SnowflakeId.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/SnowflakeId.java index 1f85006..520ba77 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/SnowflakeId.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/SnowflakeId.java @@ -3,9 +3,9 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.utils.SnowflakeIdUtils; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import java.io.Serializable; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/StringJoiner.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/StringJoiner.java index 34ea391..1adb68d 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/StringJoiner.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/StringJoiner.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import java.util.List; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java index 7b419f7..04e0cfe 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeScalarFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeScalarFunction.java index 6037b1a..be2f15d 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeScalarFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeScalarFunction.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CommonConfig; import com.geedgenetworks.common.config.KnowledgeBaseConfig; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeWithRuleScalarFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeWithRuleScalarFunction.java index d7b5045..fd32559 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeWithRuleScalarFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AbstractKnowledgeWithRuleScalarFunction.java @@ -3,7 +3,7 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CommonConfig; import com.geedgenetworks.common.config.KnowledgeBaseConfig; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AnonymityLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AnonymityLookup.java index 1b7ceb2..eb6d66b 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AnonymityLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AnonymityLookup.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.IocDarkwebKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.handler.RuleKnowledgeBaseHandler; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AppCategoryLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AppCategoryLookup.java index 849ceff..89b3a6a 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AppCategoryLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/AppCategoryLookup.java @@ -2,8 +2,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.AppCategoryKnowledgeBaseHandler; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/ArrayElementsPrepend.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/ArrayElementsPrepend.java index 2ad8f1b..dcc2da2 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/ArrayElementsPrepend.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/ArrayElementsPrepend.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import java.util.ArrayList; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/BaseStationLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/BaseStationLookup.java index 3c4ea5b..09e141d 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/BaseStationLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/BaseStationLookup.java @@ -4,8 +4,8 @@ import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.BaseStationKnowledgeBaseHandler; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import java.util.List; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookup.java index 3588053..dc9d27f 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookup.java @@ -2,7 +2,7 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.DnsServerInfoKnowledgeBaseHandler; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import java.util.List; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FieldsMerge.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FieldsMerge.java index 5d5631a..0f705b7 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FieldsMerge.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FieldsMerge.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import java.util.ArrayList; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookup.java index bc4ff46..2e88d99 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookup.java @@ -2,8 +2,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.FqdnCategoryKnowledgeBaseHandler; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookup.java index 1384b41..e06714e 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookup.java @@ -2,7 +2,7 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.FqdnWhoisKnowledgeBaseHandler; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; /** * @author gujinkai diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/H3CellLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/H3CellLookup.java index bda310f..c6f4e62 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/H3CellLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/H3CellLookup.java @@ -2,10 +2,10 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import com.uber.h3core.H3Core; -import com.geedgenetworks.spi.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.ScalarFunction; import org.apache.flink.api.common.functions.RuntimeContext; import java.io.IOException; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IcpLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IcpLookup.java index 3eb24af..8eaadc6 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IcpLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IcpLookup.java @@ -2,7 +2,7 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.FqdnIcpKnowledgeBaseHandler; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; /** * @author gujinkai diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IdcRenterLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IdcRenterLookup.java index 3733bce..7269845 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IdcRenterLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IdcRenterLookup.java @@ -2,7 +2,7 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.IdcRenterKnowledgeBaseHandler; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; /** * @author gujinkai diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookup.java index b6253d0..d6f95ee 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookup.java @@ -2,8 +2,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.IntelligenceIndicatorKnowledgeBaseHandler; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IocLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IocLookup.java index a10a134..9cc88cd 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IocLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IocLookup.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.IocMalwareKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.handler.RuleKnowledgeBaseHandler; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IpZoneLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IpZoneLookup.java index 1ce4a94..8803197 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IpZoneLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/IpZoneLookup.java @@ -2,7 +2,7 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.InternalIpKnowledgeBaseHandler; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; /** * @author gujinkai diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/L7ProtocolAndAppExtract.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/L7ProtocolAndAppExtract.java index feb38e9..fcbb53a 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/L7ProtocolAndAppExtract.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/L7ProtocolAndAppExtract.java @@ -1,9 +1,9 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookup.java index c825636..77de477 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookup.java @@ -2,7 +2,7 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.LinkDirectionKnowledgeBaseHandler; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; /** * @author gujinkai diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookup.java index 7456a3b..d926996 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookup.java @@ -2,8 +2,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.*; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.apache.flink.metrics.Counter; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/VpnLookup.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/VpnLookup.java index 0f5041f..50e6586 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/VpnLookup.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/cn/VpnLookup.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.cn; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; import com.geedgenetworks.core.udf.knowlegdebase.handler.DomainVpnKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.handler.IpVpnKnowledgeBaseHandler; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java index 89075be..3624527 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java @@ -3,9 +3,9 @@ package com.geedgenetworks.core.udf.udaf; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import java.util.ArrayList; import java.util.List; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java index c8bb84e..b16ae7e 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java @@ -4,9 +4,9 @@ package com.geedgenetworks.core.udf.udaf; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import java.util.HashSet; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java index f438599..1f8698f 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java @@ -19,9 +19,9 @@ package com.geedgenetworks.core.udf.udaf; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; /** * Collects elements within a group and returns the list of aggregated objects diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java index c67ac3a..ee31f58 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java @@ -2,7 +2,7 @@ package com.geedgenetworks.core.udf.udaf.HdrHistogram; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.sketch.util.StringUtils; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; import org.HdrHistogram.Histogramer; import java.util.Map; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java index 575e561..1e26896 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf.udaf.HdrHistogram; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.sketch.util.StringUtils; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.HdrHistogram.ArrayHistogram; import org.HdrHistogram.DirectMapHistogram; import org.HdrHistogram.Histogramer; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java index 39d464d..724e21b 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.udaf.HdrHistogram; import com.geedgenetworks.common.config.Accumulator; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; import org.HdrHistogram.Histogramer; import java.util.Map; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java index 08ed5b9..e4e9b09 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java @@ -2,7 +2,7 @@ package com.geedgenetworks.core.udf.udaf.HdrHistogram; import com.alibaba.fastjson2.JSON; import com.geedgenetworks.common.config.Accumulator; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; import org.HdrHistogram.Histogramer; import java.util.ArrayList; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java index eda3673..a3b8382 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java @@ -19,9 +19,9 @@ package com.geedgenetworks.core.udf.udaf; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; /** * Collects elements within a group and returns the list of aggregated objects diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java index 953c4fe..9cffe70 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java @@ -3,9 +3,9 @@ package com.geedgenetworks.core.udf.udaf; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; public class LongCount implements AggregateFunction { diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Max.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Max.java index 4c08748..decb770 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Max.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Max.java @@ -3,9 +3,9 @@ package com.geedgenetworks.core.udf.udaf; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.connector.event.Event; import java.time.LocalDateTime; import java.util.Map; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java index 82ca786..c61a908 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java @@ -4,9 +4,9 @@ import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.pojo.OnlineStatistics; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import java.text.DecimalFormat; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Min.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Min.java index 6893b34..acf4ba8 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Min.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Min.java @@ -4,9 +4,9 @@ package com.geedgenetworks.core.udf.udaf; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import java.time.LocalDateTime; import java.util.Map; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java index 476f045..80adf67 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java @@ -3,9 +3,9 @@ package com.geedgenetworks.core.udf.udaf; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; public class NumberSum implements AggregateFunction { private String lookupField; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java index 367be64..4c1a340 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java @@ -3,7 +3,7 @@ package com.geedgenetworks.core.udf.udaf.hlld; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.sketch.hlld.Hll; import com.geedgenetworks.sketch.util.StringUtils; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; import java.util.Map; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java index 6cffa8a..9d46ed9 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java @@ -4,9 +4,9 @@ import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.sketch.hlld.Hll; import com.geedgenetworks.sketch.hlld.HllUnion; import com.geedgenetworks.sketch.hlld.HllUtils; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.commons.collections.CollectionUtils; import java.util.Map; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java index b33fff5..e50dd12 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java @@ -6,9 +6,9 @@ import com.alibaba.fastjson2.JSONObject; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.utils.JsonPathUtil; -import com.geedgenetworks.spi.common.udf.TableFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.TableFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/PathUnroll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/PathUnroll.java index 9a34006..dcc8bfb 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/PathUnroll.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/PathUnroll.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.udtf; -import com.geedgenetworks.spi.common.udf.TableFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.TableFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java index 4a61851..61bfff1 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf.udtf; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.TableFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.TableFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUID.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUID.java index 5f2dc5d..d5cd7ed 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUID.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUID.java @@ -4,11 +4,11 @@ import com.fasterxml.uuid.Generators; import com.fasterxml.uuid.impl.RandomBasedGenerator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; -import com.geedgenetworks.spi.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.ScalarFunction; @Slf4j diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUIDv5.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUIDv5.java index 961d2bd..6f52928 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUIDv5.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUIDv5.java @@ -4,9 +4,9 @@ import com.fasterxml.uuid.Generators; import com.fasterxml.uuid.impl.NameBasedGenerator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUIDv7.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUIDv7.java index d9c69a7..af03755 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUIDv7.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/uuid/UUIDv7.java @@ -4,9 +4,9 @@ import com.fasterxml.uuid.Generators; import com.fasterxml.uuid.impl.TimeBasedEpochGenerator; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; -import com.geedgenetworks.spi.common.udf.ScalarFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.ScalarFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import lombok.extern.slf4j.Slf4j; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory new file mode 100644 index 0000000..cbb5266 --- /dev/null +++ b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory @@ -0,0 +1,7 @@ +com.geedgenetworks.core.connector.inline.InlineTableFactory +com.geedgenetworks.core.connector.print.PrintTableFactory +com.geedgenetworks.core.processor.filter.AviatorFilterProcessorFactory +com.geedgenetworks.core.processor.split.SplitProcessorFactory +com.geedgenetworks.core.processor.projection.ProjectionProcessorFactory +com.geedgenetworks.core.processor.table.TableProcessorFactory +com.geedgenetworks.core.processor.aggregate.AggregateProcessorFactory
\ No newline at end of file diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.filter.Filter b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.filter.Filter deleted file mode 100644 index 2268533..0000000 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.filter.Filter +++ /dev/null @@ -1 +0,0 @@ -com.geedgenetworks.core.filter.AviatorFilter
\ No newline at end of file diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.processor.Processor b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.processor.Processor deleted file mode 100644 index a7eef58..0000000 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.processor.Processor +++ /dev/null @@ -1,3 +0,0 @@ -com.geedgenetworks.core.processor.aggregate.AggregateProcessor -com.geedgenetworks.core.processor.projection.ProjectionProcessor -com.geedgenetworks.core.processor.table.TableProcessor
\ No newline at end of file diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.processor.ProcessorFactory b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.processor.ProcessorFactory deleted file mode 100644 index fa21381..0000000 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.processor.ProcessorFactory +++ /dev/null @@ -1 +0,0 @@ -com.geedgenetworks.core.processor.projection.ProjectionProcessFactory
\ No newline at end of file diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.split.Split b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.split.Split deleted file mode 100644 index 500c367..0000000 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.split.Split +++ /dev/null @@ -1 +0,0 @@ -com.geedgenetworks.core.split.SplitOperator
\ No newline at end of file diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory deleted file mode 100644 index 6d6a1bb..0000000 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ /dev/null @@ -1,2 +0,0 @@ -com.geedgenetworks.core.connector.inline.InlineTableFactory -com.geedgenetworks.core.connector.print.PrintTableFactory diff --git a/groot-core/src/test/java/com/geedgenetworks/core/connector/schema/SchemaParserTest.java b/groot-core/src/test/java/com/geedgenetworks/core/connector/schema/SchemaParserTest.java index 9baf738..4d42f85 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/connector/schema/SchemaParserTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/connector/schema/SchemaParserTest.java @@ -1,14 +1,12 @@ package com.geedgenetworks.core.connector.schema; -import com.geedgenetworks.spi.table.schema.SchemaParser; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.schema.SchemaParser; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.commons.io.FileUtils; import java.io.File; import java.nio.charset.StandardCharsets; -import static org.junit.jupiter.api.Assertions.*; - class SchemaParserTest { public static void main(String[] args) throws Exception{ diff --git a/groot-core/src/test/java/com/geedgenetworks/core/connector/schema/utils/DynamicSchemaManagerTest.java b/groot-core/src/test/java/com/geedgenetworks/core/connector/schema/utils/DynamicSchemaManagerTest.java index f1607b3..252c2e1 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/connector/schema/utils/DynamicSchemaManagerTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/connector/schema/utils/DynamicSchemaManagerTest.java @@ -1,11 +1,11 @@ package com.geedgenetworks.core.connector.schema.utils; -import com.geedgenetworks.spi.table.schema.DynamicSchema; -import com.geedgenetworks.spi.table.schema.HttpDynamicSchema; -import com.geedgenetworks.spi.table.schema.SchemaChangeAware; -import com.geedgenetworks.spi.table.schema.SchemaParser; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.schema.DynamicSchema; +import com.geedgenetworks.api.connector.schema.HttpDynamicSchema; +import com.geedgenetworks.api.connector.schema.SchemaChangeAware; +import com.geedgenetworks.api.connector.schema.SchemaParser; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.Types; import org.apache.flink.util.TimeUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/types/TypesTest.java b/groot-core/src/test/java/com/geedgenetworks/core/types/TypesTest.java index 8ec0051..f144abc 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/types/TypesTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/types/TypesTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.types; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.table.type.*; +import com.geedgenetworks.api.connector.type.*; import org.junit.jupiter.api.Test; import java.util.Arrays; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/AnonymityLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/AnonymityLookupTest.java index e24c16f..bfcd7a0 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/AnonymityLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/AnonymityLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/AppCategoryLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/AppCategoryLookupTest.java index 6efe966..d32d027 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/AppCategoryLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/AppCategoryLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/BaseStationLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/BaseStationLookupTest.java index 514b608..f1936f6 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/BaseStationLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/BaseStationLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookupTest.java index 2ba329c..996302c 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/DnsServerInfoLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookupTest.java index a02edcd..2ff200a 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/FqdnCategoryLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookupTest.java index de5927c..ba0b757 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/FqdnWhoisLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/H3CellLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/H3CellLookupTest.java index 6cd5087..22431c3 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/H3CellLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/H3CellLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IcpLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IcpLookupTest.java index 0fb8c0f..a27c36b 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IcpLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IcpLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IdcRenterLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IdcRenterLookupTest.java index 8a1d9f2..73a9660 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IdcRenterLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IdcRenterLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookupTest.java index 9e6335b..1fde9a8 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IntelligenceIndicatorLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IocLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IocLookupTest.java index dee6843..e7cfb7e 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IocLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IocLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IpZoneLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IpZoneLookupTest.java index da29b72..46f8ddd 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IpZoneLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/IpZoneLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookupTest.java index ab51c5a..f504e43 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/LinkDirectionLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookupTest.java index 5d34b5c..eccad83 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/UserDefineTagLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/VpnLookupTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/VpnLookupTest.java index a9f96b3..16bcd67 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/VpnLookupTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/cn/VpnLookupTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.udf.cn; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.functions.RuntimeContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/AsnLookupFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/AsnLookupFunctionTest.java index 6c71607..84d5cf5 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/AsnLookupFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/AsnLookupFunctionTest.java @@ -6,7 +6,7 @@ import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.udf.AsnLookup; import com.geedgenetworks.core.udf.knowlegdebase.handler.AsnKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java index d68ef55..90e39f8 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java @@ -6,7 +6,7 @@ import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.udf.GeoIpLookup; import com.geedgenetworks.core.udf.knowlegdebase.handler.GeoIpKnowledgeBaseHandler; import com.geedgenetworks.core.udf.knowlegdebase.KnowledgeBaseUpdateJob; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java index 263ae05..0bd3d4a 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.test.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.core.udf.udaf.CollectList; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Test; import java.text.ParseException; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java index 45fb143..edfd81a 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.test.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.core.udf.udaf.CollectSet; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Test; import java.text.ParseException; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java index 33df56d..1e51c8a 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.test.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.core.udf.udaf.FirstValue; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Test; import java.text.ParseException; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java index 4ed273c..b6934b7 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.test.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.core.udf.udaf.LastValue; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Test; import java.text.ParseException; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java index 1ca64c7..a02ba0f 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.test.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.core.udf.udaf.LongCount; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Test; import java.text.ParseException; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MaxTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MaxTest.java index 34f0ce3..9c98f08 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MaxTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MaxTest.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.test.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.core.udf.udaf.Max; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java index d53aea9..330c70b 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.test.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.core.udf.udaf.Mean; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import com.ibm.icu.text.NumberFormat; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MinTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MinTest.java index d29d0cc..4bcda37 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MinTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MinTest.java @@ -3,8 +3,8 @@ package com.geedgenetworks.core.udf.test.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.core.udf.udaf.Min; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java index 802cb14..7a931d9 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java @@ -18,8 +18,8 @@ package com.geedgenetworks.core.udf.test.aggregate; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.core.udf.udaf.NumberSum; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import com.ibm.icu.text.NumberFormat; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DecodeBase64FunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DecodeBase64FunctionTest.java index b8ecdc8..4b20ff2 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DecodeBase64FunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DecodeBase64FunctionTest.java @@ -2,8 +2,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.udf.DecodeBase64; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DomainFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DomainFunctionTest.java index 4215e5f..55f76b4 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DomainFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DomainFunctionTest.java @@ -2,8 +2,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.udf.Domain; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DropFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DropFunctionTest.java index dec6a81..ef6cebd 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DropFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/DropFunctionTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.core.udf.Drop; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.util.HashMap; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/EncodeBase64FunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/EncodeBase64FunctionTest.java index f2eb45d..07203b5 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/EncodeBase64FunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/EncodeBase64FunctionTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.core.udf.EncodeBase64; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Test; import java.nio.charset.StandardCharsets; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/EncryptFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/EncryptFunctionTest.java index 4142fcf..c0a3ef9 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/EncryptFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/EncryptFunctionTest.java @@ -13,8 +13,8 @@ import com.geedgenetworks.core.udf.Encrypt; import com.geedgenetworks.core.udf.encrypt.Crypto; import com.geedgenetworks.core.utils.CryptoProvider; import com.geedgenetworks.core.utils.KMSUtils; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import io.github.jopenlibs.vault.VaultException; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.functions.RuntimeContext; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/FlattenFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/FlattenFunctionTest.java index ab958ac..a73948f 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/FlattenFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/FlattenFunctionTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.core.udf.Flatten; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/FromUnixTimestampTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/FromUnixTimestampTest.java index 7639654..bc97a41 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/FromUnixTimestampTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/FromUnixTimestampTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.core.udf.FromUnixTimestamp; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/GenerateStringArrayFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/GenerateStringArrayFunctionTest.java index 79f3232..2156bb7 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/GenerateStringArrayFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/GenerateStringArrayFunctionTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.core.udf.GenerateStringArray; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/HmacFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/HmacFunctionTest.java index 2ccce51..841ae35 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/HmacFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/HmacFunctionTest.java @@ -2,8 +2,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.udf.Hmac; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/JsonExtractFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/JsonExtractFunctionTest.java index 8da0e86..2460b77 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/JsonExtractFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/JsonExtractFunctionTest.java @@ -4,8 +4,8 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.TypeReference; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.udf.JsonExtract; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/RenameFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/RenameFunctionTest.java index 9c914ea..a9a147b 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/RenameFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/RenameFunctionTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.core.udf.Rename; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/StringJoinerFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/StringJoinerFunctionTest.java index 13e83d8..68ea214 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/StringJoinerFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/StringJoinerFunctionTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.core.udf.StringJoiner; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/UUIDTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/UUIDTest.java index e74173d..5f1715f 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/UUIDTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/UUIDTest.java @@ -4,8 +4,8 @@ import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.core.udf.uuid.UUID; import com.geedgenetworks.core.udf.uuid.UUIDv5; import com.geedgenetworks.core.udf.uuid.UUIDv7; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/UnixTimestampConverterTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/UnixTimestampConverterTest.java index f1bf92a..396fb93 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/UnixTimestampConverterTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/simple/UnixTimestampConverterTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.test.simple; import com.geedgenetworks.core.udf.UnixTimestampConverter; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java index 2597d57..288483a 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java @@ -1,10 +1,10 @@ package com.geedgenetworks.core.udf.test.table; import com.geedgenetworks.core.udf.udtf.JsonUnroll; -import com.geedgenetworks.spi.common.udf.UDFContext; +import com.geedgenetworks.api.common.udf.UDFContext; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import java.util.HashMap; import java.util.List; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java index 24f5608..8774210 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.test.table; import com.geedgenetworks.core.udf.udtf.Unroll; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java index 1c11ccd..830f4c8 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf.udaf.HdrHistogram; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.sketch.util.StringUtils; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.HdrHistogram.ArrayHistogram; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java index c9f6fb5..ec761f2 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf.udaf.HdrHistogram; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.sketch.util.StringUtils; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.HdrHistogram.ArrayHistogram; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java index 6b9c6d7..3c4ac41 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java @@ -2,9 +2,9 @@ package com.geedgenetworks.core.udf.udaf.HdrHistogram; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.sketch.util.StringUtils; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.HdrHistogram.ArrayHistogram; import org.junit.jupiter.api.Test; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java index ec9e09e..77556d9 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java @@ -4,9 +4,9 @@ package com.geedgenetworks.core.udf.udaf.hlld; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.sketch.hlld.Hll; import com.geedgenetworks.sketch.util.StringUtils; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Test; import java.util.Collections; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java index c54847e..67c9527 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java @@ -3,9 +3,9 @@ package com.geedgenetworks.core.udf.udaf.hlld; import com.geedgenetworks.common.config.Accumulator; import com.geedgenetworks.sketch.hlld.Hll; import com.geedgenetworks.sketch.util.StringUtils; -import com.geedgenetworks.spi.common.udf.AggregateFunction; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.AggregateFunction; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Test; import java.nio.charset.StandardCharsets; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udtf/UnrollTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udtf/UnrollTest.java index ea4d759..0a46028 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/udtf/UnrollTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udtf/UnrollTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.udf.udtf; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.common.udf.UDFContext; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.common.udf.UDFContext; +import com.geedgenetworks.api.connector.event.Event; import org.junit.jupiter.api.Test; import java.util.Arrays; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/utils/LoadIntervalDataUtilTest.java b/groot-core/src/test/java/com/geedgenetworks/core/utils/LoadIntervalDataUtilTest.java index 37aab01..ef9a21e 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/utils/LoadIntervalDataUtilTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/utils/LoadIntervalDataUtilTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.core.utils; -import com.geedgenetworks.spi.configuration.util.LoadIntervalDataOptions; -import com.geedgenetworks.spi.configuration.util.LoadIntervalDataUtil; +import com.geedgenetworks.api.configuration.util.LoadIntervalDataOptions; +import com.geedgenetworks.api.configuration.util.LoadIntervalDataUtil; import java.sql.Timestamp; diff --git a/groot-core/src/test/java/com/geedgenetworks/core/utils/SingleValueMapTest.java b/groot-core/src/test/java/com/geedgenetworks/core/utils/SingleValueMapTest.java index ffadb40..5d6994d 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/utils/SingleValueMapTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/utils/SingleValueMapTest.java @@ -1,7 +1,7 @@ package com.geedgenetworks.core.utils; -import com.geedgenetworks.spi.configuration.util.LoadIntervalDataOptions; -import com.geedgenetworks.spi.configuration.util.LoadIntervalDataUtil; +import com.geedgenetworks.api.configuration.util.LoadIntervalDataOptions; +import com.geedgenetworks.api.configuration.util.LoadIntervalDataUtil; import org.junit.jupiter.api.Assertions; import java.sql.Timestamp; diff --git a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java index 9b58289..5e64962 100644 --- a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java +++ b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java @@ -14,7 +14,7 @@ import java.util.List; public class GrootStreamExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException { - String configPath = args.length > 0 ? args[0] : "/examples/inline_to_print.yaml"; + String configPath = args.length > 0 ? args[0] : "/examples/inline_to_print_test.yaml"; String configFile = getTestConfigFile(configPath); ExecuteCommandArgs executeCommandArgs = new ExecuteCommandArgs(); executeCommandArgs.setConfigFile(configFile); diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/kafka_to_print.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/kafka_to_print.yaml index aafd642..8eaa567 100644 --- a/groot-examples/end-to-end-example/src/main/resources/examples/kafka_to_print.yaml +++ b/groot-examples/end-to-end-example/src/main/resources/examples/kafka_to_print.yaml @@ -4,7 +4,7 @@ sources: schema: properties: # [object] Kafka source properties topic: DATAPATH-TELEMETRY-RECORD - kafka.bootstrap.servers: "192.168.40.147:9094" + kafka.bootstrap.servers: "192.168.44.12:9094" kafka.session.timeout.ms: 60000 kafka.max.poll.records: 3000 kafka.max.partition.fetch.bytes: 31457280 diff --git a/groot-examples/pom.xml b/groot-examples/pom.xml index bbc9c65..d6cf698 100644 --- a/groot-examples/pom.xml +++ b/groot-examples/pom.xml @@ -18,7 +18,7 @@ </modules> <properties> - <scope>provided</scope> + <scope>compile</scope> </properties> <dependencies> diff --git a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvEventDeserializationSchema.java b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvEventDeserializationSchema.java index 58278bb..8c73d9d 100644 --- a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvEventDeserializationSchema.java +++ b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvEventDeserializationSchema.java @@ -1,8 +1,8 @@ package com.geedgenetworks.formats.csv; -import com.geedgenetworks.spi.table.connector.MapDeserialization; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.MapDeserialization; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; diff --git a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvEventSerializationSchema.java b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvEventSerializationSchema.java index 72feb78..bd1b69d 100644 --- a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvEventSerializationSchema.java +++ b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvEventSerializationSchema.java @@ -1,7 +1,7 @@ package com.geedgenetworks.formats.csv; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema; diff --git a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvFormatFactory.java b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvFormatFactory.java index 435a91e..c501cb0 100644 --- a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvFormatFactory.java +++ b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvFormatFactory.java @@ -1,12 +1,12 @@ package com.geedgenetworks.formats.csv; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.table.connector.EncodingFormat; -import com.geedgenetworks.spi.table.factory.DecodingFormatFactory; -import com.geedgenetworks.spi.table.factory.EncodingFormatFactory; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.factory.TableFactory; -import com.geedgenetworks.spi.table.type.*; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; +import com.geedgenetworks.api.factory.DecodingFormatFactory; +import com.geedgenetworks.api.factory.EncodingFormatFactory; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.factory.ConnectorFactory; +import com.geedgenetworks.api.connector.type.*; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ReadableConfig; @@ -25,12 +25,12 @@ public class CsvFormatFactory implements DecodingFormatFactory, EncodingFormatFa public static final String IDENTIFIER = "csv"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } @Override - public DecodingFormat createDecodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public DecodingFormat createDecodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { FactoryUtil.validateFactoryOptions(this, formatOptions); validateFormatOptions(formatOptions); final boolean ignoreParseErrors = formatOptions.get(IGNORE_PARSE_ERRORS); @@ -42,7 +42,7 @@ public class CsvFormatFactory implements DecodingFormatFactory, EncodingFormatFa } @Override - public EncodingFormat createEncodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public EncodingFormat createEncodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { FactoryUtil.validateFactoryOptions(this, formatOptions); validateFormatOptions(formatOptions); return dataType -> { diff --git a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvSerializer.java b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvSerializer.java index 6805894..a39b0ee 100644 --- a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvSerializer.java +++ b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvSerializer.java @@ -1,7 +1,7 @@ package com.geedgenetworks.formats.csv; -import com.geedgenetworks.spi.table.type.*; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.type.*; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectWriter; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.*; diff --git a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvToMapDataConverter.java b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvToMapDataConverter.java index a55981b..71fdbf3 100644 --- a/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvToMapDataConverter.java +++ b/groot-formats/format-csv/src/main/java/com/geedgenetworks/formats/csv/CsvToMapDataConverter.java @@ -1,6 +1,6 @@ package com.geedgenetworks.formats.csv; -import com.geedgenetworks.spi.table.type.*; +import com.geedgenetworks.api.connector.type.*; import org.apache.commons.lang3.StringUtils; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectReader; diff --git a/groot-formats/format-csv/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory b/groot-formats/format-csv/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory new file mode 100644 index 0000000..e0ac788 --- /dev/null +++ b/groot-formats/format-csv/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory @@ -0,0 +1 @@ +com.geedgenetworks.formats.csv.CsvFormatFactory
\ No newline at end of file diff --git a/groot-formats/format-csv/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-formats/format-csv/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory deleted file mode 100644 index e417fa4..0000000 --- a/groot-formats/format-csv/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ /dev/null @@ -1 +0,0 @@ -com.geedgenetworks.formats.csv.CsvFormatFactory diff --git a/groot-formats/format-csv/src/test/java/com/geedgenetworks/formats/csv/CsvEventSerDeSchemaTest.java b/groot-formats/format-csv/src/test/java/com/geedgenetworks/formats/csv/CsvEventSerDeSchemaTest.java index 9bcafac..bf65a36 100644 --- a/groot-formats/format-csv/src/test/java/com/geedgenetworks/formats/csv/CsvEventSerDeSchemaTest.java +++ b/groot-formats/format-csv/src/test/java/com/geedgenetworks/formats/csv/CsvEventSerDeSchemaTest.java @@ -1,14 +1,14 @@ package com.geedgenetworks.formats.csv; -import com.geedgenetworks.spi.table.connector.MapDeserialization; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.DecodingFormatFactory; -import com.geedgenetworks.spi.table.factory.EncodingFormatFactory; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.factory.TableFactory; -import com.geedgenetworks.spi.table.schema.Schema; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.serialization.MapDeserialization; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.DecodingFormatFactory; +import com.geedgenetworks.api.factory.EncodingFormatFactory; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.factory.ConnectorFactory; +import com.geedgenetworks.api.connector.schema.Schema; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.Types; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; @@ -24,7 +24,7 @@ public class CsvEventSerDeSchemaTest { public void testSimpleSerializeDeserialize() throws Exception { StructType dataType = Types.parseStructType("int:int,bigint:bigint,double:double,string:string"); Map<String, String> options = new HashMap<>(); - TableFactory.Context context = new TableFactory.Context(Schema.newSchema(dataType), options, Configuration.fromMap(options)); + ConnectorFactory.Context context = new ConnectorFactory.Context(Schema.newSchema(dataType), options, Configuration.fromMap(options)); // 获取deserialization和serialization DeserializationSchema<Event> deserialization = FactoryUtil.discoverDecodingFormatFactory(DecodingFormatFactory.class, "csv") @@ -61,7 +61,7 @@ public class CsvEventSerDeSchemaTest { public void testSerializeDeserialize() throws Exception { StructType dataType = Types.parseStructType("int:int,bigint:bigint,double:double,string:string,int_array:array<int>,struct:struct<int:int,string:string>"); Map<String, String> options = new HashMap<>(); - TableFactory.Context context = new TableFactory.Context(Schema.newSchema(dataType), options, Configuration.fromMap(options)); + ConnectorFactory.Context context = new ConnectorFactory.Context(Schema.newSchema(dataType), options, Configuration.fromMap(options)); DeserializationSchema<Event> deserialization = FactoryUtil.discoverDecodingFormatFactory(DecodingFormatFactory.class, "csv") .createDecodingFormat(context, context.getConfiguration()).createRuntimeDecoder(dataType); @@ -142,7 +142,7 @@ public class CsvEventSerDeSchemaTest { Map<String, String> options = new HashMap<>(); options.put(CsvFormatOptions.NULL_LITERAL.key(), "null"); options.put(CsvFormatOptions.IGNORE_PARSE_ERRORS.key(), "true"); - TableFactory.Context context = new TableFactory.Context(Schema.newSchema(dataType), options, Configuration.fromMap(options)); + ConnectorFactory.Context context = new ConnectorFactory.Context(Schema.newSchema(dataType), options, Configuration.fromMap(options)); DeserializationSchema<Event> deserialization = FactoryUtil.discoverDecodingFormatFactory(DecodingFormatFactory.class, "csv") .createDecodingFormat(context, context.getConfiguration()).createRuntimeDecoder(dataType); diff --git a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonEventDeserializationSchema.java b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonEventDeserializationSchema.java index 7c69024..11ce443 100644 --- a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonEventDeserializationSchema.java +++ b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonEventDeserializationSchema.java @@ -1,9 +1,9 @@ package com.geedgenetworks.formats.json; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.table.connector.MapDeserialization; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.MapDeserialization; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.slf4j.Logger; diff --git a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonEventSerializationSchema.java b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonEventSerializationSchema.java index 6bb3473..de5c4a1 100644 --- a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonEventSerializationSchema.java +++ b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonEventSerializationSchema.java @@ -2,9 +2,9 @@ package com.geedgenetworks.formats.json; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.filter.PropertyFilter; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.SerializationSchema; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; public class JsonEventSerializationSchema implements SerializationSchema<Event> { // __开头字段为内部字段，过滤掉 diff --git a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonFormatFactory.java b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonFormatFactory.java index 4cb42aa..15e48d6 100644 --- a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonFormatFactory.java +++ b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonFormatFactory.java @@ -1,12 +1,12 @@ package com.geedgenetworks.formats.json; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.table.connector.EncodingFormat; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.DecodingFormatFactory; -import com.geedgenetworks.spi.table.factory.EncodingFormatFactory; -import com.geedgenetworks.spi.table.factory.TableFactory; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.DecodingFormatFactory; +import com.geedgenetworks.api.factory.EncodingFormatFactory; +import com.geedgenetworks.api.factory.ConnectorFactory; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.configuration.ConfigOption; @@ -22,12 +22,12 @@ public class JsonFormatFactory implements DecodingFormatFactory, EncodingFormatF public static final String IDENTIFIER = "json"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } @Override - public DecodingFormat createDecodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public DecodingFormat createDecodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { final boolean ignoreParseErrors = formatOptions.get(IGNORE_PARSE_ERRORS); return new DecodingFormat(){ @Override @@ -39,7 +39,7 @@ public class JsonFormatFactory implements DecodingFormatFactory, EncodingFormatF } @Override - public EncodingFormat createEncodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public EncodingFormat createEncodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { return new EncodingFormat() { @Override public SerializationSchema<Event> createRuntimeEncoder(StructType dataType) { diff --git a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonSerializer.java b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonSerializer.java index 44e3d2d..625f8f4 100644 --- a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonSerializer.java +++ b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonSerializer.java @@ -1,7 +1,7 @@ package com.geedgenetworks.formats.json; import com.alibaba.fastjson2.JSONWriter; -import com.geedgenetworks.spi.table.type.*; +import com.geedgenetworks.api.connector.type.*; import java.io.Serializable; import java.util.Arrays; diff --git a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonToMapDataConverter.java b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonToMapDataConverter.java index 6bbaff5..f5a6848 100644 --- a/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonToMapDataConverter.java +++ b/groot-formats/format-json/src/main/java/com/geedgenetworks/formats/json/JsonToMapDataConverter.java @@ -2,13 +2,12 @@ package com.geedgenetworks.formats.json; import com.alibaba.fastjson2.JSONException; import com.alibaba.fastjson2.JSONReader; -import com.geedgenetworks.spi.table.type.*; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.type.*; +import com.geedgenetworks.api.connector.type.StructType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.Serializable; -import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; diff --git a/groot-formats/format-json/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-formats/format-json/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index c965152..c965152 100644 --- a/groot-formats/format-json/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-formats/format-json/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-formats/format-json/src/test/java/com/geedgenetworks/formats/json/JsonSerializerTest.java b/groot-formats/format-json/src/test/java/com/geedgenetworks/formats/json/JsonSerializerTest.java index 97f8220..356608b 100644 --- a/groot-formats/format-json/src/test/java/com/geedgenetworks/formats/json/JsonSerializerTest.java +++ b/groot-formats/format-json/src/test/java/com/geedgenetworks/formats/json/JsonSerializerTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.formats.json; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.Types; import org.junit.jupiter.api.Test; import java.nio.charset.StandardCharsets; diff --git a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackDeserializer.java b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackDeserializer.java index 6c3a243..0745a0a 100644 --- a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackDeserializer.java +++ b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackDeserializer.java @@ -1,7 +1,7 @@ package com.geedgenetworks.formats.msgpack; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.*; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.*; import org.msgpack.core.MessageFormat; import org.msgpack.core.MessagePack; import org.msgpack.core.MessageUnpacker; diff --git a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackEventDeserializationSchema.java b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackEventDeserializationSchema.java index c936f2c..8791682 100644 --- a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackEventDeserializationSchema.java +++ b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackEventDeserializationSchema.java @@ -1,9 +1,9 @@ package com.geedgenetworks.formats.msgpack; -import com.geedgenetworks.spi.table.connector.MapDeserialization; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.MapDeserialization; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.util.StringUtils; diff --git a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackEventSerializationSchema.java b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackEventSerializationSchema.java index d8423e5..149a751 100644 --- a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackEventSerializationSchema.java +++ b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackEventSerializationSchema.java @@ -1,7 +1,7 @@ package com.geedgenetworks.formats.msgpack; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.SerializationSchema; public class MessagePackEventSerializationSchema implements SerializationSchema<Event> { diff --git a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackFormatFactory.java b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackFormatFactory.java index cab5e4f..cfb47f6 100644 --- a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackFormatFactory.java +++ b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackFormatFactory.java @@ -1,13 +1,13 @@ package com.geedgenetworks.formats.msgpack; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.table.connector.EncodingFormat; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.DecodingFormatFactory; -import com.geedgenetworks.spi.table.factory.EncodingFormatFactory; -import com.geedgenetworks.spi.table.factory.TableFactory; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.DecodingFormatFactory; +import com.geedgenetworks.api.factory.EncodingFormatFactory; +import com.geedgenetworks.api.factory.ConnectorFactory; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.configuration.ConfigOption; @@ -20,12 +20,12 @@ public class MessagePackFormatFactory implements DecodingFormatFactory, Encoding public static final String IDENTIFIER = "msgpack"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } @Override - public DecodingFormat createDecodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public DecodingFormat createDecodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { return new DecodingFormat() { @Override @@ -36,7 +36,7 @@ public class MessagePackFormatFactory implements DecodingFormatFactory, Encoding } @Override - public EncodingFormat createEncodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public EncodingFormat createEncodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { return new EncodingFormat() { @Override diff --git a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackSerializer.java b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackSerializer.java index 45a1e22..4dc8316 100644 --- a/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackSerializer.java +++ b/groot-formats/format-msgpack/src/main/java/com/geedgenetworks/formats/msgpack/MessagePackSerializer.java @@ -1,7 +1,7 @@ package com.geedgenetworks.formats.msgpack; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.table.type.*; +import com.geedgenetworks.api.connector.type.*; import org.apache.commons.io.IOUtils; import org.msgpack.core.MessageBufferPacker; import org.msgpack.core.MessagePack; diff --git a/groot-formats/format-msgpack/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-formats/format-msgpack/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index 83ace6c..83ace6c 100644 --- a/groot-formats/format-msgpack/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-formats/format-msgpack/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackDeserializerTest.java b/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackDeserializerTest.java index 23164fa..f0603f5 100644 --- a/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackDeserializerTest.java +++ b/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackDeserializerTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.formats.msgpack; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.Types; import org.junit.jupiter.api.Test; import org.msgpack.core.MessageBufferPacker; import org.msgpack.core.MessagePack; diff --git a/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackFormatFactoryTest.java b/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackFormatFactoryTest.java index fced05e..9119317 100644 --- a/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackFormatFactoryTest.java +++ b/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackFormatFactoryTest.java @@ -1,12 +1,12 @@ package com.geedgenetworks.formats.msgpack; -import com.geedgenetworks.spi.sink.SinkProvider; -import com.geedgenetworks.spi.sink.SinkTableFactory; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.source.SourceTableFactory; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.factory.TableFactory; +import com.geedgenetworks.api.connector.sink.SinkProvider; +import com.geedgenetworks.api.connector.sink.SinkTableFactory; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.source.SourceTableFactory; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.factory.ConnectorFactory; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.datastream.DataStreamSink; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; @@ -66,7 +66,7 @@ public class MessagePackFormatFactoryTest { public static void main(String[] args) throws Exception{ byte[] bytes = getTestBytes(); - SourceTableFactory tableFactory = FactoryUtil.discoverTableFactory(SourceTableFactory.class, "inline"); + SourceTableFactory tableFactory = FactoryUtil.discoverConnectorFactory(SourceTableFactory.class, "inline"); Map<String, String> options = new HashMap<>(); options.put("data", Base64.getEncoder().encodeToString(bytes)); options.put("repeat.count", "3"); @@ -74,15 +74,15 @@ public class MessagePackFormatFactoryTest { options.put("format", "msgpack"); Configuration configuration = Configuration.fromMap(options); - TableFactory.Context context = new TableFactory.Context( null, options, configuration); + ConnectorFactory.Context context = new ConnectorFactory.Context( null, options, configuration); SourceProvider sourceProvider = tableFactory.getSourceProvider(context); - SinkTableFactory sinkTableFactory = FactoryUtil.discoverTableFactory(SinkTableFactory.class, "print"); + SinkTableFactory sinkTableFactory = FactoryUtil.discoverConnectorFactory(SinkTableFactory.class, "print"); options = new HashMap<>(); options.put("format", "msgpack"); configuration = Configuration.fromMap(options); - context = new TableFactory.Context( null, options, configuration); + context = new ConnectorFactory.Context( null, options, configuration); SinkProvider sinkProvider = sinkTableFactory.getSinkProvider(context); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); diff --git a/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackSerializerTest.java b/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackSerializerTest.java index d1b4289..767301d 100644 --- a/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackSerializerTest.java +++ b/groot-formats/format-msgpack/src/test/java/com/geedgenetworks/formats/msgpack/MessagePackSerializerTest.java @@ -1,8 +1,8 @@ package com.geedgenetworks.formats.msgpack; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.Types; import org.junit.jupiter.api.Test; import org.msgpack.core.MessageBufferPacker; import org.msgpack.core.MessagePack; diff --git a/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufEventDeserializationSchema.java b/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufEventDeserializationSchema.java index d02ea0d..c2e4437 100644 --- a/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufEventDeserializationSchema.java +++ b/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufEventDeserializationSchema.java @@ -1,9 +1,9 @@ package com.geedgenetworks.formats.protobuf; import com.geedgenetworks.shaded.com.google.protobuf.Descriptors.Descriptor; -import com.geedgenetworks.spi.table.connector.MapDeserialization; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.MapDeserialization; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.slf4j.Logger; diff --git a/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufEventSerializationSchema.java b/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufEventSerializationSchema.java index 25d18c2..ccfe850 100644 --- a/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufEventSerializationSchema.java +++ b/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufEventSerializationSchema.java @@ -2,7 +2,7 @@ package com.geedgenetworks.formats.protobuf; import com.alibaba.fastjson2.JSON; import com.geedgenetworks.shaded.com.google.protobuf.Descriptors; -import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.api.connector.event.Event; import org.apache.flink.api.common.serialization.SerializationSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufFormatFactory.java b/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufFormatFactory.java index 572a67a..9f008e9 100644 --- a/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufFormatFactory.java +++ b/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/ProtobufFormatFactory.java @@ -1,13 +1,13 @@ package com.geedgenetworks.formats.protobuf; import com.geedgenetworks.shaded.com.google.protobuf.Descriptors; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.table.connector.EncodingFormat; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.DecodingFormatFactory; -import com.geedgenetworks.spi.table.factory.EncodingFormatFactory; -import com.geedgenetworks.spi.table.factory.TableFactory; -import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.DecodingFormatFactory; +import com.geedgenetworks.api.factory.EncodingFormatFactory; +import com.geedgenetworks.api.factory.ConnectorFactory; +import com.geedgenetworks.api.connector.type.StructType; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.configuration.ConfigOption; @@ -22,12 +22,12 @@ public class ProtobufFormatFactory implements DecodingFormatFactory, EncodingFor public static final String IDENTIFIER = "protobuf"; @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } @Override - public DecodingFormat createDecodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public DecodingFormat createDecodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { final String messageName = formatOptions.get(MESSAGE_NAME); final String descFilePath = formatOptions.get(DESC_FILE_PATH); final boolean ignoreParseErrors = formatOptions.get(IGNORE_PARSE_ERRORS); @@ -52,7 +52,7 @@ public class ProtobufFormatFactory implements DecodingFormatFactory, EncodingFor } @Override - public EncodingFormat createEncodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public EncodingFormat createEncodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { final String messageName = formatOptions.get(MESSAGE_NAME); final String descFilePath = formatOptions.get(DESC_FILE_PATH); final byte[] fileContent = ProtobufUtils.readDescriptorFileContent(descFilePath); diff --git a/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/SchemaConverters.java b/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/SchemaConverters.java index 89736de..44a8140 100644 --- a/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/SchemaConverters.java +++ b/groot-formats/format-protobuf/src/main/java/com/geedgenetworks/formats/protobuf/SchemaConverters.java @@ -5,8 +5,8 @@ import com.geedgenetworks.shaded.com.google.protobuf.CodedInputStream; import com.geedgenetworks.shaded.com.google.protobuf.Descriptors.Descriptor; import com.geedgenetworks.shaded.com.google.protobuf.Descriptors.FieldDescriptor; import com.geedgenetworks.shaded.com.google.protobuf.WireFormat; -import com.geedgenetworks.spi.table.type.*; -import com.geedgenetworks.spi.table.type.StructType.StructField; +import com.geedgenetworks.api.connector.type.*; +import com.geedgenetworks.api.connector.type.StructType.StructField; import org.apache.commons.lang3.StringUtils; import org.apache.flink.util.Preconditions; import org.slf4j.Logger; diff --git a/groot-formats/format-protobuf/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-formats/format-protobuf/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index b6c459c..b6c459c 100644 --- a/groot-formats/format-protobuf/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-formats/format-protobuf/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-formats/format-protobuf/src/test/java/com/geedgenetworks/formats/protobuf/ProtobufFormatFactoryTest.java b/groot-formats/format-protobuf/src/test/java/com/geedgenetworks/formats/protobuf/ProtobufFormatFactoryTest.java index 95941e4..1359f85 100644 --- a/groot-formats/format-protobuf/src/test/java/com/geedgenetworks/formats/protobuf/ProtobufFormatFactoryTest.java +++ b/groot-formats/format-protobuf/src/test/java/com/geedgenetworks/formats/protobuf/ProtobufFormatFactoryTest.java @@ -1,13 +1,13 @@ package com.geedgenetworks.formats.protobuf; import com.alibaba.fastjson2.JSON; -import com.geedgenetworks.spi.sink.SinkProvider; -import com.geedgenetworks.spi.sink.SinkTableFactory; -import com.geedgenetworks.spi.source.SourceProvider; -import com.geedgenetworks.spi.source.SourceTableFactory; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.FactoryUtil; -import com.geedgenetworks.spi.table.factory.TableFactory; +import com.geedgenetworks.api.connector.sink.SinkProvider; +import com.geedgenetworks.api.connector.sink.SinkTableFactory; +import com.geedgenetworks.api.connector.source.SourceProvider; +import com.geedgenetworks.api.connector.source.SourceTableFactory; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.FactoryUtil; +import com.geedgenetworks.api.factory.ConnectorFactory; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.datastream.DataStreamSink; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; @@ -25,7 +25,7 @@ class ProtobufFormatFactoryTest { String path = ProtobufFormatFactoryTest.class.getResource("/proto3_types.desc").getPath(); String messageName = "Proto3Types"; - SourceTableFactory tableFactory = FactoryUtil.discoverTableFactory(SourceTableFactory.class, "inline"); + SourceTableFactory tableFactory = FactoryUtil.discoverConnectorFactory(SourceTableFactory.class, "inline"); Map<String, String> options = new HashMap<>(); options.put("repeat.count", "3"); options.put("data", Base64.getEncoder().encodeToString(inputDatas.msg.toByteArray())); @@ -35,14 +35,14 @@ class ProtobufFormatFactoryTest { options.put("protobuf.message.name", messageName); Configuration configuration = Configuration.fromMap(options); - TableFactory.Context context = new TableFactory.Context( null, options, configuration); + ConnectorFactory.Context context = new ConnectorFactory.Context( null, options, configuration); SourceProvider sourceProvider = tableFactory.getSourceProvider(context); - SinkTableFactory sinkTableFactory = FactoryUtil.discoverTableFactory(SinkTableFactory.class, "print"); + SinkTableFactory sinkTableFactory = FactoryUtil.discoverConnectorFactory(SinkTableFactory.class, "print"); options = new HashMap<>(); options.put("format", "json"); configuration = Configuration.fromMap(options); - context = new TableFactory.Context( null, options, configuration); + context = new ConnectorFactory.Context( null, options, configuration); SinkProvider sinkProvider = sinkTableFactory.getSinkProvider(context); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); diff --git a/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawEventDeserializationSchema.java b/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawEventDeserializationSchema.java index 9ca8710..b299535 100644 --- a/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawEventDeserializationSchema.java +++ b/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawEventDeserializationSchema.java @@ -1,8 +1,8 @@ package com.geedgenetworks.formats.raw; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.Types; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.util.Preconditions; diff --git a/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawEventSerializationSchema.java b/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawEventSerializationSchema.java index 81f0835..c964a5c 100644 --- a/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawEventSerializationSchema.java +++ b/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawEventSerializationSchema.java @@ -1,9 +1,9 @@ package com.geedgenetworks.formats.raw; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.Types; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.util.Preconditions; diff --git a/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawFormatFactory.java b/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawFormatFactory.java index 38fcc23..6e493bb 100644 --- a/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawFormatFactory.java +++ b/groot-formats/format-raw/src/main/java/com/geedgenetworks/formats/raw/RawFormatFactory.java @@ -1,14 +1,14 @@ package com.geedgenetworks.formats.raw; -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import com.geedgenetworks.spi.table.connector.EncodingFormat; -import com.geedgenetworks.spi.table.event.Event; -import com.geedgenetworks.spi.table.factory.DecodingFormatFactory; -import com.geedgenetworks.spi.table.factory.EncodingFormatFactory; -import com.geedgenetworks.spi.table.factory.TableFactory; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.StructType.StructField; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.serialization.DecodingFormat; +import com.geedgenetworks.api.connector.serialization.EncodingFormat; +import com.geedgenetworks.api.connector.event.Event; +import com.geedgenetworks.api.factory.DecodingFormatFactory; +import com.geedgenetworks.api.factory.EncodingFormatFactory; +import com.geedgenetworks.api.factory.ConnectorFactory; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.StructType.StructField; +import com.geedgenetworks.api.connector.type.Types; import org.apache.flink.api.common.serialization.DeserializationSchema; import org.apache.flink.api.common.serialization.SerializationSchema; import org.apache.flink.configuration.ConfigOption; @@ -22,12 +22,12 @@ public class RawFormatFactory implements DecodingFormatFactory, EncodingFormatFa public static final StructType DEFAULT_DATATYPE = new StructType(new StructField[]{new StructField("raw", Types.BINARY)}); @Override - public String factoryIdentifier() { + public String type() { return IDENTIFIER; } @Override - public DecodingFormat createDecodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public DecodingFormat createDecodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { return new DecodingFormat(){ @Override public DeserializationSchema<Event> createRuntimeDecoder(StructType dataType) { @@ -41,7 +41,7 @@ public class RawFormatFactory implements DecodingFormatFactory, EncodingFormatFa } @Override - public EncodingFormat createEncodingFormat(TableFactory.Context context, ReadableConfig formatOptions) { + public EncodingFormat createEncodingFormat(ConnectorFactory.Context context, ReadableConfig formatOptions) { return new EncodingFormat() { @Override public SerializationSchema<Event> createRuntimeEncoder(StructType dataType) { diff --git a/groot-formats/format-raw/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory b/groot-formats/format-raw/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory index f4523f2..f4523f2 100644 --- a/groot-formats/format-raw/src/main/resources/META-INF/services/com.geedgenetworks.spi.table.factory.Factory +++ b/groot-formats/format-raw/src/main/resources/META-INF/services/com.geedgenetworks.api.factory.Factory diff --git a/groot-formats/pom.xml b/groot-formats/pom.xml index 1d42523..5f20e42 100644 --- a/groot-formats/pom.xml +++ b/groot-formats/pom.xml @@ -24,7 +24,7 @@ <dependency> <groupId>com.geedgenetworks</groupId> - <artifactId>groot-spi</artifactId> + <artifactId>groot-api</artifactId> <version>${revision}</version> <scope>provided</scope> </dependency> diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/FilterConfigOptions.java b/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/FilterConfigOptions.java deleted file mode 100644 index 214258f..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/configuration/FilterConfigOptions.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.geedgenetworks.spi.configuration; - -import com.geedgenetworks.common.config.Option; -import com.geedgenetworks.common.config.Options; - -import java.util.Map; - -public interface FilterConfigOptions { - Option<String> TYPE = Options.key("type") - .stringType() - .noDefaultValue() - .withDescription("The type of filter ."); - - Option<Map<String, String>> PROPERTIES = Options.key("properties") - .mapType() - .noDefaultValue() - .withDescription("Custom properties for filter."); -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/filter/Filter.java b/groot-spi/src/main/java/com/geedgenetworks/spi/filter/Filter.java deleted file mode 100644 index b8d9b86..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/filter/Filter.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.geedgenetworks.spi.filter; - -import com.geedgenetworks.spi.table.event.Event; -import com.typesafe.config.Config; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.io.Serializable; -import java.util.Map; - -public interface Filter<T extends FilterConfig> extends Serializable { - - DataStream<Event> filterFunction( - DataStream<Event> singleOutputStreamOperator, T FilterConfig) - throws Exception; - String type(); - - T checkConfig(String name, Map<String, Object> configProperties, Config typeSafeConfig); - -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/filter/FilterConfig.java b/groot-spi/src/main/java/com/geedgenetworks/spi/filter/FilterConfig.java deleted file mode 100644 index b78ea13..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/filter/FilterConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.geedgenetworks.spi.filter; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -public class FilterConfig implements Serializable { - - private List<String> output_fields; - private String type; - private Map<String, Object> properties; - private int parallelism; - private String name; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public List<String> getOutput_fields() { - return output_fields; - } - - public void setOutput_fields(List<String> output_fields) { - this.output_fields = output_fields; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public Map<String, Object> getProperties() { - return properties; - } - - public void setProperties(Map<String, Object> properties) { - this.properties = properties; - } - - public int getParallelism() { - return parallelism; - } - - public void setParallelism(int parallelism) { - this.parallelism = parallelism; - } -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProcessorFactory.java b/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProcessorFactory.java deleted file mode 100644 index f7f0076..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProcessorFactory.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.geedgenetworks.spi.processor; - -public interface ProcessorFactory { - String type(); - Processor<?> createProcessor(); - -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProcessorProvider.java b/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProcessorProvider.java deleted file mode 100644 index f71a560..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProcessorProvider.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.geedgenetworks.spi.processor; - -import java.util.ServiceLoader; - -public class ProcessorProvider { - - public static Processor<?> load(String type) { - ServiceLoader<ProcessorFactory> loader = ServiceLoader.load(ProcessorFactory.class); - for (ProcessorFactory factory : loader) { - if (factory.type().equals(type)) { - return factory.createProcessor(); - } - } - throw new IllegalArgumentException("Processor type not found: " + type); - } - -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProjectionConfig.java b/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProjectionConfig.java deleted file mode 100644 index f7fbfb7..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/ProjectionConfig.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.geedgenetworks.spi.processor; - -import com.geedgenetworks.spi.common.udf.UDFContext; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.List; - -@EqualsAndHashCode(callSuper = true) -@Data -public class ProjectionConfig extends ProcessorConfig { - - private List<UDFContext> functions; - private String format; - -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/TableConfig.java b/groot-spi/src/main/java/com/geedgenetworks/spi/processor/TableConfig.java deleted file mode 100644 index 8948144..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/processor/TableConfig.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.geedgenetworks.spi.processor; - -import com.geedgenetworks.spi.common.udf.UDFContext; -import lombok.Data; -import lombok.EqualsAndHashCode; - -import java.util.List; - -@EqualsAndHashCode(callSuper = true) -@Data -public class TableConfig extends ProcessorConfig { - - private List<UDFContext> functions; - private String format; -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkConfig.java b/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkConfig.java deleted file mode 100644 index 1c57410..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.geedgenetworks.spi.sink; - -import java.io.Serializable; -import java.util.Map; - -public class SinkConfig implements Serializable { - private String type; - private Map<String, Object> schema; - private Map<String, String> properties; - private String name; - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public Map<String, Object> getSchema() { - return schema; - } - - public void setSchema(Map<String, Object> schema) { - this.schema = schema; - } - - public Map<String, String> getProperties() { - return properties; - } - - public void setProperties(Map<String, String> properties) { - this.properties = properties; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkTableFactory.java b/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkTableFactory.java deleted file mode 100644 index ecbb72c..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/sink/SinkTableFactory.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.geedgenetworks.spi.sink; - - -import com.geedgenetworks.spi.table.factory.TableFactory; - -public interface SinkTableFactory extends TableFactory { - SinkProvider getSinkProvider(Context context); -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/source/SourceConfig.java b/groot-spi/src/main/java/com/geedgenetworks/spi/source/SourceConfig.java deleted file mode 100644 index 38dd98c..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/source/SourceConfig.java +++ /dev/null @@ -1,69 +0,0 @@ -package com.geedgenetworks.spi.source; - -import java.io.Serializable; -import java.util.Map; - -public class SourceConfig implements Serializable { - private String type; - private Map<String, Object> schema; - private String watermark_timestamp; - private String watermark_timestamp_unit = "ms"; - private Long watermark_lag; - private Map<String, String> properties; - private String name; - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public Map<String, Object> getSchema() { - return schema; - } - - public void setSchema(Map<String, Object> schema) { - this.schema = schema; - } - - public String getWatermark_timestamp() { - return watermark_timestamp; - } - - public void setWatermark_timestamp(String watermark_timestamp) { - this.watermark_timestamp = watermark_timestamp; - } - - public String getWatermark_timestamp_unit() { - return watermark_timestamp_unit; - } - - public void setWatermark_timestamp_unit(String watermark_timestamp_unit) { - this.watermark_timestamp_unit = watermark_timestamp_unit; - } - - public Long getWatermark_lag() { - return watermark_lag; - } - - public void setWatermark_lag(Long watermark_lag) { - this.watermark_lag = watermark_lag; - } - - public Map<String, String> getProperties() { - return properties; - } - - public void setProperties(Map<String, String> properties) { - this.properties = properties; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/source/SourceTableFactory.java b/groot-spi/src/main/java/com/geedgenetworks/spi/source/SourceTableFactory.java deleted file mode 100644 index 8e56e41..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/source/SourceTableFactory.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.geedgenetworks.spi.source; - -import com.geedgenetworks.spi.table.factory.TableFactory; - -public interface SourceTableFactory extends TableFactory { - SourceProvider getSourceProvider(Context context); -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/split/Split.java b/groot-spi/src/main/java/com/geedgenetworks/spi/split/Split.java deleted file mode 100644 index 278a8c3..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/split/Split.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.geedgenetworks.spi.split; - -import com.geedgenetworks.spi.table.event.Event; -import com.typesafe.config.Config; -import org.apache.flink.streaming.api.datastream.DataStream; - -import java.io.Serializable; -import java.util.Map; - -public interface Split<T extends SplitConfig> extends Serializable { - - DataStream<Event> splitFunction( - DataStream<Event> dataStream, T splitConfig) - throws Exception; - String type(); - - T checkConfig(String name, Map<String, Object> configProperties, Config typeSafeConfig); - -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/split/SplitConfig.java b/groot-spi/src/main/java/com/geedgenetworks/spi/split/SplitConfig.java deleted file mode 100644 index 937951a..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/split/SplitConfig.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.geedgenetworks.spi.split; - -import com.geedgenetworks.spi.common.udf.RuleContext; -import lombok.Data; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; - -@Data -public class SplitConfig implements Serializable { - - private String type; - private Map<String, Object> properties; - private int parallelism; - private String name; - private List<RuleContext> rules; -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/DecodingFormatFactory.java b/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/DecodingFormatFactory.java deleted file mode 100644 index d6fff1c..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/DecodingFormatFactory.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.geedgenetworks.spi.table.factory; - -import com.geedgenetworks.spi.table.connector.DecodingFormat; -import org.apache.flink.configuration.ReadableConfig; - -public interface DecodingFormatFactory extends FormatFactory { - DecodingFormat createDecodingFormat(TableFactory.Context context, ReadableConfig formatOptions); -} - - diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/EncodingFormatFactory.java b/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/EncodingFormatFactory.java deleted file mode 100644 index 0f620af..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/EncodingFormatFactory.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.geedgenetworks.spi.table.factory; - -import com.geedgenetworks.spi.table.connector.EncodingFormat; -import org.apache.flink.configuration.ReadableConfig; - -public interface EncodingFormatFactory extends FormatFactory { - EncodingFormat createEncodingFormat(TableFactory.Context context, ReadableConfig formatOptions); -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/Factory.java b/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/Factory.java deleted file mode 100644 index 21fbe48..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/factory/Factory.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.geedgenetworks.spi.table.factory; - -import org.apache.flink.configuration.ConfigOption; - -import java.util.Set; - -/** - * 用于注册source、 sink、format factory的基础接口，Factory从key、value配置创建实例。 - * 可用的factory列表使用java Service Provider Interfaces (SPI)发现，把实现类添加到META_INF/services/com.geedgenetworks.core.factories.Factory即可。 - * factory实现和配置参考flink sql - */ -public interface Factory { - // 返回Factory的唯一标识 - String factoryIdentifier(); - - // 必须配置的参数 - Set<ConfigOption<?>> requiredOptions(); - - // 可选的参数 - Set<ConfigOption<?>> optionalOptions(); -} diff --git a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/SchemaChangeAware.java b/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/SchemaChangeAware.java deleted file mode 100644 index 5ec9d30..0000000 --- a/groot-spi/src/main/java/com/geedgenetworks/spi/table/schema/SchemaChangeAware.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.geedgenetworks.spi.table.schema; - -import com.geedgenetworks.spi.table.type.StructType; - -public interface SchemaChangeAware { - void schemaChange(StructType dataType); -} diff --git a/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java b/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java index 0304128..e60d34d 100644 --- a/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java +++ b/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java @@ -1,8 +1,8 @@ package com.geedgenetworks.test.e2e.kafka; import com.geedgenetworks.formats.json.JsonSerializer; -import com.geedgenetworks.spi.table.type.StructType; -import com.geedgenetworks.spi.table.type.Types; +import com.geedgenetworks.api.connector.type.StructType; +import com.geedgenetworks.api.connector.type.Types; import com.geedgenetworks.test.common.TestResource; import com.geedgenetworks.test.common.TestSuiteBase; import com.geedgenetworks.test.common.container.TestContainer; @@ -20,7 +20,7 @@ <module>groot-examples</module> <module>groot-formats</module> <module>groot-tests</module> - <module>groot-spi</module> + <module>groot-api</module> </modules> <properties> |