diff options
Diffstat (limited to 'groot-bootstrap')
35 files changed, 176 insertions, 174 deletions
diff --git a/groot-bootstrap/pom.xml b/groot-bootstrap/pom.xml index 24a202a..a6c72e6 100644 --- a/groot-bootstrap/pom.xml +++ b/groot-bootstrap/pom.xml @@ -18,9 +18,8 @@ <dependencies> <dependency> - <groupId>com.geedgenetworks</groupId> - <artifactId>groot-common</artifactId> - <version>${revision}</version> + <groupId>com.beust</groupId> + <artifactId>jcommander</artifactId> </dependency> <dependency> @@ -99,6 +98,13 @@ <scope>${scope}</scope> </dependency> + <dependency> + <groupId>com.geedgenetworks</groupId> + <artifactId>format-raw</artifactId> + <version>${revision}</version> + <scope>${scope}</scope> + </dependency> + <!-- Idea debug dependencies --> <dependency> <groupId>org.apache.flink</groupId> @@ -121,12 +127,7 @@ <scope>${scope}</scope> </dependency> - <dependency> - <groupId>com.geedgenetworks</groupId> - <artifactId>format-raw</artifactId> - <version>${revision}</version> - <scope>${scope}</scope> - </dependency> + <dependency> <groupId>org.apache.flink</groupId> @@ -164,10 +165,7 @@ <scope>${scope}</scope> </dependency> - <dependency> - <groupId>com.typesafe</groupId> - <artifactId>config</artifactId> - </dependency> + <dependency> @@ -177,10 +175,7 @@ <scope>test</scope> </dependency> - <dependency> - <groupId>com.beust</groupId> - <artifactId>jcommander</artifactId> - </dependency> + </dependencies> diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AES128GCM96Shade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AES128GCM96Shade.java index 03ed1af..c91c733 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AES128GCM96Shade.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AES128GCM96Shade.java @@ -1,7 +1,6 @@ package com.geedgenetworks.bootstrap.command; import cn.hutool.core.util.RandomUtil; -import com.geedgenetworks.common.crypto.CryptoShade; import javax.crypto.Cipher; import javax.crypto.spec.GCMParameterSpec; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AES256GCM96Shade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AES256GCM96Shade.java index efee134..4eadc28 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AES256GCM96Shade.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AES256GCM96Shade.java @@ -1,7 +1,6 @@ package com.geedgenetworks.bootstrap.command; import cn.hutool.core.util.RandomUtil; -import com.geedgenetworks.common.crypto.CryptoShade; import javax.crypto.Cipher; import javax.crypto.spec.GCMParameterSpec; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESShade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESShade.java index 91e05d0..b593937 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESShade.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESShade.java @@ -2,7 +2,6 @@ package com.geedgenetworks.bootstrap.command; import cn.hutool.crypto.SecureUtil; import cn.hutool.crypto.symmetric.SymmetricAlgorithm; -import com.geedgenetworks.common.crypto.CryptoShade; import java.nio.charset.StandardCharsets; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64Shade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64Shade.java index d07c372..6cdce0f 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64Shade.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64Shade.java @@ -1,6 +1,5 @@ package com.geedgenetworks.bootstrap.command; -import com.geedgenetworks.common.crypto.CryptoShade; import java.nio.charset.StandardCharsets; import java.util.Base64; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/CommandArgs.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/CommandArgs.java index 6ee5151..bd7882a 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/CommandArgs.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/CommandArgs.java @@ -2,7 +2,7 @@ package com.geedgenetworks.bootstrap.command; import com.beust.jcommander.Parameter; import com.geedgenetworks.bootstrap.enums.DeployMode; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/CryptoShade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/CryptoShade.java new file mode 100644 index 0000000..78515a7 --- /dev/null +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/CryptoShade.java @@ -0,0 +1,29 @@ +package com.geedgenetworks.bootstrap.command; + +public interface CryptoShade { + + /** + * The unique identifier of the current interface, used it to select the correct {@link + * CryptoShade} + */ + String getIdentifier(); + + /** + * Encrypt the content + * + * @param content The content to encrypt + */ + String encrypt(String content); + + /** + * Decrypt the content + * + * @param content The content to decrypt + */ + String decrypt(String content); + + /** To expand the options that user want to encrypt */ + default String[] sensitiveOptions() { + return new String[0]; + } +} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ExecuteCommand.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ExecuteCommand.java index c3538b0..01f3bdd 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ExecuteCommand.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ExecuteCommand.java @@ -7,7 +7,7 @@ import com.geedgenetworks.bootstrap.execution.ExecutionConfigKeyName; import com.geedgenetworks.bootstrap.execution.JobExecution; import com.geedgenetworks.bootstrap.utils.ConfigBuilder; import com.geedgenetworks.bootstrap.utils.ConfigFileUtils; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.ConfigProvider; import com.geedgenetworks.common.config.GrootStreamConfig; import com.typesafe.config.Config; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4GCM96Shade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4GCM96Shade.java index a6d27e4..7fa84b4 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4GCM96Shade.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4GCM96Shade.java @@ -1,7 +1,6 @@ package com.geedgenetworks.bootstrap.command; import cn.hutool.core.util.RandomUtil; -import com.geedgenetworks.common.crypto.CryptoShade; import javax.crypto.Cipher; import javax.crypto.spec.GCMParameterSpec; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4Shade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4Shade.java index e274716..4e04d9e 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4Shade.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4Shade.java @@ -3,7 +3,6 @@ package com.geedgenetworks.bootstrap.command; import cn.hutool.crypto.KeyUtil; import cn.hutool.crypto.SmUtil; import cn.hutool.crypto.symmetric.SM4; -import com.geedgenetworks.common.crypto.CryptoShade; import java.nio.charset.StandardCharsets; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java index f5b1a5d..e0828a0 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java @@ -1,13 +1,9 @@ package com.geedgenetworks.bootstrap.execution; -import com.geedgenetworks.common.Event; import com.geedgenetworks.common.utils.ReflectionUtils; -import com.geedgenetworks.core.filter.Filter; -import com.geedgenetworks.core.processor.Processor; -import com.geedgenetworks.core.split.Split; +import com.geedgenetworks.spi.table.event.Event; import com.typesafe.config.Config; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import java.net.URL; import java.net.URLClassLoader; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java index 42a3a11..ec748cc 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java @@ -1,12 +1,13 @@ package com.geedgenetworks.bootstrap.execution; import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.Event; import com.geedgenetworks.common.config.*; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.core.pojo.ProcessorConfig; -import com.geedgenetworks.core.processor.Processor; +import com.geedgenetworks.spi.configuration.ProjectionConfigOptions; +import com.geedgenetworks.spi.processor.Processor; +import com.geedgenetworks.spi.processor.ProcessorConfig; +import com.geedgenetworks.spi.table.event.Event; import com.typesafe.config.Config; import org.apache.flink.streaming.api.datastream.DataStream; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/FilterExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/FilterExecutor.java index f3c81c2..1ea19f8 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/FilterExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/FilterExecutor.java @@ -1,24 +1,21 @@ package com.geedgenetworks.bootstrap.execution; -import com.alibaba.fastjson.JSONObject; import com.geedgenetworks.bootstrap.enums.ProcessorType; import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CheckConfigUtil; import com.geedgenetworks.common.config.CheckResult; -import com.geedgenetworks.common.config.FilterConfigOptions; -import com.geedgenetworks.common.config.ProjectionConfigOptions; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.core.filter.Filter; -import com.geedgenetworks.core.pojo.FilterConfig; +import com.geedgenetworks.spi.configuration.FilterConfigOptions; +import com.geedgenetworks.spi.filter.Filter; +import com.geedgenetworks.spi.filter.FilterConfig; +import com.geedgenetworks.spi.table.event.Event; import com.google.common.collect.Maps; import com.typesafe.config.Config; import lombok.extern.slf4j.Slf4j; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import java.net.URL; import java.util.List; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobExecution.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobExecution.java index 706fc18..325f8a4 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobExecution.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobExecution.java @@ -4,9 +4,9 @@ import com.alibaba.fastjson2.JSONObject; import com.geedgenetworks.bootstrap.enums.ProcessorType; import com.geedgenetworks.bootstrap.exception.JobExecuteException; import com.geedgenetworks.bootstrap.main.GrootStreamRunner; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.GrootStreamConfig; +import com.geedgenetworks.spi.table.event.Event; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.typesafe.config.Config; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobRuntimeEnvironment.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobRuntimeEnvironment.java index e23d446..7b4d66b 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobRuntimeEnvironment.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/JobRuntimeEnvironment.java @@ -3,7 +3,7 @@ package com.geedgenetworks.bootstrap.execution; import com.alibaba.fastjson2.JSON; import com.geedgenetworks.bootstrap.enums.TargetType; import com.geedgenetworks.bootstrap.utils.EnvironmentUtil; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CheckResult; import com.geedgenetworks.common.config.GrootStreamConfig; import com.geedgenetworks.common.utils.ReflectionUtils; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PostprocessingExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PostprocessingExecutor.java index 03e5bd5..e73b7dd 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PostprocessingExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PostprocessingExecutor.java @@ -2,13 +2,12 @@ package com.geedgenetworks.bootstrap.execution; import com.geedgenetworks.bootstrap.enums.ProcessorType; import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.Event; -import com.geedgenetworks.core.pojo.ProcessorConfig; +import com.geedgenetworks.common.config.Constants; +import com.geedgenetworks.spi.processor.ProcessorConfig; +import com.geedgenetworks.spi.table.event.Event; import com.google.common.collect.Maps; import com.typesafe.config.Config; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import java.net.URL; import java.util.List; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PreprocessingExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PreprocessingExecutor.java index da8dc62..6179265 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PreprocessingExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/PreprocessingExecutor.java @@ -2,14 +2,13 @@ package com.geedgenetworks.bootstrap.execution; import com.geedgenetworks.bootstrap.enums.ProcessorType; import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.Event; -import com.geedgenetworks.core.pojo.ProcessorConfig; +import com.geedgenetworks.common.config.Constants; +import com.geedgenetworks.spi.processor.ProcessorConfig; +import com.geedgenetworks.spi.table.event.Event; import com.google.common.collect.Maps; import com.typesafe.config.Config; import lombok.extern.slf4j.Slf4j; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import java.net.URL; import java.util.List; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessingExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessingExecutor.java index cf6b496..bc6a09e 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessingExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/ProcessingExecutor.java @@ -2,13 +2,12 @@ package com.geedgenetworks.bootstrap.execution; import com.geedgenetworks.bootstrap.enums.ProcessorType; import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.Event; -import com.geedgenetworks.core.pojo.ProcessorConfig; +import com.geedgenetworks.common.config.Constants; +import com.geedgenetworks.spi.processor.ProcessorConfig; +import com.geedgenetworks.spi.table.event.Event; import com.google.common.collect.Maps; import com.typesafe.config.Config; import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import java.net.URL; import java.util.List; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/RuntimeEnvironment.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/RuntimeEnvironment.java index b177e40..023ba65 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/RuntimeEnvironment.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/RuntimeEnvironment.java @@ -1,5 +1,5 @@ package com.geedgenetworks.bootstrap.execution; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CheckResult; import com.typesafe.config.Config; import com.typesafe.config.ConfigUtil; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SinkExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SinkExecutor.java index 70934b8..b61b6f9 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SinkExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SinkExecutor.java @@ -4,26 +4,25 @@ import com.alibaba.fastjson.JSONObject; import com.geedgenetworks.bootstrap.enums.ProcessorType; import com.geedgenetworks.bootstrap.exception.JobExecuteException; import com.geedgenetworks.bootstrap.utils.SchemaConfigParse; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CheckConfigUtil; import com.geedgenetworks.common.config.CheckResult; -import com.geedgenetworks.common.config.SinkConfigOptions; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.core.connector.schema.Schema; -import com.geedgenetworks.core.connector.sink.SinkProvider; -import com.geedgenetworks.core.factories.FactoryUtil; -import com.geedgenetworks.core.factories.SinkTableFactory; -import com.geedgenetworks.core.factories.TableFactory; -import com.geedgenetworks.core.pojo.SinkConfig; +import com.geedgenetworks.spi.sink.SinkConfig; +import com.geedgenetworks.spi.sink.SinkConfigOptions; +import com.geedgenetworks.spi.table.connector.SinkProvider; +import com.geedgenetworks.spi.table.connector.SinkTableFactory; +import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.spi.table.factory.FactoryUtil; +import com.geedgenetworks.spi.table.factory.TableFactory; +import com.geedgenetworks.spi.table.schema.Schema; import com.google.common.collect.Maps; import com.typesafe.config.Config; import lombok.extern.slf4j.Slf4j; import org.apache.flink.configuration.Configuration; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; import java.net.URL; import java.util.List; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SourceExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SourceExecutor.java index 9dff6b0..3eeaad6 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SourceExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SourceExecutor.java @@ -5,19 +5,19 @@ import com.geedgenetworks.bootstrap.enums.ProcessorType; import com.geedgenetworks.bootstrap.exception.ConfigCheckException; import com.geedgenetworks.bootstrap.exception.JobExecuteException; import com.geedgenetworks.bootstrap.utils.SchemaConfigParse; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CheckConfigUtil; import com.geedgenetworks.common.config.CheckResult; -import com.geedgenetworks.common.config.SourceConfigOptions; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.core.connector.schema.Schema; -import com.geedgenetworks.core.connector.source.SourceProvider; -import com.geedgenetworks.core.factories.FactoryUtil; -import com.geedgenetworks.core.factories.SourceTableFactory; -import com.geedgenetworks.core.factories.TableFactory; -import com.geedgenetworks.core.pojo.SourceConfig; +import com.geedgenetworks.spi.configuration.SourceConfigOptions; +import com.geedgenetworks.spi.source.SourceConfig; +import com.geedgenetworks.spi.table.connector.SourceProvider; +import com.geedgenetworks.spi.table.connector.SourceTableFactory; +import com.geedgenetworks.spi.table.event.Event; +import com.geedgenetworks.spi.table.factory.FactoryUtil; +import com.geedgenetworks.spi.table.factory.TableFactory; +import com.geedgenetworks.spi.table.schema.Schema; import com.google.common.collect.Maps; import com.typesafe.config.*; import lombok.extern.slf4j.Slf4j; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SplitExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SplitExecutor.java index 7fe93b5..3d6f264 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SplitExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/SplitExecutor.java @@ -2,18 +2,15 @@ package com.geedgenetworks.bootstrap.execution; import com.alibaba.fastjson.JSONObject; import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CheckConfigUtil; import com.geedgenetworks.common.config.CheckResult; -import com.geedgenetworks.common.config.SplitConfigOptions; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; -import com.geedgenetworks.common.udf.RuleContext; -import com.geedgenetworks.core.filter.Filter; -import com.geedgenetworks.core.pojo.FilterConfig; -import com.geedgenetworks.core.pojo.SplitConfig; -import com.geedgenetworks.core.split.Split; +import com.geedgenetworks.spi.split.Split; +import com.geedgenetworks.spi.split.SplitConfig; +import com.geedgenetworks.spi.split.SplitConfigOptions; +import com.geedgenetworks.spi.table.event.Event; import com.google.common.collect.Maps; import com.typesafe.config.Config; import lombok.extern.slf4j.Slf4j; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CommandLineUtils.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CommandLineUtils.java index 6c7d546..b87d05f 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CommandLineUtils.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CommandLineUtils.java @@ -4,7 +4,7 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.ParameterException; import com.geedgenetworks.bootstrap.command.CommandArgs; import com.geedgenetworks.bootstrap.command.UsageFormatter; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; public final class CommandLineUtils { private CommandLineUtils() { diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CryptoShadeUtils.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CryptoShadeUtils.java index 94dda4d..238bc07 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CryptoShadeUtils.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CryptoShadeUtils.java @@ -2,8 +2,8 @@ package com.geedgenetworks.bootstrap.utils; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONObject; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.crypto.CryptoShade; +import com.geedgenetworks.bootstrap.command.CryptoShade; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.TypesafeConfigUtils; import com.google.common.base.Preconditions; import com.typesafe.config.*; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/EnvironmentUtil.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/EnvironmentUtil.java index 8028608..472aab9 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/EnvironmentUtil.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/EnvironmentUtil.java @@ -1,7 +1,7 @@ package com.geedgenetworks.bootstrap.utils; import com.geedgenetworks.bootstrap.execution.ExecutionConfigKeyName; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.CheckResult; import com.typesafe.config.Config; import com.typesafe.config.ConfigUtil; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/SchemaConfigParse.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/SchemaConfigParse.java index c3076b4..f6b4292 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/SchemaConfigParse.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/SchemaConfigParse.java @@ -1,73 +1,73 @@ -package com.geedgenetworks.bootstrap.utils;
-
-import com.alibaba.fastjson2.JSON;
-import com.geedgenetworks.common.exception.CommonErrorCode;
-import com.geedgenetworks.common.exception.ConfigValidationException;
-import com.geedgenetworks.core.connector.schema.Schema;
-import com.geedgenetworks.core.connector.schema.SchemaParser;
-import com.geedgenetworks.core.types.StructType;
-import com.geedgenetworks.core.types.Types;
-import org.apache.commons.io.FileUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-import java.util.Map;
-
-public class SchemaConfigParse {
- static final String KEY_BUILTIN = "fields";
- static final String KEY_LOCAL_FILE = "local_file";
- static final String KEY_HTTP = "url";
-
- public static Schema parseSchemaConfig(Map<String, Object> schemaConfig){
- if(schemaConfig == null && schemaConfig.isEmpty()){
- return null;
- }
-
- int builtin = 0, localFile = 0, http = 0;
- if(schemaConfig.containsKey(KEY_BUILTIN)){
- builtin = 1;
- }
- if(schemaConfig.containsKey(KEY_LOCAL_FILE)){
- localFile = 1;
- }
- if(schemaConfig.containsKey(KEY_HTTP)){
- http = 1;
- }
- if(builtin + localFile + http > 1){
- throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, "only support one type schema:" + schemaConfig);
- }
-
- if(builtin == 1){
- Object fields = schemaConfig.get(KEY_BUILTIN);
- if(fields instanceof List){
- StructType dataType = Types.parseSchemaFromJson(JSON.toJSONString(fields));
- return Schema.newSchema(dataType);
- }else if(fields instanceof String){
- StructType dataType = Types.parseStructType((String) fields);
- return Schema.newSchema(dataType);
- }else{
- throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, "only support schema fields:" + fields);
- }
- }
-
- if(localFile == 1){
- String path = schemaConfig.get(KEY_LOCAL_FILE).toString();
- try {
- String content = FileUtils.readFileToString(new File(path), StandardCharsets.UTF_8);
- StructType dataType = SchemaParser.PARSER_AVRO.parser(content);
- return Schema.newSchema(dataType);
- } catch (IOException e) {
- throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, "schema path read error:" + path, e);
- }
- }
-
- if(http == 1){
- String url = schemaConfig.get(KEY_HTTP).toString();
- return Schema.newHttpDynamicSchema(url);
- }
-
- return null;
- }
-}
+package com.geedgenetworks.bootstrap.utils; + +import com.alibaba.fastjson2.JSON; +import com.geedgenetworks.common.exception.CommonErrorCode; +import com.geedgenetworks.common.exception.ConfigValidationException; +import com.geedgenetworks.spi.table.schema.Schema; +import com.geedgenetworks.spi.table.schema.SchemaParser; +import com.geedgenetworks.spi.table.type.StructType; +import com.geedgenetworks.spi.table.type.Types; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +public class SchemaConfigParse { + static final String KEY_BUILTIN = "fields"; + static final String KEY_LOCAL_FILE = "local_file"; + static final String KEY_HTTP = "url"; + + public static Schema parseSchemaConfig(Map<String, Object> schemaConfig){ + if(schemaConfig == null && schemaConfig.isEmpty()){ + return null; + } + + int builtin = 0, localFile = 0, http = 0; + if(schemaConfig.containsKey(KEY_BUILTIN)){ + builtin = 1; + } + if(schemaConfig.containsKey(KEY_LOCAL_FILE)){ + localFile = 1; + } + if(schemaConfig.containsKey(KEY_HTTP)){ + http = 1; + } + if(builtin + localFile + http > 1){ + throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, "only support one type schema:" + schemaConfig); + } + + if(builtin == 1){ + Object fields = schemaConfig.get(KEY_BUILTIN); + if(fields instanceof List){ + StructType dataType = Types.parseSchemaFromJson(JSON.toJSONString(fields)); + return Schema.newSchema(dataType); + }else if(fields instanceof String){ + StructType dataType = Types.parseStructType((String) fields); + return Schema.newSchema(dataType); + }else{ + throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, "only support schema fields:" + fields); + } + } + + if(localFile == 1){ + String path = schemaConfig.get(KEY_LOCAL_FILE).toString(); + try { + String content = FileUtils.readFileToString(new File(path), StandardCharsets.UTF_8); + StructType dataType = SchemaParser.PARSER_AVRO.parser(content); + return Schema.newSchema(dataType); + } catch (IOException e) { + throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, "schema path read error:" + path, e); + } + } + + if(http == 1){ + String url = schemaConfig.get(KEY_HTTP).toString(); + return Schema.newHttpDynamicSchema(url); + } + + return null; + } +} diff --git a/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade b/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.bootstrap.command.CryptoShade index 273b40d..273b40d 100644 --- a/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade +++ b/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.bootstrap.command.CryptoShade diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/LogoTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/LogoTest.java index d101bb2..aeb9d4b 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/LogoTest.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/LogoTest.java @@ -1,6 +1,6 @@ package com.geedgenetworks.bootstrap.main; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import org.junit.Assert; import org.junit.Test; diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobAggTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobAggTest.java index e33998c..fa9c2dd 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobAggTest.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobAggTest.java @@ -8,7 +8,7 @@ import com.geedgenetworks.bootstrap.execution.JobExecution; import com.geedgenetworks.bootstrap.main.simple.collect.CollectSink; import com.geedgenetworks.bootstrap.utils.CommandLineUtils; import com.geedgenetworks.bootstrap.utils.ConfigFileUtils; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.ConfigProvider; import com.geedgenetworks.common.config.GrootStreamConfig; import com.typesafe.config.Config; diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobDosTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobDosTest.java index ea3793e..bd4f9d8 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobDosTest.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobDosTest.java @@ -8,7 +8,7 @@ import com.geedgenetworks.bootstrap.execution.JobExecution; import com.geedgenetworks.bootstrap.main.simple.collect.CollectSink; import com.geedgenetworks.bootstrap.utils.CommandLineUtils; import com.geedgenetworks.bootstrap.utils.ConfigFileUtils; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.ConfigProvider; import com.geedgenetworks.common.config.GrootStreamConfig; import com.typesafe.config.Config; diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobEtlTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobEtlTest.java index 80b7129..1fc62d0 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobEtlTest.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobEtlTest.java @@ -8,7 +8,7 @@ import com.geedgenetworks.bootstrap.execution.JobExecution; import com.geedgenetworks.bootstrap.main.simple.collect.CollectSink; import com.geedgenetworks.bootstrap.utils.CommandLineUtils; import com.geedgenetworks.bootstrap.utils.ConfigFileUtils; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.ConfigProvider; import com.geedgenetworks.common.config.GrootStreamConfig; import com.typesafe.config.Config; @@ -93,7 +93,6 @@ public class JobEtlTest { executeCommandArgs.buildCommand(); - GrootStreamConfig grootStreamConfig = ConfigProvider.locateAndGetGrootStreamConfig(); Path configFile = ConfigFileUtils.getConfigPath(executeCommandArgs); // check config file exist diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobSplitTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobSplitTest.java index 352bad2..577b293 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobSplitTest.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/JobSplitTest.java @@ -3,13 +3,12 @@ package com.geedgenetworks.bootstrap.main.simple; import cn.hutool.setting.yaml.YamlUtil; import com.geedgenetworks.bootstrap.command.ExecuteCommandArgs; import com.geedgenetworks.bootstrap.enums.EngineType; -import com.geedgenetworks.bootstrap.exception.JobExecuteException; import com.geedgenetworks.bootstrap.execution.ExecutionConfigKeyName; import com.geedgenetworks.bootstrap.execution.JobExecution; import com.geedgenetworks.bootstrap.main.simple.collect.CollectSink; import com.geedgenetworks.bootstrap.utils.CommandLineUtils; import com.geedgenetworks.bootstrap.utils.ConfigFileUtils; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import com.geedgenetworks.common.config.ConfigProvider; import com.geedgenetworks.common.config.GrootStreamConfig; import com.typesafe.config.Config; @@ -23,7 +22,6 @@ import org.junit.ClassRule; import org.junit.Test; import java.nio.file.Path; -import java.util.List; import java.util.Map; import static org.junit.Assert.assertTrue; diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectSink.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectSink.java index c5806ed..74b40a4 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectSink.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectSink.java @@ -1,6 +1,6 @@ package com.geedgenetworks.bootstrap.main.simple.collect; -import com.geedgenetworks.common.Event; +import com.geedgenetworks.spi.table.event.Event; import org.apache.flink.streaming.api.functions.sink.SinkFunction; import java.util.*; diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectTableFactory.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectTableFactory.java index 32a0acd..e52fd3b 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectTableFactory.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/main/simple/collect/CollectTableFactory.java @@ -1,8 +1,8 @@ package com.geedgenetworks.bootstrap.main.simple.collect; -import com.geedgenetworks.common.Event; -import com.geedgenetworks.core.connector.sink.SinkProvider; -import com.geedgenetworks.core.factories.SinkTableFactory; +import com.geedgenetworks.spi.table.connector.SinkProvider; +import com.geedgenetworks.spi.table.connector.SinkTableFactory; +import com.geedgenetworks.spi.table.event.Event; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStreamSink; diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/CryptoShadeTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/CryptoShadeTest.java index f77ba44..a3d2bd5 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/CryptoShadeTest.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/CryptoShadeTest.java @@ -3,7 +3,7 @@ package com.geedgenetworks.bootstrap.utils; import cn.hutool.setting.yaml.YamlUtil; import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONObject; -import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.config.Constants; import com.typesafe.config.Config; import com.typesafe.config.ConfigObject; import com.typesafe.config.ConfigRenderOptions; |