From 339c92a61b95156b8de292f0b8a50bfad5831c23 Mon Sep 17 00:00:00 2001 From: doufenghu Date: Fri, 13 Sep 2024 19:20:00 +0800 Subject: [Improve][Common] Rename ConfigShade to CryptoShade for better describe data encryption. --- docs/connector/config-encryption-decryption.md | 8 +- .../bootstrap/command/AESConfigShade.java | 36 ------ .../geedgenetworks/bootstrap/command/AESShade.java | 37 ++++++ .../bootstrap/command/Base64ConfigShade.java | 28 ----- .../bootstrap/command/Base64Shade.java | 28 +++++ .../bootstrap/command/ConfDecryptCommand.java | 2 - .../bootstrap/command/ConfEncryptCommand.java | 5 +- .../bootstrap/command/SM4ConfigShade.java | 37 ------ .../geedgenetworks/bootstrap/command/SM4Shade.java | 37 ++++++ .../bootstrap/utils/ConfigBuilder.java | 4 +- .../bootstrap/utils/ConfigShadeUtils.java | 138 --------------------- .../bootstrap/utils/CryptoShadeUtils.java | 138 +++++++++++++++++++++ .../com.geedgenetworks.common.config.ConfigShade | 3 - .../com.geedgenetworks.common.crypto.CryptoShade | 3 + .../bootstrap/utils/ConfigShadeTest.java | 81 ------------ .../bootstrap/utils/CryptoShadeTest.java | 81 ++++++++++++ .../com.geedgenetworks.common.config.ConfigShade | 2 - .../com.geedgenetworks.common.crypto.CryptoShade | 2 + .../geedgenetworks/common/config/ConfigShade.java | 29 ----- .../geedgenetworks/common/crypto/CryptoShade.java | 29 +++++ .../geedgenetworks/example/GrootStreamExample.java | 4 +- 21 files changed, 365 insertions(+), 367 deletions(-) delete mode 100644 groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESConfigShade.java create mode 100644 groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESShade.java delete mode 100644 groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64ConfigShade.java create mode 100644 groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64Shade.java delete mode 100644 groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4ConfigShade.java create mode 100644 groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4Shade.java delete mode 100644 groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/ConfigShadeUtils.java create mode 100644 groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CryptoShadeUtils.java delete mode 100644 groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.config.ConfigShade create mode 100644 groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade delete mode 100644 groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java create mode 100644 groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/CryptoShadeTest.java delete mode 100644 groot-bootstrap/src/test/resources/META-INF/services/com.geedgenetworks.common.config.ConfigShade create mode 100644 groot-bootstrap/src/test/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade delete mode 100644 groot-common/src/main/java/com/geedgenetworks/common/config/ConfigShade.java create mode 100644 groot-common/src/main/java/com/geedgenetworks/common/crypto/CryptoShade.java diff --git a/docs/connector/config-encryption-decryption.md b/docs/connector/config-encryption-decryption.md index c2b05f6..91ca80e 100644 --- a/docs/connector/config-encryption-decryption.md +++ b/docs/connector/config-encryption-decryption.md @@ -127,13 +127,13 @@ Next, I'll show how to quickly use groot-stream's own `aes` encryption: ## How to implement user-defined encryption and decryption -1. Create a new class and implement interface `ConfigShade`, this interface has the following methods: +1. Create a new class and implement interface `CryptoShade`, this interface has the following methods: ```java - public interface ConfigShade { + public interface CryptoShade { /** * The unique identifier of the current interface, used it to select the correct {@link - * ConfigShade} + * CryptoShade} */ String getIdentifier(); @@ -157,6 +157,6 @@ Next, I'll show how to quickly use groot-stream's own `aes` encryption: } } ``` -2. Add `com.geedgenetworks.common.config.ConfigShade` in `resources/META-INF/services` +2. Add `com.geedgenetworks.common.crypto.CryptoShade` in `resources/META-INF/services` 3. Change the option `shade.identifier` to the value that you defined in `ConfigShade#getIdentifier`of you config file. diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESConfigShade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESConfigShade.java deleted file mode 100644 index 76b4944..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESConfigShade.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.geedgenetworks.bootstrap.command; - -import cn.hutool.crypto.SecureUtil; -import cn.hutool.crypto.symmetric.SymmetricAlgorithm; -import com.geedgenetworks.common.config.ConfigShade; -import java.nio.charset.StandardCharsets; - -public class AESConfigShade implements ConfigShade { - private static final String IDENTIFIER = "aes"; - - private static final byte[] SECURITY_KEY = - SecureUtil.generateKey(SymmetricAlgorithm.AES.getValue(), ".geedgenetworks.".getBytes(StandardCharsets.UTF_8)).getEncoded() ; - - private static final String[] SENSITIVE_OPTIONS = - new String[] {"connection.user", "connection.password", "kafka.sasl.jaas.config","kafka.ssl.keystore.password","kafka.ssl.truststore.password","kafka.ssl.key.password"}; - - @Override - public String[] sensitiveOptions() { - return SENSITIVE_OPTIONS; - } - - @Override - public String getIdentifier() { - return IDENTIFIER; - } - - @Override - public String encrypt(String content) { - return SecureUtil.aes(SECURITY_KEY).encryptHex(content, StandardCharsets.UTF_8); - } - - @Override - public String decrypt(String content) { - return SecureUtil.aes(SECURITY_KEY).decryptStr(content, StandardCharsets.UTF_8); - } -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESShade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESShade.java new file mode 100644 index 0000000..a2f4f56 --- /dev/null +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/AESShade.java @@ -0,0 +1,37 @@ +package com.geedgenetworks.bootstrap.command; + +import cn.hutool.crypto.SecureUtil; +import cn.hutool.crypto.symmetric.SymmetricAlgorithm; +import com.geedgenetworks.common.crypto.CryptoShade; + +import java.nio.charset.StandardCharsets; + +public class AESShade implements CryptoShade { + private static final String IDENTIFIER = "aes"; + + private static final byte[] SECURITY_KEY = + SecureUtil.generateKey(SymmetricAlgorithm.AES.getValue(), ".geedgenetworks.".getBytes(StandardCharsets.UTF_8)).getEncoded() ; + + private static final String[] SENSITIVE_OPTIONS = + new String[] {"connection.user", "connection.password", "kafka.sasl.jaas.config","kafka.ssl.keystore.password","kafka.ssl.truststore.password","kafka.ssl.key.password"}; + + @Override + public String[] sensitiveOptions() { + return SENSITIVE_OPTIONS; + } + + @Override + public String getIdentifier() { + return IDENTIFIER; + } + + @Override + public String encrypt(String content) { + return SecureUtil.aes(SECURITY_KEY).encryptHex(content, StandardCharsets.UTF_8); + } + + @Override + public String decrypt(String content) { + return SecureUtil.aes(SECURITY_KEY).decryptStr(content, StandardCharsets.UTF_8); + } +} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64ConfigShade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64ConfigShade.java deleted file mode 100644 index 4ae2b5c..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64ConfigShade.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.geedgenetworks.bootstrap.command; - -import com.geedgenetworks.common.config.ConfigShade; -import java.nio.charset.StandardCharsets; -import java.util.Base64; - -public class Base64ConfigShade implements ConfigShade { - private static final Base64.Encoder ENCODER = Base64.getEncoder(); - - private static final Base64.Decoder DECODER = Base64.getDecoder(); - - private static final String IDENTIFIER = "base64"; - - @Override - public String getIdentifier() { - return IDENTIFIER; - } - - @Override - public String encrypt(String content) { - return ENCODER.encodeToString(content.getBytes(StandardCharsets.UTF_8)); - } - - @Override - public String decrypt(String content) { - return new String(DECODER.decode(content)); - } -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64Shade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64Shade.java new file mode 100644 index 0000000..d07c372 --- /dev/null +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/Base64Shade.java @@ -0,0 +1,28 @@ +package com.geedgenetworks.bootstrap.command; + +import com.geedgenetworks.common.crypto.CryptoShade; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +public class Base64Shade implements CryptoShade { + private static final Base64.Encoder ENCODER = Base64.getEncoder(); + + private static final Base64.Decoder DECODER = Base64.getDecoder(); + + private static final String IDENTIFIER = "base64"; + + @Override + public String getIdentifier() { + return IDENTIFIER; + } + + @Override + public String encrypt(String content) { + return ENCODER.encodeToString(content.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public String decrypt(String content) { + return new String(DECODER.decode(content)); + } +} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ConfDecryptCommand.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ConfDecryptCommand.java index 75f7819..b124c9d 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ConfDecryptCommand.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ConfDecryptCommand.java @@ -5,12 +5,10 @@ import com.geedgenetworks.bootstrap.exception.CommandExecuteException; import com.geedgenetworks.bootstrap.exception.ConfigCheckException; import com.geedgenetworks.bootstrap.utils.ConfigBuilder; import com.geedgenetworks.bootstrap.utils.ConfigFileUtils; -import com.geedgenetworks.bootstrap.utils.ConfigShadeUtils; import com.typesafe.config.*; import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Map; diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ConfEncryptCommand.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ConfEncryptCommand.java index 676cba5..1d7be2e 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ConfEncryptCommand.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/ConfEncryptCommand.java @@ -5,12 +5,11 @@ import com.geedgenetworks.bootstrap.exception.CommandExecuteException; import com.geedgenetworks.bootstrap.exception.ConfigCheckException; import com.geedgenetworks.bootstrap.utils.ConfigBuilder; import com.geedgenetworks.bootstrap.utils.ConfigFileUtils; -import com.geedgenetworks.bootstrap.utils.ConfigShadeUtils; +import com.geedgenetworks.bootstrap.utils.CryptoShadeUtils; import com.typesafe.config.*; import lombok.extern.slf4j.Slf4j; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Map; import static com.geedgenetworks.bootstrap.utils.ConfigFileUtils.checkConfigExist; @@ -32,7 +31,7 @@ public class ConfEncryptCommand implements Command{ checkConfigExist(configPath); Map configMap = YamlUtil.loadByPath(configPath.toString()); Config config = ConfigBuilder.of(configMap, false); - Config encryptConfig = ConfigShadeUtils.encryptConfig(config); + Config encryptConfig = CryptoShadeUtils.encryptConfig(config); System.out.println(String.format( "Encrypt config: %s", encryptConfig.root().render(ConfigRenderOptions.defaults().setOriginComments(false)))); log.info( diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4ConfigShade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4ConfigShade.java deleted file mode 100644 index 05d3e52..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4ConfigShade.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.geedgenetworks.bootstrap.command; - -import cn.hutool.crypto.KeyUtil; -import cn.hutool.crypto.SmUtil; -import cn.hutool.crypto.symmetric.SM4; -import com.geedgenetworks.common.config.ConfigShade; - -import java.nio.charset.StandardCharsets; - -public class SM4ConfigShade implements ConfigShade { - private static final String IDENTIFIER = "sm4"; - - private static final String[] SENSITIVE_OPTIONS = - new String[] {"connection.user", "connection.password", "kafka.sasl.jaas.config","kafka.ssl.keystore.password","kafka.ssl.truststore.password","kafka.ssl.key.password"}; - - private static final byte[] SECURITY_KEY = KeyUtil.generateKey(SM4.ALGORITHM_NAME, ".geedgenetworks.".getBytes(StandardCharsets.UTF_8)).getEncoded(); - - @Override - public String[] sensitiveOptions() { - return SENSITIVE_OPTIONS; - } - - @Override - public String getIdentifier() { - return IDENTIFIER; - } - - @Override - public String encrypt(String content) { - return SmUtil.sm4(SECURITY_KEY).encryptHex(content, StandardCharsets.UTF_8); - } - - @Override - public String decrypt(String content) { - return SmUtil.sm4(SECURITY_KEY).decryptStr(content, StandardCharsets.UTF_8); - } -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4Shade.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4Shade.java new file mode 100644 index 0000000..6fd15bd --- /dev/null +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/command/SM4Shade.java @@ -0,0 +1,37 @@ +package com.geedgenetworks.bootstrap.command; + +import cn.hutool.crypto.KeyUtil; +import cn.hutool.crypto.SmUtil; +import cn.hutool.crypto.symmetric.SM4; +import com.geedgenetworks.common.crypto.CryptoShade; + +import java.nio.charset.StandardCharsets; + +public class SM4Shade implements CryptoShade { + private static final String IDENTIFIER = "sm4"; + + private static final String[] SENSITIVE_OPTIONS = + new String[] {"connection.user", "connection.password", "kafka.sasl.jaas.config","kafka.ssl.keystore.password","kafka.ssl.truststore.password","kafka.ssl.key.password"}; + + private static final byte[] SECURITY_KEY = KeyUtil.generateKey(SM4.ALGORITHM_NAME, ".geedgenetworks.".getBytes(StandardCharsets.UTF_8)).getEncoded(); + + @Override + public String[] sensitiveOptions() { + return SENSITIVE_OPTIONS; + } + + @Override + public String getIdentifier() { + return IDENTIFIER; + } + + @Override + public String encrypt(String content) { + return SmUtil.sm4(SECURITY_KEY).encryptHex(content, StandardCharsets.UTF_8); + } + + @Override + public String decrypt(String content) { + return SmUtil.sm4(SECURITY_KEY).decryptStr(content, StandardCharsets.UTF_8); + } +} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/ConfigBuilder.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/ConfigBuilder.java index 954c058..a83506d 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/ConfigBuilder.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/ConfigBuilder.java @@ -25,7 +25,7 @@ public class ConfigBuilder { Map configMap = YamlUtil.loadByPath(filePath.toString()); ConfigObject configObject = ConfigValueFactory.fromMap(configMap); Config config = configObject.toConfig(); - return ConfigShadeUtils.decryptConfig(config); + return CryptoShadeUtils.decryptConfig(config); } public static Config of(@NonNull Map objectMap) { @@ -36,7 +36,7 @@ public class ConfigBuilder { ConfigObject configObject = ConfigValueFactory.fromMap(objectMap); Config config = configObject.toConfig(); if (needDecrypt) { - return ConfigShadeUtils.decryptConfig(config); + return CryptoShadeUtils.decryptConfig(config); } return config; } diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/ConfigShadeUtils.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/ConfigShadeUtils.java deleted file mode 100644 index 98db59c..0000000 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/ConfigShadeUtils.java +++ /dev/null @@ -1,138 +0,0 @@ -package com.geedgenetworks.bootstrap.utils; - -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONObject; -import com.geedgenetworks.common.Constants; -import com.geedgenetworks.common.config.ConfigShade; -import com.geedgenetworks.common.config.TypesafeConfigUtils; -import com.google.common.base.Preconditions; -import com.typesafe.config.*; -import lombok.extern.slf4j.Slf4j; -import java.util.*; -import java.util.function.BiFunction; - -/** Config shade utilities */ -@Slf4j -public final class ConfigShadeUtils { - - private static final String SHADE_IDENTIFIER_OPTION = "shade.identifier"; - - private static final String[] DEFAULT_SENSITIVE_OPTIONS = - new String[] {"password", "username", "auth"}; - - private static final Map CONFIG_SHADES = new HashMap<>(); - - private static final ConfigShade DEFAULT_SHADE = new DefaultConfigShade(); - - static { - ServiceLoader serviceLoader = ServiceLoader.load(ConfigShade.class); - Iterator it = serviceLoader.iterator(); - it.forEachRemaining( - configShade -> { - CONFIG_SHADES.put(configShade.getIdentifier(), configShade); - }); - - log.info("Load config shade: {}", CONFIG_SHADES.keySet()); - } - - public static String encryptOption(String identifier, String content) { - ConfigShade configShade = CONFIG_SHADES.getOrDefault(identifier, DEFAULT_SHADE); - return configShade.encrypt(content); - } - - public static String decryptOption(String identifier, String content) { - ConfigShade configShade = CONFIG_SHADES.getOrDefault(identifier, DEFAULT_SHADE); - return configShade.decrypt(content); - } - - public static Config decryptConfig(Config config) { - String identifier = - TypesafeConfigUtils.getConfig( - config.hasPath(ConfigUtil.joinPath(Constants.APPLICATION, Constants.APPLICATION_ENV)) - ? config.getConfig(ConfigUtil.joinPath(Constants.APPLICATION, Constants.APPLICATION_ENV)) - : ConfigFactory.empty(), - ConfigUtil.joinPath(SHADE_IDENTIFIER_OPTION), - DEFAULT_SHADE.getIdentifier()); - return decryptConfig(identifier, config); - } - - public static Config encryptConfig(Config config) { - String identifier = - TypesafeConfigUtils.getConfig( - config.hasPath(ConfigUtil.joinPath(Constants.APPLICATION, Constants.APPLICATION_ENV)) - ? config.getConfig(ConfigUtil.joinPath(Constants.APPLICATION, Constants.APPLICATION_ENV)) - : ConfigFactory.empty(), - ConfigUtil.joinPath(SHADE_IDENTIFIER_OPTION), - DEFAULT_SHADE.getIdentifier()); - return encryptConfig(identifier, config); - } - - public static Config decryptConfig(String identifier, Config config) { - return processConfig(identifier, config, true); - } - - public static Config encryptConfig(String identifier, Config config) { - return processConfig(identifier, config, false); - } - - @SuppressWarnings("unchecked") - private static Config processConfig(String identifier, Config config, boolean isDecrypted) { - ConfigShade configShade = CONFIG_SHADES.getOrDefault(identifier, DEFAULT_SHADE); - List sensitiveOptions = new ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_OPTIONS)); - sensitiveOptions.addAll(Arrays.asList(configShade.sensitiveOptions())); - BiFunction processFunction = - (key, value) -> { - if (isDecrypted) { - return configShade.decrypt(value.toString()); - } else { - return configShade.encrypt(value.toString()); - } - }; - String jsonString = config.root().render(ConfigRenderOptions.concise()); - JSONObject configMap = JSON.parseObject(jsonString); - JSONObject sources = configMap.getJSONObject(Constants.SOURCES); - JSONObject sinks = configMap.getJSONObject(Constants.SINKS); - Preconditions.checkArgument( - !sources.isEmpty(), "Miss config! Please check the config file."); - Preconditions.checkArgument( - !sinks.isEmpty(), "Miss config! Please check the config file."); - - sources.values().forEach((value) -> { - JSONObject source = (JSONObject) value; - JSONObject propertiesObject = source.getJSONObject("properties"); - for (String sensitiveOption : sensitiveOptions) { - propertiesObject.computeIfPresent(sensitiveOption, processFunction); - } - }); - - sinks.values().forEach((value) -> { - JSONObject sink = (JSONObject) value; - JSONObject propertiesObject = sink.getJSONObject("properties"); - for (String sensitiveOption : sensitiveOptions) { - propertiesObject.computeIfPresent(sensitiveOption, processFunction); - } - }); - return ConfigValueFactory.fromMap(configMap).toConfig(); - } - - - private static class DefaultConfigShade implements ConfigShade { - private static final String IDENTIFIER = "default"; - - @Override - public String getIdentifier() { - return IDENTIFIER; - } - - @Override - public String encrypt(String content) { - return content; - } - - @Override - public String decrypt(String content) { - return content; - } - } - -} diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CryptoShadeUtils.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CryptoShadeUtils.java new file mode 100644 index 0000000..94dda4d --- /dev/null +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/utils/CryptoShadeUtils.java @@ -0,0 +1,138 @@ +package com.geedgenetworks.bootstrap.utils; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; +import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.crypto.CryptoShade; +import com.geedgenetworks.common.config.TypesafeConfigUtils; +import com.google.common.base.Preconditions; +import com.typesafe.config.*; +import lombok.extern.slf4j.Slf4j; +import java.util.*; +import java.util.function.BiFunction; + +/** Crypto shade utilities */ +@Slf4j +public final class CryptoShadeUtils { + + private static final String SHADE_IDENTIFIER_OPTION = "shade.identifier"; + + private static final String[] DEFAULT_SENSITIVE_OPTIONS = + new String[] {"password", "username", "auth"}; + + private static final Map CRYPTO_SHADES = new HashMap<>(); + + private static final CryptoShade DEFAULT_SHADE = new DefaultCryptoShade(); + + static { + ServiceLoader serviceLoader = ServiceLoader.load(CryptoShade.class); + Iterator it = serviceLoader.iterator(); + it.forEachRemaining( + configShade -> { + CRYPTO_SHADES.put(configShade.getIdentifier(), configShade); + }); + + log.info("Load config shade: {}", CRYPTO_SHADES.keySet()); + } + + public static String encryptOption(String identifier, String content) { + CryptoShade cryptoShade = CRYPTO_SHADES.getOrDefault(identifier, DEFAULT_SHADE); + return cryptoShade.encrypt(content); + } + + public static String decryptOption(String identifier, String content) { + CryptoShade cryptoShade = CRYPTO_SHADES.getOrDefault(identifier, DEFAULT_SHADE); + return cryptoShade.decrypt(content); + } + + public static Config decryptConfig(Config config) { + String identifier = + TypesafeConfigUtils.getConfig( + config.hasPath(ConfigUtil.joinPath(Constants.APPLICATION, Constants.APPLICATION_ENV)) + ? config.getConfig(ConfigUtil.joinPath(Constants.APPLICATION, Constants.APPLICATION_ENV)) + : ConfigFactory.empty(), + ConfigUtil.joinPath(SHADE_IDENTIFIER_OPTION), + DEFAULT_SHADE.getIdentifier()); + return decryptConfig(identifier, config); + } + + public static Config encryptConfig(Config config) { + String identifier = + TypesafeConfigUtils.getConfig( + config.hasPath(ConfigUtil.joinPath(Constants.APPLICATION, Constants.APPLICATION_ENV)) + ? config.getConfig(ConfigUtil.joinPath(Constants.APPLICATION, Constants.APPLICATION_ENV)) + : ConfigFactory.empty(), + ConfigUtil.joinPath(SHADE_IDENTIFIER_OPTION), + DEFAULT_SHADE.getIdentifier()); + return encryptConfig(identifier, config); + } + + public static Config decryptConfig(String identifier, Config config) { + return processConfig(identifier, config, true); + } + + public static Config encryptConfig(String identifier, Config config) { + return processConfig(identifier, config, false); + } + + @SuppressWarnings("unchecked") + private static Config processConfig(String identifier, Config config, boolean isDecrypted) { + CryptoShade cryptoShade = CRYPTO_SHADES.getOrDefault(identifier, DEFAULT_SHADE); + List sensitiveOptions = new ArrayList<>(Arrays.asList(DEFAULT_SENSITIVE_OPTIONS)); + sensitiveOptions.addAll(Arrays.asList(cryptoShade.sensitiveOptions())); + BiFunction processFunction = + (key, value) -> { + if (isDecrypted) { + return cryptoShade.decrypt(value.toString()); + } else { + return cryptoShade.encrypt(value.toString()); + } + }; + String jsonString = config.root().render(ConfigRenderOptions.concise()); + JSONObject configMap = JSON.parseObject(jsonString); + JSONObject sources = configMap.getJSONObject(Constants.SOURCES); + JSONObject sinks = configMap.getJSONObject(Constants.SINKS); + Preconditions.checkArgument( + !sources.isEmpty(), "Miss config! Please check the config file."); + Preconditions.checkArgument( + !sinks.isEmpty(), "Miss config! Please check the config file."); + + sources.values().forEach((value) -> { + JSONObject source = (JSONObject) value; + JSONObject propertiesObject = source.getJSONObject("properties"); + for (String sensitiveOption : sensitiveOptions) { + propertiesObject.computeIfPresent(sensitiveOption, processFunction); + } + }); + + sinks.values().forEach((value) -> { + JSONObject sink = (JSONObject) value; + JSONObject propertiesObject = sink.getJSONObject("properties"); + for (String sensitiveOption : sensitiveOptions) { + propertiesObject.computeIfPresent(sensitiveOption, processFunction); + } + }); + return ConfigValueFactory.fromMap(configMap).toConfig(); + } + + + private static class DefaultCryptoShade implements CryptoShade { + private static final String IDENTIFIER = "default"; + + @Override + public String getIdentifier() { + return IDENTIFIER; + } + + @Override + public String encrypt(String content) { + return content; + } + + @Override + public String decrypt(String content) { + return content; + } + } + +} diff --git a/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.config.ConfigShade b/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.config.ConfigShade deleted file mode 100644 index f490f28..0000000 --- a/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.config.ConfigShade +++ /dev/null @@ -1,3 +0,0 @@ -com.geedgenetworks.bootstrap.command.Base64ConfigShade -com.geedgenetworks.bootstrap.command.AESConfigShade -com.geedgenetworks.bootstrap.command.SM4ConfigShade \ No newline at end of file diff --git a/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade b/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade new file mode 100644 index 0000000..9c0d60c --- /dev/null +++ b/groot-bootstrap/src/main/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade @@ -0,0 +1,3 @@ +com.geedgenetworks.bootstrap.command.Base64Shade +com.geedgenetworks.bootstrap.command.AESShade +com.geedgenetworks.bootstrap.command.SM4Shade \ No newline at end of file diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java deleted file mode 100644 index 17f56ce..0000000 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.geedgenetworks.bootstrap.utils; - -import cn.hutool.setting.yaml.YamlUtil; -import com.alibaba.fastjson2.JSON; -import com.alibaba.fastjson2.JSONObject; -import com.geedgenetworks.common.Constants; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigObject; -import com.typesafe.config.ConfigRenderOptions; -import com.typesafe.config.ConfigValueFactory; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Paths; -import java.util.Map; - -@Slf4j -public class ConfigShadeTest { - - private static final String USERNAME = "grootstream"; - - private static final String PASSWORD = "grootstream_password"; - - @Test - public void testParseConfig() throws URISyntaxException { - URL resource = ConfigShadeTest.class.getResource("/inline_to_clickhouse.yaml"); - Assertions.assertNotNull(resource); - Map configMap = YamlUtil.loadByPath(Paths.get(resource.toURI()).toString()); - ConfigObject configObject = ConfigValueFactory.fromMap(configMap); - Config decryptConfig = ConfigShadeUtils.decryptConfig(configObject.toConfig()); - String jsonString = decryptConfig.root().render(ConfigRenderOptions.concise()); - JSONObject jsonObject = JSON.parseObject(jsonString); - JSONObject sinkObject = jsonObject.getJSONObject(Constants.SINKS); - log.info("Decrypt config: {}", decryptConfig.root().render(ConfigRenderOptions.concise())); - Assertions.assertEquals(sinkObject.keySet().size(), 1); - Assertions.assertNotNull(sinkObject); - Assertions.assertNotNull(sinkObject.getJSONObject("clickhouse_sink")); - Assertions.assertNotNull(sinkObject.getJSONObject("clickhouse_sink") - .getJSONObject("properties")); - Assertions.assertEquals(sinkObject.getJSONObject("clickhouse_sink") - .getJSONObject("properties").isEmpty(), false); - Assertions.assertEquals(sinkObject.getJSONObject("clickhouse_sink") - .getJSONObject("properties").get("connection.user"),USERNAME); - Assertions.assertNotNull(sinkObject.getJSONObject("clickhouse_sink") - .getJSONObject("properties").get("connection.password"), PASSWORD); - } - - @Test - public void testDecryptAndEncrypt() { - String encryptUsername = ConfigShadeUtils.encryptOption("base64", USERNAME); - String decryptUsername = ConfigShadeUtils.decryptOption("base64", encryptUsername); - String encryptPassword = ConfigShadeUtils.encryptOption("base64", PASSWORD); - String decryptPassword = ConfigShadeUtils.decryptOption("base64", encryptPassword); - Assertions.assertEquals("Z3Jvb3RzdHJlYW0=", encryptUsername); - Assertions.assertEquals("Z3Jvb3RzdHJlYW1fcGFzc3dvcmQ=", encryptPassword); - Assertions.assertEquals(decryptUsername, USERNAME); - Assertions.assertEquals(decryptPassword, PASSWORD); - encryptUsername = ConfigShadeUtils.encryptOption("aes", USERNAME); - decryptUsername = ConfigShadeUtils.decryptOption("aes", encryptUsername); - encryptPassword = ConfigShadeUtils.encryptOption("aes", PASSWORD); - decryptPassword = ConfigShadeUtils.decryptOption("aes", encryptPassword); - Assertions.assertEquals("ed986337dfdbe341be1d29702e6ae619", encryptUsername); - Assertions.assertEquals("159c7da83d988a9ec041d10a6bfbe221bcbaed6b62d9cc1b04ff51e633ebd105", encryptPassword); - Assertions.assertEquals(decryptUsername, USERNAME); - Assertions.assertEquals(decryptPassword, PASSWORD); - encryptUsername = ConfigShadeUtils.encryptOption("sm4", USERNAME); - decryptUsername = ConfigShadeUtils.decryptOption("sm4", encryptUsername); - Assertions.assertEquals("72ea74367a15cb96b0d1d42104149519", encryptUsername); - Assertions.assertEquals(decryptUsername, USERNAME); - encryptPassword = ConfigShadeUtils.encryptOption("sm4", PASSWORD); - decryptPassword = ConfigShadeUtils.decryptOption("sm4", encryptPassword); - Assertions.assertEquals("3876c7088d395bbbfa826e3648b6c9a022e7f80941c132313bde6dc8a7f2351f", encryptPassword); - Assertions.assertEquals(decryptPassword, PASSWORD); - System.out.println( ConfigShadeUtils.encryptOption("sm4", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"galaxy2019\";")); - System.out.println( ConfigShadeUtils.decryptOption("sm4", "f76480be84a8ee1b009504c6c56a5bed48239c348a468f94b4029a6a3148f51530b025d6dfa140af93b4c7c6fe0e3dce543773e779d272b5579555fbd3271e7fdbee088673a901b3f3b28e914a25f30a4a859d97594c5ea7d7c1dcefe8c62560baea32b6da0b767232ed8aca17af2dc6")); - System.out.println( ConfigShadeUtils.encryptOption("aes", "testuser")); - System.out.println( ConfigShadeUtils.encryptOption("aes", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"olap\" password=\"galaxy2019\";")); - } -} diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/CryptoShadeTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/CryptoShadeTest.java new file mode 100644 index 0000000..18e84ae --- /dev/null +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/CryptoShadeTest.java @@ -0,0 +1,81 @@ +package com.geedgenetworks.bootstrap.utils; + +import cn.hutool.setting.yaml.YamlUtil; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; +import com.geedgenetworks.common.Constants; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigObject; +import com.typesafe.config.ConfigRenderOptions; +import com.typesafe.config.ConfigValueFactory; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.Map; + +@Slf4j +public class CryptoShadeTest { + + private static final String USERNAME = "grootstream"; + + private static final String PASSWORD = "grootstream_password"; + + @Test + public void testParseConfig() throws URISyntaxException { + URL resource = CryptoShadeTest.class.getResource("/inline_to_clickhouse.yaml"); + Assertions.assertNotNull(resource); + Map configMap = YamlUtil.loadByPath(Paths.get(resource.toURI()).toString()); + ConfigObject configObject = ConfigValueFactory.fromMap(configMap); + Config decryptConfig = CryptoShadeUtils.decryptConfig(configObject.toConfig()); + String jsonString = decryptConfig.root().render(ConfigRenderOptions.concise()); + JSONObject jsonObject = JSON.parseObject(jsonString); + JSONObject sinkObject = jsonObject.getJSONObject(Constants.SINKS); + log.info("Decrypt config: {}", decryptConfig.root().render(ConfigRenderOptions.concise())); + Assertions.assertEquals(sinkObject.keySet().size(), 1); + Assertions.assertNotNull(sinkObject); + Assertions.assertNotNull(sinkObject.getJSONObject("clickhouse_sink")); + Assertions.assertNotNull(sinkObject.getJSONObject("clickhouse_sink") + .getJSONObject("properties")); + Assertions.assertEquals(sinkObject.getJSONObject("clickhouse_sink") + .getJSONObject("properties").isEmpty(), false); + Assertions.assertEquals(sinkObject.getJSONObject("clickhouse_sink") + .getJSONObject("properties").get("connection.user"),USERNAME); + Assertions.assertNotNull(sinkObject.getJSONObject("clickhouse_sink") + .getJSONObject("properties").get("connection.password"), PASSWORD); + } + + @Test + public void testDecryptAndEncrypt() { + String encryptUsername = CryptoShadeUtils.encryptOption("base64", USERNAME); + String decryptUsername = CryptoShadeUtils.decryptOption("base64", encryptUsername); + String encryptPassword = CryptoShadeUtils.encryptOption("base64", PASSWORD); + String decryptPassword = CryptoShadeUtils.decryptOption("base64", encryptPassword); + Assertions.assertEquals("Z3Jvb3RzdHJlYW0=", encryptUsername); + Assertions.assertEquals("Z3Jvb3RzdHJlYW1fcGFzc3dvcmQ=", encryptPassword); + Assertions.assertEquals(decryptUsername, USERNAME); + Assertions.assertEquals(decryptPassword, PASSWORD); + encryptUsername = CryptoShadeUtils.encryptOption("aes", USERNAME); + decryptUsername = CryptoShadeUtils.decryptOption("aes", encryptUsername); + encryptPassword = CryptoShadeUtils.encryptOption("aes", PASSWORD); + decryptPassword = CryptoShadeUtils.decryptOption("aes", encryptPassword); + Assertions.assertEquals("ed986337dfdbe341be1d29702e6ae619", encryptUsername); + Assertions.assertEquals("159c7da83d988a9ec041d10a6bfbe221bcbaed6b62d9cc1b04ff51e633ebd105", encryptPassword); + Assertions.assertEquals(decryptUsername, USERNAME); + Assertions.assertEquals(decryptPassword, PASSWORD); + encryptUsername = CryptoShadeUtils.encryptOption("sm4", USERNAME); + decryptUsername = CryptoShadeUtils.decryptOption("sm4", encryptUsername); + Assertions.assertEquals("72ea74367a15cb96b0d1d42104149519", encryptUsername); + Assertions.assertEquals(decryptUsername, USERNAME); + encryptPassword = CryptoShadeUtils.encryptOption("sm4", PASSWORD); + decryptPassword = CryptoShadeUtils.decryptOption("sm4", encryptPassword); + Assertions.assertEquals("3876c7088d395bbbfa826e3648b6c9a022e7f80941c132313bde6dc8a7f2351f", encryptPassword); + Assertions.assertEquals(decryptPassword, PASSWORD); + System.out.println( CryptoShadeUtils.encryptOption("sm4", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"galaxy2019\";")); + System.out.println( CryptoShadeUtils.decryptOption("sm4", "f76480be84a8ee1b009504c6c56a5bed48239c348a468f94b4029a6a3148f51530b025d6dfa140af93b4c7c6fe0e3dce543773e779d272b5579555fbd3271e7fdbee088673a901b3f3b28e914a25f30a4a859d97594c5ea7d7c1dcefe8c62560baea32b6da0b767232ed8aca17af2dc6")); + System.out.println( CryptoShadeUtils.encryptOption("aes", "testuser")); + System.out.println( CryptoShadeUtils.encryptOption("aes", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"olap\" password=\"galaxy2019\";")); + } +} diff --git a/groot-bootstrap/src/test/resources/META-INF/services/com.geedgenetworks.common.config.ConfigShade b/groot-bootstrap/src/test/resources/META-INF/services/com.geedgenetworks.common.config.ConfigShade deleted file mode 100644 index 6654db5..0000000 --- a/groot-bootstrap/src/test/resources/META-INF/services/com.geedgenetworks.common.config.ConfigShade +++ /dev/null @@ -1,2 +0,0 @@ -com.geedgenetworks.bootstrap.command.Base64ConfigShade -com.geedgenetworks.bootstrap.command.AESConfigShade \ No newline at end of file diff --git a/groot-bootstrap/src/test/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade b/groot-bootstrap/src/test/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade new file mode 100644 index 0000000..04adf41 --- /dev/null +++ b/groot-bootstrap/src/test/resources/META-INF/services/com.geedgenetworks.common.crypto.CryptoShade @@ -0,0 +1,2 @@ +com.geedgenetworks.bootstrap.command.Base64Shade +com.geedgenetworks.bootstrap.command.AESShade \ No newline at end of file diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigShade.java b/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigShade.java deleted file mode 100644 index 2943bc8..0000000 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/ConfigShade.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.geedgenetworks.common.config; - -public interface ConfigShade { - - /** - * The unique identifier of the current interface, used it to select the correct {@link - * ConfigShade} - */ - String getIdentifier(); - - /** - * Encrypt the content - * - * @param content The content to encrypt - */ - String encrypt(String content); - - /** - * Decrypt the content - * - * @param content The content to decrypt - */ - String decrypt(String content); - - /** To expand the options that user want to encrypt */ - default String[] sensitiveOptions() { - return new String[0]; - } -} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/crypto/CryptoShade.java b/groot-common/src/main/java/com/geedgenetworks/common/crypto/CryptoShade.java new file mode 100644 index 0000000..985f4df --- /dev/null +++ b/groot-common/src/main/java/com/geedgenetworks/common/crypto/CryptoShade.java @@ -0,0 +1,29 @@ +package com.geedgenetworks.common.crypto; + +public interface CryptoShade { + + /** + * The unique identifier of the current interface, used it to select the correct {@link + * CryptoShade} + */ + String getIdentifier(); + + /** + * Encrypt the content + * + * @param content The content to encrypt + */ + String encrypt(String content); + + /** + * Decrypt the content + * + * @param content The content to decrypt + */ + String decrypt(String content); + + /** To expand the options that user want to encrypt */ + default String[] sensitiveOptions() { + return new String[0]; + } +} diff --git a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java index 9b58289..a79919b 100644 --- a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java +++ b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java @@ -14,13 +14,13 @@ import java.util.List; public class GrootStreamExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException { - String configPath = args.length > 0 ? args[0] : "/examples/inline_to_print.yaml"; + String configPath = args.length > 0 ? args[0] : "/examples/inline_to_kafka.yaml"; String configFile = getTestConfigFile(configPath); ExecuteCommandArgs executeCommandArgs = new ExecuteCommandArgs(); executeCommandArgs.setConfigFile(configFile); executeCommandArgs.setCheckConfig(false); executeCommandArgs.setEncrypt(false); - executeCommandArgs.setDecrypt(false); + executeCommandArgs.setDecrypt(true); executeCommandArgs.setVersion(false); executeCommandArgs.setVariables(List.of("hos.bucket.name.traffic_file=user_define_traffic_file_bucket", "scheduler.knowledge_base.update.interval.minutes=1")); -- cgit v1.2.3