diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | docs/develop-guide.md | 6 | ||||
| -rw-r--r-- | docs/user-guide.md | 2 | ||||
| -rw-r--r-- | groot-bootstrap/src/test/resources/grootstream_job_test.yaml (renamed from groot-bootstrap/src/main/resources/grootstream_job_test.yaml) | 0 | ||||
| -rw-r--r-- | groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java | 3 | ||||
| -rw-r--r-- | groot-examples/cn-udf-example/pom.xml (renamed from groot-examples/cn-example/pom.xml) | 4 | ||||
| -rw-r--r-- | groot-examples/cn-udf-example/src/main/java/com/geedgenetworks/example/CnExample.java (renamed from groot-examples/cn-example/src/main/java/com/geedgenetworks/example/CnExample.java) | 0 | ||||
| -rw-r--r-- | groot-examples/cn-udf-example/src/main/resources/example/cn_grootstream_job_local_template.yaml (renamed from groot-examples/cn-example/src/main/resources/example/cn_grootstream_job_local_template.yaml) | 0 | ||||
| -rw-r--r-- | groot-examples/cn-udf-example/src/main/resources/example/cn_grootstream_job_template.yaml (renamed from groot-examples/cn-example/src/main/resources/example/cn_grootstream_job_template.yaml) | 0 | ||||
| -rw-r--r-- | groot-examples/cn-udf-example/src/main/resources/grootstream.yaml (renamed from groot-examples/cn-example/src/main/resources/grootstream.yaml) | 0 | ||||
| -rw-r--r-- | groot-examples/cn-udf-example/src/main/resources/udf.plugins (renamed from groot-examples/cn-example/src/main/resources/udf.plugins) | 0 | ||||
| -rw-r--r-- | groot-examples/end-to-end-example/pom.xml (renamed from groot-examples/inline-to-print-example/pom.xml) | 4 | ||||
| -rw-r--r-- | groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java (renamed from groot-examples/inline-to-print-example/src/main/java/com/geedgenetworks/example/InlineToPrintExample.java) | 7 | ||||
| -rw-r--r-- | groot-examples/end-to-end-example/src/main/resources/examples/inline_to_clickhouse.yaml (renamed from groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_clickhouse.yaml) | 0 | ||||
| -rw-r--r-- | groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml (renamed from groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_kafka.yaml) | 0 | ||||
| -rw-r--r-- | groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print.yaml (renamed from groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_print.yaml) | 0 | ||||
| -rw-r--r-- | groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_test.yaml (renamed from groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_print_test.yaml) | 0 | ||||
| -rw-r--r-- | groot-examples/end-to-end-example/src/main/resources/examples/kafka_to_print.yaml (renamed from groot-examples/inline-to-print-example/src/main/resources/examples/kafka_to_print.yaml) | 0 | ||||
| -rw-r--r-- | groot-examples/end-to-end-example/src/main/resources/log4j2.properties (renamed from groot-examples/inline-to-print-example/src/main/resources/log4j2.properties) | 0 | ||||
| -rw-r--r-- | groot-examples/pom.xml | 22 | ||||
| -rw-r--r-- | pom.xml | 4 |
21 files changed, 38 insertions, 18 deletions
@@ -41,7 +41,7 @@ Configure a job, you'll set up Sources, Filters, Processing Pipeline, and Sinks, ## Minimum Requirements - Git installed -- Java (JDK8/JDK11 are supported) installed and `JAVA_HOME` set +- JAVA(JDK/JRE11 are required)installed and `JAVA_HOME` set - Maven 3.5.4 - Scala 2.12 - Flink 1.13.1 @@ -75,7 +75,7 @@ FLINK_JOB_MANAGER_ADDRESS=${FLINK_JOB_MANAGER_ADDRESS:-localhost:8081} YARN_ADDRESS=${YARN_ADDRESS:-yarn-cluster} ``` #### 3. Configure the groot-stream job -You need to configure the groot-stream job in `config/inline_to_print_template.yaml` file. More information about config please check [config concept](docs/user-guide.md) +You need to configure the groot-stream job in `config/template/grootstream_job_template.yaml` file. More information about config please check [config concept](docs/user-guide.md) #### 4. Submit a job to flink engine Can be started by a daemon with `-d`. diff --git a/docs/develop-guide.md b/docs/develop-guide.md index dc8b44f..621616a 100644 --- a/docs/develop-guide.md +++ b/docs/develop-guide.md @@ -43,12 +43,12 @@ When throwing an exception with a hint message and ensure that the exception has ## Design Principles 1. Package structure: `com.geedgenetworks.[module].[sub-module]`. `groot-stream` is the parent module, and other modules are dependent on it. -2. Module naming: `groot-[module]`. e.g. `groot-common`, `groot-core`, `groot-connectors`, `groot-bootstrap`, `groot-example`, etc. +2. Module naming: `groot-[module]`. e.g. `groot-common`, `groot-core`, `groot-connectors`, `groot-bootstrap`, `groot-examples`, etc. 3. For unchecked exception (RuntimeException) within the 'groot-common' module, a global exception handling class named 'GrootRuntimeException' is defined. ## Run a job example -All examples are in module `groot-stream-example`. You can run the example [ running or debugging a job in IDEA]. -e.g. we use `groot-stream-example/src/main/java/com/geedgenetworks/example/InlineToConsoleExample.java` as the example, when you produce some sample data in `Inline` and you could see the result in console. as follows: +All examples are in module `end-to-end-examples`-. You can run the example [ running or debugging a job in IDEA]. +e.g. we use `end-to-end-examples/src/main/java/com/geedgenetworks/example/GrootStreamExample.java` as the example, when you produce some sample data in `Inline` and you could see the result in console. as follows: ```json {"log_id":155652727148914688,"decoded_as":"BASE","recv_time":111,"fqdn_string":"baidu.com","server_ip":"120.233.20.242","additional_field_subdomain":"baidu.com","client_ip":"192.168.0.1"} ``` diff --git a/docs/user-guide.md b/docs/user-guide.md index 5b06a52..fa05547 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -146,6 +146,8 @@ Used to define job environment configuration information. For more details, you Usage: start.sh [options] Options: --check Whether check config (default: false) + --encrypt Show encrypted config file. If config file is palin text, it will be encrypted and printed to the console. + --decrypt Show decrypted config file. If config file is encrypted, it will be decrypted and printed to the console. -c, --config <config file> Config file path, must be specified -e, --deploy-mode <deploy mode> Deploy mode, only support [run] (default: run) --target <target> Submitted target type, support [local, remote, yarn-session, yarn-per-job] diff --git a/groot-bootstrap/src/main/resources/grootstream_job_test.yaml b/groot-bootstrap/src/test/resources/grootstream_job_test.yaml index 45c8f56..45c8f56 100644 --- a/groot-bootstrap/src/main/resources/grootstream_job_test.yaml +++ b/groot-bootstrap/src/test/resources/grootstream_job_test.yaml diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java index ae5cc31..faa65f4 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/GeoIpLookupFunctionTest.java @@ -1,5 +1,6 @@ package com.geedgenetworks.core.udf.test; +import com.alibaba.fastjson2.JSONObject; import com.geedgenetworks.common.config.CommonConfig; import com.geedgenetworks.common.config.KnowledgeBaseConfig; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; @@ -89,7 +90,7 @@ public class GeoIpLookupFunctionTest { String ispLookup = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_PROVIDER","2600:1015:b002::"); assertEquals("", ispLookup); String infoLookupToJSONString = GeoIpKnowledgeBaseHandler.lookUp("tsg_ip_location","IP_TO_JSON","2600:1015:b002::"); - assertEquals("{\"administrativeArea\":\"Unknown\",\"areaCode\":\" \",\"asn\":\" \",\"country\":\"美国\",\"countryCode\":\"\",\"dependentLocality\":\" \",\"doubleDependentLocality\":\" \",\"isp\":\" \",\"latitude\":\"39.9388\",\"locality\":\" \",\"longitude\":\"-86.1137\",\"organization\":\"\",\"privateIP\":false,\"subAdministrativeArea\":\" \",\"superAdministrativeArea\":\"印第安纳州\"}", infoLookupToJSONString); + assertEquals(JSONObject.parseObject(infoLookupToJSONString).getString("country"), "美国"); String country = GeoIpKnowledgeBaseHandler.lookUpObject("tsg_ip_location","2600:1015:b002::").getCountry(); assertEquals("美国", country); assertNull(GeoIpKnowledgeBaseHandler.lookUpObject("tsg_ip_location", "255.255.255.255")); diff --git a/groot-examples/cn-example/pom.xml b/groot-examples/cn-udf-example/pom.xml index e428311..38ae4ea 100644 --- a/groot-examples/cn-example/pom.xml +++ b/groot-examples/cn-udf-example/pom.xml @@ -9,8 +9,8 @@ <version>${revision}</version> </parent> - <artifactId>cn-example</artifactId> - <name>Groot : Examples : CN</name> + <artifactId>cn-udf-example</artifactId> + <name>Groot : Examples : CN-UDF</name> <properties> <maven.install.skip>true</maven.install.skip> <maven.deploy.skip>true</maven.deploy.skip> diff --git a/groot-examples/cn-example/src/main/java/com/geedgenetworks/example/CnExample.java b/groot-examples/cn-udf-example/src/main/java/com/geedgenetworks/example/CnExample.java index 2bc91db..2bc91db 100644 --- a/groot-examples/cn-example/src/main/java/com/geedgenetworks/example/CnExample.java +++ b/groot-examples/cn-udf-example/src/main/java/com/geedgenetworks/example/CnExample.java diff --git a/groot-examples/cn-example/src/main/resources/example/cn_grootstream_job_local_template.yaml b/groot-examples/cn-udf-example/src/main/resources/example/cn_grootstream_job_local_template.yaml index 5a8fcb0..5a8fcb0 100644 --- a/groot-examples/cn-example/src/main/resources/example/cn_grootstream_job_local_template.yaml +++ b/groot-examples/cn-udf-example/src/main/resources/example/cn_grootstream_job_local_template.yaml diff --git a/groot-examples/cn-example/src/main/resources/example/cn_grootstream_job_template.yaml b/groot-examples/cn-udf-example/src/main/resources/example/cn_grootstream_job_template.yaml index 7c448f6..7c448f6 100644 --- a/groot-examples/cn-example/src/main/resources/example/cn_grootstream_job_template.yaml +++ b/groot-examples/cn-udf-example/src/main/resources/example/cn_grootstream_job_template.yaml diff --git a/groot-examples/cn-example/src/main/resources/grootstream.yaml b/groot-examples/cn-udf-example/src/main/resources/grootstream.yaml index 558030c..558030c 100644 --- a/groot-examples/cn-example/src/main/resources/grootstream.yaml +++ b/groot-examples/cn-udf-example/src/main/resources/grootstream.yaml diff --git a/groot-examples/cn-example/src/main/resources/udf.plugins b/groot-examples/cn-udf-example/src/main/resources/udf.plugins index 22804f6..22804f6 100644 --- a/groot-examples/cn-example/src/main/resources/udf.plugins +++ b/groot-examples/cn-udf-example/src/main/resources/udf.plugins diff --git a/groot-examples/inline-to-print-example/pom.xml b/groot-examples/end-to-end-example/pom.xml index 10135cd..0839e8e 100644 --- a/groot-examples/inline-to-print-example/pom.xml +++ b/groot-examples/end-to-end-example/pom.xml @@ -9,8 +9,8 @@ <version>${revision}</version> </parent> - <artifactId>inline-to-print-example</artifactId> - <name>Groot : Examples : Inline-to-print</name> + <artifactId>end-to-end-example</artifactId> + <name>Groot : Examples : End-to-end</name> <properties> <maven.install.skip>true</maven.install.skip> diff --git a/groot-examples/inline-to-print-example/src/main/java/com/geedgenetworks/example/InlineToPrintExample.java b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java index c139879..825f79a 100644 --- a/groot-examples/inline-to-print-example/src/main/java/com/geedgenetworks/example/InlineToPrintExample.java +++ b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java @@ -11,9 +11,9 @@ import java.net.URL; import java.nio.file.Paths; -public class InlineToPrintExample { +public class GrootStreamExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException { - String configPath = args.length > 0 ? args[0] : "/examples/inline_to_print_test.yaml"; + String configPath = args.length > 0 ? args[0] : "/examples/kafka_to_print.yaml"; String configFile = getTestConfigFile(configPath); ExecuteCommandArgs executeCommandArgs = new ExecuteCommandArgs(); executeCommandArgs.setConfigFile(configFile); @@ -25,7 +25,8 @@ public class InlineToPrintExample { public static String getTestConfigFile(String configFile) throws FileNotFoundException, URISyntaxException { - URL resource = InlineToPrintExample.class.getResource(configFile); + URL resource = GrootStreamExample.class.getResource(configFile); + if (resource == null) { throw new FileNotFoundException("Can't find config file: " + configFile); } diff --git a/groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_clickhouse.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_clickhouse.yaml index 370b7a8..370b7a8 100644 --- a/groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_clickhouse.yaml +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_clickhouse.yaml diff --git a/groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_kafka.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml index 44355c7..44355c7 100644 --- a/groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_kafka.yaml +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml diff --git a/groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_print.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print.yaml index daf6e32..daf6e32 100644 --- a/groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_print.yaml +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print.yaml diff --git a/groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_print_test.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_test.yaml index d42c05a..d42c05a 100644 --- a/groot-examples/inline-to-print-example/src/main/resources/examples/inline_to_print_test.yaml +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_test.yaml diff --git a/groot-examples/inline-to-print-example/src/main/resources/examples/kafka_to_print.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/kafka_to_print.yaml index 523d529..523d529 100644 --- a/groot-examples/inline-to-print-example/src/main/resources/examples/kafka_to_print.yaml +++ b/groot-examples/end-to-end-example/src/main/resources/examples/kafka_to_print.yaml diff --git a/groot-examples/inline-to-print-example/src/main/resources/log4j2.properties b/groot-examples/end-to-end-example/src/main/resources/log4j2.properties index 2dc1b8c..2dc1b8c 100644 --- a/groot-examples/inline-to-print-example/src/main/resources/log4j2.properties +++ b/groot-examples/end-to-end-example/src/main/resources/log4j2.properties diff --git a/groot-examples/pom.xml b/groot-examples/pom.xml index 2d7ea4d..fc08086 100644 --- a/groot-examples/pom.xml +++ b/groot-examples/pom.xml @@ -13,8 +13,8 @@ <name>Groot : Examples :</name> <packaging>pom</packaging> <modules> - <module>inline-to-print-example</module> - <module>cn-example</module> + <module>end-to-end-example</module> + <module>cn-udf-example</module> </modules> <properties> @@ -33,10 +33,26 @@ <version>${revision}</version> </dependency> + + <dependency> + <groupId>com.geedgenetworks</groupId> + <artifactId>connector-kafka</artifactId> + <version>${revision}</version> + <scope>${scope}</scope> + </dependency> + <dependency> <groupId>com.geedgenetworks</groupId> - <artifactId>groot-common</artifactId> + <artifactId>connector-clickhouse</artifactId> <version>${revision}</version> + <scope>${scope}</scope> + </dependency> + + <dependency> + <groupId>com.geedgenetworks</groupId> + <artifactId>connector-ipfix-collector</artifactId> + <version>${revision}</version> + <scope>${scope}</scope> </dependency> <dependency> @@ -23,7 +23,7 @@ </modules> <properties> - <revision>1.0.0</revision> + <revision>1.1.0</revision> <java.version>11</java.version> <flink.scope>provided</flink.scope> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> @@ -52,7 +52,7 @@ <jsonpath.version>2.4.0</jsonpath.version> <fastjson2.version>2.0.32</fastjson2.version> <hutool.version>5.8.22</hutool.version> - <galaxy.version>1.2.3</galaxy.version> + <galaxy.version>2.0.1</galaxy.version> <guava-retrying.version>2.0.0</guava-retrying.version> <ipaddress.version>5.3.3</ipaddress.version> <aviator.version>5.2.6</aviator.version> |
