summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordoufenghu <[email protected]>2024-09-09 16:47:35 +0800
committerdoufenghu <[email protected]>2024-09-09 16:47:35 +0800
commit202cf0557b3120b3599ff34015166ae8bd6c5a1b (patch)
treea175c151ca3425f43c2a7c6b486673b9a4d8f292
parent6e2564bef00eb0ca640f38209260b0b0fd172c95 (diff)
[Improve][mock-data] Add array(String) mock data
-rw-r--r--config/template/grootstream_job_template.yaml1
-rw-r--r--config/template/mock_schema/session_record_mock_desc.json76
-rw-r--r--docs/connector/connector.md27
3 files changed, 94 insertions, 10 deletions
diff --git a/config/template/grootstream_job_template.yaml b/config/template/grootstream_job_template.yaml
index 0ca2d68..3110162 100644
--- a/config/template/grootstream_job_template.yaml
+++ b/config/template/grootstream_job_template.yaml
@@ -448,6 +448,7 @@ application: # [object] Application Configuration
shade.identifier: default # [string] Shade Identifier, Using to encrypt and decrypt sensitive configuration. Support enum: default, aes, base64. if set default, it will not encrypt and decrypt sensitive configuration.
pipeline:
object-reuse: true # [boolean] Object Reuse, default is false
+
topology: # [array of object] Node List. It will be used build data flow for job dag graph.
- name: inline_source # [string] Node Name, must be unique. It will be used as the name of the corresponding Flink operator. eg. kafka_source the processor type as SOURCE.
#parallelism: 1 # [number] Operator-Level Parallelism.
diff --git a/config/template/mock_schema/session_record_mock_desc.json b/config/template/mock_schema/session_record_mock_desc.json
index c8c4acf..90060a6 100644
--- a/config/template/mock_schema/session_record_mock_desc.json
+++ b/config/template/mock_schema/session_record_mock_desc.json
@@ -115,12 +115,43 @@
"end": "103.144.108.255"
},
{
+ "name": "client_ip_tags",
+ "type": "String",
+ "array": true,
+ "options": [
+ "Country:United States",
+ "ASN:63278",
+ "Cloud Provider:IBM Cloud",
+ "Country Code:US",
+ "CDN Provider:Light CDN",
+ "ASN:6423"
+
+ ],
+ "arrayLenMin":1,
+ "arrayLenMax":5
+ },
+ {
"name": "server_ip",
"type": "IPv4",
"start": "1.0.0.0",
"end": "162.105.10.255"
},
{
+ "name": "server_ip_tags",
+ "type": "String",
+ "array": true,
+ "options": [
+ "Country:China",
+ "ASN:15169",
+ "Cloud Provider:Alibaba Cloud",
+ "Country Code:CN",
+ "CDN Provider:Akamai",
+ "Super Administrative Area:Guangdong"
+ ],
+ "arrayLenMin":1,
+ "arrayLenMax":5
+ },
+ {
"name": "c2s_ttl",
"type": "Number",
"options": [
@@ -167,12 +198,43 @@
"end": "162.105.10.255"
},
{
+ "name": "client_ip_tags",
+ "type": "String",
+ "array": true,
+ "options": [
+ "Country:China",
+ "ASN:15169",
+ "Cloud Provider:Alibaba Cloud",
+ "Country Code:CN",
+ "CDN Provider:Akamai",
+ "Super Administrative Area:Guangdong"
+ ],
+ "arrayLenMin":1,
+ "arrayLenMax":5
+ },
+ {
"name": "server_ip",
"type": "IPv4",
"start": "103.144.108.1",
"end": "103.144.108.255"
},
{
+ "name": "server_ip_tags",
+ "type": "String",
+ "array": true,
+ "options": [
+ "Country:United States",
+ "ASN:63278",
+ "Cloud Provider:IBM Cloud",
+ "Country Code:US",
+ "CDN Provider:Light CDN",
+ "ASN:6423"
+
+ ],
+ "arrayLenMin":1,
+ "arrayLenMax":5
+ },
+ {
"name": "c2s_ttl",
"type": "Number",
"options": [
@@ -340,6 +402,20 @@
"nullRate": 0.1
},
{
+ "name": "server_fqdn_tags",
+ "type": "String",
+ "array": true,
+ "options": [
+ "Category Name:Entertainment and Arts",
+ "IoC:Malware",
+ "Category Name:Home and Garden",
+ "Category Name:Translation",
+ "IoC:Spam"
+ ],
+ "arrayLenMin":1,
+ "arrayLenMax":5
+ },
+ {
"name": "server_port",
"type": "Number",
"options": [
diff --git a/docs/connector/connector.md b/docs/connector/connector.md
index 766b73e..93d64b0 100644
--- a/docs/connector/connector.md
+++ b/docs/connector/connector.md
@@ -1,3 +1,12 @@
+# Table of Contents
+- [Source Connector](#source-connector)
+ - [Common Source Options](#common-source-options)
+ - [Schema Field Projection](#schema-field-projection)
+ - [Schema Config](#schema-config)
+ - [Mock Data Type](#mock-data-type)
+- [Sink Connector](#sink-connector)
+ - [Common Sink Options](#common-sink-options)
+
# Source Connector
Source Connector contains some common core features, and each source connector supports them to varying degrees.
@@ -62,13 +71,12 @@ schema:
To retrieve the schema from a local file using its absolute path.
> Ensures that the file path is accessible to all nodes in your Flink cluster.
->
-> ```yaml
-> schema:
-> # by array
-> fields:
-> local_file: "/path/to/schema.json"
-> ```
+
+ ```yaml
+schema:
+ # Note: Only support avro schema format
+ local_file: "/path/to/schema.json"
+```
### URL
@@ -76,9 +84,8 @@ Some connectors support periodically fetching and updating the schema from a URL
```yaml
schema:
- # by array
- fields:
- url: "https://localhost:8080/schema.json"
+ # Note: Only support avro schema format
+ url: "https://localhost:8080/schema.json"
```
## Mock Data Type