diff options
| author | 窦凤虎 <[email protected]> | 2024-08-27 12:48:22 +0000 |
|---|---|---|
| committer | 窦凤虎 <[email protected]> | 2024-08-27 12:48:22 +0000 |
| commit | 62e969df69b28a9f435c925669cf6dfe018aa74f (patch) | |
| tree | 606443be4ac26c10f97a5e5b705aa652e43c9706 | |
| parent | 9a98dfe4e2d316a33cf388ad6d55380082f8f663 (diff) | |
| parent | b66b0c65844bdb531d2f5a4017a712708179c649 (diff) | |
Merge branch 'release/1.5.0' into 'master'v1.5.0
Release/1.5.0
See merge request galaxy/platform/groot-stream!97
96 files changed, 3894 insertions, 454 deletions
@@ -27,7 +27,7 @@ Groot Stream Platform helps you process netflow data - logs, metrics etc. - in r Configure a job, you'll set up Sources, Filters, Processing Pipeline, and Sinks, and will assemble several built-in functions into a Processing Pipeline. The job will then be deployed to a Flink cluster for execution. - **Source**: The data source of the job, which can be a Kafka topic, a IPFIX Collector, or a file. - **Filter**: Filters data based on specified conditions. -- **Types of Pipelines**: The fundamental unit of data stream processing is the processor, categorized by functionality into stateless and stateful processors. Each processor can be assemble `UDFs`(User-defined functions) or `UDAFs`(User-defined aggregation functions) into a pipeline. There are 3 types of pipelines at different stages of the data processing process: +- **Types of Pipelines**: The fundamental unit of data stream processing is the processor, categorized by functionality into stateless and stateful processors. Each processor can be assemble `UDFs`(User-defined functions) into a pipeline. There are 3 types of pipelines at different stages of the data processing process: - **Pre-processing Pipeline**: Optional. These pipelines that are attached to a source to normalize the events before they enter the processing pipeline. - **Processing Pipeline**: Event processing pipeline. - **Post-processing Pipeline**: Optional. These pipelines that are attached to a sink to normalize the events before they're written to the sink. diff --git a/config/grootstream_job_example.yaml b/config/grootstream_job_example.yaml index 74239c9..4726af0 100644 --- a/config/grootstream_job_example.yaml +++ b/config/grootstream_job_example.yaml @@ -24,6 +24,7 @@ processing_pipelines: output_fields: group_by_fields: [server_ip,server_port] window_type: tumbling_processing_time # tumbling_event_time,sliding_processing_time,sliding_event_time + window_timestamp_field: recv_time window_size: 60 window_slide: 10 #滑动窗口步长 functions: diff --git a/config/template/grootstream_job_template.yaml b/config/template/grootstream_job_template.yaml index 9f64abe..7cf50c8 100644 --- a/config/template/grootstream_job_template.yaml +++ b/config/template/grootstream_job_template.yaml @@ -151,7 +151,7 @@ preprocessing_pipelines: # [object] Define Processors for preprocessing pipeline # It will be accomplished the common processing for the event by the user-defined functions. # processing_pipelines: # [object] Define Processors for processing pipelines. - processor: # [object] Define projection processor name, must be unique. + projection_processor: # [object] Define projection processor name, must be unique. type: projection # [string] Processor Type remove_fields: output_fields: @@ -276,7 +276,7 @@ processing_pipelines: # [object] Define Processors for processing pipelines. lookup_fields: [ client_asn,server_asn ] output_fields: [ asn_list ] - metrics_processor: # [object] metrics processing Pipeline + projection_metrics_processor: # [object] metrics processing Pipeline type: projection output_fields: properties: @@ -323,6 +323,38 @@ processing_pipelines: # [object] Define Processors for processing pipelines. parameters: data_center_id_num: 1 + aggregate_processor: # [object] Define aggregate processor name, must be unique. + type: aggregate + group_by_fields: [ recv_time, sled_ip ] # [array of string] Group By Fields + window_type: tumbling_processing_time # [string] Window Type, tumbling_processing_time, tumbling_event_time, sliding_processing_time, sliding_event_time + window_size: 60 + functions: + - function: NUMBER_SUM + lookup_fields: [ received_bytes, sent_bytes ] + output_fields: [ received_bytes_sum ] + + - function: LONG_COUNT + lookup_fields: [ received_bytes ] + output_fields: [ sessions ] + + - function: MEAN + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_mean ] + parameters: + precision: 2 + + - function: FIRST_VALUE + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_first ] + + - function: LAST_VALUE + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_last ] + + - function: COLLECT_LIST + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_set ] + postprocessing_pipelines: # [object] Define Processors for postprocessing pipelines. postprocessor: # [object] Define projection processor name, must be unique. type: projection diff --git a/config/template/mock_schema/object_statistics_mock_desc.json b/config/template/mock_schema/object_statistics_mock_desc.json index 8542767..fbe5eb5 100644 --- a/config/template/mock_schema/object_statistics_mock_desc.json +++ b/config/template/mock_schema/object_statistics_mock_desc.json @@ -1,77 +1,113 @@ [ - {"name": "name", "type": "String", "options":["object_statistics"]}, - {"name": "timestamp_ms", "type": "Timestamp", "unit":"millis"}, - {"name": "tags", "type": "Object", "fields": [ - {"name": "object_id", "type":"Number","min":1,"max":100}, - {"name": "item_id", "type":"Number","min":1,"max":100000}, - {"name": "object_type", "type":"String","options":["ip","asn","port","url","fqdn","account","subscriberid","keywords","application","fqdn_category"]}, - { - "name": "device_id", - "type": "String", - "options": [ - "9800165603191151", - "9800165603247024", - "9800165802621377", - "9800165603191148" - ] - }, - { - "name": "device_group", - "type": "String", - "options": [ - "OLAP-MOCK-DG-1", - "OLAP-MOCK-DG-2", - "OLAP-MOCK-DG-3", - "OLAP-MOCK-DG-4" - ] - },{ - "name": "data_center", - "type": "String", - "options": [ - "OLAP-MOCK-DC-1", - "OLAP-MOCK-DC-2", - "OLAP-MOCK-DC-3", - "OLAP-MOCK-DC-4" - ] - },{ - "name": "vsys_id", - "type": "Number", - "options": [ - 2048 - ] - } - ] }, - {"name": "fields", "type": "Object", "fields": [ - { - "name": "out_bytes", - "type": "Number", - "min": 1, - "max": 15000 - }, - { - "name": "in_bytes", - "type": "Number", - "min": 1, - "max": 60000 - },{ - "name": "bytes", - "type": "Eval", - "expression": "in_bytes+out_bytes" - },{ - "name": "new_in_sessions", - "type": "Number", - "min": 1, - "max": 10 - },{ - "name": "new_out_sessions", - "type": "Number", - "min": 1, - "max": 100 - },{ - "name": "sessions", - "type": "Eval", - "expression": "new_in_sessions+new_out_sessions" - } - ] } - + { + "name": "name", + "type": "String", + "options": [ + "object_statistics" + ] + }, + { + "name": "timestamp_ms", + "type": "Timestamp", + "unit": "millis" + }, + { + "name": "object_id", + "type": "Number", + "min": 1, + "max": 100 + }, + { + "name": "item_id", + "type": "Number", + "min": 1, + "max": 100000 + }, + { + "name": "object_type", + "type": "String", + "options": [ + "ip", + "asn", + "port", + "url", + "fqdn", + "account", + "subscriberid", + "keywords", + "application", + "fqdn_category" + ] + }, + { + "name": "device_id", + "type": "String", + "options": [ + "9800165603191151", + "9800165603247024", + "9800165802621377", + "9800165603191148" + ] + }, + { + "name": "device_group", + "type": "String", + "options": [ + "OLAP-MOCK-DG-1", + "OLAP-MOCK-DG-2", + "OLAP-MOCK-DG-3", + "OLAP-MOCK-DG-4" + ] + }, + { + "name": "data_center", + "type": "String", + "options": [ + "OLAP-MOCK-DC-1", + "OLAP-MOCK-DC-2", + "OLAP-MOCK-DC-3", + "OLAP-MOCK-DC-4" + ] + }, + { + "name": "vsys_id", + "type": "Number", + "options": [ + 2048 + ] + }, + { + "name": "out_bytes", + "type": "Number", + "min": 1, + "max": 15000 + }, + { + "name": "in_bytes", + "type": "Number", + "min": 1, + "max": 60000 + }, + { + "name": "bytes", + "type": "Eval", + "expression": "in_bytes+out_bytes" + }, + { + "name": "new_in_sessions", + "type": "Number", + "min": 1, + "max": 10 + }, + { + "name": "new_out_sessions", + "type": "Number", + "min": 1, + "max": 100 + }, + { + "name": "sessions", + "type": "Eval", + "expression": "new_in_sessions+new_out_sessions" + } ]
\ No newline at end of file diff --git a/config/template/mock_schema/statistics_rule_mock_desc.json b/config/template/mock_schema/statistics_rule_mock_desc.json index c3207ec..91f54e9 100644 --- a/config/template/mock_schema/statistics_rule_mock_desc.json +++ b/config/template/mock_schema/statistics_rule_mock_desc.json @@ -1,122 +1,218 @@ [ - {"name": "name", "type": "String", "options":["statistics_rule"]}, - {"name": "timestamp_ms", "type": "Timestamp", "unit":"millis"}, - {"name": "tags", "type": "Object", "fields": [ - { "name": "unionFields", "type": "Union", "random": false, "unionFields": [ - { "weight": 2, "fields": [ - {"name": "rule_id", "type":"Number", "random": false, "options": [ - 1 - ]}, - {"name": "chart_id", "type":"Number", "random": false,"options": [ - 1 - ]}, - {"name": "template_id", "type":"Number", "random": false, "options": [ - 1 - ]} - ] + { + "name": "name", + "type": "String", + "options": [ + "statistics_rule" + ] + }, + { + "name": "timestamp_ms", + "type": "Timestamp", + "unit": "millis" + }, + { + "name": "unionFields", + "type": "Union", + "random": false, + "unionFields": [ + { + "weight": 2, + "fields": [ + { + "name": "rule_id", + "type": "Number", + "random": false, + "options": [ + 1 + ] + }, + { + "name": "chart_id", + "type": "Number", + "random": false, + "options": [ + 1 + ] + }, + { + "name": "template_id", + "type": "Number", + "random": false, + "options": [ + 1 + ] + } + ] }, - { "weight": 2, "fields": [ - {"name": "rule_id", "type":"Number", "random": false,"options": [ - 4 - ]}, - {"name": "chart_id", "type":"Number", "random": false,"options": [ - 4 - ]}, - {"name": "template_id", "type":"Number", "random": false,"options": [ - 4 - ]}, - { "name": "server_ip", - "type": "IPv4", - "start": "1.0.0.0", - "end": "162.105.10.255"} - ] + { + "weight": 2, + "fields": [ + { + "name": "rule_id", + "type": "Number", + "random": false, + "options": [ + 4 + ] + }, + { + "name": "chart_id", + "type": "Number", + "random": false, + "options": [ + 4 + ] + }, + { + "name": "template_id", + "type": "Number", + "random": false, + "options": [ + 4 + ] + }, + { + "name": "server_ip", + "type": "IPv4", + "start": "1.0.0.0", + "end": "162.105.10.255" + } + ] }, - { "weight": 2, "fields": [ - {"name": "rule_id", "type":"Number", "random": false,"options": [ - 7 - ]}, - {"name": "chart_id", "type":"Number", "random": false,"options": [ - 7 - ]}, - {"name": "template_id", "type":"Number", "random": false,"options": [ - 7 - ]}, - { - "name": "application", - "type": "String", - "options": [ - "ntp", - "stun", - "unknown", - "teredo", - "qq_web_qq", - "kugou", - "quic" - ] - } - ] + { + "weight": 2, + "fields": [ + { + "name": "rule_id", + "type": "Number", + "random": false, + "options": [ + 7 + ] + }, + { + "name": "chart_id", + "type": "Number", + "random": false, + "options": [ + 7 + ] + }, + { + "name": "template_id", + "type": "Number", + "random": false, + "options": [ + 7 + ] + }, + { + "name": "application", + "type": "String", + "options": [ + "ntp", + "stun", + "unknown", + "teredo", + "qq_web_qq", + "kugou", + "quic" + ] + } + ] } ] - }, - { "name": "version", "type": "Number", "options": [1] }, - { - "name": "device_group", - "type": "String", - "options": [ - "OLAP-MOCK-DG-1", - "OLAP-MOCK-DG-2", - "OLAP-MOCK-DG-3", - "OLAP-MOCK-DG-4" - ] - },{ - "name": "data_center", - "type": "String", - "options": [ - "OLAP-MOCK-DC-1", - "OLAP-MOCK-DC-2", - "OLAP-MOCK-DC-3", - "OLAP-MOCK-DC-4" - ] - },{ - "name": "vsys_id", - "type": "Number", - "options": [ - 2048 - ] - } - ] }, - {"name": "fields", "type": "Object", "fields": [ - { "name": "unionFields", "type": "Union", "random": false, "unionFields": [ - { "weight": 2, "fields": [ - { "name": "client_ip_sketch", "type": "String", "options": [ - "AQwAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAACAAAAAAAAAAAAAAAACAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAwAAAIAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAA==" - ,"AQwAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ,"AQwAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ,"AQwAABAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ,"AQwAABAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" - ] } - ] + }, + { + "name": "version", + "type": "Number", + "options": [ + 1 + ] + }, + { + "name": "device_group", + "type": "String", + "options": [ + "OLAP-MOCK-DG-1", + "OLAP-MOCK-DG-2", + "OLAP-MOCK-DG-3", + "OLAP-MOCK-DG-4" + ] + }, + { + "name": "data_center", + "type": "String", + "options": [ + "OLAP-MOCK-DC-1", + "OLAP-MOCK-DC-2", + "OLAP-MOCK-DC-3", + "OLAP-MOCK-DC-4" + ] + }, + { + "name": "vsys_id", + "type": "Number", + "options": [ + 2048 + ] + }, + { + "name": "unionFields", + "type": "Union", + "random": false, + "unionFields": [ + { + "weight": 2, + "fields": [ + { + "name": "client_ip_sketch", + "type": "String", + "options": [ + "AQwAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAACAAAAAAAAAAAAAAAACAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAIAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAwAAAIAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAA==", + "AQwAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "AQwAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "AQwAABAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", + "AQwAABAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAABAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAQBQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==" + ] + }, + { + "name": "unique_count_1", + "type": "Hlld", + "itemCount": 100000, + "batchCount": 1000 + }, + { + "name": "distribution_1", + "type": "HdrHistogram", + "max": 100000, + "batchCount": 1000 + } + ] }, - { "weight": 2, "fields": [ - { - "name": "bytes", - "type": "Number", - "min": 1, - "max": 15000 - } - ] + { + "weight": 2, + "fields": [ + { + "name": "bytes", + "type": "Number", + "min": 1, + "max": 15000 + } + ] }, - { "weight": 2, "fields": [ - { - "name": "sessions", - "type": "Number", - "min": 1, - "max": 200 - } - ] + { + "weight": 2, + "fields": [ + { + "name": "sessions", + "type": "Number", + "min": 1, + "max": 200 + } + ] } ] - } - ] } - + } ]
\ No newline at end of file diff --git a/config/udf.plugins b/config/udf.plugins index 2978bbe..e4f940f 100644 --- a/config/udf.plugins +++ b/config/udf.plugins @@ -21,4 +21,11 @@ com.geedgenetworks.core.udf.udaf.CollectSet com.geedgenetworks.core.udf.udaf.LongCount com.geedgenetworks.core.udf.udaf.Mean com.geedgenetworks.core.udf.udaf.LastValue -com.geedgenetworks.core.udf.udaf.FirstValue
\ No newline at end of file +com.geedgenetworks.core.udf.udaf.FirstValue +com.geedgenetworks.core.udf.udaf.hlld.Hlld +com.geedgenetworks.core.udf.udaf.hlld.HlldApproxCountDistinct +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogram +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantile +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantiles +com.geedgenetworks.core.udf.udtf.JsonUnroll +com.geedgenetworks.core.udf.udtf.Unroll
\ No newline at end of file diff --git a/docs/connector/connector.md b/docs/connector/connector.md index 08ec673..766b73e 100644 --- a/docs/connector/connector.md +++ b/docs/connector/connector.md @@ -19,14 +19,14 @@ sources: ${prop_key}: ${prop_value} ``` -| Name | Type | Required | Default | Description | -|--------------------------|---------------|----------|---------|--------------------------------------------------------------------------------------------------------------------------| -| type | String | Yes | (none) | The type of the source connector. The `SourceTableFactory` will use this value as identifier to create source connector. | -| schema | Map | No | (none) | The source table schema, config through fields or local_file or url. | -| watermark_timestamp | String | No | (none) | watermark timestamp field name, if need use eventTime. | -| watermark_timestamp_unit | String | No | ms | watermark field timestamp unit, options:ms(milliseconds),s(seconds). is required if watermark_timestamp is not none. | -| watermark_lag | Long | No | (none) | watermark out-of-order milliseconds. is required if watermark_timestamp is not none. | -| properties | Map of String | Yes | (none) | The source connector customize properties, more details see the [Source](source) documentation. | +| Name | Type | Required | Default | Description | +|--------------------------|---------------|----------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| type | String | Yes | (none) | The type of the source connector. The `SourceTableFactory` will use this value as identifier to create source connector. | +| schema | Map | No | (none) | The source table schema, config through fields or local_file or url. | +| watermark_timestamp | String | No | (none) | Specify the field name as the watermark field. It is used to track event time and generate watermarks. | +| watermark_timestamp_unit | String | No | ms | The watermark field timestamp unit. The optional values are `ms`, `s`. | +| watermark_lag | Long | No | (none) | The watermark out-of-order milliseconds (Allowed Latenness). It defines the maximum amount of time (in milliseconds) by which events can be late but still be considered for processing. | +| properties | Map of String | Yes | (none) | The source connector customize properties, more details see the [Source](source) documentation. | ## Schema Field Projection @@ -85,41 +85,49 @@ schema: The mock data type is used to define the template of the mock data. -| Mock Type | Parameter | Result Type | Default | Description | -|-----------------------------------------|-------------|-----------------------|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| **[Number](#Number)** | - | **int/bigint/double** | - | **Randomly generate a number.** | -| - | min | number | 0 | The minimum value (include). | -| - | max | number | int32.max | The maximum value (exclusive). | -| - | options | array of number | (none) | The optional values. If set, the random value will be selected from the options and `start` and `end` will be ignored. | -| - | random | boolean | true | Default is random mode. If set to false, the value will be generated in order. | -| **[Sequence](#Sequence)** | - | **bigint** | - | **Generate a sequence number based on a specific step value .** | -| - | start | bigint | 0 | The first number in the sequence (include). | -| - | step | bigint | 1 | The number to add to each subsequent value. | -| **[UniqueSequence](#UniqueSequence)** | - | **bigint** | - | **Generate a global unique sequence number.** | -| - | start | bigint | 0 | The first number in the sequence (include). | -| **[String](#String)** | - | string | - | **Randomly generate a string.** | -| - | regex | string | [a-zA-Z]{0,5} | The regular expression. | -| - | options | array of string | (none) | The optional values. If set, the random value will be selected from the options and `regex` will be ignored. | -| - | random | boolean | true | Default is random mode. If set to false, the options value will be generated in order. | -| **[Timestamp](#Timestamp)** | - | **bigint** | - | **Generate a unix timestamp in milliseconds or seconds.** | -| - | unit | string | second | The unit of the timestamp. The optional values are `second`, `millis`. | -| **[FormatTimestamp](#FormatTimestamp)** | - | **string** | - | **Generate a formatted timestamp.** | -| - | format | string | yyyy-MM-dd HH:mm:ss | The format to output. | -| - | utc | boolean | false | Default is local time. If set to true, the time will be converted to UTC time. | -| **[IPv4](#IPv4)** | - | **string** | - | **Randomly generate a IPv4 address.** | -| - | start | string | 0.0.0.0 | The minimum value of the IPv4 address(include). | -| - | end | string | 255.255.255.255 | The maximum value of the IPv4 address(include). | -| **[Expression](#Expression)** | - | string | - | **Use library [Datafaker](https://www.datafaker.net/documentation/expressions/) expressions to generate fake data.** | -| - | expression | string | (none) | The datafaker expression used #{expression}. | -| **[Eval](#Eval)** | - | **string** | - | **Use AviatorScript value expression to generate data.** | -| - | expression | string | (none) | Support basic arithmetic operations and function calls. More details sess [AviatorScript](https://www.yuque.com/boyan-avfmj/aviatorscript). | -| **[Object](#Object)** | - | **struct/object** | - | **Generate a object data structure. It used to define the nested structure of the mock data.** | -| - | fields | array of object | (none) | The fields of the object. | -| **[Union](#Union)** | - | - | - | **Generate a union data structure with multiple mock data type fields.** | -| - | unionFields | array of object | (none) | The fields of the object. | -| - | - fields | - array of object | (none) | | -| - | - weight | - int | 0 | The weight of the generated object. | -| | random | boolean | true | Default is random mode. If set to false, the options value will be generated in order. | +| Mock Type | Parameter | Result Type | Default | Description | +|-----------------------------------------|---------------------------------|-----------------------|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **[Number](#Number)** | - | **int/bigint/double** | - | **Randomly generate a number.** | +| | min | number | 0 | The minimum value (inclusive). | +| | max | number | int32.max | The maximum value (exclusive). | +| | options | array of number | (none) | The optional values. If set, the random value will be selected from the options and `min` and `max` will be ignored. | +| | random | boolean | true | Default is random mode. If set to false, the value will be generated in order. | +| **[Sequence](#Sequence)** | - | **bigint** | - | **Generate a sequence number based on a specific step value.** | +| | start | bigint | 0 | The first number in the sequence (inclusive). | +| | step | bigint | 1 | The number to add to each subsequent value. | +| **[UniqueSequence](#UniqueSequence)** | - | **bigint** | - | **Generate a globally unique sequence number.** | +| | start | bigint | 0 | The first number in the sequence (inclusive). | +| **[String](#String)** | - | string | - | **Randomly generate a string.** | +| | regex | string | [a-zA-Z]{0,5} | The regular expression used to generate the string. | +| | options | array of string | (none) | The optional values. If set, the random value will be selected from the options and `regex` will be ignored. | +| | random | boolean | true | Default is random mode. If set to false, the options value will be generated in order. | +| **[Timestamp](#Timestamp)** | - | **bigint** | - | **Generate a Unix timestamp in milliseconds or seconds.** | +| | unit | string | second | The unit of the timestamp. Options are `second` or `millis`. | +| **[FormatTimestamp](#FormatTimestamp)** | - | **string** | - | **Generate a formatted timestamp.** | +| | format | string | yyyy-MM-dd HH:mm:ss | The format to output the timestamp in. | +| | utc | boolean | false | Default is local time. If set to true, the time will be converted to UTC time. | +| **[IPv4](#IPv4)** | - | **string** | - | **Randomly generate an IPv4 address.** | +| | start | string | 0.0.0.0 | The minimum value of the IPv4 address (inclusive). | +| | end | string | 255.255.255.255 | The maximum value of the IPv4 address (inclusive). | +| **[Expression](#Expression)** | - | string | - | **Use library [Datafaker](https://www.datafaker.net/documentation/expressions/) expressions to generate fake data.** | +| | expression | string | (none) | The Datafaker expression to use, in the format `#{expression}`. | +| **[Hlld](#HLLD)** | - | **string** | - | **Generate a IP Address HyperLogLog data structure and store it as a base64 string. Use library [HLLD](https://github.com/armon/hlld).** | +| | itemCount | bigint | 1000000 | The total number of items. | +| | batchCount | int | 10000 | The number of items in each batch. | +| | precision | int | 12 | The precision of the HyperLogLog data structure. Allowed range is [4, 18]. | +| **[HdrHistogram](#HdrHistogram)** | - | **string** | - | **Generate a Latency HdrHistogram data structure and store it as a base64 string. Use library [HdrHistogram](https://github.com/HdrHistogram/HdrHistogram).** | +| | max | bigint | 100000 | The maximum value of the histogram. | +| | batchCount | int | 1000 | The random number of items in each batch. | +| | numberOfSignificantValueDigits | int | 1 | The precision of the histogram data structure. Allowed range is [1, 5]. | +| **[Eval](#Eval)** | - | **string** | - | **Use AviatorScript value expression to generate data.** | +| | expression | string | (none) | Support basic arithmetic operations and function calls. More details in [AviatorScript](https://www.yuque.com/boyan-avfmj/aviatorscript). | +| **[Object](#Object)** | - | **struct/object** | - | **Generate an object data structure. Used to define the nested structure of the mock data.** | +| | fields | array of object | (none) | The fields of the object. | +| **[Union](#Union)** | - | - | - | **Generate a union data structure with multiple mock data type fields.** | +| | unionFields | array of object | (none) | The fields of the union. | +| | weight | int | 0 | The weight of the generated object. | +| | random | boolean | true | Default is random mode. If set to false, the options value will be generated in order. | + ### Common Parameters @@ -250,6 +258,22 @@ Mock data type supports some common parameters. {"name":"phoneNumber","type":"Expression","expression":"#{phoneNumber.phoneNumber}"} ``` +### HLLD + +- Generate a IP Address HyperLogLog data structure, stored as a base64 string. At most 1000 IP addresses are generated in each batch. + +```json +{"name":"hll","type":"Hlld","itemCount":1000000,"batchCount":1000,"precision":12} +``` + +### HdrHistogram + +- Generate a Latency HdrHistogram data structure, stored as a base64 string. The maximum value of the histogram is 100000, and at most 1000 items are generated in each batch. + +```json +{"name":"distribution","type":"HdrHistogram","max":100000,"batchCount":1000,"numberOfSignificantValueDigits":1} +``` + ### Eval - Generate a value by using AviatorScript expression. Commonly used for arithmetic operations. diff --git a/docs/filter/aviator.md b/docs/filter/aviator.md index 54fe24b..e7f6c2b 100644 --- a/docs/filter/aviator.md +++ b/docs/filter/aviator.md @@ -30,7 +30,7 @@ This example read data from inline source and print to console. It will filter t filters: # [object] Define filter operator filter_operator: # [object] AviatorFilter operator name - type: com.geedgenetworks.core.filter.AviatorFilter + type: aviator properties: expression: event.server_ip == '8.8.8.8' || event.decoded_as == 'HTTP' # [string] Aviator expression, it return true or false. diff --git a/docs/images/groot_stream_architecture.jpg b/docs/images/groot_stream_architecture.jpg Binary files differindex 1fff0e5..d8f1d4b 100644 --- a/docs/images/groot_stream_architecture.jpg +++ b/docs/images/groot_stream_architecture.jpg diff --git a/docs/processor/aggregate-processor.md b/docs/processor/aggregate-processor.md new file mode 100644 index 0000000..5ab0ae0 --- /dev/null +++ b/docs/processor/aggregate-processor.md @@ -0,0 +1,71 @@ +# Aggregate Processor + +> Processing pipelines for aggregate processors using UDAFs + +## Description + +Aggregate processor is used to aggregate the data from source to sink. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDAFs(User-defined Aggregate functions) into a pipeline. +Within the pipeline, events are processed by each Function in order, top‑>down. More details can be found in user-defined aggregate functions [(UDAFs)](udaf.md). + +## Options +Note：Default will output internal fields `__window_start_timestamp` and `__window_end_timestamp` if not set output_fields. + +| name | type | required | default value | +|--------------------------|--------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| type | String | Yes | The type of the processor, now only support `com.geedgenetworks.core.processor.aggregate.AggregateProcessor` | +| output_fields | Array | No | Array of String. The list of fields that need to be kept. Fields not in the list will be removed. | +| remove_fields | Array | No | Array of String. The list of fields that need to be removed. | +| group_by_fields | Array | yes | Array of String. The list of fields that need to be grouped. | +| window_type | String | yes | The type of window, now only support `tumbling_processing_time`, `tumbling_event_time`, `sliding_processing_time`, `sliding_event_time`. if window_type is `tumbling/sliding_event_time,` you need to set watermark. | +| window_size | Long | yes | The duration of the window in seconds. | +| window_slide | Long | yes | The duration of the window slide in seconds. | +| window_timestamp_field | String | No | Set the output timestamp field name, with the unit in seconds. It is mapped to the internal field __window_start_timestamp. | +| functions | Array | No | Array of Object. The list of functions that need to be applied to the data. | + +## Usage Example + +This example use aggregate processor to aggregate the fields `received_bytes` by `client_ip` and using NUMBER_SUM function to sum all `received_bytes` in 10 seconds window. + +```yaml +sources: + inline_source: + type: inline + properties: + data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.168.10.198","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931}]' + format: json + json.ignore.parse.errors: false + + +processing_pipelines: + aggregate_processor: + type: aggregate + group_by_fields: [ client_ip ] + window_type: tumbling_processing_time + window_size: 10 + functions: + - function: NUMBER_SUM + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_sum ] + +sinks: + print_sink: + type: print + properties: + format: json + mode: log_warn + +application: + env: + name: example-inline-to-print-with-aggregation + parallelism: 3 + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [aggregate_processor] + - name: aggregate_processor + downstream: [ print_sink ] + - name: print_sink + downstream: [] +``` + diff --git a/docs/processor/projection-processor.md b/docs/processor/projection-processor.md index bc4b249..4319f36 100644 --- a/docs/processor/projection-processor.md +++ b/docs/processor/projection-processor.md @@ -1,12 +1,12 @@ # Projection Processor -> Processing pipelines for projection processor +> Processing pipelines for projection processors using scalar UDFs ## Description Projection processor is used to project the data from source to sink. It can be used to filter, remove, and transform fields. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDFs(User-defined functions) into a pipeline. -Within the pipeline, events are processed by each Function in order, top‑>down. The UDF usage detail can be found in [UDF](udf.md). +Within the pipeline, events are processed by each Function in order, top‑>down. More details can be found in User Defined Functions [(UDFs)](udf.md). ## Options diff --git a/docs/processor/table-processor.md b/docs/processor/table-processor.md new file mode 100644 index 0000000..7b3066c --- /dev/null +++ b/docs/processor/table-processor.md @@ -0,0 +1,61 @@ +# Table Processor + +> Processing pipelines for table processors using UDTFs + +## Description + +Table processor is used to process the data from source to sink. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDTFs(User-defined Table functions) into a pipeline. Within the pipeline, events are processed by each Function in order, top‑>down. More details can be found in user-defined table functions [(UDTFs)](udtf.md). + +## Options + +| name | type | required | default value | +|-----------------|--------|----------|------------------------------------------------------------------------------------------------------| +| type | String | Yes | The type of the processor, now only support `com.geedgenetworks.core.processor.table.TableProcessor` | +| output_fields | Array | No | Array of String. The list of fields that ne ed to be kept. Fields not in the list will be removed. | +| remove_fields | Array | No | Array of String. The list of fields that need to be removed. | +| functions | Array | No | Array of Object. The list of functions that need to be applied to the data. | + +## Usage Example +This example uses a table processor to unroll the encapsulation field, converting one row into multiple rows. + +```yaml +sources: + inline_source: + type: inline + properties: + data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931,"encapsulation":"[{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"c2s_source_mac\":\"48:73:97:96:38:27\",\"c2s_destination_mac\":\"58:b3:8f:fa:3b:11\",\"s2c_source_mac\":\"58:b3:8f:fa:3b:11\",\"s2c_destination_mac\":\"48:73:97:96:38:27\"}]"},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.168.10.198","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931,"device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"center-xxg-tsgx\"},{\"tag\":\"device_group\",\"value\":\"group-xxg-tsgx\"}]}"}]' + format: json + json.ignore.parse.errors: false + +processing_pipelines: + table_processor: + type: table + functions: + - function: JSON_UNROLL + lookup_fields: [ encapsulation] + output_fields: [ encapsulation ] + +sinks: + print_sink: + type: print + properties: + format: json + mode: log_warn + +application: + env: + name: example-inline-to-print-use-udtf + parallelism: 3 + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [table_processor] + - name: table_processor + downstream: [ print_sink ] + - name: print_sink + downstream: [] + +``` + + diff --git a/docs/processor/udaf.md b/docs/processor/udaf.md new file mode 100644 index 0000000..dd1dd70 --- /dev/null +++ b/docs/processor/udaf.md @@ -0,0 +1,325 @@ +# UDAF + +> The functions for aggregate processors. + +## Function of content + +- [Collect List](#Collect-List) +- [Collect Set](#Collect-Set) +- [First Value](#First-Value) +- [Last Value](#Last-Value) +- [Long Count](#Long-Count) +- [MEAN](#Mean) +- [Number SUM](#Number-SUM) +- [HLLD](#HLLD) +- [Approx Count Distinct HLLD](#Approx-Count-Distinct-HLLD) +- [HDR Histogram](#HDR-Histogram) +- [Approx Quantile HDR](#APPROX_QUANTILE_HDR) +- [Approx Quantiles HDR](#APPROX_QUANTILES_HDR) + +## Description + +UDF(User Defined Aggregate Function) is used to extend the functions of aggregate processor. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDAFs into a pipeline. Within the pipeline, events are processed by each Function in order, top‑>down. +The deference between UDF and UDAF is: +- UDF is used to process each event, and the output is also an event. UDAF is used to process a group of events, and the output is also an event. +- A UDF is designed to perform a transformation or calculation on a single event. A UDAF is designed to perform an aggregation over a group of events, such as summing values, calculating an average, or finding a maximum. It processes multiple events of input data and produces a single aggregated result. + +## UDAF Definition + The UDAF basic properties are the same as UDF, such as `name`, `event`, `context`，more detail can be found in [UDF](udf.md). But Aggregate Processor have some methods to process the data is: +- `void add()`: Add a new event to the aggregation. +- `void getResult()`: Get the result of the aggregation. + +## Functions + +### Collect List + +COLLECT_LIST is used to collect the value of the field in the group of events. + +```COLLECT_LIST(filter, lookup_fields, output_fields)``` + +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: COLLECT_LIST + lookup_fields: [client_ip] + output_fields: [client_ip_list] +``` + +### Collect Set + +COLLECT_SET is used to collect the unique value of the field in the group of events. + +```COLLECT_SET(filter, lookup_fields, output_fields)``` + +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: COLLECT_SET + lookup_fields: [client_ip] + output_fields: [client_ip_set] +``` + +### First Value + +FIRST_VALUE is used to get the first value of the field in the group of events. + +```FIRST_VALUE(filter, lookup_fields, output_fields)``` +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: FIRST_VALUE + lookup_fields: [client_ip] + output_fields: [first_client_ip] +``` +### Last Value + +LAST_VALUE is used to get the last value of the field in the group of events. + +```LAST_VALUE(filter, lookup_fields, output_fields)``` +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: LAST_VALUE + lookup_fields: [client_ip] + output_fields: [last_client_ip] +``` + +### Long Count + +LONG_COUNT is used to count the number of events in the group of events. + +```LONG_COUNT(filter, lookup_fields, output_fields)``` +- filter: optional +- lookup_fields: optional. +- output_fields: required. + +### Example + +```yaml +- function: LONG_COUNT + output_fields: [sessions] +``` + +### Mean + +MEAN is used to calculate the mean value of the field in the group of events. The lookup field value must be a number. + +```MEAN(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. +- parameters: optional. + - precision: `<Integer>` required. The precision of the mean value. Default is 2. + +### Example + +```yaml +- function: MEAN + lookup_fields: [received_bytes] + output_fields: [received_bytes_mean] +``` + +### Number SUM + +NUMBER_SUM is used to sum the value of the field in the group of events. The lookup field value must be a number. + +```NUMBER_SUM(filter, lookup_fields, output_fields)``` +- filter: optional +- lookup_fields: required. Now only support one field. +- output_fields: optional. If not set, the output field name is `lookup_field_name`. + +### Example + +```yaml +- function: NUMBER_SUM + lookup_fields: [received_bytes] + output_fields: [received_bytes_sum] +``` + +### HLLD +hlld is a high-performance C server which is used to expose HyperLogLog sets and operations over them to networked clients. More details can be found in [hlld](https://github.com/armon/hlld). + +```HLLD(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `<String>` optional. input field type can be `regular` or `sketch`. Default is `sketch`. regular field data type includes `string`, `int`, `long`, `float`, `double` etc. + - precision: `<Integer>` optional. The precision of the hlld value. Default is 12. + - output_format: `<String>` optional. The output format can be either `base64(encoded string)` or `binary(byte[])`. The default is `base64`. + +### Example + Merge multiple string field into a HyperLogLog data structure. +```yaml + - function: HLLD + lookup_fields: [client_ip] + output_fields: [client_ip_hlld] + parameters: + input_type: regular + +``` + Merge multiple `unique_count ` metric type fields into a HyperLogLog data structure +```yaml + - function: HLLD + lookup_fields: [client_ip_hlld] + output_fields: [client_ip_hlld] + parameters: + input_type: sketch +``` + +### Approx Count Distinct HLLD +Approx Count Distinct HLLD is used to count the approximate number of distinct values in the group of events. + +```APPROX_COUNT_DISTINCT_HLLD(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `<String>` optional. Refer to `HLLD` function. + - precision: `<Integer>` optional. Refer to `HLLD` function. + +### Example + +```yaml +- function: APPROX_COUNT_DISTINCT_HLLD + lookup_fields: [client_ip] + output_fields: [unique_client_ip] + parameters: + input_type: regular +``` + +```yaml +- function: APPROX_COUNT_DISTINCT_HLLD + lookup_fields: [client_ip_hlld] + output_fields: [unique_client_ip] + parameters: + input_type: sketch +``` + +### HDR Histogram + +A High Dynamic Range (HDR) Histogram. More details can be found in [HDR Histogram](https://github.com/HdrHistogram/HdrHistogram). + +```HDR_HISTOGRAM(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `<String>` optional. input field type can be `regular` or `sketch`. Default is `sketch`. regular field is a number. + - lowestDiscernibleValue: `<Integer>` optional. The lowest trackable value. Default is 1. + - highestTrackableValue: `<Integer>` optional. The highest trackable value. Default is 2. + - numberOfSignificantValueDigits: `<Integer>` optional. The number of significant value digits. Default is 1. The range is 1 to 5. + - autoResize: `<Boolean>` optional. If true, the highestTrackableValue will auto-resize. Default is true. + - output_format: `<String>` optional. The output format can be either `base64(encoded string)` or `binary(byte[])`. The default is `base64`. + +### Example + + ```yaml + - function: HDR_HISTOGRAM + lookup_fields: [latency_ms] + output_fields: [latency_ms_histogram] + parameters: + input_type: regular + lowestDiscernibleValue: 1 + highestTrackableValue: 3600000 + numberOfSignificantValueDigits: 3 + ``` + ```yaml + - function: HDR_HISTOGRAM + lookup_fields: [latency_ms_histogram] + output_fields: [latency_ms_histogram] + parameters: + input_type: sketch + ``` + +### Approx Quantile HDR + +Approx Quantile HDR is used to calculate the approximate quantile value of the field in the group of events. + +```APPROX_QUANTILE_HDR(filter, lookup_fields, output_fields, quantile[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `<String>` optional. Refer to `HDR_HISTOGRAM` function. + - lowestDiscernibleValue: `<Integer>` optional. Refer to `HDR_HISTOGRAM` function. + - highestTrackableValue: `<Integer>` required. Refer to `HDR_HISTOGRAM` function. + - numberOfSignificantValueDigits: `<Integer>` optional. Refer to `HDR_HISTOGRAM` function. + - autoResize: `<Boolean>` optional. Refer to `HDR_HISTOGRAM` function. + - probability: `<Double>` optional. The probability of the quantile. Default is 0.5. + +### Example + + ```yaml + - function: APPROX_QUANTILE_HDR + lookup_fields: [latency_ms] + output_fields: [latency_ms_p95] + parameters: + input_type: regular + probability: 0.95 + ``` + + ```yaml + - function: APPROX_QUANTILE_HDR + lookup_fields: [latency_ms_HDR] + output_fields: [latency_ms_p95] + parameters: + input_type: sketch + probability: 0.95 + + ``` + +### Approx Quantiles HDR + +Approx Quantiles HDR is used to calculate the approximate quantile values of the field in the group of events. + +```APPROX_QUANTILES_HDR(filter, lookup_fields, output_fields, quantiles[, parameters])``` +- filter: optional +- lookup_fields: required. +- output_fields: required. +- parameters: optional. + - input_type: `<String>` optional. Refer to `HDR_HISTOGRAM` function. + - lowestDiscernibleValue: `<Integer>` optional. Refer to `HDR_HISTOGRAM` function. + - highestTrackableValue: `<Integer>` required. Refer to `HDR_HISTOGRAM` function. + - numberOfSignificantValueDigits: `<Integer>` optional. Refer to `HDR_HISTOGRAM` function. + - autoResize: `<Boolean>` optional. Refer to `HDR_HISTOGRAM` function. + - probabilities: `<Array<Double>>` required. The list of probabilities of the quantiles. Range is 0 to 1. + +### Example + +```yaml +- function: APPROX_QUANTILES_HDR + lookup_fields: [latency_ms] + output_fields: [latency_ms_quantiles] + parameters: + input_type: regular + probabilities: [0.5, 0.95, 0.99] +``` + +```yaml +- function: APPROX_QUANTILES_HDR + lookup_fields: [latency_ms_HDR] + output_fields: [latency_ms_quantiles] + parameters: + input_type: sketch + probabilities: [0.5, 0.95, 0.99] +``` + + + diff --git a/docs/processor/udf.md b/docs/processor/udf.md index 2a705fd..170d86f 100644 --- a/docs/processor/udf.md +++ b/docs/processor/udf.md @@ -1,7 +1,7 @@ # UDF -> The functions for projection processors. -> +> The functions for projection processors. + ## Function of content - [Asn Lookup](#asn-lookup) @@ -24,7 +24,7 @@ ## Description -UDF(User Defined Function) is used to extend the functions of projection processor. The UDF is a part of the processing pipeline. It can be used in the pre-processing pipeline, processing pipeline, and post-processing pipeline. +Scalar UDF(User Defined Function) is used to extend the functions of projection processor. The UDF is a part of the processing pipeline. It can be used in the pre-processing pipeline, processing pipeline, and post-processing pipeline. ## UDF Definition @@ -40,7 +40,7 @@ A UDF includes the following parts: name, event(processing data), context, evalu - open function: Initialize the resources used by the function. - close function: Release the resources used by the function. -### Functions +## Functions Function define common parameters: `filter`, `lookup_fields`, `output_fields`, `parameters`, and will return a Map<String, Object> value of the event. ``` FUNCTION_NAME(filter, lookup_fields, output_fields[, parameters])``` @@ -54,8 +54,8 @@ Asn lookup function is used to lookup the asn information by ip address. You nee - lookup_fields: required - output_fields: required - parameters: required -- kb_name: required. The name of the knowledge base. -- option: required. Now only support `IP_TO_ASN`. + - kb_name: required. The name of the knowledge base. + - option: required. Now only support `IP_TO_ASN`. Example: @@ -77,8 +77,8 @@ Base64 decode function is used to decode the base64 encoded string. - lookup_fields: not required - output_fields: required - parameters: required -- value_field: `<String>` required. -- charset_field:`<String>` optional. Default is `UTF-8`. + - value_field: `<String>` required. + - charset_field:`<String>` optional. Default is `UTF-8`. Example: @@ -99,7 +99,7 @@ Base64 encode function is commonly used to encode the binary data to base64 stri - lookup_fields: not required - output_fields: required - parameters: required -- value_field: `<String>` required. + - value_field: `<String>` required. Example: @@ -119,7 +119,7 @@ Current unix timestamp function is used to get the current unix timestamp. - lookup_fields: not required - output_fields: required - parameters: optional -- precision: `<String>` optional. Default is `seconds`. Enum: `milliseconds`, `seconds`. + - precision: `<String>` optional. Default is `seconds`. Enum: `milliseconds`, `seconds`. Example: @@ -139,7 +139,7 @@ Domain function is used to extract the domain from the url. - lookup_fields: required. Support more than one fields. All fields will be processed from left to right, and the result will be overwritten if the field processed value is not null. - output_fields: required - parameters: required -- option: `<String>` required. Enum: `TOP_LEVEL_DOMAIN`, `FIRST_SIGNIFICANT_SUBDOMAIN`. + - option: `<String>` required. Enum: `TOP_LEVEL_DOMAIN`, `FIRST_SIGNIFICANT_SUBDOMAIN`. #### Option @@ -182,7 +182,7 @@ Eval function is used to adds or removes fields from events by evaluating an val - lookup_fields: not required - output_fields: required - parameters: required -- value_expression: `<String>` required. Enter a value expression to set the field’s value – this can be a constant. + - value_expression: `<String>` required. Enter a value expression to set the field’s value – this can be a constant. Example 1: Add a field `ingestion_time` with value `recv_time`: @@ -213,10 +213,10 @@ Flatten the fields of nested structure to the top level. The new fields name are - lookup_fields: optional - output_fields: not required - parameters: optional -- prefix: `<String>` optional. Prefix string for flattened field names. Default is empty. -- depth: `<Integer>` optional. Number representing the nested levels to consider for flattening. Minimum 1. Default is `5`. -- delimiter: `<String>` optional. The string used to join nested keys Default is `.`. -- json_string_keys: `<Array>` optional. The keys of the json string fields. It indicates keys that contain JSON strings and should be parsed and flattened. Default is empty. + - prefix: `<String>` optional. Prefix string for flattened field names. Default is empty. + - depth: `<Integer>` optional. Number representing the nested levels to consider for flattening. Minimum 1. Default is `5`. + - delimiter: `<String>` optional. The string used to join nested keys Default is `.`. + - json_string_keys: `<Array>` optional. The keys of the json string fields. It indicates keys that contain JSON strings and should be parsed and flattened. Default is empty. Example 1: @@ -259,7 +259,7 @@ From unix timestamp function is used to convert the unix timestamp to date time - lookup_fields: required - output_fields: required - parameters: optional -- precision: `<String>` optional. Default is `seconds`. Enum: `milliseconds`, `seconds`. + - precision: `<String>` optional. Default is `seconds`. Enum: `milliseconds`, `seconds`. #### Precision @@ -303,16 +303,16 @@ GeoIP lookup function is used to lookup the geoip information by ip address. You - lookup_fields: required - output_fields: optional - parameters: required -- kb_name: `<String>` required. The name of the knowledge base. -- option: `<String>` required. Enum: `IP_TO_COUNTRY`, `IP_TO_PROVINCE`, `IP_TO_CITY`, `IP_TO_SUBDIVISION_ADDR`, `IP_TO_DETAIL`, `IP_TO_LATLNG`, `IP_TO_PROVIDER`, `IP_TO_JSON`, `IP_TO_OBJECT`. -- geolocation_field_mapping : `<Map<String, String>>` optional. The option is required when the option is `IP_TO_OBJECT`. The mapping of the geolocation fields. The key is the field name of the knowledge base , and the value is the field name of the event. -- COUNTRY: `<String>` optional. -- PROVINCE: `<String>` optional. -- CITY: `<String>` optional. -- LONGITUDE: `<String>` optional. -- LATITUDE: `<String>` optional. -- ISP: `<String>` optional. -- ORGANIZATION: `<String>` optional. + - kb_name: `<String>` required. The name of the knowledge base. + - option: `<String>` required. Enum: `IP_TO_COUNTRY`, `IP_TO_PROVINCE`, `IP_TO_CITY`, `IP_TO_SUBDIVISION_ADDR`, `IP_TO_DETAIL`, `IP_TO_LATLNG`, `IP_TO_PROVIDER`, `IP_TO_JSON`, `IP_TO_OBJECT`. + - geolocation_field_mapping : `<Map<String, String>>` optional. The option is required when the option is `IP_TO_OBJECT`. The mapping of the geolocation fields. The key is the field name of the knowledge base , and the value is the field name of the event. + - COUNTRY: `<String>` optional. + - PROVINCE: `<String>` optional. + - CITY: `<String>` optional. + - LONGITUDE: `<String>` optional. + - LATITUDE: `<String>` optional. + - ISP: `<String>` optional. + - ORGANIZATION: `<String>` optional. #### Option @@ -369,7 +369,7 @@ JSON extract function is used to extract the value from json string. - lookup_fields: required - output_fields: required - parameters: required -- value_expression: `<String>` required. The json path expression. + - value_expression: `<String>` required. The json path expression. Example: @@ -390,7 +390,7 @@ Path combine function is used to combine the file path. The path value can be co - lookup_fields: required - output_fields: required - parameters: required -- path: `<Array>` required. + - path: `<Array>` required. Example: @@ -411,11 +411,11 @@ Rename function is used to rename or reformat(e.g. by replacing character unders - lookup_fields: not required - output_fields: not required - parameters: required -- parent_fields: `<Array>` optional. Specify fields whose children will inherit the Rename fields and Rename expression operations. -- rename_fields: `Map<String, String>` required. The key is the original field name, and the value is the new field name. -- current_field_name: `<String>` required. The original field name. -- new_field_name: `<String>` required. The new field name. -- rename_expression: `<String>` optional. AviatorScript expression whose returned value will be used to rename fields. + - parent_fields: `<Array>` optional. Specify fields whose children will inherit the Rename fields and Rename expression operations. + - rename_fields: `Map<String, String>` required. The key is the original field name, and the value is the new field name. + - current_field_name: `<String>` required. The original field name. + - new_field_name: `<String>` required. The new field name. + - rename_expression: `<String>` optional. AviatorScript expression whose returned value will be used to rename fields. ``` A single Function can include both rename_fields (to rename specified field names) and rename_expression (to globally rename fields). However, the Rename fields strategy will execute first. @@ -462,7 +462,7 @@ Snowflake ID function is used to generate the snowflake id. The snowflake id is - lookup_fields: not required - output_fields: required - parameters: optional -- data_center_id_num: `<Integer>` optional. Default is `0`, range is `0-31`. + - data_center_id_num: `<Integer>` optional. Default is `0`, range is `0-31`. Example: @@ -482,9 +482,9 @@ String joiner function joins multiple string fields using a delimiter, prefix, a - lookup_fields: required. Support more than one fields. - output_fields: required - parameters: optional -- delimiter: `<String>` optional. Default is `,`. -- prefix: `<String>` optional. Default is empty string. -- suffix: `<String>` optional. Default is empty string. + - delimiter: `<String>` optional. Default is `,`. + - prefix: `<String>` optional. Default is empty string. + - suffix: `<String>` optional. Default is empty string. Example: @@ -507,7 +507,7 @@ Unix timestamp converter function is used to convert the unix timestamp precisio - lookup_fields: required - output_fields: required - parameters: required -- precision: `<String>` required. Enum: `milliseconds`, `seconds`, `minutes`. The minutes precision is used to generate Unix timestamp, round it to the minute level, and output it in seconds format. + - precision: `<String>` required. Enum: `milliseconds`, `seconds`, `minutes`. The minutes precision is used to generate Unix timestamp, round it to the minute level, and output it in seconds format. - Example: _`__timestamp` Internal field, from source ingestion time or current unix timestamp. diff --git a/docs/processor/udtf.md b/docs/processor/udtf.md new file mode 100644 index 0000000..a6e8444 --- /dev/null +++ b/docs/processor/udtf.md @@ -0,0 +1,66 @@ +# UDTF + +> The functions for table processors. + +## Function of content + +- [UNROLL](#unroll) +- [JSON_UNROLL](#json_unroll) + +## Description + +The UDTFs(user-defined table functions) are used to process the data from source to sink. It is a part of the processing pipeline. It can be used in the pre-processing, processing, and post-processing pipeline. Each processor can assemble UDTFs into a pipeline. Within the pipeline, events are processed by each Function in order, top‑>down. +Unlike scalar functions, which return a single value, UDTFs are particularly useful when you need to explode or unroll data, transforming a single input row into multiple output rows. + +## UDTF Definition + + The UDTFs and UDFs share similar input and context structures, please refer to [UDF](udf.md). + +## Functions + +### UNROLL + +The Unroll Function handles an array field—or an expression evaluating to an array—and unrolls it into individual events. + +```UNROLL(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required +- output_fields: required +- parameters: optional + - regex: `<String>` optional. If lookup_fields is a string, the regex parameter is used to split the string into an array. The default value is a comma. + +#### Example + +```yaml +functions: + - function: UNROLL + lookup_fields: [ monitor_rule_list ] + output_fields: [ monitor_rule ] +``` + +### JSON_UNROLL + +The JSON Unroll Function handles a JSON object, unrolls/explodes an array of objects therein into individual events, while also inheriting top level fields. + +```JSON_UNROLL(filter, lookup_fields, output_fields[, parameters])``` +- filter: optional +- lookup_fields: required +- output_fields: required +- parameters: optional + - path: `<String>` optional. Path to array to unroll, default is the root of the JSON object. + - new_path: `<String>` optional. Rename path to new_path, default is the same as path. + +#### Example + +```yaml +functions: + - function: JSON_UNROLL + lookup_fields: [ device_tag ] + output_fields: [ device_tag ] + parameters: + - path: tags + - new_path: tag +``` + + + diff --git a/docs/user-guide.md b/docs/user-guide.md index 9d5b1c7..e35616f 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -137,7 +137,7 @@ Based on the filter expression, the event will be passed to downstream if the ex ## Processing Pipelines Processing pipelines are used to define the event processing logic of the job. It can be categorized by functionality into stateless and stateful processors. Based processing order, it can be categorized into pre-processing pipeline, processing pipeline and post-processing pipeline. Each processor can assemble `UDFs`(User-defined functions) into a pipeline. The detail of processor is listed in [Processor](processor). - +UDF supports [scalar UDFs](processor/udf.md) , user-defined aggregate functions [(UDAFs)](processor/udaf.md), and user-defined table functions (UDTFs). ## Sinks Sink is used to define where GrootStream needs to output data. Multiple sinks can be defined in a job. The supported sinks are listed in [Sink Connectors](connector/sink). Each sink has its own specific parameters to define how to output data, and GrootStream also extracts the properties that each sink will use, such as the `topic` and `kafka.bootstrap.servers` of the `Kafka` sink. diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java index 04a6a94..64c66b6 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractExecutor.java @@ -2,6 +2,7 @@ package com.geedgenetworks.bootstrap.execution; import com.geedgenetworks.common.utils.ReflectionUtils; import com.geedgenetworks.core.filter.Filter; +import com.geedgenetworks.core.processor.Processor; import com.geedgenetworks.core.processor.aggregate.AggregateProcessor; import com.geedgenetworks.core.processor.projection.ProjectionProcessor; import com.typesafe.config.Config; @@ -20,8 +21,7 @@ public abstract class AbstractExecutor<K, V> protected final Config operatorConfig; protected final Map<K,V> operatorMap; protected final Map<String,Filter> filterMap = new HashMap<>(); - protected final Map<String,ProjectionProcessor> projectionProcessorMap = new HashMap<>(); - protected final Map<String, AggregateProcessor> aggregateProcessorMap = new HashMap<>(); + protected final Map<String, Processor> processorMap = new HashMap<>(); protected AbstractExecutor(List<URL> jarPaths, Config operatorConfig) { this.operatorConfig = operatorConfig; @@ -30,13 +30,9 @@ public abstract class AbstractExecutor<K, V> for (Filter filter : filters) { this.filterMap.put(filter.type(), filter); } - ServiceLoader<ProjectionProcessor> projectionProcessors = ServiceLoader.load(ProjectionProcessor.class); - for (ProjectionProcessor projectionProcessor : projectionProcessors) { - this.projectionProcessorMap.put(projectionProcessor.type(), projectionProcessor); - } - ServiceLoader<AggregateProcessor> aggregateProcessors = ServiceLoader.load(AggregateProcessor.class); - for (AggregateProcessor aggregateProcessor : aggregateProcessors) { - this.aggregateProcessorMap.put(aggregateProcessor.type(), aggregateProcessor); + ServiceLoader<Processor> processors = ServiceLoader.load(Processor.class); + for (Processor processor : processors) { + this.processorMap.put(processor.type(), processor); } } diff --git a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java index 42a4828..66c0b0f 100644 --- a/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java +++ b/groot-bootstrap/src/main/java/com/geedgenetworks/bootstrap/execution/AbstractProcessorExecutor.java @@ -2,15 +2,14 @@ package com.geedgenetworks.bootstrap.execution; import com.alibaba.fastjson.JSONObject; import com.geedgenetworks.bootstrap.exception.JobExecuteException; -import com.geedgenetworks.common.config.AggregateConfigOptions; -import com.geedgenetworks.common.config.CheckConfigUtil; -import com.geedgenetworks.common.config.CheckResult; -import com.geedgenetworks.common.config.ProjectionConfigOptions; +import com.geedgenetworks.common.config.*; import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.ConfigValidationException; import com.geedgenetworks.core.pojo.AggregateConfig; import com.geedgenetworks.core.pojo.ProcessorConfig; import com.geedgenetworks.core.pojo.ProjectionConfig; +import com.geedgenetworks.core.pojo.TableConfig; +import com.geedgenetworks.core.processor.table.TableProcessor; import com.geedgenetworks.core.processor.aggregate.AggregateProcessor; import com.geedgenetworks.core.processor.projection.ProjectionProcessor; import com.typesafe.config.Config; @@ -35,6 +34,9 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor<String, case "aggregate": dataStream = executeAggregateProcessor(dataStream, node, (AggregateConfig) processorConfig); break; + case "table": + dataStream = executeTableProcessor(dataStream, node, (TableConfig) processorConfig); + break; case "projection": dataStream = executeProjectionProcessor(dataStream, node, (ProjectionConfig) processorConfig); break; @@ -43,12 +45,37 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor<String, } return dataStream; } + protected SingleOutputStreamOperator executeTableProcessor(SingleOutputStreamOperator dataStream, Node node, TableConfig tableConfig) throws JobExecuteException { + TableProcessor tableProcessor; + if (processorMap.containsKey(tableConfig.getType())) { + tableProcessor = (TableProcessor) processorMap.get(tableConfig.getType()); + } else { + Class cls; + try { + cls = Class.forName(tableConfig.getType()); + tableProcessor = (TableProcessor) cls.newInstance(); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | RuntimeException e) { + throw new JobExecuteException("get processing pipeline instance failed!", e); + } + } + if (node.getParallelism() > 0) { + tableConfig.setParallelism(node.getParallelism()); + } + try { + dataStream = + tableProcessor.processorFunction( + dataStream, tableConfig, jobRuntimeEnvironment.getStreamExecutionEnvironment().getConfig()); + } catch (Exception e) { + throw new JobExecuteException("Create orderby pipeline instance failed!", e); + } + return dataStream; + } protected SingleOutputStreamOperator executeAggregateProcessor(SingleOutputStreamOperator dataStream, Node node, AggregateConfig aggregateConfig) throws JobExecuteException { AggregateProcessor aggregateProcessor; - if (aggregateProcessorMap.containsKey(aggregateConfig.getType())) { - aggregateProcessor = aggregateProcessorMap.get(aggregateConfig.getType()); + if (processorMap.containsKey(aggregateConfig.getType())) { + aggregateProcessor = (AggregateProcessor) processorMap.get(aggregateConfig.getType()); } else { Class cls; try { @@ -63,7 +90,7 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor<String, } try { dataStream = - aggregateProcessor.aggregateProcessorFunction( + aggregateProcessor.processorFunction( dataStream, aggregateConfig, jobRuntimeEnvironment.getStreamExecutionEnvironment().getConfig()); } catch (Exception e) { throw new JobExecuteException("Create aggregate pipeline instance failed!", e); @@ -74,8 +101,8 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor<String, protected SingleOutputStreamOperator executeProjectionProcessor(SingleOutputStreamOperator dataStream, Node node, ProjectionConfig projectionConfig) throws JobExecuteException { ProjectionProcessor projectionProcessor; - if (projectionProcessorMap.containsKey(projectionConfig.getType())) { - projectionProcessor = projectionProcessorMap.get(projectionConfig.getType()); + if (processorMap.containsKey(projectionConfig.getType())) { + projectionProcessor = (ProjectionProcessor) processorMap.get(projectionConfig.getType()); } else { Class cls; try { @@ -90,8 +117,8 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor<String, } try { dataStream = - projectionProcessor.projectionProcessorFunction( - dataStream, projectionConfig); + projectionProcessor.processorFunction( + dataStream, projectionConfig,jobRuntimeEnvironment.getStreamExecutionEnvironment().getConfig()); } catch (Exception e) { throw new JobExecuteException("Create processing pipeline instance failed!", e); } @@ -114,6 +141,9 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor<String, case "aggregate": projectionConfig = checkAggregateProcessorConfig(key, value, processorsConfig); break; + case "table": + projectionConfig = checkTableProcessorConfig(key, value, processorsConfig); + break; default://兼容历史版本 projectionConfig = checkProjectionProcessorConfig(key, value, processorsConfig); } @@ -164,5 +194,24 @@ public abstract class AbstractProcessorExecutor extends AbstractExecutor<String, return aggregateConfig; } + protected TableConfig checkTableProcessorConfig(String key, Map<String, Object> value, Config config) { + + CheckResult result = CheckConfigUtil.checkAtLeastOneExists(config.getConfig(key), + TableConfigOptions.OUTPUT_FIELDS.key(), + TableConfigOptions.REMOVE_FIELDS.key(), + TableConfigOptions.FUNCTIONS.key()); + if (!result.isSuccess()) { + throw new ConfigValidationException(CommonErrorCode.CONFIG_VALIDATION_FAILED, String.format( + "Table processor: %s, At least one of [%s] should be specified.", + key, String.join(",", + TableConfigOptions.OUTPUT_FIELDS.key(), + TableConfigOptions.REMOVE_FIELDS.key(), + TableConfigOptions.FUNCTIONS.key()))); + } + + TableConfig tableConfig = new JSONObject(value).toJavaObject(TableConfig.class); + tableConfig.setName(key); + return tableConfig; + } } diff --git a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java index ccdd224..c3746a4 100644 --- a/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java +++ b/groot-bootstrap/src/test/java/com/geedgenetworks/bootstrap/utils/ConfigShadeTest.java @@ -67,5 +67,7 @@ public class ConfigShadeTest { Assertions.assertEquals(decryptPassword, PASSWORD); System.out.println( ConfigShadeUtils.encryptOption("aes", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"admin\" password=\"galaxy2019\";")); System.out.println( ConfigShadeUtils.decryptOption("aes", "454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817")); + System.out.println( ConfigShadeUtils.encryptOption("aes", "testuser")); + System.out.println( ConfigShadeUtils.encryptOption("aes", "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"olap\" password=\"galaxy2019\";")); } } diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/AggregateConfigOptions.java b/groot-common/src/main/java/com/geedgenetworks/common/config/AggregateConfigOptions.java index 3998a3b..0b0379d 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/config/AggregateConfigOptions.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/AggregateConfigOptions.java @@ -5,6 +5,8 @@ import com.geedgenetworks.common.udf.UDFContext; import java.util.List; +import static com.geedgenetworks.common.Event.WINDOW_START_TIMESTAMP; + public interface AggregateConfigOptions { Option<String> TYPE = Options.key("type") .stringType() @@ -46,4 +48,8 @@ public interface AggregateConfigOptions { .noDefaultValue() .withDescription("The size of sliding window."); + Option<String> WINDOW_TIMESTAMP_FIELD = Options.key("window_timestamp_field") + .stringType() + .noDefaultValue() + .withDescription("which field to be set the start time of window."); } diff --git a/groot-common/src/main/java/com/geedgenetworks/common/config/TableConfigOptions.java b/groot-common/src/main/java/com/geedgenetworks/common/config/TableConfigOptions.java new file mode 100644 index 0000000..480496d --- /dev/null +++ b/groot-common/src/main/java/com/geedgenetworks/common/config/TableConfigOptions.java @@ -0,0 +1,34 @@ +package com.geedgenetworks.common.config; + +import com.alibaba.fastjson2.TypeReference; +import com.geedgenetworks.common.udf.UDFContext; + +import java.util.List; + +public interface TableConfigOptions { + Option<String> TYPE = Options.key("type") + .stringType() + .noDefaultValue() + .withDescription("The type of processor."); + + Option<List<String>> OUTPUT_FIELDS = Options.key("output_fields") + .listType() + .noDefaultValue() + .withDescription("The fields to be outputted."); + + Option<List<String>> REMOVE_FIELDS = Options.key("remove_fields") + .listType() + .noDefaultValue() + .withDescription("The fields to be removed."); + + Option<List<UDFContext>> FUNCTIONS = Options.key("functions") + .type(new TypeReference<List<UDFContext>>() {}) + .noDefaultValue() + .withDescription("The functions to be executed."); + + + + + + +} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/udf/AggregateFunction.java b/groot-common/src/main/java/com/geedgenetworks/common/udf/AggregateFunction.java index 98450fd..455073f 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/udf/AggregateFunction.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/udf/AggregateFunction.java @@ -7,7 +7,9 @@ import java.io.Serializable; public interface AggregateFunction extends Serializable { - Accumulator open(UDFContext udfContext,Accumulator acc); + void open(UDFContext udfContext); + + Accumulator initAccumulator(Accumulator acc); Accumulator add(Event val, Accumulator acc); @@ -15,6 +17,5 @@ public interface AggregateFunction extends Serializable { Accumulator getResult(Accumulator acc); - void close(); - + default void close(){}; } diff --git a/groot-common/src/main/java/com/geedgenetworks/common/udf/TableFunction.java b/groot-common/src/main/java/com/geedgenetworks/common/udf/TableFunction.java new file mode 100644 index 0000000..e602291 --- /dev/null +++ b/groot-common/src/main/java/com/geedgenetworks/common/udf/TableFunction.java @@ -0,0 +1,20 @@ +package com.geedgenetworks.common.udf; + +import com.geedgenetworks.common.Event; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.util.Collector; + +import java.io.Serializable; +import java.util.List; + +public interface TableFunction extends Serializable { + + void open(RuntimeContext runtimeContext, UDFContext udfContext); + + List<Event> evaluate(Event event); + + String functionName(); + + void close(); + +} diff --git a/groot-common/src/main/java/com/geedgenetworks/common/utils/JsonPathUtil.java b/groot-common/src/main/java/com/geedgenetworks/common/utils/JsonPathUtil.java index dcba58c..0823ddc 100644 --- a/groot-common/src/main/java/com/geedgenetworks/common/utils/JsonPathUtil.java +++ b/groot-common/src/main/java/com/geedgenetworks/common/utils/JsonPathUtil.java @@ -2,6 +2,8 @@ package com.geedgenetworks.common.utils; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; import com.alibaba.fastjson2.JSONPath; import com.alibaba.fastjson2.JSONReader; @@ -42,4 +44,35 @@ public class JsonPathUtil { } return flattenResult; } + public static Object get(JSONObject jsonObject, String expr) { + Object Result = ""; + try { + Result = jsonObject.getByPath(expr); + } catch (Exception e) { + logger.error( + "The label resolution exception or [expr] analytic expression error" + + e.getMessage()); + } + return Result; + } + public static JSONObject set(JSONObject jsonObject, String expr,Object value) { + try { + JSONPath.set(jsonObject, expr, value); + } catch (Exception e) { + logger.error( + "JSONObject set value exception or [expr] expression error" + + e.getMessage()); + } + return jsonObject; + } + public static JSONObject remove(JSONObject jsonObject, String expr) { + try { + JSONPath.remove(jsonObject, expr); + } catch (Exception e) { + logger.error( + "JSONObject remove value exception or [expr] expression error" + + e.getMessage()); + } + return jsonObject; + } } diff --git a/groot-common/src/main/resources/udf.plugins b/groot-common/src/main/resources/udf.plugins index 0eb24cb..7544cc7 100644 --- a/groot-common/src/main/resources/udf.plugins +++ b/groot-common/src/main/resources/udf.plugins @@ -18,4 +18,13 @@ com.geedgenetworks.core.udf.udaf.NumberSum com.geedgenetworks.core.udf.udaf.CollectList com.geedgenetworks.core.udf.udaf.CollectSet com.geedgenetworks.core.udf.udaf.LongCount -com.geedgenetworks.core.udf.udaf.Mean
\ No newline at end of file +com.geedgenetworks.core.udf.udaf.Mean +com.geedgenetworks.core.udf.udaf.LastValue +com.geedgenetworks.core.udf.udaf.FirstValue +com.geedgenetworks.core.udf.udaf.hlld.Hlld +com.geedgenetworks.core.udf.udaf.hlld.HlldApproxCountDistinct +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogram +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantile +com.geedgenetworks.core.udf.udaf.HdrHistogram.HdrHistogramQuantiles +com.geedgenetworks.core.udf.udtf.JsonUnroll +com.geedgenetworks.core.udf.udtf.Unroll
\ No newline at end of file diff --git a/groot-connectors/connector-kafka/pom.xml b/groot-connectors/connector-kafka/pom.xml index 7ec7d86..448383b 100644 --- a/groot-connectors/connector-kafka/pom.xml +++ b/groot-connectors/connector-kafka/pom.xml @@ -15,6 +15,17 @@ <groupId>org.apache.flink</groupId> <artifactId>flink-connector-kafka_${scala.version}</artifactId> <version>${flink.version}</version> + <exclusions> + <exclusion> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + <version>1.1.8.3</version> </dependency> </dependencies> diff --git a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java index 09cc8f8..5101fa1 100644 --- a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java +++ b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/FakerUtils.java @@ -46,6 +46,10 @@ public class FakerUtils { return wrapFaker(parseIPv4Faker(obj), obj);
} else if ("Expression".equalsIgnoreCase(type)) {
return wrapFaker(parseExpressionFaker(obj), obj);
+ } else if ("Hlld".equalsIgnoreCase(type)) {
+ return wrapFaker(parseHlldFaker(obj), obj);
+ } else if ("HdrHistogram".equalsIgnoreCase(type)) {
+ return wrapFaker(parseHdrHistogramFaker(obj), obj);
} else if ("Object".equalsIgnoreCase(type)) {
return wrapFaker(parseObjectFaker(obj.getJSONArray("fields")), obj);
} else if ("Union".equalsIgnoreCase(type)) {
@@ -109,6 +113,20 @@ public class FakerUtils { return new ExpressionFaker(expression);
}
+ private static Faker<?> parseHlldFaker(JSONObject obj) {
+ long itemCount = obj.getLongValue("itemCount", 1000000L);
+ int batchCount = obj.getIntValue("batchCount", 10000);
+ int precision = obj.getIntValue("precision", 12);
+ return new HlldFaker(itemCount, batchCount, precision);
+ }
+
+ private static Faker<?> parseHdrHistogramFaker(JSONObject obj) {
+ int max = obj.getIntValue("max", 100000);
+ int batchCount = obj.getIntValue("batchCount", 1000);
+ int numberOfSignificantValueDigits = obj.getIntValue("numberOfSignificantValueDigits", 1);
+ return new HdrHistogramFaker(max, batchCount, numberOfSignificantValueDigits);
+ }
+
private static Faker<?> parseIPv4Faker(JSONObject obj) {
String start = obj.getString("start");
String end = obj.getString("end");
diff --git a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HdrHistogramFaker.java b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HdrHistogramFaker.java new file mode 100644 index 0000000..393bf8e --- /dev/null +++ b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HdrHistogramFaker.java @@ -0,0 +1,35 @@ +package com.geedgenetworks.connectors.mock.faker;
+
+import com.geedgenetworks.sketch.util.StringUtils;
+import org.HdrHistogram.HistogramSketch;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+public class HdrHistogramFaker extends Faker<Object> {
+ private final int max;
+ private final int batchCount;
+ private final int numberOfSignificantValueDigits;
+ private HistogramSketch his;
+
+ public HdrHistogramFaker(int max, int batchCount, int numberOfSignificantValueDigits) {
+ this.max = max;
+ this.batchCount = batchCount;
+ this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
+ }
+
+ @Override
+ public void init(int instanceCount, int instanceIndex) throws Exception {
+ his = new HistogramSketch(1L, max, numberOfSignificantValueDigits, false);
+ }
+
+ @Override
+ public Object geneValue() throws Exception {
+ his.reset();
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+ for (int i = 0; i < batchCount; i++) {
+ his.recordValue(random.nextInt(max));
+ }
+ return StringUtils.encodeBase64String(his.toBytes());
+ }
+
+}
diff --git a/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HlldFaker.java b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HlldFaker.java new file mode 100644 index 0000000..5af2b35 --- /dev/null +++ b/groot-connectors/connector-mock/src/main/java/com/geedgenetworks/connectors/mock/faker/HlldFaker.java @@ -0,0 +1,39 @@ +package com.geedgenetworks.connectors.mock.faker;
+
+import com.geedgenetworks.sketch.hlld.Hll;
+import com.geedgenetworks.sketch.util.StringUtils;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+public class HlldFaker extends Faker<Object> {
+ private final long itemCount;
+ private final int batchCount;
+ private final int precision;
+ private Hll hll;
+
+ public HlldFaker(long itemCount, int batchCount, int precision) {
+ this.itemCount = itemCount;
+ this.batchCount = batchCount;
+ this.precision = precision;
+ }
+
+ public HlldFaker(long itemCount, int batchCount) {
+ this(itemCount, batchCount, 12);
+ }
+
+ @Override
+ public void init(int instanceCount, int instanceIndex) throws Exception {
+ hll = new Hll(precision);
+ }
+
+ @Override
+ public Object geneValue() throws Exception {
+ hll.reset();
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+ for (int i = 0; i < batchCount; i++) {
+ hll.add(random.nextLong(itemCount));
+ }
+ return StringUtils.encodeBase64String(hll.toBytes());
+ }
+
+}
diff --git a/groot-core/pom.xml b/groot-core/pom.xml index 18ae33b..e723fa5 100644 --- a/groot-core/pom.xml +++ b/groot-core/pom.xml @@ -66,6 +66,11 @@ </dependency> <dependency> + <groupId>com.geedgenetworks</groupId> + <artifactId>sketches</artifactId> + </dependency> + + <dependency> <groupId>com.alibaba.nacos</groupId> <artifactId>nacos-client</artifactId> <exclusions> diff --git a/groot-core/src/main/java/com/geedgenetworks/core/pojo/AggregateConfig.java b/groot-core/src/main/java/com/geedgenetworks/core/pojo/AggregateConfig.java index 8cccbbd..d3cbaac 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/pojo/AggregateConfig.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/pojo/AggregateConfig.java @@ -13,10 +13,9 @@ public class AggregateConfig extends ProcessorConfig { private List<String> group_by_fields; - private String timestamp_field; + private String window_timestamp_field; private String window_type; private Integer window_size; - private Integer max_out_of_orderness; private Integer window_slide; private List<UDFContext> functions; diff --git a/groot-core/src/main/java/com/geedgenetworks/core/pojo/TableConfig.java b/groot-core/src/main/java/com/geedgenetworks/core/pojo/TableConfig.java new file mode 100644 index 0000000..3efb8e1 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/pojo/TableConfig.java @@ -0,0 +1,15 @@ +package com.geedgenetworks.core.pojo; + +import com.geedgenetworks.common.udf.UDFContext; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +public class TableConfig extends ProcessorConfig { + + private List<UDFContext> functions; + private String format; +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/Processor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/Processor.java new file mode 100644 index 0000000..172b368 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/Processor.java @@ -0,0 +1,14 @@ +package com.geedgenetworks.core.processor; + +import com.geedgenetworks.common.Event; +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; + +public interface Processor<T> { + + SingleOutputStreamOperator<Event> processorFunction( + SingleOutputStreamOperator<Event> singleOutputStreamOperator, + T processorConfig, ExecutionConfig config) + throws Exception; + String type(); +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java index 9acf8fc..0846ffe 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessor.java @@ -1,16 +1,7 @@ package com.geedgenetworks.core.processor.aggregate; import com.geedgenetworks.core.pojo.AggregateConfig; -import com.geedgenetworks.common.Event; +import com.geedgenetworks.core.processor.Processor; +public interface AggregateProcessor extends Processor<AggregateConfig> { -import org.apache.flink.api.common.ExecutionConfig; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; - -public interface AggregateProcessor { - - SingleOutputStreamOperator<Event> aggregateProcessorFunction( - SingleOutputStreamOperator<Event> singleOutputStreamOperator, - AggregateConfig aggregateConfig, ExecutionConfig config) - throws Exception; - String type(); } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java index b535faf..803fefc 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorFunction.java @@ -37,19 +37,11 @@ public class AggregateProcessorFunction implements org.apache.flink.api.common.f public AggregateProcessorFunction(AggregateConfig aggregateConfig, ExecutionConfig config) { udfClassNameLists = JSON.parseObject(config.getGlobalJobParameters().toMap().get(Constants.SYSPROP_UDF_PLUGIN_CONFIG), List.class); udfContexts = aggregateConfig.getFunctions(); - groupByFields = aggregateConfig.getGroup_by_fields(); - } - - @Override - public Accumulator createAccumulator() { - - functions = Lists.newLinkedList(); if (udfContexts == null || udfContexts.isEmpty()) { throw new RuntimeException(); } - Map<String, Object> map = new HashMap<>(); - Accumulator accumulator = new Accumulator(); - accumulator.setMetricsFields(map); + groupByFields = aggregateConfig.getGroup_by_fields(); + functions = Lists.newLinkedList(); Map<String, String> udfClassReflect = getClassReflect(udfClassNameLists); try { for (UDFContext udfContext : udfContexts) { @@ -59,7 +51,6 @@ public class AggregateProcessorFunction implements org.apache.flink.api.common.f if (udfClassReflect.containsKey(udfContext.getFunction())) { Class<?> cls = Class.forName(udfClassReflect.get(udfContext.getFunction())); AggregateFunction aggregateFunction = (AggregateFunction) cls.getConstructor().newInstance(); - aggregateFunction.open(udfContext, accumulator); // 函数如果包含filter,对表达式进行编译 if (udfContext.getFilter() != null) { AviatorEvaluatorInstance instance = AviatorEvaluator.getInstance(); @@ -72,6 +63,7 @@ public class AggregateProcessorFunction implements org.apache.flink.api.common.f udfEntity.setFilterExpression(filterExpression); udfEntity.setName(udfContext.getFunction()); udfEntity.setClassName(udfClassReflect.get(udfContext.getFunction())); + udfEntity.setUdfContext(udfContext); functions.add(udfEntity); } else { throw new GrootStreamRuntimeException(CommonErrorCode.UNSUPPORTED_OPERATION, @@ -79,14 +71,24 @@ public class AggregateProcessorFunction implements org.apache.flink.api.common.f } } + for (UdfEntity udfEntity : functions) { + udfEntity.getAggregateFunction().open(udfEntity.getUdfContext()); + } } catch (Exception e) { throw new GrootStreamRuntimeException(CommonErrorCode.UNSUPPORTED_OPERATION, "Initialization UDAF failed!", e); } - + } + @Override + public Accumulator createAccumulator() { + Map<String, Object> map = new HashMap<>(); + Accumulator accumulator = new Accumulator(); + accumulator.setMetricsFields(map); + for (UdfEntity udfEntity : functions) { + udfEntity.getAggregateFunction().initAccumulator(accumulator); + } return accumulator; } - @Override public Accumulator add(Event event, Accumulator accumulator) { diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorImpl.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorImpl.java index 2f086b0..bc87c32 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorImpl.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/AggregateProcessorImpl.java @@ -11,17 +11,12 @@ import org.apache.flink.streaming.api.windowing.assigners.SlidingProcessingTimeW import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; import org.apache.flink.streaming.api.windowing.assigners.TumblingProcessingTimeWindows; import org.apache.flink.streaming.api.windowing.time.Time; - import static com.geedgenetworks.common.Constants.*; -public class AggregateProcessorImpl implements AggregateProcessor { +public class AggregateProcessorImpl implements AggregateProcessor { @Override - public SingleOutputStreamOperator<Event> aggregateProcessorFunction( - SingleOutputStreamOperator<Event> grootEventSingleOutputStreamOperator, - AggregateConfig aggregateConfig, ExecutionConfig config) - throws Exception { - + public SingleOutputStreamOperator<Event> processorFunction(SingleOutputStreamOperator<Event> grootEventSingleOutputStreamOperator, AggregateConfig aggregateConfig, ExecutionConfig config) throws Exception { if (aggregateConfig.getParallelism() != 0) { switch (aggregateConfig.getWindow_type()) { case TUMBLING_PROCESSING_TIME: diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunctionImpl.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunctionImpl.java index eaa712a..cd5c485 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunctionImpl.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/aggregate/ProcessWindowFunctionImpl.java @@ -44,7 +44,9 @@ public class ProcessWindowFunctionImpl extends org.apache.flink.streaming.api.fu internalMetrics.incrementOutEvents(); internalMetrics.incrementErrorEvents(accumulator.getErrorCount()); internalMetrics.incrementInEvents(accumulator.getInEvents()); - + if (aggregateConfig.getWindow_timestamp_field() != null) { + event.getExtractedFields().put(aggregateConfig.getWindow_timestamp_field(), context.window().getStart()); + } if (aggregateConfig.getOutput_fields() != null && !aggregateConfig.getOutput_fields().isEmpty()) { event.setExtractedFields( diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java index 862ba5e..f15d481 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessor.java @@ -1,15 +1,8 @@ package com.geedgenetworks.core.processor.projection; -import com.geedgenetworks.common.Event; import com.geedgenetworks.core.pojo.ProjectionConfig; +import com.geedgenetworks.core.processor.Processor; -import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +public interface ProjectionProcessor extends Processor<ProjectionConfig>{ -public interface ProjectionProcessor { - - SingleOutputStreamOperator<Event> projectionProcessorFunction( - SingleOutputStreamOperator<Event> grootEventSingleOutputStreamOperator, - ProjectionConfig projectionConfig) - throws Exception; - String type(); } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorImpl.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorImpl.java index 79b0e0d..6b46a7b 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorImpl.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/ProjectionProcessorImpl.java @@ -3,16 +3,14 @@ package com.geedgenetworks.core.processor.projection; import com.geedgenetworks.common.Event; import com.geedgenetworks.core.pojo.ProjectionConfig; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; public class ProjectionProcessorImpl implements ProjectionProcessor { - @Override - public SingleOutputStreamOperator<Event> projectionProcessorFunction( - SingleOutputStreamOperator<Event> grootEventSingleOutputStreamOperator, - ProjectionConfig projectionConfig) - throws Exception{ + @Override + public SingleOutputStreamOperator<Event> processorFunction(SingleOutputStreamOperator<Event> grootEventSingleOutputStreamOperator, ProjectionConfig projectionConfig, ExecutionConfig config) throws Exception { if (projectionConfig.getParallelism() != 0) { return grootEventSingleOutputStreamOperator .process(new ProjectionProcessFunction(projectionConfig)) diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java index c36a785..ab6a6f5 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/projection/UdfEntity.java @@ -3,6 +3,8 @@ package com.geedgenetworks.core.processor.projection; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.ScalarFunction; +import com.geedgenetworks.common.udf.TableFunction; +import com.geedgenetworks.common.udf.UDFContext; import com.googlecode.aviator.Expression; import lombok.Data; @@ -12,7 +14,9 @@ import java.io.Serializable; public class UdfEntity implements Serializable { private ScalarFunction scalarFunction; private AggregateFunction aggregateFunction; + private TableFunction tableFunction; private Expression filterExpression; private String name; private String className; + private UDFContext udfContext; } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java new file mode 100644 index 0000000..4078997 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessor.java @@ -0,0 +1,7 @@ +package com.geedgenetworks.core.processor.table; + +import com.geedgenetworks.core.pojo.TableConfig; +import com.geedgenetworks.core.processor.Processor; +public interface TableProcessor extends Processor<TableConfig> { + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java new file mode 100644 index 0000000..7b6a5e2 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorFunction.java @@ -0,0 +1,138 @@ +package com.geedgenetworks.core.processor.table; + +import com.alibaba.fastjson.JSON; +import com.geedgenetworks.common.Constants; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.exception.CommonErrorCode; +import com.geedgenetworks.common.exception.GrootStreamRuntimeException; +import com.geedgenetworks.common.udf.TableFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.common.utils.ColumnUtil; +import com.geedgenetworks.core.metrics.InternalMetrics; +import com.geedgenetworks.core.pojo.TableConfig; +import com.geedgenetworks.core.processor.projection.UdfEntity; +import com.google.common.collect.Lists; +import com.googlecode.aviator.AviatorEvaluator; +import com.googlecode.aviator.AviatorEvaluatorInstance; +import com.googlecode.aviator.Expression; +import com.googlecode.aviator.Options; +import com.googlecode.aviator.exception.ExpressionRuntimeException; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.common.functions.RichFlatMapFunction; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.util.Collector; +import org.checkerframework.checker.units.qual.A; + +import java.util.*; + +import static com.geedgenetworks.core.utils.UDFUtils.filterExecute; +import static com.geedgenetworks.core.utils.UDFUtils.getClassReflect; +@Slf4j +public class TableProcessorFunction extends RichFlatMapFunction<Event, Event> { + private LinkedList<UdfEntity> functions; + private final TableConfig tableConfig; + private transient InternalMetrics internalMetrics; + + public TableProcessorFunction(TableConfig tableConfig) { + this.tableConfig = tableConfig; + } + + @Override + public void open(Configuration parameters) { + functions = Lists.newLinkedList(); + try { + this.internalMetrics = new InternalMetrics(getRuntimeContext()); + List<UDFContext> udfContexts = tableConfig.getFunctions(); + if (udfContexts == null || udfContexts.isEmpty()) { + return; + } + Configuration configuration = (Configuration) getRuntimeContext().getExecutionConfig().getGlobalJobParameters(); + List<String> udfClassNameLists = JSON.parseObject(configuration.toMap().get(Constants.SYSPROP_UDF_PLUGIN_CONFIG), List.class); + Map<String, String> udfClassReflect = getClassReflect(udfClassNameLists); + for (UDFContext udfContext : udfContexts) { + Expression filterExpression = null; + UdfEntity udfEntity = new UdfEntity(); + // 平台注册的函数包含任务中配置的函数则对函数进行实例化 + if (udfClassReflect.containsKey(udfContext.getFunction())) { + Class<?> cls = Class.forName(udfClassReflect.get(udfContext.getFunction())); + TableFunction tableFunction = (TableFunction) cls.getConstructor().newInstance(); + tableFunction.open(getRuntimeContext(), udfContext); + // 函数如果包含filter,对表达式进行编译 + if (udfContext.getFilter() != null) { + AviatorEvaluatorInstance instance = AviatorEvaluator.getInstance(); + instance.setCachedExpressionByDefault(true); + instance.setOption(Options.OPTIMIZE_LEVEL, AviatorEvaluator.EVAL); + instance.setFunctionMissing(null); + filterExpression = instance.compile(udfContext.getFilter(), true); + } + udfEntity.setTableFunction(tableFunction); + udfEntity.setFilterExpression(filterExpression); + udfEntity.setName(udfContext.getFunction()); + udfEntity.setClassName(udfClassReflect.get(udfContext.getFunction())); + functions.add(udfEntity); + } else { + throw new GrootStreamRuntimeException(CommonErrorCode.UNSUPPORTED_OPERATION, + "Unsupported UDTF: " + udfContext.getFunction()); + } + + } + } catch (Exception e) { + throw new GrootStreamRuntimeException(CommonErrorCode.UNSUPPORTED_OPERATION, "Initialization UDTF failed!", e); + } + } + + + @Override + public void flatMap(Event event, Collector<Event> out) throws Exception { + internalMetrics.incrementInEvents(); + int errorCount = 0; + List<Event> events = new ArrayList<>(); + events.add(event); + for (UdfEntity udfEntity : functions) { + List<Event> newEvents = new ArrayList<>(); + for(int i=0;i<events.size();i++) { + try { + boolean result = udfEntity.getFilterExpression() != null ? filterExecute(udfEntity.getFilterExpression(), udfEntity.getFilterExpression().newEnv("event", events.get(i).getExtractedFields())) : true; + if (!events.get(i).isDropped() && result) { + newEvents.addAll(udfEntity.getTableFunction().evaluate(events.get(i))); + } + } catch (ExpressionRuntimeException ignore) { + log.error("Function " + udfEntity.getName() + " Invalid filter ! "); + errorCount++; + } catch (Exception e) { + log.error("Function " + udfEntity.getName() + " execute exception !", e); + errorCount++; + } + + } + events.clear(); + events.addAll(newEvents); + } + if(errorCount>0){ + internalMetrics.incrementErrorEvents(); + } + for(Event newEvent:events){ + if (tableConfig.getOutput_fields() != null + && !tableConfig.getOutput_fields().isEmpty()) { + newEvent.setExtractedFields( + ColumnUtil.columnSelector( + newEvent.getExtractedFields(), tableConfig.getOutput_fields())); + } + if (tableConfig.getRemove_fields() != null + && !tableConfig.getRemove_fields().isEmpty()) { + newEvent.setExtractedFields( + ColumnUtil.columnRemover( + newEvent.getExtractedFields(), tableConfig.getRemove_fields())); + } + if (!event.isDropped()) { + out.collect(newEvent); + internalMetrics.incrementOutEvents(); + } else { + internalMetrics.incrementDroppedEvents(); + } + + } + + + } +}
\ No newline at end of file diff --git a/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorImpl.java b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorImpl.java new file mode 100644 index 0000000..f36f8db --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/processor/table/TableProcessorImpl.java @@ -0,0 +1,33 @@ +package com.geedgenetworks.core.processor.table; + +import com.geedgenetworks.common.Event; +import com.geedgenetworks.core.pojo.TableConfig; +import com.geedgenetworks.core.processor.projection.ProjectionProcessFunction; +import org.apache.flink.api.common.ExecutionConfig; +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; + +import java.time.Duration; + +public class TableProcessorImpl implements TableProcessor { + + @Override + public SingleOutputStreamOperator<Event> processorFunction(SingleOutputStreamOperator<Event> grootEventSingleOutputStreamOperator, TableConfig tableConfig, ExecutionConfig config) throws Exception { + + if (tableConfig.getParallelism() != 0) { + return grootEventSingleOutputStreamOperator + .flatMap(new TableProcessorFunction(tableConfig)) + .setParallelism(tableConfig.getParallelism()) + .name(tableConfig.getName()); + } else { + return grootEventSingleOutputStreamOperator + .flatMap(new TableProcessorFunction(tableConfig)) + .name(tableConfig.getName()); + } + } + + @Override + public String type() { + return "table"; + } +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java index 3153ef7..84c2c2a 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Flatten.java @@ -24,6 +24,9 @@ public class Flatten implements ScalarFunction { @Override public void open(RuntimeContext runtimeContext, UDFContext udfContext) { + if(udfContext.getParameters()==null){ + throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); + } prefix = udfContext.getParameters().getOrDefault("prefix", "").toString(); delimiter = udfContext.getParameters().getOrDefault("delimiter", ".").toString(); flattenKeys = new HashSet<>(); diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java index e48a503..874735d 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/PathCombine.java @@ -31,7 +31,6 @@ public class PathCombine implements ScalarFunction { if (udfContext.getParameters() != null && !udfContext.getParameters().isEmpty()) { String paths = udfContext.getParameters().getOrDefault("path","").toString(); - // 使用逗号分隔项并转换为数组 if (!paths.isEmpty()) { List<String> pathList; try { diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java index ba9b4d2..6a77c3a 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/Rename.java @@ -26,7 +26,9 @@ public class Rename implements ScalarFunction { @Override public void open(RuntimeContext runtimeContext, UDFContext udfContext) { - + if(udfContext.getParameters()==null ){ + throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); + } String parentFields = udfContext.getParameters().getOrDefault("parent_fields", "").toString(); this.parentFields = new HashSet<>(); if (!parentFields.isEmpty()) { diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java index a8171b3..bdb41e0 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/UnixTimestampConverter.java @@ -32,9 +32,6 @@ public class UnixTimestampConverter implements ScalarFunction { if(udfContext.getLookup_fields().size() != 1){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "The function output fields only support 1 value"); } - if(udfContext.getParameters() == null){ - throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "The function must contain parameters"); - } if(!udfContext.getParameters().containsKey("precision")){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "parameters must containkey precision"); } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java index 4a43163..423eff9 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectList.java @@ -22,6 +22,8 @@ import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.processor.projection.UdfEntity; + import java.util.*; /** @@ -32,9 +34,8 @@ public class CollectList implements AggregateFunction { private String lookupField; private String outputField; - @Override - public Accumulator open(UDFContext udfContext,Accumulator acc) { + public void open(UDFContext udfContext) { if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -45,11 +46,14 @@ public class CollectList implements AggregateFunction { else { outputField = lookupField; } + + } + @Override + public Accumulator initAccumulator(Accumulator acc) { acc.getMetricsFields().put(outputField, new ArrayList<>()); return acc; } - @Override public Accumulator add(Event event, Accumulator acc) { if(event.getExtractedFields().containsKey(lookupField)){ @@ -71,8 +75,4 @@ public class CollectList implements AggregateFunction { return acc; } - @Override - public void close() { - - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java index a425118..b4dfb14 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/CollectSet.java @@ -8,6 +8,7 @@ import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.UDFContext; +import java.util.ArrayList; import java.util.HashSet; import java.util.Set; @@ -21,7 +22,7 @@ public class CollectSet implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc) { + public void open(UDFContext udfContext) { if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -32,11 +33,13 @@ public class CollectSet implements AggregateFunction { else { outputField = lookupField; } + } + @Override + public Accumulator initAccumulator(Accumulator acc) { acc.getMetricsFields().put(outputField, new HashSet<>()); return acc; } - @Override public Accumulator add(Event event, Accumulator acc) { if(event.getExtractedFields().containsKey(lookupField)){ @@ -58,8 +61,5 @@ public class CollectSet implements AggregateFunction { return acc; } - @Override - public void close() { - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java index 27490ef..6301a01 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/FirstValue.java @@ -23,6 +23,8 @@ import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.UDFContext; +import java.util.ArrayList; + /** * Collects elements within a group and returns the list of aggregated objects */ @@ -33,7 +35,7 @@ public class FirstValue implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc) { + public void open(UDFContext udfContext) { if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -44,9 +46,12 @@ public class FirstValue implements AggregateFunction { else { outputField = lookupField; } - return acc; } + @Override + public Accumulator initAccumulator(Accumulator acc) { + return acc; + } @Override public Accumulator add(Event event, Accumulator acc) { @@ -66,8 +71,4 @@ public class FirstValue implements AggregateFunction { return acc; } - @Override - public void close() { - - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java new file mode 100644 index 0000000..368e8c1 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogram.java @@ -0,0 +1,42 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.udf.UDFContext;
+import com.geedgenetworks.sketch.util.StringUtils;
+import org.HdrHistogram.Histogramer;
+
+import java.util.Map;
+
+public class HdrHistogram extends HdrHistogramBaseAggregate {
+ boolean outputBase64;
+
+ @Override
+ public void open(UDFContext c) {
+ super.open(c);
+ Map<String, Object> params = c.getParameters();
+ outputBase64 = "base64".equalsIgnoreCase(params.getOrDefault("output_format", "base64").toString());
+ }
+
+ @Override
+ public Accumulator getResult(Accumulator acc) {
+ Object agg = acc.getMetricsFields().get(outputField);
+ if (agg == null) {
+ return acc;
+ }
+
+ byte[] bytes = ((Histogramer) agg).toBytes();
+ if (outputBase64) {
+ acc.getMetricsFields().put(outputField, StringUtils.encodeBase64String(bytes));
+ } else {
+ acc.getMetricsFields().put(outputField, bytes);
+ }
+
+ return acc;
+ }
+
+ @Override
+ public String functionName() {
+ return "HDR_HISTOGRAM";
+ }
+
+}
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java new file mode 100644 index 0000000..1648fa5 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramBaseAggregate.java @@ -0,0 +1,101 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.Event;
+import com.geedgenetworks.common.udf.AggregateFunction;
+import com.geedgenetworks.common.udf.UDFContext;
+import com.geedgenetworks.sketch.util.StringUtils;
+import org.HdrHistogram.ArrayHistogram;
+import org.HdrHistogram.DirectMapHistogram;
+import org.HdrHistogram.Histogramer;
+import org.apache.commons.collections.CollectionUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+
+public abstract class HdrHistogramBaseAggregate implements AggregateFunction {
+ protected String inputField;
+ protected String outputField;
+ protected boolean inputSketch;
+ protected long lowestDiscernibleValue;
+ protected long highestTrackableValue;
+ protected int numberOfSignificantValueDigits;
+ protected boolean autoResize;
+
+ @Override
+ public void open(UDFContext c) {
+ inputField = c.getLookup_fields().get(0);
+ if (CollectionUtils.isNotEmpty(c.getOutput_fields())) {
+ outputField = c.getOutput_fields().get(0);
+ } else {
+ outputField = inputField;
+ }
+ Map<String, Object> params = c.getParameters();
+ lowestDiscernibleValue = Long.parseLong(params.getOrDefault("lowestDiscernibleValue", "1").toString());
+ highestTrackableValue = Long.parseLong(params.getOrDefault("highestTrackableValue", "2").toString());
+ numberOfSignificantValueDigits = Integer.parseInt(params.getOrDefault("numberOfSignificantValueDigits", "1").toString());
+ autoResize = Boolean.valueOf(params.getOrDefault("autoResize", "true").toString());
+ inputSketch = "sketch".equalsIgnoreCase(params.getOrDefault("input_type", "sketch").toString());
+ }
+
+ @Override
+ public Accumulator initAccumulator(Accumulator acc) {
+ return acc;
+ }
+
+ @Override
+ public Accumulator add(Event event, Accumulator acc) {
+ Object value = event.getExtractedFields().get(inputField);
+ if (value == null) {
+ return acc;
+ }
+
+ if (inputSketch) {
+ updateHdrMerge(acc, value);
+ } else {
+ updateHdr(acc, value);
+ }
+
+ return acc;
+ }
+
+ protected void updateHdr(Accumulator acc, Object value) {
+ Map<String, Object> aggs = acc.getMetricsFields();
+ ArrayHistogram his = (ArrayHistogram) aggs.get(outputField);
+ if (his == null) {
+ his = new ArrayHistogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
+ his.setAutoResize(autoResize);
+ aggs.put(outputField, his);
+ }
+
+ his.recordValue(((Number) value).longValue());
+ }
+
+
+ protected void updateHdrMerge(Accumulator acc, Object value) {
+ Map<String, Object> aggs = acc.getMetricsFields();
+ ArrayHistogram his = (ArrayHistogram) aggs.get(outputField);
+ if (his == null) {
+ his = new ArrayHistogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
+ his.setAutoResize(autoResize);
+ aggs.put(outputField, his);
+ }
+
+ Histogramer h;
+ if (value instanceof String) {
+ byte[] bytes = StringUtils.decodeBase64(((String) value).getBytes(StandardCharsets.UTF_8));
+ h = DirectMapHistogram.wrapBytes(bytes);
+ } else if (value instanceof byte[]) {
+ h = DirectMapHistogram.wrapBytes((byte[]) value);
+ } else if (value instanceof Histogramer) {
+ h = (Histogramer) value;
+ } else {
+ throw new IllegalArgumentException("Unsupported type " + value.getClass());
+ }
+
+ his.merge(h);
+ }
+
+ @Override
+ public void close() {}
+}
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java new file mode 100644 index 0000000..b9f7d5b --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantile.java @@ -0,0 +1,36 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.udf.UDFContext;
+import org.HdrHistogram.Histogramer;
+
+import java.util.Map;
+
+public class HdrHistogramQuantile extends HdrHistogramBaseAggregate {
+ Double probability;
+
+ @Override
+ public void open(UDFContext c) {
+ super.open(c);
+ Map<String, Object> params = c.getParameters();
+ probability = Double.parseDouble(params.getOrDefault("probability", "0.5").toString());
+ }
+
+ @Override
+ public Accumulator getResult(Accumulator acc) {
+ Object agg = acc.getMetricsFields().get(outputField);
+ if (agg == null) {
+ return acc;
+ }
+
+ long percentile = ((Histogramer) agg).getValueAtPercentile(probability * 100);
+ acc.getMetricsFields().put(outputField, percentile);
+ return acc;
+ }
+
+ @Override
+ public String functionName() {
+ return "APPROX_QUANTILE_HDR";
+ }
+
+}
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java new file mode 100644 index 0000000..ccfffd3 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantiles.java @@ -0,0 +1,51 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram;
+
+import com.alibaba.fastjson2.JSON;
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.udf.UDFContext;
+import org.HdrHistogram.Histogramer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class HdrHistogramQuantiles extends HdrHistogramBaseAggregate {
+ double[] probabilities;
+
+ @Override
+ public void open(UDFContext c) {
+ super.open(c);
+ Map<String, Object> params = c.getParameters();
+ Object ps = params.get("probabilities");
+ if(ps == null){
+ throw new IllegalArgumentException("probabilities param is requested");
+ }
+ List<Double> floats = JSON.parseArray(ps instanceof String ? ps.toString(): JSON.toJSONString(ps), Double.class);
+ probabilities = new double[floats.size()];
+ for (int i = 0; i < floats.size(); i++) {
+ probabilities[i] = floats.get(i);
+ }
+ }
+
+ @Override
+ public Accumulator getResult(Accumulator acc) {
+ Object agg = acc.getMetricsFields().get(outputField);
+ if (agg == null) {
+ return acc;
+ }
+
+ Histogramer his = ((Histogramer) agg);
+ final List<Long> counts = new ArrayList<>(probabilities.length);
+ for (int i = 0; i < probabilities.length; i++) {
+ counts.add(his.getValueAtPercentile(probabilities[i] * 100));
+ }
+ acc.getMetricsFields().put(outputField, counts);
+ return acc;
+ }
+
+ @Override
+ public String functionName() {
+ return "APPROX_QUANTILES_HDR";
+ }
+
+}
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java index 4adafd4..f27a2e6 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LastValue.java @@ -36,7 +36,7 @@ public class LastValue implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc) { + public void open(UDFContext udfContext) { if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -47,10 +47,12 @@ public class LastValue implements AggregateFunction { else { outputField = lookupField; } + } + @Override + public Accumulator initAccumulator(Accumulator acc) { return acc; } - @Override public Accumulator add(Event event, Accumulator acc) { if(event.getExtractedFields().containsKey(lookupField)){ @@ -69,8 +71,4 @@ public class LastValue implements AggregateFunction { return acc; } - @Override - public void close() { - - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java index 5662935..ea33271 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/LongCount.java @@ -13,15 +13,16 @@ public class LongCount implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc){ + public void open(UDFContext udfContext){ if(udfContext.getOutput_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } outputField = udfContext.getOutput_fields().get(0); + } + @Override + public Accumulator initAccumulator(Accumulator acc) { return acc; } - - @Override public Accumulator add(Event event, Accumulator acc) { @@ -39,9 +40,5 @@ public class LongCount implements AggregateFunction { return acc; } - @Override - public void close() { - - } } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java index 380f598..2a615ef 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/Mean.java @@ -17,7 +17,7 @@ public class Mean implements AggregateFunction { private Integer precision; private DecimalFormat df; @Override - public Accumulator open(UDFContext udfContext,Accumulator acc){ + public void open(UDFContext udfContext){ if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); @@ -29,7 +29,7 @@ public class Mean implements AggregateFunction { else { outputField = lookupField; } - if(!udfContext.getParameters().isEmpty()) { + if(udfContext.getParameters()!= null && !udfContext.getParameters().isEmpty()) { precision = Integer.parseInt(udfContext.getParameters().getOrDefault("precision", "-1").toString()); if (precision > 0) { StringBuilder pattern = new StringBuilder("#."); @@ -41,11 +41,14 @@ public class Mean implements AggregateFunction { }else { precision = -1; } + + } + @Override + public Accumulator initAccumulator(Accumulator acc) { acc.getMetricsFields().put(outputField,new OnlineStatistics()); return acc; } - @Override public Accumulator add(Event event, Accumulator acc) { @@ -76,9 +79,4 @@ public class Mean implements AggregateFunction { return acc; } - @Override - public void close() { - - } - } diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java index 4ed3143..01e9a5b 100644 --- a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/NumberSum.java @@ -6,6 +6,7 @@ import com.geedgenetworks.common.exception.CommonErrorCode; import com.geedgenetworks.common.exception.GrootStreamRuntimeException; import com.geedgenetworks.common.udf.AggregateFunction; import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.pojo.OnlineStatistics; public class NumberSum implements AggregateFunction { @@ -14,7 +15,7 @@ public class NumberSum implements AggregateFunction { @Override - public Accumulator open(UDFContext udfContext,Accumulator acc){ + public void open(UDFContext udfContext){ if(udfContext.getLookup_fields()==null ){ throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); } @@ -25,10 +26,12 @@ public class NumberSum implements AggregateFunction { else { outputField = lookupField; } - return acc; } - + @Override + public Accumulator initAccumulator(Accumulator acc) { + return acc; + } @Override public Accumulator add(Event event, Accumulator acc) { diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java new file mode 100644 index 0000000..e373a7a --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/Hlld.java @@ -0,0 +1,40 @@ +package com.geedgenetworks.core.udf.udaf.hlld;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.udf.UDFContext;
+import com.geedgenetworks.sketch.hlld.Hll;
+import com.geedgenetworks.sketch.util.StringUtils;
+
+import java.util.Map;
+
+public class Hlld extends HlldBaseAggregate {
+ boolean outputBase64;
+
+ @Override
+ public void open(UDFContext c) {
+ super.open(c);
+ Map<String, Object> params = c.getParameters();
+ outputBase64 = "base64".equalsIgnoreCase(params.getOrDefault("output_format", "base64").toString());
+ }
+
+ @Override
+ public Accumulator getResult(Accumulator acc) {
+ Hll hll = getResultHll(acc);
+ if (hll == null) {
+ return acc;
+ }
+
+ if (outputBase64) {
+ acc.getMetricsFields().put(outputField, StringUtils.encodeBase64String(hll.toBytes()));
+ } else {
+ acc.getMetricsFields().put(outputField, hll.toBytes());
+ }
+
+ return acc;
+ }
+
+ @Override
+ public String functionName() {
+ return "HLLD";
+ }
+}
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinct.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinct.java new file mode 100644 index 0000000..ec003f8 --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinct.java @@ -0,0 +1,25 @@ +package com.geedgenetworks.core.udf.udaf.hlld;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.sketch.hlld.Hll;
+
+public class HlldApproxCountDistinct extends HlldBaseAggregate {
+
+ @Override
+ public Accumulator getResult(Accumulator acc) {
+ Hll hll = getResultHll(acc);
+ if (hll == null) {
+ return acc;
+ }
+
+ acc.getMetricsFields().put(outputField, (long)hll.size());
+
+ return acc;
+ }
+
+ @Override
+ public String functionName() {
+ return "APPROX_COUNT_DISTINCT_HLLD";
+ }
+
+}
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java new file mode 100644 index 0000000..71d61dc --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udaf/hlld/HlldBaseAggregate.java @@ -0,0 +1,98 @@ +package com.geedgenetworks.core.udf.udaf.hlld;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.Event;
+import com.geedgenetworks.common.udf.AggregateFunction;
+import com.geedgenetworks.common.udf.UDFContext;
+import com.geedgenetworks.sketch.hlld.Hll;
+import com.geedgenetworks.sketch.hlld.HllUnion;
+import com.geedgenetworks.sketch.hlld.HllUtils;
+import org.apache.commons.collections.CollectionUtils;
+
+import java.util.Map;
+
+public abstract class HlldBaseAggregate implements AggregateFunction {
+ protected String inputField;
+ protected String outputField;
+ protected boolean inputSketch;
+ protected int precision = 12;
+
+ @Override
+ public void open(UDFContext c) {
+ inputField = c.getLookup_fields().get(0);
+ if (CollectionUtils.isNotEmpty(c.getOutput_fields())) {
+ outputField = c.getOutput_fields().get(0);
+ } else {
+ outputField = inputField;
+ }
+ Map<String, Object> params = c.getParameters();
+ precision = Integer.parseInt(params.getOrDefault("precision", "12").toString());
+ inputSketch = "sketch".equalsIgnoreCase(params.getOrDefault("input_type", "sketch").toString());
+ }
+
+ @Override
+ public Accumulator initAccumulator(Accumulator acc) {
+ return acc;
+ }
+
+ @Override
+ public Accumulator add(Event event, Accumulator acc) {
+ Object value = event.getExtractedFields().get(inputField);
+ if (value == null) {
+ return acc;
+ }
+
+ if (inputSketch) {
+ updateHllUnion(acc, value);
+ } else {
+ updateHll(acc, value);
+ }
+
+ return acc;
+ }
+
+ protected Hll getResultHll(Accumulator acc){
+ Object agg = acc.getMetricsFields().get(outputField);
+ if (agg == null) {
+ return null;
+ }
+
+ return inputSketch ? ((HllUnion) agg).getResult() : (Hll) agg;
+ }
+
+ protected void updateHll(Accumulator acc, Object value) {
+ Map<String, Object> aggs = acc.getMetricsFields();
+ Hll hll = (Hll) aggs.get(outputField);
+ if (hll == null) {
+ hll = new Hll(precision);
+ aggs.put(outputField, hll);
+ }
+
+ if (value instanceof Integer || value instanceof Long) {
+ hll.add(((Number) value).longValue());
+ } else if (value instanceof Float || value instanceof Double) {
+ hll.add(((Number) value).doubleValue());
+ } else if (value instanceof String) {
+ hll.add((String) value);
+ } else if (value instanceof byte[]) {
+ hll.add((byte[]) value);
+ } else {
+ throw new IllegalArgumentException("Unsupported type " + value.getClass());
+ }
+ }
+
+ protected void updateHllUnion(Accumulator acc, Object value) {
+ Map<String, Object> aggs = acc.getMetricsFields();
+ HllUnion hllUnion = (HllUnion) aggs.get(outputField);
+ if (hllUnion == null) {
+ hllUnion = new HllUnion(precision);
+ aggs.put(outputField, hllUnion);
+ }
+
+ Hll hll = HllUtils.deserializeHll(value);
+ hllUnion.update(hll);
+ }
+
+ @Override
+ public void close() {}
+}
diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java new file mode 100644 index 0000000..2e8eb7e --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/JsonUnroll.java @@ -0,0 +1,125 @@ +package com.geedgenetworks.core.udf.udtf; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.exception.CommonErrorCode; +import com.geedgenetworks.common.exception.GrootStreamRuntimeException; +import com.geedgenetworks.common.udf.TableFunction; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.common.utils.JsonPathUtil; +import com.googlecode.aviator.AviatorEvaluator; +import com.googlecode.aviator.AviatorEvaluatorInstance; +import com.googlecode.aviator.Expression; +import com.googlecode.aviator.Options; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.common.functions.RuntimeContext; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + + +@Slf4j +public class JsonUnroll implements TableFunction { + + private String lookupFieldName; + private String outputFieldName; + private String path; + private String new_Path; + + + @Override + public void open(RuntimeContext runtimeContext, UDFContext udfContext) { + if(udfContext.getLookup_fields()==null ){ + throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); + } + this.lookupFieldName = udfContext.getLookup_fields().get(0); + if(udfContext.getOutput_fields()!=null && !udfContext.getOutput_fields().isEmpty()) { + this.outputFieldName = udfContext.getOutput_fields().get(0); + } + else { + outputFieldName = lookupFieldName; + } + if(udfContext.getParameters()==null ){ + path=""; + new_Path=""; + } + else { + path=udfContext.getParameters().getOrDefault("path", "").toString().trim(); + new_Path=udfContext.getParameters().getOrDefault("new_path", path).toString().trim(); + } + } + + @Override + public List<Event> evaluate(Event event) { + try { + if(event.getExtractedFields().containsKey(lookupFieldName) ){ + try { + if(path.isEmpty()){ + JSONArray jsonArray = JSONArray.parseArray(event.getExtractedFields().get(lookupFieldName).toString()); + return parseList(jsonArray,event); + }else { + JSONObject jsonObject = JSONObject.parseObject(event.getExtractedFields().get(lookupFieldName).toString()); + Object obj = JsonPathUtil.get(jsonObject,path); + if(obj instanceof List || obj instanceof Array) { + List list = (List) obj; + List<Event> eventList = new ArrayList<>(); + for (Object o : list) { + JSONObject newJsonObject = new JSONObject(); + newJsonObject.putAll(jsonObject); + JsonPathUtil.remove(newJsonObject,path); + JsonPathUtil.set(newJsonObject,new_Path,o); + String jsonString = JSON.toJSONString(newJsonObject); + Event newEvent = new Event(); + newEvent.setExtractedFields(new HashMap<>()); + newEvent.getExtractedFields().putAll(event.getExtractedFields()); + newEvent.getExtractedFields().remove(lookupFieldName); + newEvent.getExtractedFields().put(outputFieldName, jsonString); + eventList.add(newEvent); + } + return eventList; + } + else { + log.error("Invalid unroll ! expression=" +path + " Exception :" + " expression should return a list or array"); + } + } + + }catch (Exception e) { + log.error("Invalid unroll ! expression=" +path + " Exception :" + e.getMessage()); + } + } + }catch (Exception e) { + log.error("Invalid parseObject ! expression=" +path + " Exception :" + e.getMessage()); + } + return Collections.singletonList(event); + } + + private List<Event> parseList(Object object,Event event) { + List list = (List) object; + List<Event> eventList = new ArrayList<>(); + for (Object obj : list) { + Event newEvent = new Event(); + newEvent.setExtractedFields(new HashMap<>()); + newEvent.getExtractedFields().putAll(event.getExtractedFields()); + newEvent.getExtractedFields().remove(lookupFieldName); + newEvent.getExtractedFields().put(outputFieldName, JSON.toJSONString(obj)); + eventList.add(newEvent); + } + return eventList; + } + + @Override + public String functionName() { + return "JSON_UNROLL"; + } + + @Override + public void close() { + + } + +} diff --git a/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java new file mode 100644 index 0000000..5becb8e --- /dev/null +++ b/groot-core/src/main/java/com/geedgenetworks/core/udf/udtf/Unroll.java @@ -0,0 +1,107 @@ +package com.geedgenetworks.core.udf.udtf; + +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.exception.CommonErrorCode; +import com.geedgenetworks.common.exception.GrootStreamRuntimeException; +import com.geedgenetworks.common.udf.TableFunction; +import com.geedgenetworks.common.udf.UDFContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.flink.api.common.functions.RuntimeContext; + +import java.util.*; + + +@Slf4j +public class Unroll implements TableFunction { + + private String lookupFieldName; + private String outputFieldName; + private String regex; + + + @Override + public void open(RuntimeContext runtimeContext, UDFContext udfContext) { + if(udfContext.getLookup_fields()==null ){ + throw new GrootStreamRuntimeException(CommonErrorCode.ILLEGAL_ARGUMENT, "Missing required parameters"); + } + this.lookupFieldName = udfContext.getLookup_fields().get(0); + if(udfContext.getOutput_fields()!=null && !udfContext.getOutput_fields().isEmpty()) { + this.outputFieldName = udfContext.getOutput_fields().get(0); + } + else { + outputFieldName = lookupFieldName; + } + if(udfContext.getParameters()==null ){ + regex=""; + } + else { + this.regex=udfContext.getParameters().getOrDefault("regex", "").toString().trim(); + } + } + + @Override + public List<Event> evaluate(Event event) { + try { + if(event.getExtractedFields().containsKey(lookupFieldName)) { + + if(regex.isEmpty()){ + if (event.getExtractedFields().get(lookupFieldName) instanceof List ) { + return parseList(event.getExtractedFields().get(lookupFieldName), event); + } else if(event.getExtractedFields().get(lookupFieldName) instanceof Object[]){ + return parseArray(event.getExtractedFields().get(lookupFieldName), event); + }else { + log.error("Invalid unroll ! Object is not instance of list or array. expression=" + regex); + } + } + else { + if (event.getExtractedFields().get(lookupFieldName) instanceof String) { + String[] array =((String) event.getExtractedFields().get(lookupFieldName)).split(regex); + return parseArray(array, event); + }else { + log.error("Invalid unroll ! Object is not instance of String. expression=" + regex); + } + } + } + }catch (Exception e) { + log.error("Invalid parseObject ! expression=" +regex + " Exception :" + e.getMessage()); + } + return Collections.singletonList(event); + } + + private List<Event> parseList(Object object,Event event) { + List list = (List) object; + List<Event> eventList = new ArrayList<>(); + for (Object obj : list) { + Event newEvent = new Event(); + newEvent.setExtractedFields(new HashMap<>()); + newEvent.getExtractedFields().putAll(event.getExtractedFields()); + newEvent.getExtractedFields().remove(lookupFieldName); + newEvent.getExtractedFields().put(outputFieldName, obj); + eventList.add(newEvent); + } + return eventList; + } + private List<Event> parseArray(Object object, Event event) { + List<Event> eventList = new ArrayList<>(); + Object[] objects = (Object[]) object; + for (Object obj : objects) { + Event newEvent = new Event(); + newEvent.setExtractedFields(new HashMap<>()); + newEvent.getExtractedFields().putAll(event.getExtractedFields()); + newEvent.getExtractedFields().remove(lookupFieldName); + newEvent.getExtractedFields().put(outputFieldName, obj); + eventList.add(newEvent); + } + return eventList; + } + @Override + public String functionName() { + return "UNROLL"; + } + + @Override + public void close() { + + } + +} diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor new file mode 100644 index 0000000..1f32ffa --- /dev/null +++ b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.Processor @@ -0,0 +1,3 @@ +com.geedgenetworks.core.processor.aggregate.AggregateProcessorImpl +com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl +com.geedgenetworks.core.processor.table.TableProcessorImpl
\ No newline at end of file diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.aggregate.AggregateProcessor b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.aggregate.AggregateProcessor deleted file mode 100644 index 426a1a9..0000000 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.aggregate.AggregateProcessor +++ /dev/null @@ -1 +0,0 @@ -com.geedgenetworks.core.processor.aggregate.AggregateProcessorImpl
\ No newline at end of file diff --git a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.projection.ProjectionProcessor b/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.projection.ProjectionProcessor deleted file mode 100644 index ede2c8c..0000000 --- a/groot-core/src/main/resources/META-INF/services/com.geedgenetworks.core.processor.projection.ProjectionProcessor +++ /dev/null @@ -1 +0,0 @@ -com.geedgenetworks.core.processor.projection.ProjectionProcessorImpl
\ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java index a01edb3..b0d846b 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectListTest.java @@ -33,7 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class CollectListTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { List<String> arr = List.of("192.168.1.1", "192.168.1.2", "192.168.1.3", "192.168.1.4"); excute(arr); @@ -49,7 +49,8 @@ public class CollectListTest { Map<String, Object> metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = collectList.open(udfContext,accumulator); + collectList.open(udfContext); + Accumulator agg = collectList.initAccumulator(accumulator); for (String o : arr) { Event event = new Event(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java index ae69d7c..ea4fe8d 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/CollectSetTest.java @@ -47,8 +47,8 @@ public class CollectSetTest { Map<String, Object> metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = collectSet.open(udfContext,accumulator); - + collectSet.open(udfContext); + Accumulator agg = collectSet.initAccumulator(accumulator); for (String o : arr) { Event event = new Event(); Map<String, Object> extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java index 2c4d460..506f6de 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/FirstValueTest.java @@ -31,7 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class FirstValueTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { List<String> arr = List.of("192.168.1.1", "192.168.1.2", "192.168.1.3", "192.168.1.4","192.168.1.4"); excute(arr); @@ -47,8 +47,8 @@ public class FirstValueTest { Map<String, Object> metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = firstValue.open(udfContext,accumulator); - + firstValue.open(udfContext); + Accumulator agg = firstValue.initAccumulator(accumulator); for (String o : arr) { Event event = new Event(); Map<String, Object> extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java index e9609f7..f8306cd 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LastValueTest.java @@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class LastValueTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { List<String> arr = List.of("192.168.1.1", "192.168.1.2", "192.168.1.3", "192.168.1.4","192.168.1.4"); excute(arr); @@ -50,8 +50,8 @@ public class LastValueTest { Map<String, Object> metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = lastValue.open(udfContext,accumulator); - + lastValue.open(udfContext); + Accumulator agg = lastValue.initAccumulator(accumulator); for (String o : arr) { Event event = new Event(); Map<String, Object> extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java index 3bde558..3c02499 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/LongCountTest.java @@ -35,7 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class LongCountTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { Long[] longArr = new Long[]{1L, 2L, 3L, 4L}; excute(longArr); @@ -49,8 +49,8 @@ public class LongCountTest { Map<String, Object> metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = longCount.open(udfContext,accumulator); - + longCount.open(udfContext); + Accumulator agg = longCount.initAccumulator(accumulator); for (Number o : arr) { Event event = new Event(); Map<String, Object> extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java index 48c4e0f..6deed0f 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/MeanTest.java @@ -35,12 +35,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class MeanTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { Integer[] intArr1 = new Integer[]{1, 2, 3, 4}; Integer[] intArr2 = new Integer[]{1, 6, 3}; excute(intArr1, 0); excute2(intArr2, 2); + excute3(intArr1); } private static void excute(Number[] arr,int precision) throws ParseException { @@ -54,8 +55,8 @@ public class MeanTest { Map<String, Object> metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = mean.open(udfContext,accumulator); - + mean.open(udfContext); + Accumulator agg = mean.initAccumulator(accumulator); for (Number o : arr) { Event event = new Event(); Map<String, Object> extractedFields = new HashMap<>(); @@ -79,8 +80,8 @@ public class MeanTest { Map<String, Object> metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = mean.open(udfContext,accumulator); - + mean.open(udfContext); + Accumulator agg = mean.initAccumulator(accumulator); for (Number o : arr) { Event event = new Event(); Map<String, Object> extractedFields = new HashMap<>(); @@ -93,5 +94,26 @@ public class MeanTest { assertEquals(NumberFormat.getInstance().parse((result.getMetricsFields().get("field_mean").toString())),NumberFormat.getInstance().parse("3.33")); } + private static void excute3(Number[] arr) throws ParseException { + UDFContext udfContext = new UDFContext(); + udfContext.setLookup_fields(List.of("field")); + udfContext.setOutput_fields(Collections.singletonList("field_mean")); + Mean mean = new Mean(); + Map<String, Object> metricsFields = new HashMap<>(); + Accumulator accumulator = new Accumulator(); + accumulator.setMetricsFields(metricsFields); + mean.open(udfContext); + Accumulator agg = mean.initAccumulator(accumulator); + for (Number o : arr) { + Event event = new Event(); + Map<String, Object> extractedFields = new HashMap<>(); + extractedFields.put("field", o); + event.setExtractedFields(extractedFields); + agg = mean.add(event, agg); + + } + Accumulator result = mean.getResult(agg); + assertEquals(NumberFormat.getInstance().parse((result.getMetricsFields().get("field_mean").toString())),NumberFormat.getInstance().parse("2.5")); + } }
\ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java index a1cd54e..d0d3d2c 100644 --- a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/aggregate/NumberSumTest.java @@ -31,7 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class NumberSumTest { @Test - public void testNumberSumTest() throws ParseException { + public void test() throws ParseException { Integer[] intArr = new Integer[]{1, 2, 3, 4}; Long[] longArr = new Long[]{1L, 2L, 3L, 4L}; @@ -52,8 +52,8 @@ public class NumberSumTest { Map<String, Object> metricsFields = new HashMap<>(); Accumulator accumulator = new Accumulator(); accumulator.setMetricsFields(metricsFields); - Accumulator agg = numberSum.open(udfContext,accumulator); - + numberSum.open(udfContext); + Accumulator agg = numberSum.initAccumulator(accumulator); for (Number o : arr) { Event event = new Event(); Map<String, Object> extractedFields = new HashMap<>(); diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java new file mode 100644 index 0000000..02f0b66 --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/JsonUnrollFunctionTest.java @@ -0,0 +1,105 @@ +package com.geedgenetworks.core.udf.test.table; + +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.udf.udtf.JsonUnroll; +import com.geedgenetworks.core.udf.udtf.Unroll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class JsonUnrollFunctionTest { + + private static Map<String, Object> nestedMap; + @BeforeAll + public static void setUp() { + nestedMap = Map.of( + "k1","[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}]", + "k2","{\"k2_1\":\"[{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}]\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"destination_mac\":\"ff:ff:ff:ff:ff:ff\"}", + "k3","{\n" + + " \"k3_1\": {\n" + + " \"k3_1_1\": [\n" + + " {\n" + + " \"tunnels_schema_type\": \"ETHERNET\",\n" + + " \"source_mac\": \"52:d4:18:c7:e5:11\"\n" + + " },\n" + + " {\n" + + " \"tunnels_schema_type\": \"ETHERNET\",\n" + + " \"source_mac\": \"ff:ff:ff:ff:ff:ff\"\n" + + " }\n" + + " ],\n" + + " \"k3_1_2\": {\n" + + " \"tunnels_schema_type\": \"ETHERNET\",\n" + + " \"source_mac\": 19.95\n" + + " }\n" + + " }\n" + + "}", + "k4","" + + ); + } + + // 测试方法 + + @Test + public void testJsonUnrollFunction1() { + UDFContext udfContext = new UDFContext(); + JsonUnroll unroll = new JsonUnroll(); + Event event = new Event(); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k1")); + udfContext.setOutput_fields(List.of("newk1")); + unroll.open(null, udfContext); + List<Event> result3 = unroll.evaluate(event); + assertEquals(2, result3.size()); + assertEquals("{\"destination_mac\":\"ff:ff:ff:ff:ff:ff\",\"source_mac\":\"52:d4:18:c7:e5:11\",\"tunnels_schema_type\":\"ETHERNET\"}", result3.get(0).getExtractedFields().get("newk1").toString()); + assertEquals("{\"destination_mac\":\"ff:ff:ff:ff:ff:ff\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\",\"tunnels_schema_type\":\"ETHERNET\"}", result3.get(1).getExtractedFields().get("newk1").toString()); + + + + + } + @Test + public void testJsonUnrollFunction2() { + + + UDFContext udfContext = new UDFContext(); + JsonUnroll unroll = new JsonUnroll(); + Event event = new Event(); + Map<String, Object> params = new HashMap<>(); + udfContext.setParameters(params); + params.put("path", "$.k3_1.k3_1_1"); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k3")); + udfContext.setOutput_fields(List.of("newk3")); + unroll.open(null, udfContext); + List<Event> result2 = unroll.evaluate(event); + assertEquals(2, result2.size()); + assertEquals("{\"k3_1\":{\"k3_1_2\":{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":19.95},\"k3_1_1\":{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"52:d4:18:c7:e5:11\"}}}",result2.get(0).getExtractedFields().get("newk3").toString()); + assertEquals("{\"k3_1\":{\"k3_1_2\":{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":19.95},\"k3_1_1\":{\"tunnels_schema_type\":\"ETHERNET\",\"source_mac\":\"ff:ff:ff:ff:ff:ff\"}}}",result2.get(1).getExtractedFields().get("newk3").toString()); + + } + @Test + public void testJsonUnrollFunction3() { + + + UDFContext udfContext = new UDFContext(); + JsonUnroll unroll = new JsonUnroll(); + Event event = new Event(); + Map<String, Object> params = new HashMap<>(); + udfContext.setParameters(params); + params.put("path", "$.k4_1.k4_1_1"); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k4")); + udfContext.setOutput_fields(List.of("newk4")); + unroll.open(null, udfContext); + List<Event> result2 = unroll.evaluate(event); + assertEquals(1, result2.size()); + + } +} diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java new file mode 100644 index 0000000..2f4da76 --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/test/table/UnrollFunctionTest.java @@ -0,0 +1,86 @@ +package com.geedgenetworks.core.udf.test.table; + +import com.geedgenetworks.common.Event; +import com.geedgenetworks.common.udf.UDFContext; +import com.geedgenetworks.core.udf.udtf.Unroll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class UnrollFunctionTest { + + private static Map<String, Object> nestedMap; + @BeforeAll + public static void setUp() { + nestedMap = Map.of( + "k1", List.of( + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "52:d4:18:c7:e5:11"), + Map.of("tunnels_schema_type", "ETHERNET", "source_mac", "ff:ff:ff:ff:ff:ff") + ), + "k2","{\"source_mac\":\"52:d4:18:c7:e5:10\"},{\"source_mac\":\"ff:ff:ff:ff:ff:ff\"},{\"source_mac\":\"52:d4:18:c7:e5:11\"}", + "k3","" + + ); + } + + // 测试方法 + @Test + public void testUnrollFunction1() { + + + UDFContext udfContext = new UDFContext(); + udfContext.setLookup_fields(List.of("k1")); + udfContext.setOutput_fields(List.of("newk1")); + Unroll unroll = new Unroll(); + unroll.open(null, udfContext); + Event event = new Event(); + event.setExtractedFields(nestedMap); + List<Event> result = unroll.evaluate(event); + assertEquals(2, result.size()); + Map<String, Object> map1 = (Map<String, Object>) result.get(0).getExtractedFields().get("newk1"); + assertEquals("52:d4:18:c7:e5:11", map1.get("source_mac")); + Map<String, Object> map2 = (Map<String, Object>) result.get(1).getExtractedFields().get("newk1"); + assertEquals("ff:ff:ff:ff:ff:ff", map2.get("source_mac")); + + } + @Test + public void testUnrollFunction2() { + + + UDFContext udfContext = new UDFContext(); + Unroll unroll = new Unroll(); + Event event = new Event(); + Map<String, Object> params = new HashMap<>(); + params.put("regex", ","); + udfContext.setParameters(params); + udfContext.setParameters(params); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k2")); + udfContext.setOutput_fields(List.of("k2")); + unroll.open(null, udfContext); + List<Event> result2 = unroll.evaluate(event); + assertEquals(3, result2.size()); + assertEquals("{\"source_mac\":\"52:d4:18:c7:e5:10\"}", result2.get(0).getExtractedFields().get("k2")); + + } + @Test + public void testUnrollFunction3() { + + + UDFContext udfContext = new UDFContext(); + Unroll unroll = new Unroll(); + Event event = new Event(); + event.setExtractedFields(nestedMap); + udfContext.setLookup_fields(List.of("k3")); + udfContext.setOutput_fields(List.of("newk3")); + unroll.open(null, udfContext); + List<Event> result2 = unroll.evaluate(event); + assertEquals(1, result2.size()); + + } +} diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java new file mode 100644 index 0000000..33f7bad --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantileTest.java @@ -0,0 +1,89 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.Event;
+import com.geedgenetworks.common.udf.AggregateFunction;
+import com.geedgenetworks.common.udf.UDFContext;
+import com.geedgenetworks.sketch.util.StringUtils;
+import org.HdrHistogram.ArrayHistogram;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class HdrHistogramQuantileTest {
+ AggregateFunction agg;
+ Accumulator acc;
+ Event event;
+
+ @Test
+ public void inputRegular() {
+ double probability = 0.5;
+ initData( "regular", 2, probability);
+ long count = 100000;
+ Map<String, Object> fields = event.getExtractedFields();
+ for (int i = 1; i <= count; i++) {
+ fields.put("ms", i);
+ agg.add(event, acc);
+ }
+
+ long expect = (long) (count * probability);
+ long rst = (long)agg.getResult(acc).getMetricsFields().get("ms_his");
+ double error = Math.abs(rst - expect) / (double) expect;
+ System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error));
+ assertTrue(error <= 0.05);
+ }
+
+ @Test
+ public void inputSketch() {
+ double probability = 0.5;
+ initData( "sketch", 2, probability);
+ long count = 100000;
+ Map<String, Object> fields = event.getExtractedFields();
+
+ ArrayHistogram his = new ArrayHistogram(2);
+ for (int i = 1; i <= count; i++) {
+ his.recordValue(i);
+ }
+ fields.put("ms", StringUtils.encodeBase64String(his.toBytes()));
+ agg.add(event, acc);
+
+ his = new ArrayHistogram(2);
+ for (int i = 1; i <= count; i++) {
+ his.recordValue(i);
+ }
+ fields.put("ms", his.toBytes());
+ agg.add(event, acc);
+
+ long expect = (long) (count * probability);
+ long rst = (long)agg.getResult(acc).getMetricsFields().get("ms_his");
+ double error = Math.abs(rst - expect) / (double) expect;
+ System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error));
+ assertTrue(error <= 0.05);
+ }
+
+ private void initData(String input_type, int numberOfSignificantValueDigits, double probability){
+ agg = new HdrHistogramQuantile();
+ UDFContext c = new UDFContext();
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("input_type", input_type);
+ parameters.put("numberOfSignificantValueDigits", numberOfSignificantValueDigits);
+ parameters.put("probability", probability);
+ c.setParameters(parameters);
+ c.setLookup_fields(Collections.singletonList("ms"));
+ c.setOutput_fields(Collections.singletonList("ms_his"));
+
+ agg.open(c);
+ Map<String, Object> map = new HashMap<>();
+ acc = new Accumulator();
+ acc.setMetricsFields(map);
+ agg.initAccumulator(acc);
+
+ event = new Event();
+ Map<String, Object> fields = new HashMap<>();
+ event.setExtractedFields(fields);
+ }
+}
\ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java new file mode 100644 index 0000000..4eefd9a --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramQuantilesTest.java @@ -0,0 +1,98 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.Event;
+import com.geedgenetworks.common.udf.AggregateFunction;
+import com.geedgenetworks.common.udf.UDFContext;
+import com.geedgenetworks.sketch.util.StringUtils;
+import org.HdrHistogram.ArrayHistogram;
+import org.junit.jupiter.api.Test;
+
+import java.util.*;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class HdrHistogramQuantilesTest {
+ AggregateFunction agg;
+ Accumulator acc;
+ Event event;
+
+ @Test
+ public void inputRegular() {
+ double[] probabilities = new double[]{0, 0.1, 0.25, 0.5, 0.75, 1};
+ initData( "regular", 2, probabilities);
+ long count = 100000;
+ Map<String, Object> fields = event.getExtractedFields();
+ for (int i = 1; i <= count; i++) {
+ fields.put("ms", i);
+ agg.add(event, acc);
+ }
+
+ long[] expects = Arrays.stream(probabilities).mapToLong(x -> (long) (count * x)).toArray();
+
+ List<Long> rsts = (List<Long>)agg.getResult(acc).getMetricsFields().get("ms_his");
+ for (int i = 0; i < expects.length; i++) {
+ long rst = rsts.get(i);
+ long expect = expects[i];
+ double probability = probabilities[i];
+ double error = probability <= 0.01? 0 : Math.abs(rst - expect) / (double) expect;
+ System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error));
+ assertTrue(error <= 0.05);
+ }
+ }
+
+ @Test
+ public void inputSketch() {
+ double[] probabilities = new double[]{0, 0.1, 0.25, 0.5, 0.75, 1};
+ initData( "sketch", 2, probabilities);
+ long count = 100000;
+ Map<String, Object> fields = event.getExtractedFields();
+ long[] expects = Arrays.stream(probabilities).mapToLong(x -> (long) (count * x)).toArray();
+
+ ArrayHistogram his = new ArrayHistogram(2);
+ for (int i = 1; i <= count; i++) {
+ his.recordValue(i);
+ }
+ fields.put("ms", StringUtils.encodeBase64String(his.toBytes()));
+ agg.add(event, acc);
+
+ his = new ArrayHistogram(2);
+ for (int i = 1; i <= count; i++) {
+ his.recordValue(i);
+ }
+ fields.put("ms", his.toBytes());
+ agg.add(event, acc);
+
+ List<Long> rsts = (List<Long>)agg.getResult(acc).getMetricsFields().get("ms_his");
+ for (int i = 0; i < expects.length; i++) {
+ long rst = rsts.get(i);
+ long expect = expects[i];
+ double probability = probabilities[i];
+ double error = probability <= 0.01? 0 : Math.abs(rst - expect) / (double) expect;
+ System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error));
+ assertTrue(error <= 0.05);
+ }
+ }
+
+ private void initData(String input_type, int numberOfSignificantValueDigits, double[] probabilities){
+ agg = new HdrHistogramQuantiles();
+ UDFContext c = new UDFContext();
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("input_type", input_type);
+ parameters.put("numberOfSignificantValueDigits", numberOfSignificantValueDigits);
+ parameters.put("probabilities", probabilities);
+ c.setParameters(parameters);
+ c.setLookup_fields(Collections.singletonList("ms"));
+ c.setOutput_fields(Collections.singletonList("ms_his"));
+
+ agg.open(c);
+ Map<String, Object> map = new HashMap<>();
+ acc = new Accumulator();
+ acc.setMetricsFields(map);
+ agg.initAccumulator(acc);
+
+ event = new Event();
+ Map<String, Object> fields = new HashMap<>();
+ event.setExtractedFields(fields);
+ }
+}
\ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java new file mode 100644 index 0000000..f177ca5 --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/HdrHistogram/HdrHistogramTest.java @@ -0,0 +1,102 @@ +package com.geedgenetworks.core.udf.udaf.HdrHistogram;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.Event;
+import com.geedgenetworks.common.udf.AggregateFunction;
+import com.geedgenetworks.common.udf.UDFContext;
+import com.geedgenetworks.sketch.util.StringUtils;
+import org.HdrHistogram.ArrayHistogram;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class HdrHistogramTest {
+ AggregateFunction agg;
+ Accumulator acc;
+ Event event;
+
+ @Test
+ public void inputRegular() {
+ double[] probabilities = new double[]{0, 0.1, 0.25, 0.5, 0.75, 1};
+ initData( "regular", 2, "base64");
+ long count = 100000;
+ Map<String, Object> fields = event.getExtractedFields();
+ for (int i = 1; i <= count; i++) {
+ fields.put("ms", i);
+ agg.add(event, acc);
+ }
+
+ long[] expects = Arrays.stream(probabilities).mapToLong(x -> (long) (count * x)).toArray();
+ String str = (String) agg.getResult(acc).getMetricsFields().get("ms_his");
+ ArrayHistogram his = ArrayHistogram.fromBytes(StringUtils.decodeBase64(str.getBytes(StandardCharsets.UTF_8)));
+
+ for (int i = 0; i < expects.length; i++) {
+ long rst = his.getValueAtPercentile(probabilities[i] * 100);
+ long expect = expects[i];
+ double probability = probabilities[i];
+ double error = probability <= 0.01? 0 : Math.abs(rst - expect) / (double) expect;
+ System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error));
+ assertTrue(error <= 0.05);
+ }
+ }
+
+ @Test
+ public void inputSketch() {
+ double[] probabilities = new double[]{0, 0.1, 0.25, 0.5, 0.75, 1};
+ initData( "sketch", 2, "binary");
+ long count = 100000;
+ Map<String, Object> fields = event.getExtractedFields();
+ long[] expects = Arrays.stream(probabilities).mapToLong(x -> (long) (count * x)).toArray();
+
+ ArrayHistogram his = new ArrayHistogram(2);
+ for (int i = 1; i <= count; i++) {
+ his.recordValue(i);
+ }
+ fields.put("ms", StringUtils.encodeBase64String(his.toBytes()));
+ agg.add(event, acc);
+
+ his = new ArrayHistogram(2);
+ for (int i = 1; i <= count; i++) {
+ his.recordValue(i);
+ }
+ fields.put("ms", his.toBytes());
+ agg.add(event, acc);
+
+ byte[] bytes = (byte[]) agg.getResult(acc).getMetricsFields().get("ms_his");
+ ArrayHistogram h = ArrayHistogram.fromBytes(bytes);
+
+ for (int i = 0; i < expects.length; i++) {
+ long rst = h.getValueAtPercentile(probabilities[i] * 100);
+ long expect = expects[i];
+ double probability = probabilities[i];
+ double error = probability <= 0.01? 0 : Math.abs(rst - expect) / (double) expect;
+ System.out.println(String.format("%s:%d,%d,%.4f", probability, expect , rst , error));
+ assertTrue(error <= 0.05);
+ }
+ }
+
+ private void initData(String input_type, int numberOfSignificantValueDigits, String output_format){
+ agg = new HdrHistogram();
+ UDFContext c = new UDFContext();
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("input_type", input_type);
+ parameters.put("numberOfSignificantValueDigits", numberOfSignificantValueDigits);
+ parameters.put("output_format", output_format);
+ c.setParameters(parameters);
+ c.setLookup_fields(Collections.singletonList("ms"));
+ c.setOutput_fields(Collections.singletonList("ms_his"));
+
+ agg.open(c);
+ Map<String, Object> map = new HashMap<>();
+ acc = new Accumulator();
+ acc.setMetricsFields(map);
+ agg.initAccumulator(acc);
+
+ event = new Event();
+ Map<String, Object> fields = new HashMap<>();
+ event.setExtractedFields(fields);
+ }
+}
\ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java new file mode 100644 index 0000000..eae356d --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldApproxCountDistinctTest.java @@ -0,0 +1,87 @@ +package com.geedgenetworks.core.udf.udaf.hlld;
+
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.Event;
+import com.geedgenetworks.common.udf.AggregateFunction;
+import com.geedgenetworks.common.udf.UDFContext;
+import com.geedgenetworks.sketch.hlld.Hll;
+import com.geedgenetworks.sketch.util.StringUtils;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+
+public class HlldApproxCountDistinctTest {
+ AggregateFunction agg;
+ Accumulator acc;
+ Event event;
+
+
+ @Test
+ public void inputRegular() {
+ initData(14, "regular");
+ long count = 100000;
+ Map<String, Object> fields = event.getExtractedFields();
+ for (int i = 0; i < count; i++) {
+ fields.put("ip", i);
+ agg.add(event, acc);
+ }
+
+ long rst = (long)agg.getResult(acc).getMetricsFields().get("ip_cnt");
+ double error = Math.abs(rst - count) / (double) count;
+ System.out.println(String.format("%d,%d,%.4f", count , rst , error));
+ assertTrue(error <= 0.05);
+ }
+
+ @Test
+ public void inputSketch() {
+ initData(14, "sketch");
+ long count = 150000;
+ Map<String, Object> fields = event.getExtractedFields();
+
+ Hll hll = new Hll(12);
+ for (int i = 0; i < 100000; i++) {
+ hll.add(i);
+ }
+ fields.put("ip", StringUtils.encodeBase64String(hll.toBytes()));
+ agg.add(event, acc);
+
+ hll = new Hll(13);
+ for (int i = 50000; i < 150000; i++) {
+ hll.add(i);
+ }
+ fields.put("ip", hll.toBytes());
+ agg.add(event, acc);
+
+ long rst = (long)agg.getResult(acc).getMetricsFields().get("ip_cnt");
+ double error = Math.abs(rst - count) / (double) count;
+ System.out.println(String.format("%d,%d,%.4f", count , rst , error));
+ assertTrue(error <= 0.05);
+ }
+
+ private void initData(int precision, String input_type){
+ agg = new HlldApproxCountDistinct();
+ UDFContext c = new UDFContext();
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("precision", precision);
+ parameters.put("input_type", input_type);
+ c.setParameters(parameters);
+ c.setLookup_fields(Collections.singletonList("ip"));
+ c.setOutput_fields(Collections.singletonList("ip_cnt"));
+
+ agg.open(c);
+ Map<String, Object> map = new HashMap<>();
+ acc = new Accumulator();
+ acc.setMetricsFields(map);
+ agg.initAccumulator(acc);
+
+ event = new Event();
+ Map<String, Object> fields = new HashMap<>();
+ event.setExtractedFields(fields);
+ }
+}
\ No newline at end of file diff --git a/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java new file mode 100644 index 0000000..f489ee4 --- /dev/null +++ b/groot-core/src/test/java/com/geedgenetworks/core/udf/udaf/hlld/HlldTest.java @@ -0,0 +1,86 @@ +package com.geedgenetworks.core.udf.udaf.hlld;
+
+import com.geedgenetworks.common.Accumulator;
+import com.geedgenetworks.common.Event;
+import com.geedgenetworks.common.udf.AggregateFunction;
+import com.geedgenetworks.common.udf.UDFContext;
+import com.geedgenetworks.sketch.hlld.Hll;
+import com.geedgenetworks.sketch.util.StringUtils;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class HlldTest {
+ AggregateFunction agg;
+ Accumulator acc;
+ Event event;
+
+ @Test
+ public void inputRegular() {
+ initData(14, "regular", "base64");
+ long count = 100000;
+ Map<String, Object> fields = event.getExtractedFields();
+ for (int i = 0; i < count; i++) {
+ fields.put("ip", i);
+ agg.add(event, acc);
+ }
+
+ String hllStr = (String)agg.getResult(acc).getMetricsFields().get("ip_cnt");
+ long rst = (long) Hll.fromBytes(StringUtils.decodeBase64(hllStr.getBytes(StandardCharsets.UTF_8))).size();
+ double error = Math.abs(rst - count) / (double) count;
+ System.out.println(String.format("%d,%d,%.4f", count , rst , error));
+ assertTrue(error <= 0.05);
+ }
+
+ @Test
+ public void inputSketch() {
+ initData(14, "sketch", "binary");
+ long count = 150000;
+ Map<String, Object> fields = event.getExtractedFields();
+ for (int i = 0; i < 100000; i++) {
+ Hll hll = new Hll(12);
+ hll.add(i);
+ fields.put("ip", StringUtils.encodeBase64String(hll.toBytes()));
+ agg.add(event, acc);
+ }
+ for (int i = 50000; i < 150000; i++) {
+ Hll hll = new Hll(13);
+ hll.add(i);
+ fields.put("ip", hll.toBytes());
+ agg.add(event, acc);
+ }
+
+ byte[] hllBytes = (byte[])agg.getResult(acc).getMetricsFields().get("ip_cnt");
+ long rst = (long) Hll.fromBytes(hllBytes).size();
+ double error = Math.abs(rst - count) / (double) count;
+ System.out.println(String.format("%d,%d,%.4f", count , rst , error));
+ assertTrue(error <= 0.05);
+ }
+
+ private void initData(int precision, String input_type, String output_format){
+ agg = new Hlld();
+ UDFContext c = new UDFContext();
+ Map<String, Object> parameters = new HashMap<>();
+ parameters.put("precision", precision);
+ parameters.put("input_type", input_type);
+ parameters.put("output_format", output_format);
+ c.setParameters(parameters);
+ c.setLookup_fields(Collections.singletonList("ip"));
+ c.setOutput_fields(Collections.singletonList("ip_cnt"));
+
+ agg.open(c);
+ Map<String, Object> map = new HashMap<>();
+ acc = new Accumulator();
+ acc.setMetricsFields(map);
+ agg.initAccumulator(acc);
+
+ event = new Event();
+ Map<String, Object> fields = new HashMap<>();
+ event.setExtractedFields(fields);
+ }
+}
diff --git a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java index d927133..0eba408 100644 --- a/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java +++ b/groot-examples/end-to-end-example/src/main/java/com/geedgenetworks/example/GrootStreamExample.java @@ -13,11 +13,13 @@ import java.nio.file.Paths; public class GrootStreamExample { public static void main(String[] args) throws FileNotFoundException, URISyntaxException { - String configPath = args.length > 0 ? args[0] : "/examples/inline_to_print_test.yaml"; + String configPath = args.length > 0 ? args[0] : "/examples/inline_to_print_with_aggregation.yaml"; String configFile = getTestConfigFile(configPath); ExecuteCommandArgs executeCommandArgs = new ExecuteCommandArgs(); executeCommandArgs.setConfigFile(configFile); executeCommandArgs.setCheckConfig(false); + executeCommandArgs.setEncrypt(false); + executeCommandArgs.setDecrypt(false); executeCommandArgs.setVersion(false); executeCommandArgs.setDeployMode(DeployMode.RUN); executeCommandArgs.setTargetType(TargetType.LOCAL); diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/inline_all_data_type_to_clickhouse.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_all_data_type_to_clickhouse.yaml new file mode 100644 index 0000000..e449a08 --- /dev/null +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_all_data_type_to_clickhouse.yaml @@ -0,0 +1,78 @@ +sources: + inline_source: + type: inline + schema: + fields: + - name: id + type: bigint + - name: c_array_string + type: array<string> + - name: c_array_short + type: array<int> + - name: c_array_int + type: array<int> + - name: c_array_long + type: array<bigint> + - name: c_array_float + type: array<float> + - name: c_array_double + type: array<double> + - name: c_string + type: string + - name: c_int8 + type: int + - name: c_int16 + type: int + - name: c_int32 + type: int + - name: c_int64 + type: int + - name: c_float32 + type: float + - name: c_float64 + type: double + - name: c_decimal + type: double + - name: c_date + type: string + - name: c_datetime + type: string + - name: c_nullable + type: int + - name: c_lowcardinality + type: string + properties: + # + # [string] Event Data, it will be parsed to Map<String, Object> by the specified format. + # + data: '[{"id":0,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":0,"c_lowcardinality":"string"},{"id":1,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":1,"c_lowcardinality":"string"},{"id":2,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":2,"c_lowcardinality":"string"},{"id":3,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":3,"c_lowcardinality":"string"},{"id":4,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":4,"c_lowcardinality":"string"},{"id":5,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":5,"c_lowcardinality":"string"},{"id":6,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":6,"c_lowcardinality":"string"},{"id":7,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":7,"c_lowcardinality":"string"},{"id":8,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":8,"c_lowcardinality":"string"},{"id":9,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":9,"c_lowcardinality":"string"},{"id":10,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":10,"c_lowcardinality":"string"},{"id":11,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":11,"c_lowcardinality":"string"},{"id":12,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":12,"c_lowcardinality":"string"},{"id":13,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":13,"c_lowcardinality":"string"},{"id":14,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":14,"c_lowcardinality":"string"},{"id":15,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":15,"c_lowcardinality":"string"},{"id":16,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":16,"c_lowcardinality":"string"},{"id":17,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":17,"c_lowcardinality":"string"},{"id":18,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":18,"c_lowcardinality":"string"},{"id":19,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":19,"c_lowcardinality":"string"},{"id":20,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":20,"c_lowcardinality":"string"},{"id":21,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":21,"c_lowcardinality":"string"},{"id":22,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":22,"c_lowcardinality":"string"},{"id":23,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":23,"c_lowcardinality":"string"},{"id":24,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":24,"c_lowcardinality":"string"},{"id":25,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":25,"c_lowcardinality":"string"},{"id":26,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":26,"c_lowcardinality":"string"},{"id":27,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":27,"c_lowcardinality":"string"},{"id":28,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":28,"c_lowcardinality":"string"},{"id":29,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":29,"c_lowcardinality":"string"},{"id":30,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":30,"c_lowcardinality":"string"},{"id":31,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":31,"c_lowcardinality":"string"},{"id":32,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":32,"c_lowcardinality":"string"},{"id":33,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":33,"c_lowcardinality":"string"},{"id":34,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":34,"c_lowcardinality":"string"},{"id":35,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":35,"c_lowcardinality":"string"},{"id":36,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":36,"c_lowcardinality":"string"},{"id":37,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":37,"c_lowcardinality":"string"},{"id":38,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":38,"c_lowcardinality":"string"},{"id":39,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":39,"c_lowcardinality":"string"},{"id":40,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":40,"c_lowcardinality":"string"},{"id":41,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":41,"c_lowcardinality":"string"},{"id":42,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":42,"c_lowcardinality":"string"},{"id":43,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":43,"c_lowcardinality":"string"},{"id":44,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":44,"c_lowcardinality":"string"},{"id":45,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":45,"c_lowcardinality":"string"},{"id":46,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":46,"c_lowcardinality":"string"},{"id":47,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":47,"c_lowcardinality":"string"},{"id":48,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":48,"c_lowcardinality":"string"},{"id":49,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":49,"c_lowcardinality":"string"},{"id":50,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":50,"c_lowcardinality":"string"},{"id":51,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":51,"c_lowcardinality":"string"},{"id":52,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":52,"c_lowcardinality":"string"},{"id":53,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":53,"c_lowcardinality":"string"},{"id":54,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":54,"c_lowcardinality":"string"},{"id":55,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":55,"c_lowcardinality":"string"},{"id":56,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":56,"c_lowcardinality":"string"},{"id":57,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":57,"c_lowcardinality":"string"},{"id":58,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":58,"c_lowcardinality":"string"},{"id":59,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":59,"c_lowcardinality":"string"},{"id":60,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":60,"c_lowcardinality":"string"},{"id":61,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":61,"c_lowcardinality":"string"},{"id":62,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":62,"c_lowcardinality":"string"},{"id":63,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":63,"c_lowcardinality":"string"},{"id":64,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":64,"c_lowcardinality":"string"},{"id":65,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":65,"c_lowcardinality":"string"},{"id":66,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":66,"c_lowcardinality":"string"},{"id":67,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":67,"c_lowcardinality":"string"},{"id":68,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":68,"c_lowcardinality":"string"},{"id":69,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":69,"c_lowcardinality":"string"},{"id":70,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":70,"c_lowcardinality":"string"},{"id":71,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":71,"c_lowcardinality":"string"},{"id":72,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":72,"c_lowcardinality":"string"},{"id":73,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":73,"c_lowcardinality":"string"},{"id":74,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":74,"c_lowcardinality":"string"},{"id":75,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":75,"c_lowcardinality":"string"},{"id":76,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":76,"c_lowcardinality":"string"},{"id":77,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":77,"c_lowcardinality":"string"},{"id":78,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":78,"c_lowcardinality":"string"},{"id":79,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":79,"c_lowcardinality":"string"},{"id":80,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":80,"c_lowcardinality":"string"},{"id":81,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":81,"c_lowcardinality":"string"},{"id":82,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":82,"c_lowcardinality":"string"},{"id":83,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":83,"c_lowcardinality":"string"},{"id":84,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":84,"c_lowcardinality":"string"},{"id":85,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":85,"c_lowcardinality":"string"},{"id":86,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":86,"c_lowcardinality":"string"},{"id":87,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":87,"c_lowcardinality":"string"},{"id":88,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":88,"c_lowcardinality":"string"},{"id":89,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":89,"c_lowcardinality":"string"},{"id":90,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":90,"c_lowcardinality":"string"},{"id":91,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":91,"c_lowcardinality":"string"},{"id":92,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":92,"c_lowcardinality":"string"},{"id":93,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":93,"c_lowcardinality":"string"},{"id":94,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":94,"c_lowcardinality":"string"},{"id":95,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":95,"c_lowcardinality":"string"},{"id":96,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":96,"c_lowcardinality":"string"},{"id":97,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":97,"c_lowcardinality":"string"},{"id":98,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":98,"c_lowcardinality":"string"},{"id":99,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":99,"c_lowcardinality":"string"}]' + format: json + interval.per.row: 1s + repeat.count: 10 + json.ignore.parse.errors: false + + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: 192.168.44.12:9001 + table: default.sink_table + batch.size: 10 + batch.byte.size: 200MB + batch.interval: 1s + connection.user: e54c9568586180eede1506eecf3574e9 + connection.password: 86cf0e2ffba3f541a6c6761313e5cc7e + +application: # [object] Define job configuration + env: + name: example-inline-to-clickhouse + parallelism: 1 + shade.identifier: aes + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [ clickhouse_sink ] + - name: clickhouse_sink + downstream: []
\ No newline at end of file diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml index a5c5ece..517d29b 100644 --- a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_kafka.yaml @@ -46,7 +46,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a80bea937958aaa485c2acc2b475603495a23eb59f055e037c0b186acb22886bd0275ca91f1633441d9943e7962942252 format: json log.failures.only: true @@ -64,7 +64,7 @@ sinks: kafka.compression.type: snappy kafka.security.protocol: SASL_PLAINTEXT kafka.sasl.mechanism: PLAIN - kafka.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="admin" password="galaxy2019"; + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 format: json log.failures.only: true @@ -72,7 +72,7 @@ application: # [object] Define job configuration env: name: example-inline-to-kafka parallelism: 3 - shade.identifier: default + shade.identifier: aes pipeline: object-reuse: true topology: diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_use_udtf.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_use_udtf.yaml new file mode 100644 index 0000000..edde893 --- /dev/null +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_use_udtf.yaml @@ -0,0 +1,36 @@ +sources: + inline_source: + type: inline + properties: + data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931,"encapsulation":"[{\"tunnels_schema_type\":\"MULTIPATH_ETHERNET\",\"c2s_source_mac\":\"48:73:97:96:38:27\",\"c2s_destination_mac\":\"58:b3:8f:fa:3b:11\",\"s2c_source_mac\":\"58:b3:8f:fa:3b:11\",\"s2c_destination_mac\":\"48:73:97:96:38:27\"}]"},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.168.10.198","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931,"device_tag":"{\"tags\":[{\"tag\":\"data_center\",\"value\":\"center-xxg-tsgx\"},{\"tag\":\"device_group\",\"value\":\"group-xxg-tsgx\"}]}"}]' + format: json + json.ignore.parse.errors: false + +processing_pipelines: + table_processor: + type: table + functions: + - function: JSON_UNROLL + lookup_fields: [ encapsulation] + output_fields: [ encapsulation ] + +sinks: + print_sink: + type: print + properties: + format: json + mode: log_warn + +application: + env: + name: example-inline-to-print-with-aggregation + parallelism: 3 + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [table_processor] + - name: table_processor + downstream: [ print_sink ] + - name: print_sink + downstream: []
\ No newline at end of file diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml new file mode 100644 index 0000000..04dddf8 --- /dev/null +++ b/groot-examples/end-to-end-example/src/main/resources/examples/inline_to_print_with_aggregation.yaml @@ -0,0 +1,47 @@ +sources: + inline_source: + type: inline + properties: + data: '[{"tcp_rtt_ms":128,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.ct.cn","http_url":"www.ct.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"8.8.8.8","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":5575,"sent_pkts":97,"sent_bytes":5892,"received_pkts":250,"received_bytes":333931},{"tcp_rtt_ms":256,"decoded_as":"HTTP","http_version":"http1","http_request_line":"GET / HTTP/1.1","http_host":"www.abc.cn","http_url":"www.cabc.cn/","http_user_agent":"curl/8.0.1","http_status_code":200,"http_response_line":"HTTP/1.1 200 OK","http_response_content_type":"text/html; charset=UTF-8","http_response_latency_ms":31,"http_session_duration_ms":5451,"in_src_mac":"ba:bb:a7:3c:67:1c","in_dest_mac":"86:dd:7a:8f:ae:e2","out_src_mac":"86:dd:7a:8f:ae:e2","out_dest_mac":"ba:bb:a7:3c:67:1c","tcp_client_isn":678677906,"tcp_server_isn":1006700307,"address_type":4,"client_ip":"192.11.22.22","server_ip":"4.4.4.4","client_port":42751,"server_port":80,"in_link_id":65535,"out_link_id":65535,"start_timestamp_ms":1703646546127,"end_timestamp_ms":1703646551702,"duration_ms":2575,"sent_pkts":197,"sent_bytes":5892,"received_pkts":350,"received_bytes":533931}]' + format: json + json.ignore.parse.errors: false + + + +processing_pipelines: + aggregate_processor: + type: aggregate + group_by_fields: [ client_ip ] + window_type: tumbling_processing_time + window_size: 10 + functions: + - function: NUMBER_SUM + lookup_fields: [ received_bytes] + output_fields: [ received_bytes_sum ] + - function: APPROX_COUNT_DISTINCT_HLLD + lookup_fields: [ server_ip ] + output_fields: [ server_ip_count ] + parameters: + input_type: regular + + +sinks: + print_sink: + type: print + properties: + format: json + mode: log_warn + +application: + env: + name: example-inline-to-print-with-aggregation + parallelism: 3 + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [aggregate_processor] + - name: aggregate_processor + downstream: [ print_sink ] + - name: print_sink + downstream: []
\ No newline at end of file diff --git a/groot-examples/end-to-end-example/src/main/resources/examples/session_record_mock_to_print_with_aggregation.yaml b/groot-examples/end-to-end-example/src/main/resources/examples/session_record_mock_to_print_with_aggregation.yaml new file mode 100644 index 0000000..a3629c1 --- /dev/null +++ b/groot-examples/end-to-end-example/src/main/resources/examples/session_record_mock_to_print_with_aggregation.yaml @@ -0,0 +1,167 @@ +sources: # [object] Define connector source + mock_source: + type: mock + #watermark_timestamp: __timestamp + #watermark_timestamp_unit: s + #watermark_lag: 10000 + properties: + mock.desc.file.path: ./config/template/mock_schema/session_record_mock_desc.json + rows.per.second: 10 + +preprocessing_pipelines: + etl_processor: + type: projection + functions: + - function: SNOWFLAKE_ID + lookup_fields: [''] + output_fields: [log_id] + parameters: + data_center_id_num: 1 + - function: UNIX_TIMESTAMP_CONVERTER + lookup_fields: [ __timestamp ] + output_fields: [ recv_time ] + parameters: + precision: seconds + - function: SNOWFLAKE_ID + lookup_fields: [ '' ] + output_fields: [ session_id ] + parameters: + data_center_id_num: 2 + - function: EVAL + output_fields: [ ingestion_time ] + parameters: + value_expression: recv_time + + - function: DOMAIN + lookup_fields: [ http_host, ssl_sni, dtls_sni, quic_sni ] + output_fields: [ server_domain ] + parameters: + option: FIRST_SIGNIFICANT_SUBDOMAIN + + + - function: ASN_LOOKUP + lookup_fields: [ client_ip ] + output_fields: [ client_asn ] + parameters: + kb_name: tsg_ip_asn + option: IP_TO_ASN + + - function: ASN_LOOKUP + lookup_fields: [ server_ip ] + output_fields: [ server_asn ] + parameters: + kb_name: tsg_ip_asn + option: IP_TO_ASN + + - function: GEOIP_LOOKUP + lookup_fields: [ client_ip ] + output_fields: [] + parameters: + kb_name: tsg_ip_location + option: IP_TO_OBJECT + geolocation_field_mapping: + COUNTRY: client_country + PROVINCE: client_super_administrative_area + CITY: client_administrative_area + + - function: GEOIP_LOOKUP + lookup_fields: [ server_ip ] + output_fields: [] + parameters: + kb_name: tsg_ip_location + option: IP_TO_OBJECT + geolocation_field_mapping: + COUNTRY: server_country + PROVINCE: server_super_administrative_area + CITY: server_administrative_area + + - function: CURRENT_UNIX_TIMESTAMP + output_fields: [ processing_time ] + parameters: + precision: seconds + + - function: UNIX_TIMESTAMP_CONVERTER + lookup_fields: [recv_time] + output_fields: [recv_time] + parameters: + precision: seconds + interval: 60 + + +processing_pipelines: + aggregate_processor: + type: aggregate + group_by_fields: [recv_time, sled_ip] + window_type: tumbling_processing_time + window_size: 60 + functions: + - function: NUMBER_SUM + lookup_fields: [received_bytes, sent_bytes] + output_fields: [received_bytes_sum] + + - function: LONG_COUNT + lookup_fields: [received_bytes] + output_fields: [sessions] + + - function: MEAN + lookup_fields: [received_bytes] + output_fields: [received_bytes_mean] + parameters: + precision: 2 + + - function: FIRST_VALUE + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_first ] + + - function: LAST_VALUE + lookup_fields: [ received_bytes ] + output_fields: [ received_bytes_last ] + + - function: COLLECT_LIST + lookup_fields: [received_bytes] + output_fields: [received_bytes_set] + + +sinks: + print_sink: + type: print + properties: + mode: log_info + format: json + + kafka_sink: + type: kafka + properties: + topic: SESSION-RECORD + kafka.bootstrap.servers: 192.168.44.12:9094 + kafka.retries: 0 + kafka.linger.ms: 10 + kafka.request.timeout.ms: 30000 + kafka.batch.size: 262144 + kafka.buffer.memory: 134217728 + kafka.max.request.size: 10485760 + kafka.compression.type: snappy + kafka.security.protocol: SASL_PLAINTEXT + kafka.sasl.mechanism: PLAIN + kafka.sasl.jaas.config: 454f65ea6eef1256e3067104f82730e737b68959560966b811e7ff364116b03124917eb2b0f3596f14733aa29ebad9352644ce1a5c85991c6f01ba8a5e8f177a7ff0b2d3889a424249967b3870b50993d9644f239f0de82cdb13bdb502959e16afadffa49ef1e1d2b9c9b5113e619817 + format: json + json.ignore.parse.errors: false + log.failures.only: true + + +application: # [object] Define job configuration + env: + name: session_record_mock_to_print_with_aggregation + parallelism: 3 + shade.identifier: aes + pipeline: + object-reuse: true + topology: + - name: mock_source + downstream: [ etl_processor ] + - name: etl_processor + downstream: [ aggregate_processor ] + - name: aggregate_processor + downstream: [ print_sink ] + - name: print_sink + downstream: []
\ No newline at end of file diff --git a/groot-tests/pom.xml b/groot-tests/pom.xml index 76f533a..b46ad10 100644 --- a/groot-tests/pom.xml +++ b/groot-tests/pom.xml @@ -16,6 +16,7 @@ <module>test-common</module> <module>test-e2e-base</module> <module>test-e2e-kafka</module> + <module>test-e2e-clickhouse</module> </modules> <properties> diff --git a/groot-tests/test-e2e-clickhouse/pom.xml b/groot-tests/test-e2e-clickhouse/pom.xml new file mode 100644 index 0000000..aef4470 --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/pom.xml @@ -0,0 +1,89 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.geedgenetworks</groupId> + <artifactId>groot-tests</artifactId> + <version>${revision}</version> + </parent> + + <artifactId>test-e2e-clickhouse</artifactId> + <name>Groot : Tests : E2E : ClickHouse</name> + + <properties> + <maven.compiler.source>11</maven.compiler.source> + <maven.compiler.target>11</maven.compiler.target> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <clickhouse.jdbc.version>0.6.3</clickhouse.jdbc.version> + <hikaricp.version>4.0.3</hikaricp.version> + <apache-httpclient.version>5.2.1</apache-httpclient.version> + </properties> + + <dependencies> + + <dependency> + <groupId>com.geedgenetworks</groupId> + <artifactId>test-common</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>clickhouse</artifactId> + <version>${testcontainer.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>com.zaxxer</groupId> + <artifactId>HikariCP</artifactId> + <version>${hikaricp.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + <version>1.8.0</version> + <scope>test</scope> + </dependency> + + + <dependency> + <groupId>com.clickhouse</groupId> + <artifactId>clickhouse-jdbc</artifactId> + <version>${clickhouse.jdbc.version}</version> + <scope>test</scope> + </dependency> + + <!-- Recommended to communicate with ClickHouse server over http --> + <dependency> + <groupId>org.apache.httpcomponents.client5</groupId> + <artifactId>httpclient5</artifactId> + <version>${apache-httpclient.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>com.geedgenetworks</groupId> + <artifactId>connector-clickhouse</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + + + <dependency> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + <version>${snappy-java.version}</version> + <scope>test</scope> + </dependency> + + + </dependencies> + +</project>
\ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java new file mode 100644 index 0000000..20caace --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/java/com/geedgenetworks/test/e2e/clickhouse/ClickHouseIT.java @@ -0,0 +1,355 @@ +package com.geedgenetworks.test.e2e.clickhouse; + +import com.alibaba.fastjson2.JSON; +import com.geedgenetworks.test.common.TestResource; +import com.geedgenetworks.test.common.TestSuiteBase; +import com.geedgenetworks.test.common.container.ContainerUtil; +import com.geedgenetworks.test.common.container.TestContainer; +import com.geedgenetworks.test.common.container.TestContainerId; +import com.geedgenetworks.test.common.junit.DisabledOnContainer; +import com.google.common.collect.Maps; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.shaded.org.apache.commons.io.IOUtils; +import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.sql.*; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.awaitility.Awaitility.await; + +@Slf4j +@DisabledOnContainer( + value = {TestContainerId.FLINK_1_17}, + disabledReason = "Override TestSuiteBase @DisabledOnContainer") +public class ClickHouseIT extends TestSuiteBase implements TestResource { + private static final String CLICKHOUSE_DOCKER_IMAGE = "clickhouse/clickhouse-server:23.3.19.32"; + private static final String DRIVER_CLASS = "com.clickhouse.jdbc.ClickHouseDriver"; + private static final String INIT_CLICKHOUSE_PATH = "/init/clickhouse_test_sql.conf"; + private static final String DATABASE = "default"; + private static final String SOURCE_TABLE = "source_table"; + private static final String SINK_TABLE = "sink_table"; + private static final String INSERT_SQL = "insert_sql"; + private static final String COMPARE_SQL = "compare_sql"; + private static final String HOST = "clickhouse"; + private static final Config CONFIG = getInitClickhouseConfig(); + private ClickHouseContainer clickHouseContainer; + private Connection connection; + private static final String[] default_columns = new String[] { + "id", + "c_array_string", + "c_array_short", + "c_array_int", + "c_array_long", + "c_array_float", + "c_array_double", + "c_string", + "c_int8", + "c_int16", + "c_int32", + "c_int64", + "c_float32", + "c_float64", + "c_decimal", + "c_nullable", + "c_lowcardinality" + }; + private static final List<Map<String, Object>>TEST_DATASET = generateTestDataSet(); + + + @BeforeAll + @Override + public void startUp() throws Exception { + this.clickHouseContainer = + new ClickHouseContainer(CLICKHOUSE_DOCKER_IMAGE) + .withNetwork(NETWORK) + .withNetworkAliases(HOST) + .withCopyFileToContainer(MountableFile.forClasspathResource("init/users.xml"), "/etc/clickhouse-server/users.xml") + .withCopyFileToContainer(MountableFile.forClasspathResource("init/init-clickhouse.sql"), "/docker-entrypoint-initdb.d/init-clickhouse.sql") + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger(CLICKHOUSE_DOCKER_IMAGE))); + + Startables.deepStart(Stream.of(this.clickHouseContainer)).join(); + System.out.println("Clickhouse JDBC URL: " + this.clickHouseContainer.getJdbcUrl()); + System.out.println("Clickhouse username: " + this.clickHouseContainer.getUsername()); + System.out.println("Clickhouse password: " + this.clickHouseContainer.getPassword()); + + log.info("Clickhouse container started"); + Awaitility.given() + .ignoreExceptions() + .await() + .atMost(360L, TimeUnit.SECONDS) + .untilAsserted(this::initConnection); + this.initializeClickhouseTable(); + this.batchInsertData(); + log.info(JSON.toJSONString(TEST_DATASET)); + + } + + private void initConnection() + throws SQLException, ClassNotFoundException, InstantiationException, + IllegalAccessException { + final Properties info = new Properties(); + info.put("user", this.clickHouseContainer.getUsername()); + info.put("password", this.clickHouseContainer.getPassword()); + this.connection = + ((Driver) Class.forName(DRIVER_CLASS).newInstance()) + .connect(this.clickHouseContainer.getJdbcUrl(), info); + + } + + @TestTemplate + public void testClickHouse(TestContainer container) throws Exception { + assertHasData(SOURCE_TABLE); + clearTable(SOURCE_TABLE); + } + + @TestTemplate + public void testClickHouseDataTypeSinkTable(TestContainer container) throws Exception { + CompletableFuture.supplyAsync( + () -> { + try { + Container.ExecResult execResult = container.executeJob("/clickhouse_data_type_sink.yaml"); + Assertions.assertEquals(0, execResult.getExitCode(), execResult.getStderr()); + return execResult; + } catch (Exception e) { + log.error("Commit task exception:" + e.getMessage()); + throw new RuntimeException(e); + } + }); + + await().atMost(300000, TimeUnit.MILLISECONDS) + .untilAsserted( + () -> { + assertHasData(SINK_TABLE); + compareResult(); + clearTable(SINK_TABLE); + }); + } + + + private void assertHasData(String table) { + String sql = String.format("select * from %s.%s limit 1", DATABASE, table); + try (Statement statement = connection.createStatement(); + ResultSet source = statement.executeQuery(sql);) { + Assertions.assertTrue(source.next()); + } catch (SQLException e) { + throw new RuntimeException("test clickhouse server image error", e); + } + } + + private void clearTable(String table) { + try (Statement statement = connection.createStatement()) { + statement.execute(String.format("truncate table %s.%s", DATABASE, table)); + } catch (SQLException e) { + throw new RuntimeException("Test clickhouse server image error", e); + } + } + + private void compareResult() throws SQLException, IOException { + String sourceSql = "select * from " + SOURCE_TABLE + " order by id "; + String sinkSql = "select * from " + SINK_TABLE + " order by id"; + try (Statement sourceStatement = connection.createStatement(); + Statement sinkStatement = connection.createStatement(); + ResultSet sourceResultSet = sourceStatement.executeQuery(sourceSql); + ResultSet sinkResultSet = sinkStatement.executeQuery(sinkSql)) { + Assertions.assertEquals( + sourceResultSet.getMetaData().getColumnCount(), + sinkResultSet.getMetaData().getColumnCount()); + String columns = String.join(",", default_columns); + Assertions.assertTrue( + compare(String.format(CONFIG.getString(COMPARE_SQL), columns, columns))); + } + + sourceSql = "select count(distinct id) as count from " + SOURCE_TABLE; + sinkSql = "select count(distinct id) as count from " + SINK_TABLE; + + try (Statement sourceStatement = connection.createStatement(); + Statement sinkStatement = connection.createStatement(); + ResultSet sourceResultSet = sourceStatement.executeQuery(sourceSql); + ResultSet sinkResultSet = sinkStatement.executeQuery(sinkSql)) { + while (sourceResultSet.next()) { + if (sinkResultSet.next()) { + int sinkUniqueIds = sinkResultSet.getInt("count"); + int sourceUniqueIds = sourceResultSet.getInt("count"); + Assertions.assertEquals(sinkUniqueIds, sourceUniqueIds); + } + } + } + + + } + + private Boolean compare(String sql) { + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql);) { + return !resultSet.next(); + } catch (SQLException e) { + throw new RuntimeException("result compare error", e); + } + } + + + private void batchInsertData() { + String sql = CONFIG.getString(INSERT_SQL); + PreparedStatement preparedStatement = null; + try { + this.connection.setAutoCommit(true); + preparedStatement = this.connection.prepareStatement(sql); + for (Map<String, Object> row : TEST_DATASET) { + preparedStatement.setLong(1, (Long) row.get(default_columns[0])); + preparedStatement.setArray(2, toSqlArray(row.get(default_columns[1]))); + preparedStatement.setArray(3, toSqlArray(row.get(default_columns[2]))); + preparedStatement.setArray(4, toSqlArray(row.get(default_columns[3]))); + preparedStatement.setArray(5, toSqlArray(row.get(default_columns[4]))); + preparedStatement.setArray(6, toSqlArray(row.get(default_columns[5]))); + preparedStatement.setArray(7, toSqlArray(row.get(default_columns[6]))); + preparedStatement.setString(8, (String) row.get(default_columns[7])); + preparedStatement.setByte(9, (Byte) row.get(default_columns[8])); + preparedStatement.setShort(10, (Short) row.get(default_columns[9])); + preparedStatement.setInt(11, (Integer) row.get(default_columns[10])); + preparedStatement.setLong(12, (Long) row.get(default_columns[11])); + preparedStatement.setFloat(13, (Float) row.get(default_columns[12])); + preparedStatement.setDouble(14, (Double) row.get(default_columns[13])); + preparedStatement.setBigDecimal(15, (BigDecimal) row.get(default_columns[14])); + preparedStatement.setInt(16, (Integer) row.get(default_columns[15])); + preparedStatement.setString(17, (String) row.get(default_columns[16])); + preparedStatement.addBatch(); + } + + preparedStatement.executeBatch(); + preparedStatement.clearBatch(); + + } catch (SQLException e) { + throw new RuntimeException("Batch insert data failed!", e); + } finally { + if (preparedStatement != null) { + try { + preparedStatement.close(); + } catch (SQLException e) { + throw new RuntimeException("PreparedStatement close failed!", e); + } + } + } + + + + } + + + private Array toSqlArray(Object value) throws SQLException { + Object[] elements = null; + String sqlType = null; + if (String[].class.equals(value.getClass())) { + sqlType = "TEXT"; + elements = (String[]) value; + } else if (Boolean[].class.equals(value.getClass())) { + sqlType = "BOOLEAN"; + elements = (Boolean[]) value; + } else if (Byte[].class.equals(value.getClass())) { + sqlType = "TINYINT"; + elements = (Byte[]) value; + } else if (Short[].class.equals(value.getClass())) { + sqlType = "SMALLINT"; + elements = (Short[]) value; + } else if (Integer[].class.equals(value.getClass())) { + sqlType = "INTEGER"; + elements = (Integer[]) value; + } else if (Long[].class.equals(value.getClass())) { + sqlType = "BIGINT"; + elements = (Long[]) value; + } else if (Float[].class.equals(value.getClass())) { + sqlType = "REAL"; + elements = (Float[]) value; + } else if (Double[].class.equals(value.getClass())) { + sqlType = "DOUBLE"; + elements = (Double[]) value; + } + if (sqlType == null) { + throw new IllegalArgumentException( + "array inject error, not supported data type: " + value.getClass()); + } + return connection.createArrayOf(sqlType, elements); + } + + private static List<Map<String, Object>> generateTestDataSet() { + List<Map<String, Object>> rows = new ArrayList<>(); + for (int i = 0; i < 100; ++i) { + Map<String, Object> row = Maps.newLinkedHashMap(); + row.put(default_columns[0], (long) i); + row.put(default_columns[1], new String[] {"string"}); + row.put(default_columns[2], new Short[] {Short.parseShort("1")}); + row.put(default_columns[3], new Integer[] {Integer.parseInt("1")}); + row.put(default_columns[4], new Long[] {Long.parseLong("1")}); + row.put(default_columns[5], new Float[] {Float.parseFloat("1.1")}); + row.put(default_columns[6], new Double[] {Double.parseDouble("1.1")}); + row.put(default_columns[7], "string"); + row.put(default_columns[8], Byte.parseByte("1")); + row.put(default_columns[9], Short.parseShort("1")); + row.put(default_columns[10], Integer.parseInt("1")); + row.put(default_columns[11], Long.parseLong("1")); + row.put(default_columns[12], Float.parseFloat("1.1")); + row.put(default_columns[13], Double.parseDouble("1.1")); + row.put(default_columns[14], BigDecimal.valueOf(11L, 1)); + row.put(default_columns[15], i); + row.put(default_columns[16], "string"); + rows.add(row); + } + return rows; + } + + + + private void initializeClickhouseTable() { + try { + Statement statement = this.connection.createStatement(); + statement.execute(CONFIG.getString(SOURCE_TABLE)); + statement.execute(CONFIG.getString(SINK_TABLE)); + } catch (SQLException e) { + throw new RuntimeException("Initializing Clickhouse table failed!", e); + } + } + + private static Config getInitClickhouseConfig() { + File file = ContainerUtil.getResourcesFile(INIT_CLICKHOUSE_PATH); + Config config = ConfigFactory.parseFile(file); + assert config.hasPath(SOURCE_TABLE) + && config.hasPath(SINK_TABLE) + && config.hasPath(INSERT_SQL) + && config.hasPath(COMPARE_SQL); + return config; + } + + @AfterAll + @Override + public void tearDown() throws Exception { + if (this.connection != null) { + this.connection.close(); + } + if (this.clickHouseContainer != null) { + this.clickHouseContainer.stop(); + } + + } +} diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml b/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml new file mode 100644 index 0000000..3406a67 --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/clickhouse_data_type_sink.yaml @@ -0,0 +1,79 @@ +sources: + inline_source: + type: inline + schema: + fields: + - name: id + type: bigint + - name: c_array_string + type: array<string> + - name: c_array_short + type: array<int> + - name: c_array_int + type: array<int> + - name: c_array_long + type: array<bigint> + - name: c_array_float + type: array<float> + - name: c_array_double + type: array<double> + - name: c_string + type: string + - name: c_int8 + type: int + - name: c_int16 + type: int + - name: c_int32 + type: int + - name: c_int64 + type: int + - name: c_float32 + type: float + - name: c_float64 + type: double + - name: c_decimal + type: double + - name: c_date + type: string + - name: c_datetime + type: string + - name: c_nullable + type: int + - name: c_lowcardinality + type: string + properties: + # + # [string] Event Data, it will be parsed to Map<String, Object> by the specified format. + # + data: '[{"id":0,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":0,"c_lowcardinality":"string"},{"id":1,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":1,"c_lowcardinality":"string"},{"id":2,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":2,"c_lowcardinality":"string"},{"id":3,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":3,"c_lowcardinality":"string"},{"id":4,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":4,"c_lowcardinality":"string"},{"id":5,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":5,"c_lowcardinality":"string"},{"id":6,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":6,"c_lowcardinality":"string"},{"id":7,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":7,"c_lowcardinality":"string"},{"id":8,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":8,"c_lowcardinality":"string"},{"id":9,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":9,"c_lowcardinality":"string"},{"id":10,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":10,"c_lowcardinality":"string"},{"id":11,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":11,"c_lowcardinality":"string"},{"id":12,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":12,"c_lowcardinality":"string"},{"id":13,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":13,"c_lowcardinality":"string"},{"id":14,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":14,"c_lowcardinality":"string"},{"id":15,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":15,"c_lowcardinality":"string"},{"id":16,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":16,"c_lowcardinality":"string"},{"id":17,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":17,"c_lowcardinality":"string"},{"id":18,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":18,"c_lowcardinality":"string"},{"id":19,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":19,"c_lowcardinality":"string"},{"id":20,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":20,"c_lowcardinality":"string"},{"id":21,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":21,"c_lowcardinality":"string"},{"id":22,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":22,"c_lowcardinality":"string"},{"id":23,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":23,"c_lowcardinality":"string"},{"id":24,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":24,"c_lowcardinality":"string"},{"id":25,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":25,"c_lowcardinality":"string"},{"id":26,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":26,"c_lowcardinality":"string"},{"id":27,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":27,"c_lowcardinality":"string"},{"id":28,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":28,"c_lowcardinality":"string"},{"id":29,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":29,"c_lowcardinality":"string"},{"id":30,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":30,"c_lowcardinality":"string"},{"id":31,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":31,"c_lowcardinality":"string"},{"id":32,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":32,"c_lowcardinality":"string"},{"id":33,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":33,"c_lowcardinality":"string"},{"id":34,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":34,"c_lowcardinality":"string"},{"id":35,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":35,"c_lowcardinality":"string"},{"id":36,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":36,"c_lowcardinality":"string"},{"id":37,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":37,"c_lowcardinality":"string"},{"id":38,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":38,"c_lowcardinality":"string"},{"id":39,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":39,"c_lowcardinality":"string"},{"id":40,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":40,"c_lowcardinality":"string"},{"id":41,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":41,"c_lowcardinality":"string"},{"id":42,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":42,"c_lowcardinality":"string"},{"id":43,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":43,"c_lowcardinality":"string"},{"id":44,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":44,"c_lowcardinality":"string"},{"id":45,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":45,"c_lowcardinality":"string"},{"id":46,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":46,"c_lowcardinality":"string"},{"id":47,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":47,"c_lowcardinality":"string"},{"id":48,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":48,"c_lowcardinality":"string"},{"id":49,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":49,"c_lowcardinality":"string"},{"id":50,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":50,"c_lowcardinality":"string"},{"id":51,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":51,"c_lowcardinality":"string"},{"id":52,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":52,"c_lowcardinality":"string"},{"id":53,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":53,"c_lowcardinality":"string"},{"id":54,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":54,"c_lowcardinality":"string"},{"id":55,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":55,"c_lowcardinality":"string"},{"id":56,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":56,"c_lowcardinality":"string"},{"id":57,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":57,"c_lowcardinality":"string"},{"id":58,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":58,"c_lowcardinality":"string"},{"id":59,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":59,"c_lowcardinality":"string"},{"id":60,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":60,"c_lowcardinality":"string"},{"id":61,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":61,"c_lowcardinality":"string"},{"id":62,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":62,"c_lowcardinality":"string"},{"id":63,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":63,"c_lowcardinality":"string"},{"id":64,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":64,"c_lowcardinality":"string"},{"id":65,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":65,"c_lowcardinality":"string"},{"id":66,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":66,"c_lowcardinality":"string"},{"id":67,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":67,"c_lowcardinality":"string"},{"id":68,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":68,"c_lowcardinality":"string"},{"id":69,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":69,"c_lowcardinality":"string"},{"id":70,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":70,"c_lowcardinality":"string"},{"id":71,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":71,"c_lowcardinality":"string"},{"id":72,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":72,"c_lowcardinality":"string"},{"id":73,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":73,"c_lowcardinality":"string"},{"id":74,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":74,"c_lowcardinality":"string"},{"id":75,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":75,"c_lowcardinality":"string"},{"id":76,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":76,"c_lowcardinality":"string"},{"id":77,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":77,"c_lowcardinality":"string"},{"id":78,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":78,"c_lowcardinality":"string"},{"id":79,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":79,"c_lowcardinality":"string"},{"id":80,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":80,"c_lowcardinality":"string"},{"id":81,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":81,"c_lowcardinality":"string"},{"id":82,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":82,"c_lowcardinality":"string"},{"id":83,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":83,"c_lowcardinality":"string"},{"id":84,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":84,"c_lowcardinality":"string"},{"id":85,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":85,"c_lowcardinality":"string"},{"id":86,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":86,"c_lowcardinality":"string"},{"id":87,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":87,"c_lowcardinality":"string"},{"id":88,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":88,"c_lowcardinality":"string"},{"id":89,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":89,"c_lowcardinality":"string"},{"id":90,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":90,"c_lowcardinality":"string"},{"id":91,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":91,"c_lowcardinality":"string"},{"id":92,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":92,"c_lowcardinality":"string"},{"id":93,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":93,"c_lowcardinality":"string"},{"id":94,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":94,"c_lowcardinality":"string"},{"id":95,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":95,"c_lowcardinality":"string"},{"id":96,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":96,"c_lowcardinality":"string"},{"id":97,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":97,"c_lowcardinality":"string"},{"id":98,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":98,"c_lowcardinality":"string"},{"id":99,"c_array_string":["string"],"c_array_short":[1],"c_array_int":[1],"c_array_long":[1],"c_array_float":[1.1],"c_array_double":[1.1],"c_string":"string","c_int8":1,"c_int16":1,"c_int32":1,"c_int64":1,"c_float32":1.1,"c_float64":1.1,"c_decimal":1.1,"c_nullable":99,"c_lowcardinality":"string"}]' + format: json + interval.per.row: 10ms + repeat.count: -1 + json.ignore.parse.errors: false + + +sinks: + clickhouse_sink: + type: clickhouse + properties: + host: clickhouse:9000 + table: sink_table + connection.database: default + batch.size: 100 + batch.byte.size: 200MB + batch.interval: 1s + connection.user: ee9b0016824d59c8c191aa9633e4b61e + connection.password: ee9b0016824d59c8c191aa9633e4b61e + +application: # [object] Define job configuration + env: + name: example-inline-to-clickhouse + parallelism: 1 + shade.identifier: aes + pipeline: + object-reuse: true + topology: + - name: inline_source + downstream: [ clickhouse_sink ] + - name: clickhouse_sink + downstream: []
\ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf b/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf new file mode 100644 index 0000000..f132795 --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/init/clickhouse_test_sql.conf @@ -0,0 +1,81 @@ +source_table = """ +set allow_experimental_geo_types = 1; +create table if not exists `default`.source_table( + `id` Int64, + `c_array_string` Array(String), + `c_array_short` Array(Int16), + `c_array_int` Array(Int32), + `c_array_long` Array(Int64), + `c_array_float` Array(Float32), + `c_array_double` Array(Float64), + `c_string` String, + `c_int8` Int8, + `c_int16` Int16, + `c_int32` Int32, + `c_int64` Int64, + `c_float32` Float32, + `c_float64` Float64, + `c_decimal` Decimal(9,4), + `c_nullable` Nullable(Int32), + `c_lowcardinality` LowCardinality(String) +)engine=Memory; +""" + +sink_table = """ +create table if not exists `default`.sink_table( + `id` Int64, + `c_array_string` Array(String), + `c_array_short` Array(Int16), + `c_array_int` Array(Int32), + `c_array_long` Array(Int64), + `c_array_float` Array(Float32), + `c_array_double` Array(Float64), + `c_string` String, + `c_int8` Int8, + `c_int16` Int16, + `c_int32` Int32, + `c_int64` Int64, + `c_float32` Float32, + `c_float64` Float64, + `c_decimal` Decimal(9,4), + `c_nullable` Nullable(Int32), + `c_lowcardinality` LowCardinality(String) +)engine=Memory; +""" + +insert_sql = """ +insert into `default`.source_table +( + `id`, + `c_array_string`, + `c_array_short`, + `c_array_int`, + `c_array_long`, + `c_array_float`, + `c_array_double`, + `c_string`, + `c_int8`, + `c_int16`, + `c_int32`, + `c_int64`, + `c_float32`, + `c_float64`, + `c_decimal`, + `c_nullable`, + `c_lowcardinality` +) +values +(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) +""" + +compare_sql = """ +select + %s + from ( + select * from default.source_table +union all + select * from default.sink_table + ) +group by %s +having count(*) < 2 +"""
\ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/init/init-clickhouse.sql b/groot-tests/test-e2e-clickhouse/src/test/resources/init/init-clickhouse.sql new file mode 100644 index 0000000..fd9daac --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/init/init-clickhouse.sql @@ -0,0 +1,4 @@ +show databases; +-- ALTER USER default IDENTIFIED WITH plaintext_password BY 'testuser'; +CREATE USER testuser IDENTIFIED WITH plaintext_password BY 'testuser'; +GRANT ALL ON *.* TO testuser;
\ No newline at end of file diff --git a/groot-tests/test-e2e-clickhouse/src/test/resources/init/users.xml b/groot-tests/test-e2e-clickhouse/src/test/resources/init/users.xml new file mode 100644 index 0000000..86a590d --- /dev/null +++ b/groot-tests/test-e2e-clickhouse/src/test/resources/init/users.xml @@ -0,0 +1,29 @@ +<yandex> + <users> + <default> + <password></password> + <profile>default</profile> + <access_management>1</access_management> + <named_collection_control>1</named_collection_control> + <show_named_collections>1</show_named_collections> + <show_named_collections_secrets>1</show_named_collections_secrets> + <networks> + <ip>::/0</ip> + </networks> + <grants> + <grant>ALTER TABLE ON *.*</grant> + <grant>CREATE USER ON *.*</grant> + <grant>GRANT ON *.*</grant> + </grants> + </default> + </users> + + <profiles> + <default> + <max_memory_usage>10000000000</max_memory_usage> + <use_uncompressed_cache>1</use_uncompressed_cache> + <load_balancing>random</load_balancing> + <max_threads>8</max_threads> + </default> + </profiles> +</yandex> diff --git a/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaConnectorIT.java b/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java index 56108c5..50ff9d8 100644 --- a/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaConnectorIT.java +++ b/groot-tests/test-e2e-kafka/src/test/java/com/geedgenetworks/test/e2e/kafka/KafkaIT.java @@ -18,7 +18,6 @@ import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.SaslConfigs; -import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.serialization.StringDeserializer; @@ -48,7 +47,7 @@ import static org.awaitility.Awaitility.await; @DisabledOnContainer( value = {TestContainerId.FLINK_1_17}, disabledReason = "Override TestSuiteBase @DisabledOnContainer") -public class KafkaConnectorIT extends TestSuiteBase implements TestResource { +public class KafkaIT extends TestSuiteBase implements TestResource { private KafkaContainer kafkaContainer; @@ -164,8 +163,8 @@ public class KafkaConnectorIT extends TestSuiteBase implements TestResource { public void testKafkaAsSinkProducerQuota(TestContainer container) throws IOException, InterruptedException { //Create topic with 3 partitions executeShell("kafka-topics --create --topic SESSION-RECORD-QUOTA-TEST --bootstrap-server kafkaCluster:9092 --partitions 3 --replication-factor 1 --command-config /etc/kafka/kafka_client_jass_cli.properties"); - //Set producer quota to 5KB/s - executeShell("kafka-configs --bootstrap-server kafkaCluster:9092 --alter --add-config 'producer_byte_rate=5120' --entity-type users --entity-name admin --entity-type clients --entity-name SESSION-RECORD-QUOTA-TEST --command-config /etc/kafka/kafka_client_jass_cli.properties "); + //Set producer quota to 2KB/s + executeShell("kafka-configs --bootstrap-server kafkaCluster:9092 --alter --add-config 'producer_byte_rate=2048' --entity-type users --entity-name admin --entity-type clients --entity-name SESSION-RECORD-QUOTA-TEST --command-config /etc/kafka/kafka_client_jass_cli.properties "); CompletableFuture.supplyAsync( () -> { @@ -184,7 +183,7 @@ public class KafkaConnectorIT extends TestSuiteBase implements TestResource { .untilAsserted( () -> { data.addAll(getKafkaConsumerListData("SESSION-RECORD-QUOTA-TEST")); - Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "TimeoutException") && data.size()>1000); + Assertions.assertTrue(StringUtils.contains(container.getServerLogs(), "TimeoutException") && data.size()>100); }); } @@ -244,8 +243,6 @@ public class KafkaConnectorIT extends TestSuiteBase implements TestResource { }); } - - private void generateTestData(String topic, int start, int end) { StructType dataType = Types.parseStructType("id: int, client_ip: string, server_ip: string, flag: string"); JsonSerializer serializer = new JsonSerializer(dataType); diff --git a/plugin-mapping.properties b/plugin-mapping.properties index 4e2eb81..7cc144d 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -1,7 +1,7 @@ #Connectors grootstream.source.kafka = connector-kafka grootstream.sink.kafka = connector-kafka -grootstream.source.clickhouse= connector-clickhouse +grootstream.sink.clickhouse = connector-clickhouse grootstream.source.ipfix = connector-ipfix-collector grootstream.source.mock = connector-mock grootstream.source.file = connector-file
\ No newline at end of file @@ -36,7 +36,7 @@ <maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version> <flatten-maven-plugin.version>1.3.0</flatten-maven-plugin.version> <maven-git-commit-id-plugin.version>4.0.4</maven-git-commit-id-plugin.version> - <testcontainer.version>1.20.0</testcontainer.version> + <testcontainer.version>1.20.1</testcontainer.version> <awaitility.version>4.2.0</awaitility.version> <spotless.version>2.40.0</spotless.version> <slf4j.version>1.7.25</slf4j.version> @@ -84,6 +84,30 @@ </properties> + <repositories> + <repository> + <id>nexus3</id> + <name>Team Nexus Repository</name> + <url>http://192.168.40.153:8081/repository/public/</url> + </repository> + <repository> + <id>cloudera</id> + <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> + </repository> + </repositories> + + <distributionManagement> + <repository> + <uniqueVersion>true</uniqueVersion> + <id>platform-releases</id> + <url>http://192.168.40.153:8081/repository/platform-release/</url> + </repository> + <snapshotRepository> + <id>platform-snapshots</id> + <url>http://192.168.40.153:8081/repository/platform-snapshot/</url> + </snapshotRepository> + </distributionManagement> + <dependencyManagement> <dependencies> <dependency> @@ -245,6 +269,11 @@ <version>${fastjson2.version}</version> </dependency> <dependency> + <groupId>com.geedgenetworks</groupId> + <artifactId>sketches</artifactId> + <version>1.0.0</version> + </dependency> + <dependency> <groupId>com.alibaba.nacos</groupId> <artifactId>nacos-client</artifactId> <version>${nacos.version}</version> @@ -967,28 +996,6 @@ </build> - <repositories> - <repository> - <id>nexus3</id> - <name>Team Nexus Repository</name> - <url>http://192.168.40.153:8081/repository/public/</url> - </repository> - <repository> - <id>cloudera</id> - <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> - </repository> - </repositories> - <distributionManagement> - <repository> - <uniqueVersion>true</uniqueVersion> - <id>platform-releases</id> - <url>http://192.168.40.153:8081/repository/platform-release/</url> - </repository> - <snapshotRepository> - <id>platform-snapshots</id> - <url>http://192.168.40.153:8081/repository/platform-snapshot/</url> - </snapshotRepository> - </distributionManagement> </project> |
