过滤所有schema为BASE的原始日志

4 files changed, 9 insertions, 13 deletions

diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java
index de3735b..da2d897 100644
--- a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java
+++ b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadClickhouseData.java
@@ -265,7 +265,7 @@ public class ReadClickhouseData {
     }
 
     public static String getVertexIpSql() {
-        String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime;
+        String where = " common_recv_time >= " + minTime + " AND common_recv_time < " + maxTime + " AND common_schema_type != 'BASE'";
         String clientIpSql = "SELECT common_client_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_c2s) as common_link_info,'client' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
         String serverIpSql = "SELECT common_server_ip AS IP, MIN(common_recv_time) AS FIRST_FOUND_TIME,MAX(common_recv_time) AS LAST_FOUND_TIME,count(*) as SESSION_COUNT,sum(common_c2s_byte_num+common_s2c_byte_num) as BYTES_SUM,groupUniqArray(2)(common_link_info_s2c) as common_link_info,'server' as ip_type FROM tsg_galaxy_v3.connection_record_log where " + where + " group by IP";
         return "SELECT * FROM((" + clientIpSql + ") UNION ALL (" + serverIpSql + "))";
diff --git a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java
index d84ac45..d8ded7e 100644
--- a/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java
+++ b/IP-learning-graph/src/main/java/cn/ac/iie/service/ingestion/ReadHistoryArangoData.java
@@ -99,6 +99,8 @@ public class ReadHistoryArangoData<T extends BaseDocument> extends Thread {
         ArrayList<String> distCip = (ArrayList<String>) doc.getAttribute("DIST_CIP");
         ArrayList<Long> distCipTs = (ArrayList<Long>) doc.getAttribute("DIST_CIP_TS");
         if (distCip == null || distCip.isEmpty()){
+            doc.updateAttribute("DIST_CIP", new String[0]);
+            doc.updateAttribute("DIST_CIP_TS", new long[0]);
             return;
         }
         distCipTs.add(currentHour - RECENT_COUNT_HOUR * 3600);
diff --git a/IP-learning-graph/src/main/resources/application.properties b/IP-learning-graph/src/main/resources/application.properties
index cce0baf..fec32ff 100644
--- a/IP-learning-graph/src/main/resources/application.properties
+++ b/IP-learning-graph/src/main/resources/application.properties
@@ -1,12 +1,9 @@
 #arangoDB参数配置
-arangoDB.host=192.168.40.182
-#arangoDB.host=192.168.40.224
+arangoDB.host=192.168.44.12
 arangoDB.port=8529
 arangoDB.user=upsert
 arangoDB.password=ceiec2018
-#arangoDB.DB.name=ip-learning-test
-arangoDB.DB.name=ip-learning-test-0
-#arangoDB.DB.name=tsg_galaxy_v3
+arangoDB.DB.name=tsg_galaxy_v3
 arangoDB.batch=100000
 arangoDB.ttl=3600
 
@@ -19,8 +16,8 @@ thread.await.termination.time=10
 
 #读取clickhouse时间范围方式，0：读取过去一小时，1：指定时间范围
 time.limit.type=0
-read.clickhouse.max.time=1596684142
-read.clickhouse.min.time=1596425769
+read.clickhouse.max.time=1598323368
+read.clickhouse.min.time=1597222501
 
 update.interval=3600
 distinct.client.ip.num=10000
diff --git a/IP-learning-graph/src/main/resources/clickhouse.properties b/IP-learning-graph/src/main/resources/clickhouse.properties
index 339c5d5..4027174 100644
--- a/IP-learning-graph/src/main/resources/clickhouse.properties
+++ b/IP-learning-graph/src/main/resources/clickhouse.properties
@@ -1,10 +1,7 @@
 drivers=ru.yandex.clickhouse.ClickHouseDriver
 mdb.user=default
-#db.id=192.168.40.186:8123/tsg_galaxy_v3?socket_timeout=300000
-#mdb.password=111111
-
-db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000&compress=0
-#db.id=192.168.40.224:8123/tsg_galaxy_v3?socket_timeout=300000
+#db.id=192.168.44.10:8124/tsg_galaxy_v3?socket_timeout=300000
+db.id=192.168.44.12:8123/tsg_galaxy_v3?socket_timeout=300000&compress=0
 mdb.password=ceiec2019
 initialsize=1
 minidle=1