summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorzhanghongqing <[email protected]>2022-10-11 11:29:38 +0800
committerzhanghongqing <[email protected]>2022-10-11 11:29:38 +0800
commit16d8cda6a897ed683efd9c31a1ebfd939e07dcb7 (patch)
tree8cc2d724778e9f76f046e62f0f12872d07b85cc0
parent39cc79078ed0f6159074d79e85578145c88bab99 (diff)
增加ioc dark相关接口调度方法v2-221011
Diffstat
-rw-r--r--pom.xml7
-rw-r--r--src/main/java/cn/ac/iie/api/BrightCloud.java2
-rw-r--r--src/main/java/cn/ac/iie/controller/FileDownload.java116
-rw-r--r--src/main/java/cn/ac/iie/controller/FileLoad.java76
-rw-r--r--src/main/java/cn/ac/iie/controller/WebSketchController.java5
-rw-r--r--src/main/java/cn/ac/iie/dao/DomainCategorySchema.java83
-rw-r--r--src/main/java/cn/ac/iie/pojo/IocDarkWeb.java461
-rw-r--r--src/main/java/cn/ac/iie/service/DarkWebService.java315
-rw-r--r--src/main/java/cn/ac/iie/service/DbService.java100
-rw-r--r--src/main/java/cn/ac/iie/service/DnsServer/FileWriteInDb.java4
-rw-r--r--src/main/java/cn/ac/iie/service/FileQuery.java50
-rw-r--r--src/main/java/cn/ac/iie/service/ListQuery.java82
-rw-r--r--src/main/java/cn/ac/iie/util/ConfigUtils.java3
-rw-r--r--src/main/java/cn/ac/iie/util/MariaDBUtils.java81
-rw-r--r--src/main/java/cn/ac/iie/util/TimeUtils.java2
-rw-r--r--src/main/java/cn/ac/iie/util/TypeUtils.java1286
-rw-r--r--src/main/java/cn/ac/iie/util/ValidationUtils.java16
17 files changed, 2571 insertions, 118 deletions
diff --git a/pom.xml b/pom.xml
index f16e80d..4fa9738 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,6 +35,7 @@
<configuration>
<source>8</source>
<target>8</target>
+ <encoding>utf-8</encoding>
</configuration>
</plugin>
</plugins>
@@ -102,6 +103,12 @@
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
+
+ <dependency>
+ <groupId>cn.hutool</groupId>
+ <artifactId>hutool-all</artifactId>
+ <version>5.5.2</version>
+ </dependency>
</dependencies>
diff --git a/src/main/java/cn/ac/iie/api/BrightCloud.java b/src/main/java/cn/ac/iie/api/BrightCloud.java
index 7a12b0c..f31d396 100644
--- a/src/main/java/cn/ac/iie/api/BrightCloud.java
+++ b/src/main/java/cn/ac/iie/api/BrightCloud.java
@@ -61,7 +61,7 @@ public class BrightCloud {
param.put("urls", domains);
- // 建立实际的连接
+ //建立实际的连接
con.connect();
OutputStreamWriter writer = new OutputStreamWriter(this.con.getOutputStream(), StandardCharsets.UTF_8);
writer.write(param.toString());
diff --git a/src/main/java/cn/ac/iie/controller/FileDownload.java b/src/main/java/cn/ac/iie/controller/FileDownload.java
new file mode 100644
index 0000000..067665e
--- /dev/null
+++ b/src/main/java/cn/ac/iie/controller/FileDownload.java
@@ -0,0 +1,116 @@
+package cn.ac.iie.controller;
+
+import cn.hutool.core.date.BetweenFormater;
+import cn.hutool.core.date.DateTime;
+import cn.hutool.core.date.DateUnit;
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.io.StreamProgress;
+import cn.hutool.core.io.unit.DataSizeUtil;
+import cn.hutool.core.lang.Console;
+import cn.hutool.core.net.url.UrlBuilder;
+import cn.hutool.http.HttpUtil;
+
+import java.io.File;
+import java.util.Properties;
+import java.util.UUID;
+
+
+/**
+ * @className: FileDownload
+ * @description: TODO
+ * @author:
+ * @date: 2021/11/1 19:04
+ * @version: 1.0
+ */
+public class FileDownload {
+
+ /**
+ * @return java.lang.String
+ * @author
+ * @description 从网络URL中下载文件
+ * @date 15:33 2021/11/2
+ * @params [fileUrl, diskPath]
+ */
+ public static Long downLoadFromUrl(String fileUrl,String fileName, String diskPath) {
+ String path;
+ long size = 0;
+ if (fileUrl != null) {
+ //文件后缀
+ try {
+ String uuidName = UUID.randomUUID().toString();
+ File file = new File(diskPath);
+ if (!file.exists()) {
+ //创建文件夹
+ boolean mkdir = file.mkdir();
+ if (!mkdir) {
+ throw new RuntimeException("创建文件夹失败，路径为：" + diskPath);
+ }
+ }
+ path = diskPath + File.separator + uuidName + fileName;
+ String begin = DateUtil.now();
+ DateTime beginTime = DateUtil.parse(begin);
+ //带进度显示的文件下载
+ size = HttpUtil.downloadFile(fileUrl, FileUtil.file(path), new StreamProgress() {
+ @Override
+ public void start() {
+ Console.log("开始下载，时间为：" + begin);
+ }
+
+ @Override
+ public void progress(long progressSize) {
+ Console.log("已下载：{}", FileUtil.readableFileSize(progressSize));
+ }
+
+ @Override
+ public void finish() {
+ String end = DateUtil.now();
+ DateTime endTime = DateUtil.parse(end);
+ long between = DateUtil.between(beginTime, endTime, DateUnit.MS);
+ Console.log("下载完成，用时：" + DateUtil.formatBetween(between, BetweenFormater.Level.SECOND));
+ }
+ });
+ } catch (Exception e) {
+ Console.log("下载异常，异常信息为：" + e.getMessage());
+ }
+ }
+ return size;
+ }
+
+ final static String apikey = "9qFTSgvV1nAbIUL8GFKgwW3aolmxpI4X";
+
+ public static void main(String[] args) {
+ Properties prop = System.getProperties();
+ prop.put("proxySet", true);
+ // // HTTP代理的IP设置
+ // prop.setProperty("http.proxyHost", "localhost");
+ // // HTTP代理的端口设置
+ // prop.setProperty("http.proxyPort", "8008");
+
+ // socks代理服务器的地址与端口
+ prop.setProperty("socksProxyHost", "43.254.1.192");
+ prop.setProperty("socksProxyPort", "19040");
+
+ String[] urlList = {
+ "172.16.31.12:8000/api/tor/validity/?start=2022-01-25T14:32:01&end=2022-08-25T14:32:01",
+ "",
+ ""
+ };
+ String start = "";
+ String end = "";
+ String url = UrlBuilder.create()
+ .setScheme("http")
+ .setHost("172.16.31.12")
+ .setPort(8000)
+ .addPath("/aaa").addPath("bbb")
+ .addQuery("start", start)
+ .addQuery("end ", end)
+ .build();
+ //文件下载的URL
+// String url = "......";
+ //文件的存储位置
+ String path = "C:\\test\\h3\\";
+ long size = FileDownload.downLoadFromUrl(url,"", path);
+ System.out.println("文件大小为：" + DataSizeUtil.format(size));
+ }
+}
diff --git a/src/main/java/cn/ac/iie/controller/FileLoad.java b/src/main/java/cn/ac/iie/controller/FileLoad.java
new file mode 100644
index 0000000..cfcd94c
--- /dev/null
+++ b/src/main/java/cn/ac/iie/controller/FileLoad.java
@@ -0,0 +1,76 @@
+package cn.ac.iie.controller;
+
+import cn.hutool.core.date.DateUnit;
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.io.file.FileWriter;
+import cn.hutool.http.HttpRequest;
+
+import java.io.File;
+import java.util.Date;
+import java.util.Properties;
+
+public class FileLoad {
+ final static String apikey = "9qFTSgvV1nAbIUL8GFKgwW3aolmxpI4X";
+ static String filePath = "C:\\test\\h3\\";
+// static String server = "http://inrjoynmhpxc3hx64rpitec5z6ikx7jlo2xmouwxdey6mv6esvzlpfid.onion/api/type/?ip=104.244.79.75&apikey=9qFTSgvV1nAbIUL8GFKgwW3aolmxpI4X";
+// static String server = "http://172.16.31.12:8000/api/";
+
+ public static void socketUtil() {
+ try {
+ String proxyHost = "43.254.1.192";
+ String proxyPort = "19040";
+
+ Properties prop = System.getProperties();
+ prop.put("proxySet", true);
+
+ // socks代理服务器的地址与端口
+ prop.setProperty("socksProxyHost", proxyHost);
+ prop.setProperty("socksProxyPort", proxyPort);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static void main(String[] args) {
+ socketUtil();
+
+ String[] urlList = {
+ "tor",
+// "i2p"
+// "mtproxy"
+// "obfs4"
+// "snowflake"
+ };
+ String beginDateStr = "2022-09-20T00:00:00";
+ String stopDateStr = "2007-10-01T00:00:00";
+ Date stopDate = DateUtil.parse(stopDateStr);
+ System.err.println("开始查询");
+
+ for (String api : urlList) {
+ Date endDate = DateUtil.parse(beginDateStr);
+ long betweenDay = 800L;
+ while (betweenDay > 31) {
+ Date startDate = DateUtil.offsetDay(endDate, -60);
+ String startFormat = DateUtil.format(startDate, "yyyy-MM-dd'T'HH:mm:ss");
+ String endFormat = DateUtil.format(endDate, "yyyy-MM-dd'T'HH:mm:ss");
+ betweenDay = DateUtil.between(stopDate, startDate, DateUnit.DAY);
+ String server = "http://inrjoynmhpxc3hx64rpitec5z6ikx7jlo2xmouwxdey6mv6esvzlpfid.onion/api/";
+
+ String url = server + api + "/validity/?start=" + startFormat + "&end=" + endFormat + "&ip=104.244.79.75&apikey=9qFTSgvV1nAbIUL8GFKgwW3aolmxpI4X";
+ String body = HttpRequest.get(url)
+ .header("apikey", apikey)
+ .execute().body();
+
+ FileWriter fileWriter = new FileWriter(filePath.concat(api).concat(File.separator).concat(startFormat.substring(0, 10)).concat(".json"));
+// fileWriter.write(JSONUtil.formatJsonStr(body));
+ fileWriter.write(body);
+ System.err.println(startFormat + " " + endFormat);
+
+ endDate = startDate;
+ }
+ System.err.println("已经结束" + api);
+
+ }
+ System.err.println("全部结束");
+ }
+}
diff --git a/src/main/java/cn/ac/iie/controller/WebSketchController.java b/src/main/java/cn/ac/iie/controller/WebSketchController.java
new file mode 100644
index 0000000..6cd3399
--- /dev/null
+++ b/src/main/java/cn/ac/iie/controller/WebSketchController.java
@@ -0,0 +1,5 @@
+package cn.ac.iie.controller;
+
+public class WebSketchController {
+
+}
diff --git a/src/main/java/cn/ac/iie/dao/DomainCategorySchema.java b/src/main/java/cn/ac/iie/dao/DomainCategorySchema.java
index f19f39a..03f0a8c 100644
--- a/src/main/java/cn/ac/iie/dao/DomainCategorySchema.java
+++ b/src/main/java/cn/ac/iie/dao/DomainCategorySchema.java
@@ -35,6 +35,8 @@ public class DomainCategorySchema {
private String category_group;
private Integer category_conf;
private Boolean is_a1_cat;
+ private Integer status_code = 0;
+ private String submit_user;
// category schema
@@ -55,7 +57,7 @@ public class DomainCategorySchema {
this.query_success = query_success;
// 没有设置match_pattern，则二级域名为右匹配，其余为全匹配
- if (match_pattern == null){
+ if (match_pattern == null) {
this.match_pattern = ValidationUtils.getMatchPattern(fqdn);
} else {
this.match_pattern = match_pattern;
@@ -71,8 +73,8 @@ public class DomainCategorySchema {
}
- public static void insertRecords(List<DomainCategorySchema> categoryFiles, MariaDbBase mariaDbBase){
- for (DomainCategorySchema categoryFile: categoryFiles) {
+ public static void insertRecords(List<DomainCategorySchema> categoryFiles, MariaDbBase mariaDbBase) {
+ for (DomainCategorySchema categoryFile : categoryFiles) {
// 生成sql
String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
" (" + categoryFile.getKeys() + ") values" +
@@ -83,8 +85,8 @@ public class DomainCategorySchema {
}
}
- public void updateRecords(List<DomainCategorySchema> categoryFiles, MariaDbBase mariaDbBase){
- for (DomainCategorySchema categoryFile: categoryFiles) {
+ public void updateRecords(List<DomainCategorySchema> categoryFiles, MariaDbBase mariaDbBase) {
+ for (DomainCategorySchema categoryFile : categoryFiles) {
String resSql = "UPDATE " + dataBase + "." +
tableName + ' ' +
@@ -101,28 +103,29 @@ public class DomainCategorySchema {
String queryFqdns = fqdns.stream().map(s -> "'" + s + "'").collect(Collectors.joining(","));
String sql = "SELECT * FROM " + dataBase + "." +
tableName + ' ' +
- " WHERE fqdn in (" + queryFqdns + ") and source = '"+ source + "'";
+ " WHERE fqdn in (" + queryFqdns + ") and source = '" + source + "'";
return rs2schema(mariaDbBase.querySqlExecute(sql));
}
public static List<DomainCategorySchema> rs2schema(ResultSet rs) throws SQLException {
List<DomainCategorySchema> schemaFiles = new ArrayList<>();
- while (rs.next()){
+ while (rs.next()) {
schemaFiles.add(
- new DomainCategorySchema(
- rs.getString("fqdn"),
- rs.getString("source"),
- rs.getBoolean("query_success"),
- rs.getInt("match_pattern"),
- rs.getInt("reputation_score"),
- rs.getString("reputation_level"),
- rs.getInt("category_id"),
- rs.getString("category_name"),
- rs.getString("category_group"),
- rs.getInt("category_conf"),
- rs.getBoolean("is_a1_cat")
- ));
+ new DomainCategorySchema(
+ rs.getString("fqdn"),
+ rs.getString("source"),
+ rs.getBoolean("query_success"),
+ rs.getInt("match_pattern"),
+ rs.getInt("reputation_score"),
+ rs.getString("reputation_level"),
+ rs.getInt("category_id"),
+ rs.getString("category_name"),
+ rs.getString("category_group"),
+ rs.getInt("category_conf"),
+ rs.getBoolean("is_a1_cat")
+
+ ));
}
return schemaFiles;
}
@@ -140,6 +143,8 @@ public class DomainCategorySchema {
jsonObject.put("category_name", schema.getCategory_name());
jsonObject.put("category_conf", schema.getCategory_conf());
jsonObject.put("is_a1_cat", schema.getIs_a1_cat());
+ jsonObject.put("status_code", schema.getStatus_code());
+ jsonObject.put("submit_user", schema.getSubmit_user());
return jsonObject;
}
@@ -149,7 +154,7 @@ public class DomainCategorySchema {
", '" + source + '\'' +
", " + query_success +
", " + match_pattern +
- ", " + reputation_score +
+ ", " + reputation_score +
", '" + reputation_level + '\'' +
", " + category_id +
", '" + category_name + '\'' +
@@ -162,7 +167,7 @@ public class DomainCategorySchema {
public String getKeys() {
String resString;
- resString = "fqdn" +
+ resString = "fqdn" +
", source" +
", query_success" +
", match_pattern" +
@@ -178,15 +183,15 @@ public class DomainCategorySchema {
public String getKeyValues() {
String resString = "source='" + source + '\'' +
- ", query_success=" + query_success +
- ", match_pattern=" + match_pattern +
- ", reputation_score=" + reputation_score +
- ", reputation_level='" + reputation_level + '\'' +
- ", category_id=" + category_id +
- ", category_name='" + category_name + '\'' +
- ", category_group='" + category_group + '\'' +
- ", category_conf=" + category_conf +
- ", is_a1_cat=" + is_a1_cat;
+ ", query_success=" + query_success +
+ ", match_pattern=" + match_pattern +
+ ", reputation_score=" + reputation_score +
+ ", reputation_level='" + reputation_level + '\'' +
+ ", category_id=" + category_id +
+ ", category_name='" + category_name + '\'' +
+ ", category_group='" + category_group + '\'' +
+ ", category_conf=" + category_conf +
+ ", is_a1_cat=" + is_a1_cat;
return resString.replace("'null'", "null");
}
@@ -278,4 +283,20 @@ public class DomainCategorySchema {
public void setIs_a1_cat(Boolean is_a1_cat) {
this.is_a1_cat = is_a1_cat;
}
+
+ public Integer getStatus_code() {
+ return status_code;
+ }
+
+ public void setStatus_code(Integer status_code) {
+ this.status_code = status_code;
+ }
+
+ public String getSubmit_user() {
+ return submit_user;
+ }
+
+ public void setSubmit_user(String submit_user) {
+ this.submit_user = submit_user;
+ }
}
diff --git a/src/main/java/cn/ac/iie/pojo/IocDarkWeb.java b/src/main/java/cn/ac/iie/pojo/IocDarkWeb.java
new file mode 100644
index 0000000..42a57ac
--- /dev/null
+++ b/src/main/java/cn/ac/iie/pojo/IocDarkWeb.java
@@ -0,0 +1,461 @@
+package cn.ac.iie.pojo;
+
+
+import cn.ac.iie.config.ApplicationConfig;
+import cn.ac.iie.dao.MariaDbBase;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class IocDarkWeb {
+
+ private Long id;
+ private String iocValue;
+ private String iocType;
+ private String nodeType;
+ private String ipv6;
+ private String isp;
+ private String asn;
+ private String operatingSystem;
+ private String country;
+ private String city;
+ private String latitude;
+ private String Longtitude;
+ private String torFingerprint;
+ private Integer torDirPort;
+ private Integer torOrPort;
+ private String torFlags;
+ private String torVersion;
+ private String torNickname;
+ private String torExitPolicy;
+ private String i2PHash;
+ private String i2PMethod;
+ private String i2PVersion;
+ private Boolean i2PIsFloodfill;
+ private String i2PBandwidth;
+ private Boolean i2PIsHidden;
+ private Boolean i2PIsReachable;
+ private String i2PPublishTime;
+ private Integer mtproxyPort;
+ private String mtproxySecret;
+ private Integer obfs4Port;
+ private String obfs4Fingerprint;
+ private String obfs4Cert;
+ private Integer obfs4IatMode;
+ private Integer snowflakePort;
+ private Long firstAppearTime;
+ private Long lastAppearTime;
+ private Long createTime;
+ private Long updateTime;
+
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+
+ public String getIocValue() {
+ return iocValue;
+ }
+
+ public void setIocValue(String iocValue) {
+ this.iocValue = iocValue;
+ }
+
+
+ public String getIocType() {
+ return iocType;
+ }
+
+ public void setIocType(String iocType) {
+ this.iocType = iocType;
+ }
+
+
+ public String getIpv6() {
+ return ipv6;
+ }
+
+ public void setIpv6(String ipv6) {
+ this.ipv6 = ipv6;
+ }
+
+
+ public String getIsp() {
+ return isp;
+ }
+
+ public void setIsp(String isp) {
+ this.isp = isp;
+ }
+
+
+ public String getAsn() {
+ return asn;
+ }
+
+ public void setAsn(String asn) {
+ this.asn = asn;
+ }
+
+
+ public String getOperatingSystem() {
+ return operatingSystem;
+ }
+
+ public void setOperatingSystem(String operatingSystem) {
+ this.operatingSystem = operatingSystem;
+ }
+
+
+ public String getCountry() {
+ return country;
+ }
+
+ public void setCountry(String country) {
+ this.country = country;
+ }
+
+
+ public String getCity() {
+ return city;
+ }
+
+ public void setCity(String city) {
+ this.city = city;
+ }
+
+
+ public String getLatitude() {
+ return latitude;
+ }
+
+ public void setLatitude(String latitude) {
+ this.latitude = latitude;
+ }
+
+
+ public String getLongtitude() {
+ return Longtitude;
+ }
+
+ public void setLongtitude(String Longtitude) {
+ this.Longtitude = Longtitude;
+ }
+
+
+ public String getTorFingerprint() {
+ return torFingerprint;
+ }
+
+ public void setTorFingerprint(String torFingerprint) {
+ this.torFingerprint = torFingerprint;
+ }
+
+
+ public Integer getTorDirPort() {
+ return torDirPort;
+ }
+
+ public void setTorDirPort(Integer torDirPort) {
+ this.torDirPort = torDirPort;
+ }
+
+
+ public Integer getTorOrPort() {
+ return torOrPort;
+ }
+
+ public void setTorOrPort(Integer torOrPort) {
+ this.torOrPort = torOrPort;
+ }
+
+
+ public String getTorFlags() {
+ return torFlags;
+ }
+
+ public void setTorFlags(String torFlags) {
+ this.torFlags = torFlags;
+ }
+
+
+ public String getTorVersion() {
+ return torVersion;
+ }
+
+ public void setTorVersion(String torVersion) {
+ this.torVersion = torVersion;
+ }
+
+
+ public String getTorNickname() {
+ return torNickname;
+ }
+
+ public void setTorNickname(String torNickname) {
+ this.torNickname = torNickname;
+ }
+
+
+ public String getTorExitPolicy() {
+ return torExitPolicy;
+ }
+
+ public void setTorExitPolicy(String torExitPolicy) {
+ this.torExitPolicy = torExitPolicy;
+ }
+
+
+ public String getI2PHash() {
+ return i2PHash;
+ }
+
+ public void setI2PHash(String i2PHash) {
+ this.i2PHash = i2PHash;
+ }
+
+
+ public String getI2PMethod() {
+ return i2PMethod;
+ }
+
+ public void setI2PMethod(String i2PMethod) {
+ this.i2PMethod = i2PMethod;
+ }
+
+
+ public String getI2PVersion() {
+ return i2PVersion;
+ }
+
+ public void setI2PVersion(String i2PVersion) {
+ this.i2PVersion = i2PVersion;
+ }
+
+
+ public Boolean getI2PIsFloodfill() {
+ return i2PIsFloodfill;
+ }
+
+ public void setI2PIsFloodfill(Boolean i2PIsFloodfill) {
+ this.i2PIsFloodfill = i2PIsFloodfill;
+ }
+
+
+ public String getI2PBandwidth() {
+ return i2PBandwidth;
+ }
+
+ public void setI2PBandwidth(String i2PBandwidth) {
+ this.i2PBandwidth = i2PBandwidth;
+ }
+
+
+ public Boolean getI2PIsHidden() {
+ return i2PIsHidden;
+ }
+
+ public void setI2PIsHidden(Boolean i2PIsHidden) {
+ this.i2PIsHidden = i2PIsHidden;
+ }
+
+
+ public Boolean getI2PIsReachable() {
+ return i2PIsReachable;
+ }
+
+ public void setI2PIsReachable(Boolean i2PIsReachable) {
+ this.i2PIsReachable = i2PIsReachable;
+ }
+
+
+ public String getI2PPublishTime() {
+ return i2PPublishTime;
+ }
+
+ public void setI2PPublishTime(String i2PPublishTime) {
+ this.i2PPublishTime = i2PPublishTime;
+ }
+
+
+ public Integer getMtproxyPort() {
+ return mtproxyPort;
+ }
+
+ public void setMtproxyPort(Integer mtproxyPort) {
+ this.mtproxyPort = mtproxyPort;
+ }
+
+
+ public String getMtproxySecret() {
+ return mtproxySecret;
+ }
+
+ public void setMtproxySecret(String mtproxySecret) {
+ this.mtproxySecret = mtproxySecret;
+ }
+
+
+ public Integer getObfs4Port() {
+ return obfs4Port;
+ }
+
+ public void setObfs4Port(Integer obfs4Port) {
+ this.obfs4Port = obfs4Port;
+ }
+
+
+ public String getObfs4Fingerprint() {
+ return obfs4Fingerprint;
+ }
+
+ public void setObfs4Fingerprint(String obfs4Fingerprint) {
+ this.obfs4Fingerprint = obfs4Fingerprint;
+ }
+
+
+ public String getObfs4Cert() {
+ return obfs4Cert;
+ }
+
+ public void setObfs4Cert(String obfs4Cert) {
+ this.obfs4Cert = obfs4Cert;
+ }
+
+
+ public Integer getObfs4IatMode() {
+ return obfs4IatMode;
+ }
+
+ public void setObfs4IatMode(Integer obfs4IatMode) {
+ this.obfs4IatMode = obfs4IatMode;
+ }
+
+
+ public Integer getSnowflakePort() {
+ return snowflakePort;
+ }
+
+ public void setSnowflakePort(Integer snowflakePort) {
+ this.snowflakePort = snowflakePort;
+ }
+
+
+ public Long getFirstAppearTime() {
+ return firstAppearTime;
+ }
+
+ public void setFirstAppearTime(Long firstAppearTime) {
+ this.firstAppearTime = firstAppearTime;
+ }
+
+
+ public Long getLastAppearTime() {
+ return lastAppearTime;
+ }
+
+ public void setLastAppearTime(Long lastAppearTime) {
+ this.lastAppearTime = lastAppearTime;
+ }
+
+
+ public Long getCreateTime() {
+ return createTime;
+ }
+
+ public void setCreateTime(Long createTime) {
+ this.createTime = createTime;
+ }
+
+
+ public Long getUpdateTime() {
+ return updateTime;
+ }
+
+ public void setUpdateTime(Long updateTime) {
+ this.updateTime = updateTime;
+ }
+
+ public String getNodeType() {
+ return nodeType;
+ }
+
+ public void setNodeType(String nodeType) {
+ this.nodeType = nodeType;
+ }
+
+ public String getValues() {
+ return
+ " '" + iocValue + '\'' +
+ ", '" + iocType + '\'' +
+ ", '" + nodeType + '\'' +
+ ", '" + ipv6 + '\'' +
+ ", '" + isp + '\'' +
+ ", '" + asn + '\'' +
+ ", '" + operatingSystem + '\'' +
+ ", '" + country + '\'' +
+ ", '" + city + '\'' +
+ ", " + latitude +
+ ", " + Longtitude +
+ ", '" + torFingerprint + '\'' +
+ ", " + torDirPort +
+ ", " + torOrPort +
+ ", '" + torFlags + '\'' +
+ ", '" + torVersion + '\'' +
+ ", '" + torNickname + '\'' +
+ ", '" + torExitPolicy + '\'' +
+ ", '" + i2PHash + '\'' +
+ ", '" + i2PMethod + '\'' +
+ ", '" + i2PVersion + '\'' +
+ ", " + i2PIsFloodfill +
+ ", '" + i2PBandwidth + '\'' +
+ ", " + i2PIsHidden +
+ ", " + i2PIsReachable +
+ ", '" + i2PPublishTime + '\'' +
+ ", " + mtproxyPort +
+ ", '" + mtproxySecret + '\'' +
+ ", " + obfs4Port +
+ ", '" + obfs4Fingerprint + '\'' +
+ ", '" + obfs4Cert + '\'' +
+ ", " + obfs4IatMode +
+ ", " + snowflakePort +
+ ", " + firstAppearTime +
+ ", " + lastAppearTime +
+ ", " + createTime +
+ ", " + updateTime;
+ }
+
+ private static final String dataBase = ApplicationConfig.DATABASE;
+ private static final String tableName = "ioc_darkweb";
+ private static final String fields = "ioc_value, ioc_type,node_type, ipv6, isp, asn, operating_system, country, city, latitude, Longtitude, tor_fingerprint, " +
+ "tor_dir_port, tor_or_port, tor_flags, tor_version, tor_nickname, tor_exit_policy, i2p_hash, i2p_method, i2p_version, i2p_is_floodfill, i2p_bandwidth, " +
+ "i2p_is_hidden, i2p_is_reachable, i2p_publish_time, mtproxy_port, mtproxy_secret, obfs4_port, obfs4_fingerprint, obfs4_cert, obfs4_iat_mode, snowflake_port, " +
+ "first_appear_time, last_appear_time, create_time, update_time";
+
+ public static void insertRecords(List<IocDarkWeb> iocDarkWebs, MariaDbBase mariaDbBase) {
+ for (IocDarkWeb iocDarkWeb : iocDarkWebs) {
+ // 生成sql
+ String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
+ " (" + fields + ") values" +
+ '(' + iocDarkWeb.getValues() + ')';
+ resSql = resSql.replace("'null'", "null");
+ mariaDbBase.writeSqlExecute(resSql);
+ }
+ }
+
+ public static String getInsertSql() {
+ String[] split = fields.split(",");
+ Arrays.fill(split, "?");
+ String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
+ " (" + fields + ") values" +
+ '(' + String.join(",", split) + ')';
+
+ return resSql;
+ }
+
+}
diff --git a/src/main/java/cn/ac/iie/service/DarkWebService.java b/src/main/java/cn/ac/iie/service/DarkWebService.java
new file mode 100644
index 0000000..a4a16fa
--- /dev/null
+++ b/src/main/java/cn/ac/iie/service/DarkWebService.java
@@ -0,0 +1,315 @@
+package cn.ac.iie.service;
+
+import cn.ac.iie.pojo.IocDarkWeb;
+import cn.ac.iie.util.MariaDBUtils;
+import cn.ac.iie.util.TypeUtils;
+import cn.ac.iie.util.ValidationUtils;
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.core.io.FileUtil;
+import cn.hutool.core.io.file.FileReader;
+import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.json.JSONNull;
+import cn.hutool.json.JSONUtil;
+import cn.hutool.log.Log;
+import com.google.common.collect.Lists;
+import org.apache.commons.io.filefilter.FileFilterUtils;
+import org.apache.commons.lang3.time.StopWatch;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import static cn.ac.iie.config.ApplicationConfig.QUERY_READIN_BATCH;
+
+public class DarkWebService {
+
+ private static Log logger = Log.get();
+
+
+ public static void main(String[] args) {
+
+ //1.读取文件，转为list<Map> ，参数 文件目录，多个目录
+ //tor
+ //i2p
+ //mtproxy
+ //obfs4
+ //snowflake
+ //generalbridge
+ try {
+ String parentPath = "C:\\test\\h3\\";
+ String[] apis = {"tor", "i2p", "mtproxy", "obfs4", "snowflake"/*, "generalbridge"*/};
+ String[] t = {"07", "09", "10", "11", "12", "13", "14", "15", "17", "20"};
+
+ String apiType = apis[0];
+ for (String s : t) {
+
+ List<Map> tors = readJson(parentPath + apiType +"\\"+s+"\\");
+ System.err.println(apiType+"读取日志" + tors.size());
+
+ List<IocDarkWeb> list = new ArrayList<>();
+ //2.转成对象
+ for (Map data : tors) {
+ Map result = (Map) data.get("result");
+ List<Map> nodeList = (List<Map>) result.get("node_list");
+ for (Map node : nodeList) {
+
+ String ipv6s = "";
+ if (node.get("ipv6") instanceof List) {
+ List ipv6 = TypeUtils.castToJavaBean(node.get("ipv6"), List.class);
+ ipv6s = String.join(",", ipv6);
+ } else {
+ ipv6s = TypeUtils.castToString(Optional.ofNullable(node.get("ipv6")).orElse(""));
+ }
+
+ //location
+ Map detail = (Map) node.get("detail");
+ Map location = (Map) detail.get("location");
+ String isp = TypeUtils.castToString(location.get("isp"));
+ String asn = TypeUtils.castToString(location.get("as_number"));
+ String country = TypeUtils.castToString(location.get("country"));
+
+ String city = TypeUtils.castToString(location.get("city"));
+ String latitude = TypeUtils.castToString(location.get("latitude"));
+ String longtitude = TypeUtils.castToString(location.get("longtitude"));
+
+ Long firstAppearTime = TypeUtils.castToLong(node.get("first_appear_time")) / 1000;
+ Long lastAppearTime = TypeUtils.castToLong(node.get("last_appear_time")) / 1000;
+
+ //创建对象
+ IocDarkWeb iocDarkWeb = new IocDarkWeb();
+
+ iocDarkWeb.setNodeType(apiType);
+
+ iocDarkWeb.setIpv6(ipv6s);
+
+ //location
+ iocDarkWeb.setIsp(isp);
+ iocDarkWeb.setAsn(asn);
+ iocDarkWeb.setCountry(country);
+ iocDarkWeb.setCity(city);
+ iocDarkWeb.setLatitude(latitude);
+ iocDarkWeb.setLongtitude(longtitude);
+
+ //时间
+ iocDarkWeb.setFirstAppearTime(firstAppearTime);
+ iocDarkWeb.setLastAppearTime(lastAppearTime);
+ iocDarkWeb.setCreateTime(DateUtil.currentSeconds());
+ iocDarkWeb.setUpdateTime(DateUtil.currentSeconds());
+
+ iocDarkWeb.setIocType("ip");
+
+ switch (apiType) {
+ case "tor":
+ //tor
+ String ip = TypeUtils.castToString(node.get("ip"));
+ iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
+ torDetail(node, detail, iocDarkWeb);
+ break;
+ case "i2p":
+ ip = TypeUtils.castToString(node.get("ipv4"));
+ iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
+ i2pDetail(detail, iocDarkWeb);
+ break;
+ case "mtproxy":
+ //特殊处理，mtproxy节点中server字段支持域名和ip两种形式
+ String server = TypeUtils.castToString(node.get("server"));
+ iocDarkWeb.setIocType(ValidationUtils.isValidIp(server) ? "ip" : "domain");
+ iocDarkWeb.setIocValue(server);
+
+ mtproxyDetail(detail, iocDarkWeb);
+ break;
+ case "obfs4":
+ ip = TypeUtils.castToString(node.get("ipv4"));
+ iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
+ obfs4Detail(detail, iocDarkWeb);
+ break;
+ case "snowflake":
+ ip = TypeUtils.castToString(node.get("ipv4"));
+ iocDarkWeb.setIocValue(ObjectUtil.isEmpty(ip) ? ipv6s : ip);
+ snowflakeDetail(detail, iocDarkWeb);
+ break;
+ }
+
+ list.add(iocDarkWeb);
+ }
+ }
+
+
+ //3.写入数据库
+ List<List<IocDarkWeb>> apiPartitions = Lists.partition(list, QUERY_READIN_BATCH);
+ for (List<IocDarkWeb> partition : apiPartitions) {
+ System.err.println("开始写入" + partition.size());
+ insert(partition);
+ System.err.println("成功写入" + partition.size());
+ }
+ }
+ System.err.println(apiType+"写入完成");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ private static void snowflakeDetail(Map detail, IocDarkWeb iocDarkWeb) {
+ //snowflake
+ Integer snowflakePort = TypeUtils.castToInt(detail.get("port"));
+ //snowflake
+ iocDarkWeb.setSnowflakePort(snowflakePort);
+ }
+
+ private static void obfs4Detail(Map detail, IocDarkWeb iocDarkWeb) {
+ //obfs4
+ Integer obfs4Port = TypeUtils.castToInt(detail.get("port"));
+ String obfs4Fingerprint = TypeUtils.castToString(detail.get("fingerprint"));
+ String obfs4Cert = TypeUtils.castToString(detail.get("cert"));
+ Integer obfs4IatMode = TypeUtils.castToInt(detail.get("iatMode"));
+ //obfs4
+ iocDarkWeb.setObfs4Port(obfs4Port);
+ iocDarkWeb.setObfs4Fingerprint(obfs4Fingerprint);
+ iocDarkWeb.setObfs4Cert(obfs4Cert);
+ iocDarkWeb.setObfs4IatMode(obfs4IatMode);
+ }
+
+ private static void mtproxyDetail(Map detail, IocDarkWeb iocDarkWeb) {
+ //mtproxy
+ Integer mtproxyPort = TypeUtils.castToInt(detail.get("port"));
+ String mtproxySecret = TypeUtils.castToString(detail.get("secret"));
+ //mtproxy
+ iocDarkWeb.setMtproxyPort(mtproxyPort);
+ iocDarkWeb.setMtproxySecret(mtproxySecret);
+ }
+
+ private static void i2pDetail(Map detail, IocDarkWeb iocDarkWeb) {
+ //i2p
+ String i2pHash = TypeUtils.castToString(detail.get("hash"));
+ String i2pMethod = TypeUtils.castToString(detail.get("method"));
+ String i2pVersion = TypeUtils.castToString(detail.get("version"));
+ Boolean i2pFloodFill = TypeUtils.castToBoolean(detail.get("isFloodfill"));
+ String i2pWidth = TypeUtils.castToString(detail.get("width"));
+ Boolean i2pHidden = TypeUtils.castToBoolean(detail.get("isHidden"));
+ Boolean i2pReachable = TypeUtils.castToBoolean(detail.get("isReachable"));
+ String i2pPublish = TypeUtils.castToString(detail.get("publish"));
+ //i2p
+ iocDarkWeb.setI2PHash(i2pHash);
+ iocDarkWeb.setI2PMethod(i2pMethod);
+ iocDarkWeb.setI2PVersion(i2pVersion);
+ iocDarkWeb.setI2PIsFloodfill(i2pFloodFill);
+ iocDarkWeb.setI2PBandwidth(i2pWidth);
+ iocDarkWeb.setI2PIsHidden(i2pHidden);
+ iocDarkWeb.setI2PIsReachable(i2pReachable);
+ iocDarkWeb.setI2PPublishTime(i2pPublish);
+ }
+
+ private static void torDetail(Map node, Map detail, IocDarkWeb iocDarkWeb) {
+ String fingerprint = TypeUtils.castToString(node.get("fingerprint"));
+ Integer dirPort = node.get("dir_port") == JSONNull.NULL ? null : TypeUtils.castToInt(node.get("dir_port"));
+ Integer orPort = node.get("or_port") == JSONNull.NULL ? null : TypeUtils.castToInt(node.get("or_port"));
+ List flagsList = TypeUtils.castToJavaBean(detail.get("flags"), List.class);
+ String flags = String.join(",", flagsList);
+ String nickname = TypeUtils.castToString(detail.get("nickname"));
+ String exitPolicy = TypeUtils.castToString(detail.get("exit_policy"));
+
+ String torVirsion = TypeUtils.castToString(detail.get("tor_virsion"));
+ String operatingSystem = TypeUtils.castToString(detail.get("host_system"));
+ //tor
+ iocDarkWeb.setTorFingerprint(fingerprint);
+ iocDarkWeb.setTorDirPort(dirPort);
+ iocDarkWeb.setTorOrPort(orPort);
+ iocDarkWeb.setTorFlags(flags);
+ iocDarkWeb.setTorNickname(nickname);
+ iocDarkWeb.setTorExitPolicy(exitPolicy);
+ iocDarkWeb.setOperatingSystem(operatingSystem);
+
+ iocDarkWeb.setTorVersion(torVirsion);
+ }
+
+
+ public static List<Map> readJson(String path) {
+
+ List<File> files = FileUtil.loopFiles(Paths.get(path), 1, FileFilterUtils.suffixFileFilter("json"));
+
+ logger.info("读取文件数量 {}", files.size());
+ List<Map> jsonList = files.stream().map(f -> {
+ FileReader fr = new FileReader(f);
+ Map objMap = JSONUtil.toBean(fr.readString(), Map.class);
+
+ return objMap;
+ }).collect(Collectors.toList());
+
+ return jsonList;
+ }
+
+
+ public static void insert(List<IocDarkWeb> iocDarkWebs) throws Exception {
+
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+
+ Connection connection = MariaDBUtils.getConnection();
+ connection.setAutoCommit(false);
+
+ PreparedStatement pst = connection.prepareStatement(IocDarkWeb.getInsertSql());
+
+ for (IocDarkWeb i : iocDarkWebs) {
+
+ pst.setString(1, i.getIocValue());
+ pst.setString(2, i.getIocType());
+ pst.setString(3, i.getNodeType());
+ pst.setString(4, i.getIpv6());
+ pst.setString(5, i.getIsp());
+ pst.setString(6, i.getAsn());
+ pst.setString(7, i.getOperatingSystem());
+ pst.setString(8, i.getCountry());
+ pst.setString(9, i.getCity());
+ pst.setString(10, i.getLatitude());
+ pst.setString(11, i.getLongtitude());
+ pst.setString(12, i.getTorFingerprint());
+ pst.setObject(13, i.getTorDirPort());
+ pst.setObject(14, i.getTorOrPort());
+ pst.setString(15, i.getTorFlags());
+ pst.setString(16, i.getTorVersion());
+ pst.setString(17, i.getTorNickname());
+ pst.setString(18, i.getTorExitPolicy());
+
+ pst.setString(19, i.getI2PHash());
+ pst.setString(20, i.getI2PMethod());
+ pst.setString(21, i.getI2PVersion());
+ pst.setObject(22, i.getI2PIsFloodfill());
+ pst.setString(23, i.getI2PBandwidth());
+ pst.setObject(24, i.getI2PIsHidden());
+ pst.setObject(25, i.getI2PIsReachable());
+ pst.setString(26, i.getI2PPublishTime());
+
+ pst.setObject(27, i.getMtproxyPort());
+ pst.setString(28, i.getMtproxySecret());
+
+ pst.setObject(29, i.getObfs4Port());
+ pst.setString(30, i.getObfs4Fingerprint());
+ pst.setString(31, i.getObfs4Cert());
+ pst.setObject(32, i.getObfs4IatMode());
+
+ pst.setObject(33, i.getSnowflakePort());
+
+ pst.setLong(34, i.getFirstAppearTime());
+ pst.setLong(35, i.getLastAppearTime());
+ pst.setLong(36, i.getCreateTime());
+ pst.setLong(37, i.getUpdateTime());
+
+ pst.addBatch();
+
+ }
+ pst.executeBatch();
+ connection.commit();
+
+ stopWatch.stop();
+ logger.info("成功提交数据 {} ,花费时间 {} ms", iocDarkWebs.size(), stopWatch.getTime());
+ MariaDBUtils.close(pst, connection);
+ }
+
+
+}
diff --git a/src/main/java/cn/ac/iie/service/DbService.java b/src/main/java/cn/ac/iie/service/DbService.java
new file mode 100644
index 0000000..a6689c6
--- /dev/null
+++ b/src/main/java/cn/ac/iie/service/DbService.java
@@ -0,0 +1,100 @@
+package cn.ac.iie.service;
+
+import cn.ac.iie.config.ApplicationConfig;
+import cn.ac.iie.dao.DomainCategorySchema;
+import cn.ac.iie.dao.MariaDbBase;
+import cn.ac.iie.util.MariaDBUtils;
+import cn.hutool.core.io.IoUtil;
+import cn.hutool.log.Log;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class DbService {
+
+
+ private Log logger = Log.get();
+
+ private static final String dataBase = ApplicationConfig.DATABASE;
+ private static final String tableName = ApplicationConfig.DOMAIN_CATE_TABLENAME;
+
+ private MariaDbBase mariaDB;
+
+ public void execute(List<DomainCategorySchema> recordsFromBcApi) {
+
+ for (DomainCategorySchema categoryFile : recordsFromBcApi) {
+ // 生成sql
+ String resSql = "INSERT INTO " + dataBase + "." + tableName + ' ' +
+ " (" + categoryFile.getKeys() + ") values" +
+ '(' + categoryFile.getValues() + ')';
+ resSql = resSql.replace("'null'", "null");
+
+ mariaDB.writeSqlExecute(resSql);
+ }
+
+ }
+
+ /**
+ * @throws Exception
+ */
+ public void exec(List<Map> data, String sql) throws Exception {
+ if (Objects.isNull(data)) {
+ logger.info(" data is empty ");
+ return;
+ }
+ Connection conn = null;
+ PreparedStatement pst = null;
+ try {
+ //创建数据库连接库对象
+ conn = MariaDBUtils.getConnection();
+ long start = System.currentTimeMillis();
+ conn.setAutoCommit(false);
+ pst = conn.prepareStatement(sql);
+ int count = 0;
+ int index = 1;
+ for (int i = 1; i <= data.size(); i++) {
+ for (Object val : data.get(i - 1).values()) {
+ if (val instanceof Long) {
+ pst.setLong((index++), Long.valueOf(String.valueOf(val)));
+ } else if (val instanceof Integer) {
+ pst.setInt((index++), Integer.valueOf(String.valueOf(val)));
+ } else if (val instanceof Boolean) {
+ pst.setBoolean((index++), Boolean.valueOf(String.valueOf(val)));
+ } else {
+ pst.setString((index++), String.valueOf(val));
+ }
+ }
+ index = 1;
+ pst.addBatch();
+ //1w提交一次
+ if (i % ApplicationConfig.DB_QUERY_BATCH_SIZE == 0) {
+ int[] ints = pst.executeBatch();
+ count = count + ints.length;
+ conn.commit();
+ pst.clearBatch();
+ }
+ }
+ int[] ints = pst.executeBatch();
+ count = count + ints.length;
+ conn.commit();
+ logger.info("sql {} , count {} , take {}", sql, tableName, count, (System.currentTimeMillis() - start) + "ms");
+ } catch (Exception e) {
+ if (conn != null) {
+ try {
+ conn.rollback();
+ } catch (SQLException e1) {
+ logger.error(e1);
+ }
+ }
+ logger.error(e);
+ throw e;
+ } finally {
+ IoUtil.close(pst);
+ IoUtil.close(conn);
+ }
+ }
+}
diff --git a/src/main/java/cn/ac/iie/service/DnsServer/FileWriteInDb.java b/src/main/java/cn/ac/iie/service/DnsServer/FileWriteInDb.java
index e0f5470..04a9295 100644
--- a/src/main/java/cn/ac/iie/service/DnsServer/FileWriteInDb.java
+++ b/src/main/java/cn/ac/iie/service/DnsServer/FileWriteInDb.java
@@ -34,7 +34,7 @@ public class FileWriteInDb {
// 读取doh列表
HashMap<String, Integer>isSupportDoh = new HashMap<>();
- String listFile = "/Users/joy/work/iie/project/cyber_narrator/App/WebSketch/SourceCode/webskt-query-agent/src/main/resources/doh_support_list.csv";
+ String listFile = "C:\\code\\webskt-query-agent\\src\\main\\resources\\doh_support_list.csv";
BufferedReader brList = null;
BufferedReader bufferedReader = new BufferedReader(new FileReader(listFile));
String line;
@@ -45,7 +45,7 @@ public class FileWriteInDb {
}
}
- String srcPath = "/Users/joy/work/iie/project/cyber_narrator/App/WebSketch/SourceCode/webskt-query-agent/src/main/resources/dns_server_v1.csv";
+ String srcPath = "C:\\code\\webskt-query-agent\\src\\main\\resources\\dns_server_v1.csv";
String charset = "utf-8";
try (CSVReader csvReader = new CSVReaderBuilder(new BufferedReader(new InputStreamReader(new FileInputStream(new File(srcPath)), charset))).build()) {
Iterator<String[]> iterator = csvReader.iterator();
diff --git a/src/main/java/cn/ac/iie/service/FileQuery.java b/src/main/java/cn/ac/iie/service/FileQuery.java
index 3f94de3..9e7ebd6 100644
--- a/src/main/java/cn/ac/iie/service/FileQuery.java
+++ b/src/main/java/cn/ac/iie/service/FileQuery.java
@@ -18,6 +18,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.text.DecimalFormat;
import java.util.List;
+import java.util.Optional;
/**
* Created with IntelliJ IDEA.
@@ -34,7 +35,7 @@ public class FileQuery {
long apiResultNum = 0;
long failedQueryNum = 0;
- public String queryExecutor(String objectType, String queryType, String queryFile, Boolean isLocal) throws SQLException, IOException {
+ public String queryExecutor(String objectType, String queryType, String queryFile, String username, Boolean isLocal) throws SQLException, IOException {
Connection mariaConn = MariaDBUtils.getConnection();
Statement mariaStat = mariaConn.createStatement();
@@ -47,7 +48,7 @@ public class FileQuery {
File importFile = new File(queryFile);
String importFileName = FileUtils.getFileName(importFile);
Long fileLineNum = FileUtils.getFileLineNum(importFile);
- LOG.info("[File query]-" + importFileName + ": File Found.");
+ LOG.info("[File query]-" + importFileName + ": File Found.");
LOG.info("[File query]-" + queryType + ".");
String outputFileName = importFileName + "." + queryType + ".json";
@@ -67,11 +68,11 @@ public class FileQuery {
ListQuery listQueryUtils = new ListQuery();
int batch_index = 1;
- while ((objectList = FileUtils.getBatchLineReadIn(bufferedReader, ApplicationConfig.QUERY_READIN_BATCH))
- .size() > 0) { // 批量读取
+ // 批量读取
+ while ((objectList = FileUtils.getBatchLineReadIn(bufferedReader, ApplicationConfig.QUERY_READIN_BATCH)).size() > 0) {
// 校验
objectList = ValidationUtils.getChecked(objectList, objectType);
- JSONArray queryResults = listQueryUtils.getQueryResults(objectType, queryType, objectList, isLocal);
+ JSONArray queryResults = listQueryUtils.getQueryResults(objectType, queryType, objectList,username, isLocal);
resultJson.addAll(queryResults);
@@ -82,17 +83,17 @@ public class FileQuery {
// 写入output
// todo 写入格式优化 check
- for(Object json: queryResults){
+ for (Object json : queryResults) {
outWriter.write(JsonTool.prettyJson(json.toString()));
}
// 进度日志
- if(batch_index * ApplicationConfig.QUERY_READIN_BATCH % ApplicationConfig.QUERY_LOG_FILE_LINE_INTERVAL == 0){
+ if (batch_index * ApplicationConfig.QUERY_READIN_BATCH % ApplicationConfig.QUERY_LOG_FILE_LINE_INTERVAL == 0) {
String percent = new DecimalFormat("##.00%").format((float) queryNum / fileLineNum);
- LOG.info("[File query]-" + queryType + "-"+ importFileName + " Progress:" + percent);
+ LOG.info("[File query]-" + queryType + "-" + importFileName + " Progress:" + percent);
}
- batch_index ++;
+ batch_index++;
}
LOG.info("[File query]-" + queryType + "-" + importFileName + " "
@@ -112,27 +113,28 @@ public class FileQuery {
}
-
public static void main(String[] args) throws Exception {
-// String filePath = args[2]
-// String objectType = args[1];
-// String queryType = args[0];
-// boolean isLocal = true;
-// if (args.length >= 4){
-// isLocal = Boolean.parseBoolean(args[2]);
-// }
+ String queryType = args[0];
+ String objectType = args[1];
+ String filePath = args[2];
+ String username = args[4];
+ username = Optional.of(username).orElse("");
+ boolean isLocal = true;
+ if (args.length >= 5) {
+ isLocal = Boolean.parseBoolean(args[3]);
+ }
//
-// new FileQuery().queryExecutor(filePath, objectType, queryType, isLocal);
+ new FileQuery().queryExecutor(objectType, queryType, filePath, username, isLocal);
// JSONArray queryResults = new ListQuery()
// .getQueryResults("domain",
// "domain_category", false);
-
- new FileQuery().queryExecutor(
- "domain",
- "domain_category",
- "/Users/joy/work/iie/project/cyber_narrator/App/WebSketch/SourceCode/webskt-query-agent/files/import_file/恶意域名.txt",
- true);
+//
+// new FileQuery().queryExecutor(
+// "domain",
+// "domain_category",
+// "/Users/joy/work/iie/project/cyber_narrator/App/WebSketch/SourceCode/webskt-query-agent/files/import_file/恶意域名.txt",
+// true);
}
}
diff --git a/src/main/java/cn/ac/iie/service/ListQuery.java b/src/main/java/cn/ac/iie/service/ListQuery.java
index 4289d8d..253f794 100644
--- a/src/main/java/cn/ac/iie/service/ListQuery.java
+++ b/src/main/java/cn/ac/iie/service/ListQuery.java
@@ -45,7 +45,7 @@ public class ListQuery {
long apiResultNum;
long failedQueryNum;
- public JSONArray getQueryResults(String objectType, String queryType, List<String> queryObjects, Boolean isLocal) throws SQLException, IOException {
+ public JSONArray getQueryResults(String objectType, String queryType, List<String> queryObjects,String username, Boolean isLocal) throws SQLException, IOException {
queryNum = 0;
dbResultNum = 0;
apiResultNum = 0;
@@ -64,9 +64,9 @@ public class ListQuery {
this.queryNum = queryObjects.size();
// 执行查询
- if (queryType.equals("domain_category")){
- json = getDomainCategory(queryObjects, isLocal);
- } else if (queryType.equals("domain_whois")){
+ if (queryType.equals("domain_category")) {
+ json = getDomainCategory(queryObjects,username, isLocal);
+ } else if (queryType.equals("domain_whois")) {
json = getDomainWhois(queryObjects, isLocal);
} else {
// TODO： get dns server info
@@ -80,17 +80,16 @@ public class ListQuery {
return json;
}
- public JSONArray getDomainCategory(List<String> domains, boolean isLocal) throws SQLException, IOException {
+ public JSONArray getDomainCategory(List<String> domains,String username, boolean isLocal) throws SQLException, IOException {
JSONArray results = new JSONArray();
// 查询本地数据库
- ArrayList<String>objectsFromDB = new ArrayList<>();
+ ArrayList<String> objectsFromDB = new ArrayList<>();
List<List<String>> partitions = Lists.partition(queryObjects, MAX_DB_BATCH_SIZE);
- for (List<String> partition: partitions){ // 批量查询
- List<DomainCategorySchema> dbRecords =
- DomainCategorySchema.getDbRecord(partition, mariaDB, "brightcloud");
- for (DomainCategorySchema record : dbRecords){
+ for (List<String> partition : partitions) { // 批量查询
+ List<DomainCategorySchema> dbRecords = DomainCategorySchema.getDbRecord(partition, mariaDB, "brightcloud");
+ for (DomainCategorySchema record : dbRecords) {
objectsFromDB.add(record.getFqdn()); // 保存查询记录
JSONObject jsonObject = DomainCategorySchema.schema2json(record);
results.add(jsonObject); // 保存查询结果
@@ -102,16 +101,17 @@ public class ListQuery {
List<DomainCategorySchema> bcResults = new ArrayList<>();
List<String> objectsFromApi = new ArrayList<>(queryObjects);
objectsFromApi.removeAll(objectsFromDB);
- if(!isLocal && objectsFromApi.size()>0){
+ if (!isLocal && objectsFromApi.size() > 0) {
BrightCloud brightCloud = new BrightCloud();
List<List<String>> apiPartitions = Lists.partition(objectsFromApi, ApplicationConfig.API_BC_MAXIMUM_QUERYNUM);
- for (List<String> partition: apiPartitions) { // 批量查询
+ for (List<String> partition : apiPartitions) { // 批量查询
List<DomainCategorySchema> recordsFromBcApi = brightCloud.responseSparse(brightCloud.getQueryResults(partition));
for (DomainCategorySchema record : recordsFromBcApi) {
- if(record.getQuery_success().equals(true)){ // 查询成功的结果
+ if (record.getQuery_success().equals(true)) { //查询成功的结果
+ record.setSubmit_user(username);
bcResults.add(record);
- if(bcResults.size() > MAX_DB_BATCH_SIZE){ // 超过一定量时写入数据库
+ if (bcResults.size() > MAX_DB_BATCH_SIZE) { //超过一定量时写入数据库
DomainCategorySchema.insertRecords(bcResults, mariaDB);
bcResults.clear();
}
@@ -141,12 +141,12 @@ public class ListQuery {
JSONArray results = new JSONArray();
// 查询本地数据库
- ArrayList<String>objectsFromDB = new ArrayList<>();
+ ArrayList<String> objectsFromDB = new ArrayList<>();
List<List<String>> partitions = Lists.partition(queryObjects, MAX_DB_BATCH_SIZE);
- for (List<String> partition: partitions){ // 批量查询
+ for (List<String> partition : partitions) { // 批量查询
List<DomainWhoisSchema> dbRecords =
DomainWhoisSchema.getDbRecord(partition, mariaDB, "chinaz");
- for (DomainWhoisSchema record : dbRecords){
+ for (DomainWhoisSchema record : dbRecords) {
objectsFromDB.add(record.getFqdn()); // 保存查询记录
JSONObject jsonObject = DomainWhoisSchema.schema2json(record);
results.add(jsonObject); // 保存查询结果
@@ -156,19 +156,19 @@ public class ListQuery {
// 调用api
List<DomainWhoisSchema> chinazResults = new ArrayList<>();
- if(!isLocal){
+ if (!isLocal) {
ChinaZ chinaz = new ChinaZ();
List<String> objectsFromApi = new ArrayList<>(queryObjects);
objectsFromApi.removeAll(objectsFromDB); // 需要调用api查询的部分对象
List<List<String>> apiPartitions = Lists.partition(objectsFromApi, ApplicationConfig.API_CHINAZ_MAXIMUM_QUERYNUM);
- for (List<String> partition: apiPartitions) { // 批量查询
+ for (List<String> partition : apiPartitions) { // 批量查询
List<DomainWhoisSchema> recordsFromApi = chinaz.responseSparse(chinaz.getQueryResults(partition));
for (DomainWhoisSchema record : recordsFromApi) {
- if(record.getQuery_success().equals(true)){ // 查询成功的结果
+ if (record.getQuery_success().equals(true)) { // 查询成功的结果
chinazResults.add(record);
- if(chinazResults.size() > MAX_DB_BATCH_SIZE){ // 超过一定量时写入数据库
+ if (chinazResults.size() > MAX_DB_BATCH_SIZE) { // 超过一定量时写入数据库
DomainWhoisSchema.insertRecords(chinazResults, mariaDB);
chinazResults.clear();
}
@@ -197,22 +197,36 @@ public class ListQuery {
public static void main(String[] args) throws Exception {
-// String objectType = args[0];
-// String queryType = args[1];
-// boolean isLocal = true;
-// if (args.length >= 3){
-// isLocal = Boolean.parseBoolean(args[2]);
-// }
-// List<String> queryObjects = Arrays.asList(args[2].split(","));
-//
-// System.out.println(
-// new ListQuery().getQueryResults(objectType, queryType, queryObjects, isLocal)
-// .toString());
+ String objectType = args[0];
+ String queryType = args[1];
+ List<String> queryObjects = Arrays.asList(args[2].split(","));
+ String username = args[4];
+ boolean isLocal = true;
+ if (args.length >= 3){
+ isLocal = Boolean.parseBoolean(args[3]);
+ }
+
+ System.out.println(
+ new ListQuery().getQueryResults(objectType, queryType, queryObjects,username, isLocal)
+ .toString());
JSONArray queryResults = new ListQuery()
.getQueryResults("domain",
"domain_category",
- Arrays.asList("baidu.com", "cctv.com"), false);
-// System.out.println(queryResults.toJSONString());
+ Arrays.asList("baidu.com", "cctv.com"), username,true);
+ System.err.println(queryResults.toJSONString());
+ List<String> fields = Arrays.asList("name", "age");
+ String[] placeholder = new String[fields.size()];
+ Arrays.fill(placeholder, "?");
+
+ String format = String.format("INSERT INTO %s (%s) VALUES (%s)",
+ "table",
+ String.join(",", fields),
+ String.join(",", placeholder));
+ System.err.println(format);
+ String json = "{\"namespace\":\"log.session\",\"type\":\"record\",\"name\":\"session\",\"fields\":[{\"name\":\"common_log_id\",\"type\":\"long\"},{\"name\":\"common_service\",\"type\":\"long\"},{\"name\":\"common_recv_time\",\"type\":\"long\"},{\"name\":\"common_direction\",\"type\":\"long\"},{\"name\":\"common_l4_protocol\",\"type\":\"string\"},{\"name\":\"common_address_type\",\"type\":\"long\"},{\"name\":\"common_schema_type\",\"type\":\"string\"},{\"name\":\"common_policy_id\",\"type\":\"long\"},{\"name\":\"common_user_tags\",\"type\":\"string\"},{\"name\":\"common_action\",\"type\":\"long\"},{\"name\":\"common_sub_action\",\"type\":\"string\"},{\"name\":\"common_user_region\",\"type\":\"string\"},{\"name\":\"common_client_ip\",\"type\":\"string\"},{\"name\":\"common_client_port\",\"type\":\"long\"},{\"name\":\"common_internal_ip\",\"type\":\"string\"},{\"name\":\"common_entrance_id\",\"type\":\"long\"},{\"name\":\"common_device_id\",\"type\":\"string\"},{\"name\":\"common_egress_link_id\",\"type\":\"long\"},{\"name\":\"common_ingress_link_id\",\"type\":\"long\"},{\"name\":\"common_isp\",\"type\":\"string\"},{\"name\":\"common_device_tag\",\"type\":\"string\"},{\"name\":\"common_data_center\",\"type\":\"string\"},{\"name\":\"common_encapsulation\",\"type\":\"long\"},{\"name\":\"common_tunnels\",\"type\":\"string\"},{\"name\":\"common_sled_ip\",\"type\":\"string\"},{\"name\":\"common_device_group\",\"type\":\"string\"},{\"name\":\"common_app_behavior\",\"type\":\"string\"},{\"name\":\"common_client_location\",\"type\":\"string\"},{\"name\":\"common_client_asn\",\"type\":\"string\"},{\"name\":\"common_subscriber_id\",\"type\":\"string\"},{\"name\":\"common_imei\",\"type\":\"string\"},{\"name\":\"common_imsi\",\"type\":\"string\"},{\"name\":\"common_phone_number\",\"type\":\"string\"},{\"name\":\"common_server_ip\",\"type\":\"string\"},{\"name\":\"common_server_port\",\"type\":\"long\"},{\"name\":\"common_external_ip\",\"type\":\"string\"},{\"name\":\"common_server_location\",\"type\":\"string\"},{\"name\":\"common_server_asn\",\"type\":\"string\"},{\"name\":\"common_protocol_label\",\"type\":\"string\"},{\"name\":\"common_service_category\",\"type\":{\"type\":\"array\",\"items\":\"long\"}},{\"name\":\"common_app_label\",\"type\":\"string\"},{\"name\":\"common_app_id\",\"type\":\"string\"},{\"name\":\"common_userdefine_app_name\",\"type\":\"string\"},{\"name\":\"common_app_surrogate_id\",\"type\":\"string\"},{\"name\":\"common_l7_protocol\",\"type\":\"string\"},{\"name\":\"common_sessions\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_num\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_num\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_diff\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_diff\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_diff\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_diff\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_num\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_num\",\"type\":\"long\"},{\"name\":\"common_start_time\",\"type\":\"long\"},{\"name\":\"common_end_time\",\"type\":\"long\"},{\"name\":\"common_establish_latency_ms\",\"type\":\"long\"},{\"name\":\"common_con_duration_ms\",\"type\":\"long\"},{\"name\":\"common_stream_dir\",\"type\":\"long\"},{\"name\":\"common_address_list\",\"type\":\"string\"},{\"name\":\"common_has_dup_traffic\",\"type\":\"long\"},{\"name\":\"common_stream_error\",\"type\":\"string\"},{\"name\":\"common_stream_trace_id\",\"type\":\"long\"},{\"name\":\"common_link_info_c2s\",\"type\":\"string\"},{\"name\":\"common_link_info_s2c\",\"type\":\"string\"},{\"name\":\"common_packet_capture_file\",\"type\":\"string\"},{\"name\":\"common_c2s_ipfrag_num\",\"type\":\"long\"},{\"name\":\"common_s2c_ipfrag_num\",\"type\":\"long\"},{\"name\":\"common_c2s_tcp_lostlen\",\"type\":\"long\"},{\"name\":\"common_s2c_tcp_lostlen\",\"type\":\"long\"},{\"name\":\"common_c2s_tcp_unorder_num\",\"type\":\"long\"},{\"name\":\"common_s2c_tcp_unorder_num\",\"type\":\"long\"},{\"name\":\"common_c2s_pkt_retrans\",\"type\":\"long\"},{\"name\":\"common_s2c_pkt_retrans\",\"type\":\"long\"},{\"name\":\"common_c2s_byte_retrans\",\"type\":\"long\"},{\"name\":\"common_s2c_byte_retrans\",\"type\":\"long\"},{\"name\":\"common_tcp_client_isn\",\"type\":\"long\"},{\"name\":\"common_tcp_server_isn\",\"type\":\"long\"},{\"name\":\"common_mirrored_pkts\",\"type\":\"long\"},{\"name\":\"common_mirrored_bytes\",\"type\":\"long\"},{\"name\":\"common_first_ttl\",\"type\":\"long\"},{\"name\":\"common_processing_time\",\"type\":\"long\"},{\"name\":\"http_url\",\"type\":\"string\"},{\"name\":\"http_host\",\"type\":\"string\"},{\"name\":\"http_domain\",\"type\":\"string\"},{\"name\":\"http_request_line\",\"type\":\"string\"},{\"name\":\"http_response_line\",\"type\":\"string\"},{\"name\":\"http_request_header\",\"type\":\"string\"},{\"name\":\"http_response_header\",\"type\":\"string\"},{\"name\":\"http_request_content\",\"type\":\"string\"},{\"name\":\"http_response_content\",\"type\":\"string\"},{\"name\":\"http_request_body\",\"type\":\"string\"},{\"name\":\"http_response_body\",\"type\":\"string\"},{\"name\":\"http_request_body_key\",\"type\":\"string\"},{\"name\":\"http_response_body_key\",\"type\":\"string\"},{\"name\":\"http_proxy_flag\",\"type\":\"long\"},{\"name\":\"http_sequence\",\"type\":\"long\"},{\"name\":\"http_snapshot\",\"type\":\"string\"},{\"name\":\"http_cookie\",\"type\":\"string\"},{\"name\":\"http_referer\",\"type\":\"string\"},{\"name\":\"http_user_agent\",\"type\":\"string\"},{\"name\":\"http_request_content_length\",\"type\":\"string\"},{\"name\":\"http_request_content_type\",\"type\":\"string\"},{\"name\":\"http_response_content_length\",\"type\":\"string\"},{\"name\":\"http_response_content_type\",\"type\":\"string\"},{\"name\":\"http_content_length\",\"type\":\"string\"},{\"name\":\"http_content_type\",\"type\":\"string\"},{\"name\":\"http_set_cookie\",\"type\":\"string\"},{\"name\":\"http_version\",\"type\":\"string\"},{\"name\":\"http_response_latency_ms\",\"type\":\"long\"},{\"name\":\"http_session_duration_ms\",\"type\":\"long\"},{\"name\":\"http_action_file_size\",\"type\":\"long\"},{\"name\":\"mail_protocol_type\",\"type\":\"string\"},{\"name\":\"mail_account\",\"type\":\"string\"},{\"name\":\"mail_to_cmd\",\"type\":\"string\"},{\"name\":\"mail_from_cmd\",\"type\":\"string\"},{\"name\":\"mail_from\",\"type\":\"string\"},{\"name\":\"mail_to\",\"type\":\"string\"},{\"name\":\"mail_cc\",\"type\":\"string\"},{\"name\":\"mail_bcc\",\"type\":\"string\"},{\"name\":\"mail_subject\",\"type\":\"string\"},{\"name\":\"mail_subject_charset\",\"type\":\"string\"},{\"name\":\"mail_content\",\"type\":\"string\"},{\"name\":\"mail_content_charset\",\"type\":\"string\"},{\"name\":\"mail_attachment_name\",\"type\":\"string\"},{\"name\":\"mail_attachment_name_charset\",\"type\":\"string\"},{\"name\":\"mail_attachment_content\",\"type\":\"string\"},{\"name\":\"mail_eml_file\",\"type\":\"string\"},{\"name\":\"mail_snapshot\",\"type\":\"string\"},{\"name\":\"dns_message_id\",\"type\":\"long\"},{\"name\":\"dns_qr\",\"type\":\"long\"},{\"name\":\"dns_opcode\",\"type\":\"long\"},{\"name\":\"dns_aa\",\"type\":\"long\"},{\"name\":\"dns_tc\",\"type\":\"long\"},{\"name\":\"dns_rd\",\"type\":\"long\"},{\"name\":\"dns_ra\",\"type\":\"long\"},{\"name\":\"dns_rcode\",\"type\":\"long\"},{\"name\":\"dns_qdcount\",\"type\":\"long\"},{\"name\":\"dns_ancount\",\"type\":\"long\"},{\"name\":\"dns_nscount\",\"type\":\"long\"},{\"name\":\"dns_arcount\",\"type\":\"long\"},{\"name\":\"dns_qname\",\"type\":\"string\"},{\"name\":\"dns_qtype\",\"type\":\"long\"},{\"name\":\"dns_qclass\",\"type\":\"long\"},{\"name\":\"dns_cname\",\"type\":\"string\"},{\"name\":\"dns_sub\",\"type\":\"long\"},{\"name\":\"dns_rr\",\"type\":\"string\"},{\"name\":\"ssl_version\",\"type\":\"string\"},{\"name\":\"ssl_sni\",\"type\":\"string\"},{\"name\":\"ssl_san\",\"type\":\"string\"},{\"name\":\"ssl_cn\",\"type\":\"string\"},{\"name\":\"ssl_pinningst\",\"type\":\"long\"},{\"name\":\"ssl_intercept_state\",\"type\":\"long\"},{\"name\":\"ssl_passthrough_reason\",\"type\":\"string\"},{\"name\":\"ssl_server_side_latency\",\"type\":\"long\"},{\"name\":\"ssl_client_side_latency\",\"type\":\"long\"},{\"name\":\"ssl_server_side_version\",\"type\":\"string\"},{\"name\":\"ssl_client_side_version\",\"type\":\"string\"},{\"name\":\"ssl_cert_verify\",\"type\":\"long\"},{\"name\":\"ssl_error\",\"type\":\"string\"},{\"name\":\"ssl_con_latency_ms\",\"type\":\"long\"},{\"name\":\"ssl_ja3_fingerprint\",\"type\":\"string\"},{\"name\":\"ssl_ja3_hash\",\"type\":\"string\"},{\"name\":\"ssl_cert_issuer\",\"type\":\"string\"},{\"name\":\"ssl_cert_subject\",\"type\":\"string\"},{\"name\":\"quic_version\",\"type\":\"string\"},{\"name\":\"quic_sni\",\"type\":\"string\"},{\"name\":\"quic_user_agent\",\"type\":\"string\"},{\"name\":\"ftp_account\",\"type\":\"string\"},{\"name\":\"ftp_url\",\"type\":\"string\"},{\"name\":\"ftp_content\",\"type\":\"string\"},{\"name\":\"ftp_link_type\",\"type\":\"string\"},{\"name\":\"bgp_type\",\"type\":\"long\"},{\"name\":\"bgp_as_num\",\"type\":\"string\"},{\"name\":\"bgp_route\",\"type\":\"string\"},{\"name\":\"voip_calling_account\",\"type\":\"string\"},{\"name\":\"voip_called_account\",\"type\":\"string\"},{\"name\":\"voip_calling_number\",\"type\":\"string\"},{\"name\":\"voip_called_number\",\"type\":\"string\"},{\"name\":\"sip_call_id\",\"type\":\"string\"},{\"name\":\"sip_originator_description\",\"type\":\"string\"},{\"name\":\"sip_responder_description\",\"type\":\"string\"},{\"name\":\"sip_user_agent\",\"type\":\"string\"},{\"name\":\"sip_server\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_connect_ip\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_media_port\",\"type\":\"long\"},{\"name\":\"sip_originator_sdp_media_type\",\"type\":\"string\"},{\"name\":\"sip_originator_sdp_content\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_connect_ip\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_media_port\",\"type\":\"long\"},{\"name\":\"sip_responder_sdp_media_type\",\"type\":\"string\"},{\"name\":\"sip_responder_sdp_content\",\"type\":\"string\"},{\"name\":\"sip_duration_s\",\"type\":\"long\"},{\"name\":\"sip_bye\",\"type\":\"string\"},{\"name\":\"rtp_payload_type_c2s\",\"type\":\"long\"},{\"name\":\"rtp_payload_type_s2c\",\"type\":\"long\"},{\"name\":\"rtp_pcap_path\",\"type\":\"string\"},{\"name\":\"rtp_originator_dir\",\"type\":\"long\"},{\"name\":\"ssh_version\",\"type\":\"string\"},{\"name\":\"ssh_auth_success\",\"type\":\"string\"},{\"name\":\"ssh_client_version\",\"type\":\"string\"},{\"name\":\"ssh_server_version\",\"type\":\"string\"},{\"name\":\"ssh_cipher_alg\",\"type\":\"string\"},{\"name\":\"ssh_mac_alg\",\"type\":\"string\"},{\"name\":\"ssh_compression_alg\",\"type\":\"string\"},{\"name\":\"ssh_kex_alg\",\"type\":\"string\"},{\"name\":\"ssh_host_key_alg\",\"type\":\"string\"},{\"name\":\"ssh_host_key\",\"type\":\"string\"},{\"name\":\"ssh_hassh\",\"type\":\"string\"},{\"name\":\"stratum_cryptocurrency\",\"type\":\"string\"},{\"name\":\"stratum_mining_pools\",\"type\":\"string\"},{\"name\":\"stratum_mining_program\",\"type\":\"string\"},{\"name\":\"streaming_media_url\",\"type\":\"string\"},{\"name\":\"streaming_media_protocol\",\"type\":\"string\"},{\"name\":\"app_extra_info\",\"type\":\"string\"}]}";
+ Object parse = JSONObject.parse(json);
+
+ System.err.println(parse);
}
}
diff --git a/src/main/java/cn/ac/iie/util/ConfigUtils.java b/src/main/java/cn/ac/iie/util/ConfigUtils.java
index f169e64..6722a06 100644
--- a/src/main/java/cn/ac/iie/util/ConfigUtils.java
+++ b/src/main/java/cn/ac/iie/util/ConfigUtils.java
@@ -2,7 +2,6 @@ package cn.ac.iie.util;
import org.apache.log4j.Logger;
-import org.mariadb.jdbc.MariaDbConnection;
import java.util.Properties;
@@ -37,7 +36,7 @@ public class ConfigUtils {
static {
try {
propCommon.load(ConfigUtils.class.getClassLoader().getResourceAsStream("application.properties"));
- propCommon.load(MariaDbConnection.class.getClassLoader().getResourceAsStream("mariadb.properties"));
+// propCommon.load(MariaDbConnection.class.getClassLoader().getResourceAsStream("mariadb.properties"));
} catch (Exception e) {
propCommon = null;
diff --git a/src/main/java/cn/ac/iie/util/MariaDBUtils.java b/src/main/java/cn/ac/iie/util/MariaDBUtils.java
index e88c4b1..246c969 100644
--- a/src/main/java/cn/ac/iie/util/MariaDBUtils.java
+++ b/src/main/java/cn/ac/iie/util/MariaDBUtils.java
@@ -1,23 +1,22 @@
package cn.ac.iie.util;
+import cn.hutool.core.date.DateUtil;
import com.alibaba.druid.pool.DruidDataSourceFactory;
import me.geso.jdbcutils.JDBCUtils;
+import org.apache.commons.lang3.time.StopWatch;
import javax.sql.DataSource;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
+import java.math.BigInteger;
+import java.sql.*;
import java.util.Properties;
/**
* Druid连接池的工具类
*/
public class MariaDBUtils {
- private static DataSource ds ;
+ private static DataSource ds;
- static{
+ static {
try {
Properties pro = new Properties();
pro.load(JDBCUtils.class.getClassLoader().getResourceAsStream("druid.properties"));
@@ -37,16 +36,16 @@ public class MariaDBUtils {
/**
* 释放资源
*/
- public static void close(Statement stmt,Connection conn){
+ public static void close(Statement stmt, Connection conn) {
- close(null,stmt,conn);
+ close(null, stmt, conn);
}
- public static void close(ResultSet rs , Statement stmt, Connection conn){
+ public static void close(ResultSet rs, Statement stmt, Connection conn) {
- if(rs != null){
+ if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
@@ -55,7 +54,7 @@ public class MariaDBUtils {
}
- if(stmt != null){
+ if (stmt != null) {
try {
stmt.close();
} catch (SQLException e) {
@@ -63,7 +62,7 @@ public class MariaDBUtils {
}
}
- if(conn != null){
+ if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
@@ -75,8 +74,60 @@ public class MariaDBUtils {
/**
* 获取连接池方法
*/
- public static DataSource getDataSource(){
- return ds;
+ public static DataSource getDataSource() {
+ return ds;
+ }
+
+ public static void main(String[] args) {
+
+ try {
+ Class.forName("ru.yandex.clickhouse.ClickHouseDriver");// 驱动包
+ String url = "jdbc:clickhouse://192.168.44.85:8123/tsg_galaxy_v3";// url路径
+ String user = "tsg_insert";// 账号
+ String password = "galaxy2019";// 密码
+ Connection connection = DriverManager.getConnection(url, user, password);
+
+
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ System.err.println("开始写入ck数据 ：{}");
+
+ connection.setAutoCommit(false);
+
+ PreparedStatement preparedStatement = null;
+ long time = DateUtil.currentSeconds();
+ time = time - 1;
+// String sql = "INSERT INTO tsg_galaxy_v3.connection_record_log_local" +
+// "(cap_ip,recv_ip,src_ip,dst_ip,src_port,dst_port,addr_type,protocol,fxo_id,link_status,dir_status,total_cs_pkts,total_sc_pkts,total_cs_bytes,total_sc_bytes,log_gen_time,aa,wv,yy,user_mask,conn_start_time,app_class,app_id,http_host,http_url,http_cookie,`http_user-agent`,http_method,http_accept,http_accept_encoding,http_referer,http_rescode,tls_sni,tls_cert,phone_num,imei,imsi) VALUES " +
+// "('127.0.0.1', '192.168.44.12', 'aaa', 'String', 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 123456789, 1657526027, 123456789, 123456789, 123456789, 123456789, "+ time +", 123456789, 123456789, 'String', 'String', 'String', 'String', 'String', 'String', 'String', 'String', 123456789, 'String', 'String', 123456789, 123456789, 123456789);";
+
+// String sql ="INSERT INTO tsg_galaxy_v3.session_record\n" +
+// "(common_recv_time, common_log_id, common_stream_trace_id, common_direction, common_stream_dir, common_start_time, common_end_time, common_con_duration_ms, common_establish_latency_ms, common_processing_time, common_ingestion_time, common_entrance_id, common_device_id, common_egress_link_id, common_ingress_link_id, common_isp, common_data_center, common_sled_ip, common_device_group, common_app_behavior, common_action, common_sub_action, common_policy_id, common_user_tags, common_user_region, common_client_ip, common_internal_ip, common_client_port, common_client_location, common_client_asn, common_subscriber_id, common_imei, common_imsi, common_phone_number, common_server_ip, common_external_ip, common_server_port, common_server_location, common_server_asn, common_app_id, common_userdefine_app_name, common_app_identify_info, common_app_label, common_app_surrogate_id, common_l7_protocol, common_protocol_label, common_service_category, common_service, common_l4_protocol, common_sessions, common_c2s_pkt_num, common_s2c_pkt_num, common_c2s_pkt_diff, common_s2c_pkt_diff, common_c2s_byte_num, common_s2c_byte_num, common_c2s_byte_diff, common_s2c_byte_diff, common_c2s_ipfrag_num, common_s2c_ipfrag_num, common_c2s_tcp_lostlen, common_s2c_tcp_lostlen, common_c2s_tcp_unorder_num, common_s2c_tcp_unorder_num, common_c2s_pkt_retrans, common_s2c_pkt_retrans, common_c2s_byte_retrans, common_s2c_byte_retrans, common_first_ttl, common_tcp_client_isn, common_tcp_server_isn, common_mirrored_pkts, common_mirrored_bytes, common_address_type, common_schema_type, common_device_tag, common_encapsulation, common_tunnels, common_address_list, common_has_dup_traffic, common_stream_error, common_link_info_c2s, common_link_info_s2c, common_packet_capture_file, http_url, http_host, http_domain, http_request_line, http_response_line, http_request_header, http_response_header, http_request_content, http_response_content, http_request_body, http_response_body, http_request_body_key, http_response_body_key, http_proxy_flag, http_sequence, http_snapshot, http_cookie, http_referer, http_user_agent, http_request_content_length, http_request_content_type, http_response_content_length, http_response_content_type, http_content_length, http_content_type, http_set_cookie, http_version, http_response_latency_ms, http_session_duration_ms, http_action_file_size, mail_protocol_type, mail_account, mail_to_cmd, mail_from_cmd, mail_from, mail_to, mail_cc, mail_bcc, mail_subject, mail_subject_charset, mail_content, mail_content_charset, mail_attachment_name, mail_attachment_name_charset, mail_attachment_content, mail_eml_file, mail_snapshot, dns_message_id, dns_qr, dns_opcode, dns_aa, dns_tc, dns_rd, dns_ra, dns_rcode, dns_qdcount, dns_ancount, dns_nscount, dns_arcount, dns_qname, dns_qtype, dns_qclass, dns_cname, dns_sub, dns_rr, dns_response_latency_ms, ssl_version, ssl_sni, ssl_san, ssl_cn, ssl_pinningst, ssl_intercept_state, ssl_passthrough_reason, ssl_server_side_latency, ssl_client_side_latency, ssl_server_side_version, ssl_client_side_version, ssl_cert_verify, ssl_error, ssl_con_latency_ms, ssl_ja3_fingerprint, ssl_ja3_hash, ssl_cert_issuer, ssl_cert_subject, quic_version, quic_sni, quic_user_agent, ftp_account, ftp_url, ftp_content, ftp_link_type, bgp_type, bgp_as_num, bgp_route, voip_calling_account, voip_called_account, voip_calling_number, voip_called_number, streaming_media_url, streaming_media_protocol, app_extra_info, sip_call_id, sip_originator_description, sip_responder_description, sip_user_agent, sip_server, sip_originator_sdp_connect_ip, sip_originator_sdp_media_port, sip_originator_sdp_media_type, sip_originator_sdp_content, sip_responder_sdp_connect_ip, sip_responder_sdp_media_port, sip_responder_sdp_media_type, sip_responder_sdp_content, sip_duration_s, sip_bye, rtp_payload_type_c2s, rtp_payload_type_s2c, rtp_pcap_path, rtp_originator_dir, ssh_version, ssh_auth_success, ssh_client_version, ssh_server_version, ssh_cipher_alg, ssh_mac_alg, ssh_compression_alg, ssh_kex_alg, ssh_host_key_alg, ssh_host_key, ssh_hassh, stratum_cryptocurrency, stratum_mining_pools, stratum_mining_program, rdp_encryption_method, rdp_encryption_level, rdp_certificate_permanent, rdp_certificate_count, rdp_certificate_type, rdp_requested_color_depth, rdp_desktop_height, rdp_desktop_width, rdp_client_product_id, rdp_client_name, rdp_client_version, rdp_keyboard_layout, rdp_client_channels, rdp_security_protocol, rdp_cookie)\n" +
+// "VALUES("+time+", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', 0, 0, '', '', '', '', '', 0, '', 0, '', '', '', '', 0, '', '', '', '', '', '', '', '', 0, '', '', '', '', '', '', '', '', '', null, 0, '', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', '', 0, '', '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', 0, 0, '', 0, '', 0, '', '', '', '', 0, 0, '', 0, 0, '', '', 0, '', 0, '', '', '', '', '', '', '', '', '', '', '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, '', '', '', 0, '', '', 0, '', 0, 0, '', 0, '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', 0, 0, '', '', '', '', '', '', '', '', '', '', '');\n";
+ preparedStatement = connection.prepareStatement("INSERT INTO tsg_galaxy_v3.connection_relation_log" +
+ "(start_time, src_ip, dst_ip,bytes)" +
+ "VALUES(?, ?, ?, ?)");
+ for (int i = 0; i < 10; i++) {
+ preparedStatement.setLong(1, time);
+ preparedStatement.setString(2, "127.0.0.1");
+ preparedStatement.setString(3, "127.0.0.1");
+ BigInteger bytes = BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.valueOf(Long.MAX_VALUE));
+ System.err.println(bytes);
+ System.err.println(bytes.bitLength());
+ preparedStatement.setObject(4, bytes);
+ preparedStatement.addBatch();
+ }
+ preparedStatement.executeBatch();
+ connection.commit();
+ stopWatch.stop();
+ System.err.println("总共花费时间 {}" + stopWatch.getTime());
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ } catch (SQLException throwables) {
+ throwables.printStackTrace();
+ }
+
+
}
} \ No newline at end of file
diff --git a/src/main/java/cn/ac/iie/util/TimeUtils.java b/src/main/java/cn/ac/iie/util/TimeUtils.java
index c29ede3..4dfc8af 100644
--- a/src/main/java/cn/ac/iie/util/TimeUtils.java
+++ b/src/main/java/cn/ac/iie/util/TimeUtils.java
@@ -1,7 +1,5 @@
package cn.ac.iie.util;
-import cn.ac.iie.config.ApplicationConfig;
-
import java.util.Calendar;
import java.util.Date;
diff --git a/src/main/java/cn/ac/iie/util/TypeUtils.java b/src/main/java/cn/ac/iie/util/TypeUtils.java
new file mode 100644
index 0000000..701f99c
--- /dev/null
+++ b/src/main/java/cn/ac/iie/util/TypeUtils.java
@@ -0,0 +1,1286 @@
+//
+// Source code recreated from a .class file by IntelliJ IDEA
+// (powered by FernFlower decompiler)
+//
+
+package cn.ac.iie.util;
+
+import cn.hutool.json.JSONNull;
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONException;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.PropertyNamingStrategy;
+import com.alibaba.fastjson.annotation.JSONField;
+import com.alibaba.fastjson.annotation.JSONType;
+import com.alibaba.fastjson.parser.Feature;
+import com.alibaba.fastjson.parser.JSONScanner;
+import com.alibaba.fastjson.parser.ParserConfig;
+import com.alibaba.fastjson.serializer.SerializeBeanInfo;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.fastjson.util.FieldInfo;
+import com.alibaba.fastjson.util.IOUtils;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.*;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.security.AccessControlException;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static com.alibaba.fastjson.util.TypeUtils.cast;
+import static com.alibaba.fastjson.util.TypeUtils.getSuperMethodAnnotation;
+
+public class TypeUtils {
+ public static boolean compatibleWithJavaBean = false;
+ public static boolean compatibleWithFieldName = false;
+ private static boolean setAccessibleEnable = true;
+ private static boolean oracleTimestampMethodInited = false;
+ private static Method oracleTimestampMethod;
+ private static boolean oracleDateMethodInited = false;
+ private static Method oracleDateMethod;
+ private static boolean optionalClassInited = false;
+ private static Class<?> optionalClass;
+ private static boolean transientClassInited = false;
+ private static Class<? extends Annotation> transientClass;
+ private static ConcurrentMap<String, Class<?>> mappings;
+ private static Class<?> pathClass;
+ private static boolean pathClass_error;
+
+ public TypeUtils() {
+ }
+
+ public static String castToString(Object value) {
+ if (value == JSONNull.NULL) {
+ return "";
+ }
+ return Optional.ofNullable(String.valueOf(value)).orElse("");
+ }
+
+ public static Byte castToByte(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Number) {
+ return ((Number) value).byteValue();
+ } else if (value instanceof String) {
+ String strVal = (String) value;
+ return strVal.length() != 0 && !"null".equals(strVal) && !"NULL".equals(strVal) ? Byte.parseByte(strVal) : null;
+ } else {
+ throw new JSONException("can not cast to byte, value : " + value);
+ }
+ }
+
+ public static Character castToChar(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Character) {
+ return (Character) value;
+ } else if (value instanceof String) {
+ String strVal = (String) value;
+ if (strVal.length() == 0) {
+ return null;
+ } else if (strVal.length() != 1) {
+ throw new JSONException("can not cast to char, value : " + value);
+ } else {
+ return strVal.charAt(0);
+ }
+ } else {
+ throw new JSONException("can not cast to char, value : " + value);
+ }
+ }
+
+ public static Short castToShort(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Number) {
+ return ((Number) value).shortValue();
+ } else if (value instanceof String) {
+ String strVal = (String) value;
+ return strVal.length() != 0 && !"null".equals(strVal) && !"NULL".equals(strVal) ? Short.parseShort(strVal) : null;
+ } else {
+ throw new JSONException("can not cast to short, value : " + value);
+ }
+ }
+
+ public static BigDecimal castToBigDecimal(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof BigDecimal) {
+ return (BigDecimal) value;
+ } else if (value instanceof BigInteger) {
+ return new BigDecimal((BigInteger) value);
+ } else {
+ String strVal = value.toString();
+ return strVal.length() == 0 ? null : new BigDecimal(strVal);
+ }
+ }
+
+ public static BigInteger castToBigInteger(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof BigInteger) {
+ return (BigInteger) value;
+ } else if (!(value instanceof Float) && !(value instanceof Double)) {
+ String strVal = value.toString();
+ return strVal.length() != 0 && !"null".equals(strVal) && !"NULL".equals(strVal) ? new BigInteger(strVal) : null;
+ } else {
+ return BigInteger.valueOf(((Number) value).longValue());
+ }
+ }
+
+ public static Float castToFloat(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Number) {
+ return ((Number) value).floatValue();
+ } else if (value instanceof String) {
+ String strVal = value.toString();
+ if (strVal.length() != 0 && !"null".equals(strVal) && !"NULL".equals(strVal)) {
+ if (strVal.indexOf(44) != 0) {
+ strVal = strVal.replaceAll(",", "");
+ }
+
+ return Float.parseFloat(strVal);
+ } else {
+ return null;
+ }
+ } else {
+ throw new JSONException("can not cast to float, value : " + value);
+ }
+ }
+
+ public static Double castToDouble(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Number) {
+ return ((Number) value).doubleValue();
+ } else if (value instanceof String) {
+ String strVal = value.toString();
+ if (strVal.length() != 0 && !"null".equals(strVal) && !"NULL".equals(strVal)) {
+ if (strVal.indexOf(44) != 0) {
+ strVal = strVal.replaceAll(",", "");
+ }
+
+ return Double.parseDouble(strVal);
+ } else {
+ return null;
+ }
+ } else {
+ throw new JSONException("can not cast to double, value : " + value);
+ }
+ }
+
+ public static Date castToDate(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Date) {
+ return (Date) value;
+ } else if (value instanceof Calendar) {
+ return ((Calendar) value).getTime();
+ } else {
+ long longValue = -1L;
+ if (value instanceof Number) {
+ longValue = ((Number) value).longValue();
+ return new Date(longValue);
+ } else {
+ if (value instanceof String) {
+ String strVal = (String) value;
+ JSONScanner dateLexer = new JSONScanner(strVal);
+
+ try {
+ if (dateLexer.scanISO8601DateIfMatch(false)) {
+ Calendar calendar = dateLexer.getCalendar();
+ Date var6 = calendar.getTime();
+ return var6;
+ }
+ } finally {
+ dateLexer.close();
+ }
+
+ String format;
+ if (strVal.startsWith("/Date(") && strVal.endsWith(")/")) {
+ format = strVal.substring(6, strVal.length() - 2);
+ strVal = format;
+ }
+
+ if (strVal.indexOf(45) != -1) {
+ if (strVal.length() == JSON.DEFFAULT_DATE_FORMAT.length()) {
+ format = JSON.DEFFAULT_DATE_FORMAT;
+ } else if (strVal.length() == 10) {
+ format = "yyyy-MM-dd";
+ } else if (strVal.length() == "yyyy-MM-dd HH:mm:ss".length()) {
+ format = "yyyy-MM-dd HH:mm:ss";
+ } else {
+ format = "yyyy-MM-dd HH:mm:ss.SSS";
+ }
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(format, JSON.defaultLocale);
+ dateFormat.setTimeZone(JSON.defaultTimeZone);
+
+ try {
+ return dateFormat.parse(strVal);
+ } catch (ParseException var34) {
+ throw new JSONException("can not cast to Date, value : " + strVal);
+ }
+ }
+
+ if (strVal.length() == 0) {
+ return null;
+ }
+
+ longValue = Long.parseLong(strVal);
+ }
+
+ if (longValue < 0L) {
+ Class<?> clazz = value.getClass();
+ Object result;
+ if ("oracle.sql.TIMESTAMP".equals(clazz.getName())) {
+ if (oracleTimestampMethod == null && !oracleTimestampMethodInited) {
+ try {
+ oracleTimestampMethod = clazz.getMethod("toJdbc");
+ } catch (NoSuchMethodException var36) {
+ } finally {
+ oracleTimestampMethodInited = true;
+ }
+ }
+
+ try {
+ result = oracleTimestampMethod.invoke(value);
+ } catch (Exception var35) {
+ throw new JSONException("can not cast oracle.sql.TIMESTAMP to Date", var35);
+ }
+
+ return (Date) result;
+ } else if ("oracle.sql.DATE".equals(clazz.getName())) {
+ if (oracleDateMethod == null && !oracleDateMethodInited) {
+ try {
+ oracleDateMethod = clazz.getMethod("toJdbc");
+ } catch (NoSuchMethodException var39) {
+ } finally {
+ oracleDateMethodInited = true;
+ }
+ }
+
+ try {
+ result = oracleDateMethod.invoke(value);
+ } catch (Exception var38) {
+ throw new JSONException("can not cast oracle.sql.DATE to Date", var38);
+ }
+
+ return (Date) result;
+ } else {
+ throw new JSONException("can not cast to Date, value : " + value);
+ }
+ } else {
+ return new Date(longValue);
+ }
+ }
+ }
+ }
+
+ public static java.sql.Date castToSqlDate(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof java.sql.Date) {
+ return (java.sql.Date) value;
+ } else if (value instanceof Date) {
+ return new java.sql.Date(((Date) value).getTime());
+ } else if (value instanceof Calendar) {
+ return new java.sql.Date(((Calendar) value).getTimeInMillis());
+ } else {
+ long longValue = 0L;
+ if (value instanceof Number) {
+ longValue = ((Number) value).longValue();
+ }
+
+ if (value instanceof String) {
+ String strVal = (String) value;
+ if (strVal.length() == 0 || "null".equals(strVal) || "NULL".equals(strVal)) {
+ return null;
+ }
+
+ if (isNumber(strVal)) {
+ longValue = Long.parseLong(strVal);
+ } else {
+ JSONScanner scanner = new JSONScanner(strVal);
+ if (!scanner.scanISO8601DateIfMatch(false)) {
+ throw new JSONException("can not cast to Timestamp, value : " + strVal);
+ }
+
+ longValue = scanner.getCalendar().getTime().getTime();
+ }
+ }
+
+ if (longValue <= 0L) {
+ throw new JSONException("can not cast to Date, value : " + value);
+ } else {
+ return new java.sql.Date(longValue);
+ }
+ }
+ }
+
+ public static Timestamp castToTimestamp(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Calendar) {
+ return new Timestamp(((Calendar) value).getTimeInMillis());
+ } else if (value instanceof Timestamp) {
+ return (Timestamp) value;
+ } else if (value instanceof Date) {
+ return new Timestamp(((Date) value).getTime());
+ } else {
+ long longValue = 0L;
+ if (value instanceof Number) {
+ longValue = ((Number) value).longValue();
+ }
+
+ if (value instanceof String) {
+ String strVal = (String) value;
+ if (strVal.length() == 0 || "null".equals(strVal) || "NULL".equals(strVal)) {
+ return null;
+ }
+
+ if (isNumber(strVal)) {
+ longValue = Long.parseLong(strVal);
+ } else {
+ JSONScanner scanner = new JSONScanner(strVal);
+ if (!scanner.scanISO8601DateIfMatch(false)) {
+ throw new JSONException("can not cast to Timestamp, value : " + strVal);
+ }
+
+ longValue = scanner.getCalendar().getTime().getTime();
+ }
+ }
+
+ if (longValue <= 0L) {
+ throw new JSONException("can not cast to Timestamp, value : " + value);
+ } else {
+ return new Timestamp(longValue);
+ }
+ }
+ }
+
+ public static boolean isNumber(String str) {
+ for (int i = 0; i < str.length(); ++i) {
+ char ch = str.charAt(i);
+ if (ch != '+' && ch != '-') {
+ if (ch < '0' || ch > '9') {
+ return false;
+ }
+ } else if (i != 0) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ public static Long castToLong(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Number) {
+ return ((Number) value).longValue();
+ } else {
+ if (value instanceof String) {
+ String strVal = (String) value;
+ if (strVal.length() == 0 || "null".equals(strVal) || "NULL".equals(strVal)) {
+ return null;
+ }
+
+ if (strVal.indexOf(44) != 0) {
+ strVal = strVal.replaceAll(",", "");
+ }
+
+ try {
+ return Long.parseLong(strVal);
+ } catch (NumberFormatException var4) {
+ JSONScanner dateParser = new JSONScanner(strVal);
+ Calendar calendar = null;
+ if (dateParser.scanISO8601DateIfMatch(false)) {
+ calendar = dateParser.getCalendar();
+ }
+
+ dateParser.close();
+ if (calendar != null) {
+ return calendar.getTimeInMillis();
+ }
+ }
+ }
+
+ if (value instanceof Map) {
+ Map map = (Map) value;
+ if (map.size() == 2 && map.containsKey("andIncrement") && map.containsKey("andDecrement")) {
+ Iterator iter = map.values().iterator();
+ iter.next();
+ Object value2 = iter.next();
+ return castToLong(value2);
+ }
+ }
+
+ throw new JSONException("can not cast to long, value : " + value);
+ }
+ }
+
+ public static Integer castToInt(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Integer) {
+ return (Integer) value;
+ } else if (value instanceof Number) {
+ return ((Number) value).intValue();
+ } else if (value instanceof String) {
+ String strVal = (String) value;
+ if (strVal.length() != 0 && !"null".equals(strVal) && !"NULL".equals(strVal)) {
+ if (strVal.indexOf(44) != 0) {
+ strVal = strVal.replaceAll(",", "");
+ }
+
+ return Integer.parseInt(strVal);
+ } else {
+ return null;
+ }
+ } else if (value instanceof Boolean) {
+ return (Boolean) value ? 1 : 0;
+ } else {
+ if (value instanceof Map) {
+ Map map = (Map) value;
+ if (map.size() == 2 && map.containsKey("andIncrement") && map.containsKey("andDecrement")) {
+ Iterator iter = map.values().iterator();
+ iter.next();
+ Object value2 = iter.next();
+ return castToInt(value2);
+ }
+ }
+
+ throw new JSONException("can not cast to int, value : " + value);
+ }
+ }
+
+ public static byte[] castToBytes(Object value) {
+ if (value instanceof byte[]) {
+ return (byte[]) ((byte[]) value);
+ } else if (value instanceof String) {
+ return IOUtils.decodeBase64((String) value);
+ } else {
+ throw new JSONException("can not cast to int, value : " + value);
+ }
+ }
+
+ public static Boolean castToBoolean(Object value) {
+ if (value == null) {
+ return null;
+ } else if (value instanceof Boolean) {
+ return (Boolean) value;
+ } else if (value instanceof Number) {
+ return ((Number) value).intValue() == 1;
+ } else {
+ if (value instanceof String) {
+ String strVal = (String) value;
+ if (strVal.length() == 0 || "null".equals(strVal) || "NULL".equals(strVal)) {
+ return null;
+ }
+
+ if ("true".equalsIgnoreCase(strVal) || "1".equals(strVal)) {
+ return Boolean.TRUE;
+ }
+
+ if ("false".equalsIgnoreCase(strVal) || "0".equals(strVal)) {
+ return Boolean.FALSE;
+ }
+
+ if ("Y".equalsIgnoreCase(strVal) || "T".equals(strVal)) {
+ return Boolean.TRUE;
+ }
+
+ if ("F".equalsIgnoreCase(strVal) || "N".equals(strVal)) {
+ return Boolean.FALSE;
+ }
+ }
+
+ throw new JSONException("can not cast to boolean, value : " + value);
+ }
+ }
+
+ public static <T> T castToJavaBean(Object obj, Class<T> clazz) {
+ return cast(obj, clazz, ParserConfig.getGlobalInstance());
+ }
+
+
+ public static Locale toLocale(String strVal) {
+ String[] items = strVal.split("_");
+ if (items.length == 1) {
+ return new Locale(items[0]);
+ } else {
+ return items.length == 2 ? new Locale(items[0], items[1]) : new Locale(items[0], items[1], items[2]);
+ }
+ }
+
+ private static void addBaseClassMappings() {
+ mappings.put("byte", Byte.TYPE);
+ mappings.put("short", Short.TYPE);
+ mappings.put("int", Integer.TYPE);
+ mappings.put("long", Long.TYPE);
+ mappings.put("float", Float.TYPE);
+ mappings.put("double", Double.TYPE);
+ mappings.put("boolean", Boolean.TYPE);
+ mappings.put("char", Character.TYPE);
+ mappings.put("[byte", byte[].class);
+ mappings.put("[short", short[].class);
+ mappings.put("[int", int[].class);
+ mappings.put("[long", long[].class);
+ mappings.put("[float", float[].class);
+ mappings.put("[double", double[].class);
+ mappings.put("[boolean", boolean[].class);
+ mappings.put("[char", char[].class);
+ mappings.put("[B", byte[].class);
+ mappings.put("[S", short[].class);
+ mappings.put("[I", int[].class);
+ mappings.put("[J", long[].class);
+ mappings.put("[F", float[].class);
+ mappings.put("[D", double[].class);
+ mappings.put("[C", char[].class);
+ mappings.put("[Z", boolean[].class);
+ Class<?>[] classes = new Class[]{Object.class, Cloneable.class, loadClass("java.lang.AutoCloseable"), Exception.class, RuntimeException.class, IllegalAccessError.class, IllegalAccessException.class, IllegalArgumentException.class, IllegalMonitorStateException.class, IllegalStateException.class, IllegalThreadStateException.class, IndexOutOfBoundsException.class, InstantiationError.class, InstantiationException.class, InternalError.class, InterruptedException.class, LinkageError.class, NegativeArraySizeException.class, NoClassDefFoundError.class, NoSuchFieldError.class, NoSuchFieldException.class, NoSuchMethodError.class, NoSuchMethodException.class, NullPointerException.class, NumberFormatException.class, OutOfMemoryError.class, SecurityException.class, StackOverflowError.class, StringIndexOutOfBoundsException.class, TypeNotPresentException.class, VerifyError.class, StackTraceElement.class, HashMap.class, Hashtable.class, TreeMap.class, IdentityHashMap.class, WeakHashMap.class, LinkedHashMap.class, HashSet.class, LinkedHashSet.class, TreeSet.class, TimeUnit.class, ConcurrentHashMap.class, loadClass("java.util.concurrent.ConcurrentSkipListMap"), loadClass("java.util.concurrent.ConcurrentSkipListSet"), AtomicInteger.class, AtomicLong.class, Collections.EMPTY_MAP.getClass(), BitSet.class, Calendar.class, Date.class, Locale.class, UUID.class, Time.class, java.sql.Date.class, Timestamp.class, SimpleDateFormat.class, JSONObject.class, loadClass("java.awt.Rectangle"), loadClass("java.awt.Point"), loadClass("java.awt.Font"), loadClass("java.awt.Color"), loadClass("org.springframework.remoting.support.RemoteInvocation"), loadClass("org.springframework.remoting.support.RemoteInvocationResult")};
+ Class[] var1 = classes;
+ int var2 = classes.length;
+
+ for (int var3 = 0; var3 < var2; ++var3) {
+ Class clazz = var1[var3];
+ if (clazz != null) {
+ mappings.put(clazz.getName(), clazz);
+ }
+ }
+
+ }
+
+ public static void clearClassMapping() {
+ mappings.clear();
+ addBaseClassMappings();
+ }
+
+ public static Class<?> loadClass(String className) {
+ return loadClass(className, (ClassLoader) null);
+ }
+
+ public static boolean isPath(Class<?> clazz) {
+ if (pathClass == null && !pathClass_error) {
+ try {
+ pathClass = Class.forName("java.nio.file.Path");
+ } catch (Throwable var2) {
+ pathClass_error = true;
+ }
+ }
+
+ return pathClass != null ? pathClass.isAssignableFrom(clazz) : false;
+ }
+
+ public static Class<?> getClassFromMapping(String className) {
+ return (Class) mappings.get(className);
+ }
+
+ public static Class<?> loadClass(String className, ClassLoader classLoader) {
+ if (className != null && className.length() != 0) {
+ Class<?> clazz = (Class) mappings.get(className);
+ if (clazz != null) {
+ return clazz;
+ } else if (className.charAt(0) == '[') {
+ Class<?> componentType = loadClass(className.substring(1), classLoader);
+ return Array.newInstance(componentType, 0).getClass();
+ } else if (className.startsWith("L") && className.endsWith(";")) {
+ String newClassName = className.substring(1, className.length() - 1);
+ return loadClass(newClassName, classLoader);
+ } else {
+ try {
+ if (classLoader != null) {
+ clazz = classLoader.loadClass(className);
+ mappings.put(className, clazz);
+ return clazz;
+ }
+ } catch (Throwable var6) {
+ var6.printStackTrace();
+ }
+
+ try {
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ if (contextClassLoader != null && contextClassLoader != classLoader) {
+ clazz = contextClassLoader.loadClass(className);
+ mappings.put(className, clazz);
+ return clazz;
+ }
+ } catch (Throwable var5) {
+ }
+
+ try {
+ clazz = Class.forName(className);
+ mappings.put(className, clazz);
+ return clazz;
+ } catch (Throwable var4) {
+ return clazz;
+ }
+ }
+ } else {
+ return null;
+ }
+ }
+
+ public static SerializeBeanInfo buildBeanInfo(Class<?> beanType, Map<String, String> aliasMap, PropertyNamingStrategy propertyNamingStrategy) {
+ return buildBeanInfo(beanType, aliasMap, propertyNamingStrategy, false);
+ }
+
+ public static SerializeBeanInfo buildBeanInfo(Class<?> beanType, Map<String, String> aliasMap, PropertyNamingStrategy propertyNamingStrategy, boolean fieldBased) {
+ JSONType jsonType = (JSONType) beanType.getAnnotation(JSONType.class);
+ Map<String, Field> fieldCacheMap = new HashMap();
+ ParserConfig.parserAllFieldToCache(beanType, fieldCacheMap);
+ List<FieldInfo> fieldInfoList = fieldBased ? computeGettersWithFieldBase(beanType, aliasMap, false, propertyNamingStrategy) : computeGetters(beanType, jsonType, aliasMap, fieldCacheMap, false, propertyNamingStrategy);
+ FieldInfo[] fields = new FieldInfo[fieldInfoList.size()];
+ fieldInfoList.toArray(fields);
+ String[] orders = null;
+ String typeName = null;
+ int features;
+ if (jsonType != null) {
+ orders = jsonType.orders();
+ typeName = jsonType.typeName();
+ if (typeName.length() == 0) {
+ typeName = null;
+ }
+
+ features = SerializerFeature.of(jsonType.serialzeFeatures());
+ } else {
+ features = 0;
+ }
+
+ Object sortedFieldList;
+ if (orders != null && orders.length != 0) {
+ sortedFieldList = fieldBased ? computeGettersWithFieldBase(beanType, aliasMap, true, propertyNamingStrategy) : computeGetters(beanType, jsonType, aliasMap, fieldCacheMap, true, propertyNamingStrategy);
+ } else {
+ sortedFieldList = new ArrayList(fieldInfoList);
+ Collections.sort((List) sortedFieldList);
+ }
+
+ FieldInfo[] sortedFields = new FieldInfo[((List) sortedFieldList).size()];
+ ((List) sortedFieldList).toArray(sortedFields);
+ if (Arrays.equals(sortedFields, fields)) {
+ sortedFields = fields;
+ }
+
+ return new SerializeBeanInfo(beanType, jsonType, typeName, features, fields, sortedFields);
+ }
+
+ public static List<FieldInfo> computeGettersWithFieldBase(Class<?> clazz, Map<String, String> aliasMap, boolean sorted, PropertyNamingStrategy propertyNamingStrategy) {
+ Map<String, FieldInfo> fieldInfoMap = new LinkedHashMap();
+
+ for (Class currentClass = clazz; currentClass != null; currentClass = currentClass.getSuperclass()) {
+ Field[] fields = currentClass.getDeclaredFields();
+ computeFields(currentClass, aliasMap, propertyNamingStrategy, fieldInfoMap, fields);
+ }
+
+ return getFieldInfos(clazz, sorted, fieldInfoMap);
+ }
+
+ public static List<FieldInfo> computeGetters(Class<?> clazz, Map<String, String> aliasMap) {
+ return computeGetters(clazz, aliasMap, true);
+ }
+
+ public static List<FieldInfo> computeGetters(Class<?> clazz, Map<String, String> aliasMap, boolean sorted) {
+ JSONType jsonType = (JSONType) clazz.getAnnotation(JSONType.class);
+ Map<String, Field> fieldCacheMap = new HashMap();
+ ParserConfig.parserAllFieldToCache(clazz, fieldCacheMap);
+ return computeGetters(clazz, jsonType, aliasMap, fieldCacheMap, sorted, PropertyNamingStrategy.CamelCase);
+ }
+
+ public static List<FieldInfo> computeGetters(Class<?> clazz, JSONType jsonType, Map<String, String> aliasMap, Map<String, Field> fieldCacheMap, boolean sorted, PropertyNamingStrategy propertyNamingStrategy) {
+ Map<String, FieldInfo> fieldInfoMap = new LinkedHashMap();
+ Method[] var7 = clazz.getMethods();
+ int var8 = var7.length;
+
+ for (int var9 = 0; var9 < var8; ++var9) {
+ Method method = var7[var9];
+ String methodName = method.getName();
+ int ordinal = 0;
+ int serialzeFeatures = 0;
+ int parserFeatures = 0;
+ String label = null;
+ if (!Modifier.isStatic(method.getModifiers()) && !method.getReturnType().equals(Void.TYPE) && method.getParameterTypes().length == 0 && method.getReturnType() != ClassLoader.class && (!method.getName().equals("getMetaClass") || !method.getReturnType().getName().equals("groovy.lang.MetaClass"))) {
+ JSONField annotation = (JSONField) method.getAnnotation(JSONField.class);
+ if (annotation == null) {
+ annotation = getSuperMethodAnnotation(clazz, method);
+ }
+
+ if (annotation != null) {
+ if (!annotation.serialize()) {
+ continue;
+ }
+
+ ordinal = annotation.ordinal();
+ serialzeFeatures = SerializerFeature.of(annotation.serialzeFeatures());
+ parserFeatures = Feature.of(annotation.parseFeatures());
+ if (annotation.name().length() != 0) {
+ String propertyName = annotation.name();
+ if (aliasMap != null) {
+ propertyName = (String) aliasMap.get(propertyName);
+ if (propertyName == null) {
+ continue;
+ }
+ }
+
+ FieldInfo fieldInfo = new FieldInfo(propertyName, method, (Field) null, clazz, (Type) null, ordinal, serialzeFeatures, parserFeatures, annotation, (JSONField) null, label);
+ fieldInfoMap.put(propertyName, fieldInfo);
+ continue;
+ }
+
+ if (annotation.label().length() != 0) {
+ label = annotation.label();
+ }
+ }
+
+ char c2;
+ String propertyName;
+ if (methodName.startsWith("get")) {
+ if (methodName.length() < 4 || methodName.equals("getClass") || methodName.equals("getDeclaringClass") && clazz.isEnum()) {
+ continue;
+ }
+
+ c2 = methodName.charAt(3);
+ if (!Character.isUpperCase(c2) && c2 <= 512) {
+ if (c2 == '_') {
+ propertyName = methodName.substring(4);
+ } else if (c2 == 'f') {
+ propertyName = methodName.substring(3);
+ } else {
+ if (methodName.length() < 5 || !Character.isUpperCase(methodName.charAt(4))) {
+ continue;
+ }
+
+ propertyName = decapitalize(methodName.substring(3));
+ }
+ } else {
+ if (compatibleWithJavaBean) {
+ propertyName = decapitalize(methodName.substring(3));
+ } else {
+ propertyName = Character.toLowerCase(methodName.charAt(3)) + methodName.substring(4);
+ }
+
+ propertyName = getPropertyNameByCompatibleFieldName(fieldCacheMap, methodName, propertyName, 3);
+ }
+
+ boolean ignore = isJSONTypeIgnore(clazz, propertyName);
+ if (ignore) {
+ continue;
+ }
+
+ Field field = ParserConfig.getFieldFromCache(propertyName, fieldCacheMap);
+ if (field == null && propertyName.length() > 1) {
+ char ch = propertyName.charAt(1);
+ if (ch >= 'A' && ch <= 'Z') {
+ String javaBeanCompatiblePropertyName = decapitalize(methodName.substring(3));
+ field = ParserConfig.getFieldFromCache(javaBeanCompatiblePropertyName, fieldCacheMap);
+ }
+ }
+
+ JSONField fieldAnnotation = null;
+ if (field != null) {
+ fieldAnnotation = (JSONField) field.getAnnotation(JSONField.class);
+ if (fieldAnnotation != null) {
+ if (!fieldAnnotation.serialize()) {
+ continue;
+ }
+
+ ordinal = fieldAnnotation.ordinal();
+ serialzeFeatures = SerializerFeature.of(fieldAnnotation.serialzeFeatures());
+ parserFeatures = Feature.of(fieldAnnotation.parseFeatures());
+ if (fieldAnnotation.name().length() != 0) {
+ propertyName = fieldAnnotation.name();
+ if (aliasMap != null) {
+ propertyName = (String) aliasMap.get(propertyName);
+ if (propertyName == null) {
+ continue;
+ }
+ }
+ }
+
+ if (fieldAnnotation.label().length() != 0) {
+ label = fieldAnnotation.label();
+ }
+ }
+ }
+
+ if (aliasMap != null) {
+ propertyName = (String) aliasMap.get(propertyName);
+ if (propertyName == null) {
+ continue;
+ }
+ }
+
+ if (propertyNamingStrategy != null) {
+ propertyName = propertyNamingStrategy.translate(propertyName);
+ }
+
+ FieldInfo fieldInfo = new FieldInfo(propertyName, method, field, clazz, (Type) null, ordinal, serialzeFeatures, parserFeatures, annotation, fieldAnnotation, label);
+ fieldInfoMap.put(propertyName, fieldInfo);
+ }
+
+ if (methodName.startsWith("is") && methodName.length() >= 3 && (method.getReturnType() == Boolean.TYPE || method.getReturnType() == Boolean.class)) {
+ c2 = methodName.charAt(2);
+ if (Character.isUpperCase(c2)) {
+ if (compatibleWithJavaBean) {
+ propertyName = decapitalize(methodName.substring(2));
+ } else {
+ propertyName = Character.toLowerCase(methodName.charAt(2)) + methodName.substring(3);
+ }
+
+ propertyName = getPropertyNameByCompatibleFieldName(fieldCacheMap, methodName, propertyName, 2);
+ } else if (c2 == '_') {
+ propertyName = methodName.substring(3);
+ } else {
+ if (c2 != 'f') {
+ continue;
+ }
+
+ propertyName = methodName.substring(2);
+ }
+
+ Field field = ParserConfig.getFieldFromCache(propertyName, fieldCacheMap);
+ if (field == null) {
+ field = ParserConfig.getFieldFromCache(methodName, fieldCacheMap);
+ }
+
+ JSONField fieldAnnotation = null;
+ if (field != null) {
+ fieldAnnotation = (JSONField) field.getAnnotation(JSONField.class);
+ if (fieldAnnotation != null) {
+ if (!fieldAnnotation.serialize()) {
+ continue;
+ }
+
+ ordinal = fieldAnnotation.ordinal();
+ serialzeFeatures = SerializerFeature.of(fieldAnnotation.serialzeFeatures());
+ parserFeatures = Feature.of(fieldAnnotation.parseFeatures());
+ if (fieldAnnotation.name().length() != 0) {
+ propertyName = fieldAnnotation.name();
+ if (aliasMap != null) {
+ propertyName = (String) aliasMap.get(propertyName);
+ if (propertyName == null) {
+ continue;
+ }
+ }
+ }
+
+ if (fieldAnnotation.label().length() != 0) {
+ label = fieldAnnotation.label();
+ }
+ }
+ }
+
+ if (aliasMap != null) {
+ propertyName = (String) aliasMap.get(propertyName);
+ if (propertyName == null) {
+ continue;
+ }
+ }
+
+ if (propertyNamingStrategy != null) {
+ propertyName = propertyNamingStrategy.translate(propertyName);
+ }
+
+ if (!fieldInfoMap.containsKey(propertyName)) {
+ FieldInfo fieldInfo = new FieldInfo(propertyName, method, field, clazz, (Type) null, ordinal, serialzeFeatures, parserFeatures, annotation, fieldAnnotation, label);
+ fieldInfoMap.put(propertyName, fieldInfo);
+ }
+ }
+ }
+ }
+
+ Field[] fields = clazz.getFields();
+ computeFields(clazz, aliasMap, propertyNamingStrategy, fieldInfoMap, fields);
+ return getFieldInfos(clazz, sorted, fieldInfoMap);
+ }
+
+ private static List<FieldInfo> getFieldInfos(Class<?> clazz, boolean sorted, Map<String, FieldInfo> fieldInfoMap) {
+ List<FieldInfo> fieldInfoList = new ArrayList();
+ boolean containsAll = false;
+ String[] orders = null;
+ JSONType annotation = (JSONType) clazz.getAnnotation(JSONType.class);
+ String[] var7;
+ int var8;
+ int var9;
+ String item;
+ if (annotation != null) {
+ orders = annotation.orders();
+ if (orders != null && orders.length == fieldInfoMap.size()) {
+ containsAll = true;
+ var7 = orders;
+ var8 = orders.length;
+
+ for (var9 = 0; var9 < var8; ++var9) {
+ item = var7[var9];
+ if (!fieldInfoMap.containsKey(item)) {
+ containsAll = false;
+ break;
+ }
+ }
+ } else {
+ containsAll = false;
+ }
+ }
+
+ if (containsAll) {
+ var7 = orders;
+ var8 = orders.length;
+
+ for (var9 = 0; var9 < var8; ++var9) {
+ item = var7[var9];
+ FieldInfo fieldInfo = (FieldInfo) fieldInfoMap.get(item);
+ fieldInfoList.add(fieldInfo);
+ }
+ } else {
+ Iterator var12 = fieldInfoMap.values().iterator();
+
+ while (var12.hasNext()) {
+ FieldInfo fieldInfo = (FieldInfo) var12.next();
+ fieldInfoList.add(fieldInfo);
+ }
+
+ if (sorted) {
+ Collections.sort(fieldInfoList);
+ }
+ }
+
+ return fieldInfoList;
+ }
+
+ private static void computeFields(Class<?> clazz, Map<String, String> aliasMap, PropertyNamingStrategy propertyNamingStrategy, Map<String, FieldInfo> fieldInfoMap, Field[] fields) {
+ Field[] var5 = fields;
+ int var6 = fields.length;
+
+ for (int var7 = 0; var7 < var6; ++var7) {
+ Field field = var5[var7];
+ if (!Modifier.isStatic(field.getModifiers())) {
+ JSONField fieldAnnotation = (JSONField) field.getAnnotation(JSONField.class);
+ int ordinal = 0;
+ int serialzeFeatures = 0;
+ int parserFeatures = 0;
+ String propertyName = field.getName();
+ String label = null;
+ if (fieldAnnotation != null) {
+ if (!fieldAnnotation.serialize()) {
+ continue;
+ }
+
+ ordinal = fieldAnnotation.ordinal();
+ serialzeFeatures = SerializerFeature.of(fieldAnnotation.serialzeFeatures());
+ parserFeatures = Feature.of(fieldAnnotation.parseFeatures());
+ if (fieldAnnotation.name().length() != 0) {
+ propertyName = fieldAnnotation.name();
+ }
+
+ if (fieldAnnotation.label().length() != 0) {
+ label = fieldAnnotation.label();
+ }
+ }
+
+ if (aliasMap != null) {
+ propertyName = (String) aliasMap.get(propertyName);
+ if (propertyName == null) {
+ continue;
+ }
+ }
+
+ if (propertyNamingStrategy != null) {
+ propertyName = propertyNamingStrategy.translate(propertyName);
+ }
+
+ if (!fieldInfoMap.containsKey(propertyName)) {
+ FieldInfo fieldInfo = new FieldInfo(propertyName, (Method) null, field, clazz, (Type) null, ordinal, serialzeFeatures, parserFeatures, (JSONField) null, fieldAnnotation, label);
+ fieldInfoMap.put(propertyName, fieldInfo);
+ }
+ }
+ }
+
+ }
+
+ private static String getPropertyNameByCompatibleFieldName(Map<String, Field> fieldCacheMap, String methodName, String propertyName, int fromIdx) {
+ if (compatibleWithFieldName && !fieldCacheMap.containsKey(propertyName)) {
+ String tempPropertyName = methodName.substring(fromIdx);
+ return fieldCacheMap.containsKey(tempPropertyName) ? tempPropertyName : propertyName;
+ } else {
+ return propertyName;
+ }
+ }
+
+ private static boolean isJSONTypeIgnore(Class<?> clazz, String propertyName) {
+ JSONType jsonType = (JSONType) clazz.getAnnotation(JSONType.class);
+ if (jsonType != null) {
+ String[] fields = jsonType.includes();
+ int i;
+ if (fields.length > 0) {
+ for (i = 0; i < fields.length; ++i) {
+ if (propertyName.equals(fields[i])) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ fields = jsonType.ignores();
+
+ for (i = 0; i < fields.length; ++i) {
+ if (propertyName.equals(fields[i])) {
+ return true;
+ }
+ }
+ }
+
+ return clazz.getSuperclass() != Object.class && clazz.getSuperclass() != null && isJSONTypeIgnore(clazz.getSuperclass(), propertyName);
+ }
+
+ public static boolean isGenericParamType(Type type) {
+ if (type instanceof ParameterizedType) {
+ return true;
+ } else if (type instanceof Class) {
+ Type superType = ((Class) type).getGenericSuperclass();
+ return superType == Object.class ? false : isGenericParamType(superType);
+ } else {
+ return false;
+ }
+ }
+
+ public static Type getGenericParamType(Type type) {
+ if (type instanceof ParameterizedType) {
+ return type;
+ } else {
+ return type instanceof Class ? getGenericParamType(((Class) type).getGenericSuperclass()) : type;
+ }
+ }
+
+ public static Type unwrapOptional(Type type) {
+ if (!optionalClassInited) {
+ try {
+ optionalClass = Class.forName("java.util.Optional");
+ } catch (Exception var5) {
+ } finally {
+ optionalClassInited = true;
+ }
+ }
+
+ if (type instanceof ParameterizedType) {
+ ParameterizedType parameterizedType = (ParameterizedType) type;
+ if (parameterizedType.getRawType() == optionalClass) {
+ return parameterizedType.getActualTypeArguments()[0];
+ }
+ }
+
+ return type;
+ }
+
+ public static Class<?> getClass(Type type) {
+ if (type.getClass() == Class.class) {
+ return (Class) type;
+ } else if (type instanceof ParameterizedType) {
+ return getClass(((ParameterizedType) type).getRawType());
+ } else if (type instanceof TypeVariable) {
+ Type boundType = ((TypeVariable) type).getBounds()[0];
+ return (Class) boundType;
+ } else {
+ return Object.class;
+ }
+ }
+
+ public static Field getField(Class<?> clazz, String fieldName, Field[] declaredFields) {
+ Field[] var3 = declaredFields;
+ int var4 = declaredFields.length;
+
+ for (int var5 = 0; var5 < var4; ++var5) {
+ Field field = var3[var5];
+ if (fieldName.equals(field.getName())) {
+ return field;
+ }
+ }
+
+ Class<?> superClass = clazz.getSuperclass();
+ if (superClass != null && superClass != Object.class) {
+ return getField(superClass, fieldName, superClass.getDeclaredFields());
+ } else {
+ return null;
+ }
+ }
+
+ public static int getSerializeFeatures(Class<?> clazz) {
+ JSONType annotation = (JSONType) clazz.getAnnotation(JSONType.class);
+ return annotation == null ? 0 : SerializerFeature.of(annotation.serialzeFeatures());
+ }
+
+ public static int getParserFeatures(Class<?> clazz) {
+ JSONType annotation = (JSONType) clazz.getAnnotation(JSONType.class);
+ return annotation == null ? 0 : Feature.of(annotation.parseFeatures());
+ }
+
+ public static String decapitalize(String name) {
+ if (name != null && name.length() != 0) {
+ if (name.length() > 1 && Character.isUpperCase(name.charAt(1)) && Character.isUpperCase(name.charAt(0))) {
+ return name;
+ } else {
+ char[] chars = name.toCharArray();
+ chars[0] = Character.toLowerCase(chars[0]);
+ return new String(chars);
+ }
+ } else {
+ return name;
+ }
+ }
+
+ static void setAccessible(AccessibleObject obj) {
+ if (setAccessibleEnable) {
+ if (!obj.isAccessible()) {
+ try {
+ obj.setAccessible(true);
+ } catch (AccessControlException var2) {
+ setAccessibleEnable = false;
+ }
+
+ }
+ }
+ }
+
+ public static Type getCollectionItemType(Type fieldType) {
+ Type itemType = null;
+ Class<?> clazz = null;
+ Type actualTypeArgument;
+ if (fieldType instanceof ParameterizedType) {
+ actualTypeArgument = ((ParameterizedType) fieldType).getActualTypeArguments()[0];
+ if (actualTypeArgument instanceof WildcardType) {
+ WildcardType wildcardType = (WildcardType) actualTypeArgument;
+ Type[] upperBounds = wildcardType.getUpperBounds();
+ if (upperBounds.length == 1) {
+ actualTypeArgument = upperBounds[0];
+ }
+ }
+
+ itemType = actualTypeArgument;
+ } else if (fieldType instanceof Class && !(clazz = (Class) fieldType).getName().startsWith("java.")) {
+ actualTypeArgument = clazz.getGenericSuperclass();
+ itemType = getCollectionItemType(actualTypeArgument);
+ }
+
+ if (itemType == null) {
+ itemType = Object.class;
+ }
+
+ return (Type) itemType;
+ }
+
+ public static Class<?> getCollectionItemClass(Type fieldType) {
+ if (fieldType instanceof ParameterizedType) {
+ Type actualTypeArgument = ((ParameterizedType) fieldType).getActualTypeArguments()[0];
+ if (actualTypeArgument instanceof WildcardType) {
+ WildcardType wildcardType = (WildcardType) actualTypeArgument;
+ Type[] upperBounds = wildcardType.getUpperBounds();
+ if (upperBounds.length == 1) {
+ actualTypeArgument = upperBounds[0];
+ }
+ }
+
+ if (actualTypeArgument instanceof Class) {
+ Class<?> itemClass = (Class) actualTypeArgument;
+ if (!Modifier.isPublic(itemClass.getModifiers())) {
+ throw new JSONException("can not create ASMParser");
+ } else {
+ return itemClass;
+ }
+ } else {
+ throw new JSONException("can not create ASMParser");
+ }
+ } else {
+ return Object.class;
+ }
+ }
+
+ public static Collection createCollection(Type type) {
+ Class<?> rawClass = getRawClass(type);
+ Object list;
+ if (rawClass != AbstractCollection.class && rawClass != Collection.class) {
+ if (rawClass.isAssignableFrom(HashSet.class)) {
+ list = new HashSet();
+ } else if (rawClass.isAssignableFrom(LinkedHashSet.class)) {
+ list = new LinkedHashSet();
+ } else if (rawClass.isAssignableFrom(TreeSet.class)) {
+ list = new TreeSet();
+ } else if (rawClass.isAssignableFrom(ArrayList.class)) {
+ list = new ArrayList();
+ } else if (rawClass.isAssignableFrom(EnumSet.class)) {
+ Object itemType;
+ if (type instanceof ParameterizedType) {
+ itemType = ((ParameterizedType) type).getActualTypeArguments()[0];
+ } else {
+ itemType = Object.class;
+ }
+
+ list = EnumSet.noneOf((Class) itemType);
+ } else {
+ try {
+ list = (Collection) rawClass.newInstance();
+ } catch (Exception var4) {
+ throw new JSONException("create instance error, class " + rawClass.getName());
+ }
+ }
+ } else {
+ list = new ArrayList();
+ }
+
+ return (Collection) list;
+ }
+
+ public static Class<?> getRawClass(Type type) {
+ if (type instanceof Class) {
+ return (Class) type;
+ } else if (type instanceof ParameterizedType) {
+ return getRawClass(((ParameterizedType) type).getRawType());
+ } else {
+ throw new JSONException("TODO");
+ }
+ }
+
+ public static boolean isProxy(Class<?> clazz) {
+ Class[] var1 = clazz.getInterfaces();
+ int var2 = var1.length;
+ int var3 = 0;
+
+ while (var3 < var2) {
+ Class<?> item = var1[var3];
+ String interfaceName = item.getName();
+ if (!interfaceName.equals("net.sf.cglib.proxy.Factory") && !interfaceName.equals("org.springframework.cglib.proxy.Factory")) {
+ if (!interfaceName.equals("javassist.util.proxy.ProxyObject") && !interfaceName.equals("org.apache.ibatis.javassist.util.proxy.ProxyObject")) {
+ ++var3;
+ continue;
+ }
+
+ return true;
+ }
+
+ return true;
+ }
+
+ return false;
+ }
+
+
+ static {
+ try {
+ compatibleWithJavaBean = "true".equals(IOUtils.getStringProperty("fastjson.compatibleWithJavaBean"));
+ compatibleWithFieldName = "true".equals(IOUtils.getStringProperty("fastjson.compatibleWithFieldName"));
+ } catch (Throwable var1) {
+ }
+
+ mappings = new ConcurrentHashMap(16, 0.75F, 1);
+ addBaseClassMappings();
+ pathClass_error = false;
+ }
+}
diff --git a/src/main/java/cn/ac/iie/util/ValidationUtils.java b/src/main/java/cn/ac/iie/util/ValidationUtils.java
index 6c68222..f855bc1 100644
--- a/src/main/java/cn/ac/iie/util/ValidationUtils.java
+++ b/src/main/java/cn/ac/iie/util/ValidationUtils.java
@@ -1,16 +1,17 @@
package cn.ac.iie.util;
-import cn.ac.iie.api.BrightCloud;
import cn.ac.iie.config.ApplicationConfig;
-import org.apache.log4j.Logger;
import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
import sun.net.util.IPAddressUtil;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -21,9 +22,10 @@ public class ValidationUtils {
* 获取二级域名
**/
public static String getSecDomain(String fqdnOrUrl){
- String filePath = Objects.requireNonNull(BrightCloud.class.getClassLoader()
- .getResource(ApplicationConfig.TLD_FILE)).getFile();
- HashMap<String, HashMap<String, String>> maps = readTopDomainFile(filePath);
+// String filePath = Objects.requireNonNull(BrightCloud.class.getClassLoader()
+// .getResource(ApplicationConfig.TLD_FILE)).getFile();
+ String filePath = ApplicationConfig.TLD_FILE;
+ HashMap<String, HashMap<String, String>> maps = readTopDomainFile(filePath);
try {
String[] split = fqdnOrUrl.split("\\.");
String secDomain = null;
generated by cgit v1.2.3 (git 2.39.1) at 2025-12-30 23:37:21 +0000